diff --git a/README.md b/README.md index 9ca3e8bb9..832cec7a5 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,22 @@ using symlink trees: - Native IDE compatibility: VSCode, PyCharm, and language servers resolve jump-to-definition correctly into the Bazel sandbox +### Experimental Indexed Imports + +Large repositories can avoid creating a separate wheel-package symlink for every Python binary and test: + +```text +# .bazelrc +common --@aspect_rules_py//py:experimental_indexed_imports=true +``` + +When enabled, each private runtime virtual environment registers an import finder backed by a small ownership index. +Packages with executable `.pth` files, native-layout requirements, or ambiguous ownership keep their physical +projections. Public virtual environments remain physical so IDEs and other filesystem-based tools continue to work. + +Individual binaries and tests can opt out with `indexed_imports = False` when a tool must inspect the complete physical +package tree or pass its `sys.path` to another Python interpreter. + ### Strict Sandbox Isolation - **Isolated mode**: Python executes with `-I` flag, preventing implicit loading of user site-packages or host diff --git a/py/BUILD.bazel b/py/BUILD.bazel index d6db2885e..76a71be45 100644 --- a/py/BUILD.bazel +++ b/py/BUILD.bazel @@ -1,4 +1,11 @@ load("@bazel_lib//:bzl_library.bzl", "bzl_library") +load("@bazel_skylib//rules:common_settings.bzl", "bool_flag") + +bool_flag( + name = "experimental_indexed_imports", + build_setting_default = False, + visibility = ["//visibility:public"], +) # Users can set, e.g. --@aspect_rules_py//py:python_version=3.12 alias( diff --git a/py/private/py_venv/BUILD.bazel b/py/private/py_venv/BUILD.bazel index 8df0198f8..e9fcd907e 100644 --- a/py/private/py_venv/BUILD.bazel +++ b/py/private/py_venv/BUILD.bazel @@ -4,6 +4,8 @@ load("@bazel_skylib//rules:common_settings.bzl", "bool_flag") package(default_visibility = ["//py:__subpackages__"]) exports_files([ + "import_index.py", + "templates/_aspect_rules_py_import_index.py", "templates/console_script.tmpl.sh", "templates/link.py", "templates/venv.tmpl.sh", @@ -83,6 +85,7 @@ bzl_library( "//py/private/toolchain:types", "@bazel_lib//lib:expand_make_vars", "@bazel_lib//lib:paths", + "@bazel_skylib//rules:common_settings", ], ) diff --git a/py/private/py_venv/import_index.py b/py/private/py_venv/import_index.py new file mode 100644 index 000000000..bc1169896 --- /dev/null +++ b/py/private/py_venv/import_index.py @@ -0,0 +1,195 @@ +"""Build a runtime import index from declared Bazel artifact paths. + +Input C/H/R describe covered, known-layout, and raw roots; S/T/L/Q/A/B describe +source/symlink paths; W records virtual wheel projections and their owners. +Output I/D retain wheels, R preserves root order, and P/N map imports/namespaces +to roots. Only artifact paths are consumed; source contents are not action inputs. +""" + +import sys +from collections import defaultdict +from pathlib import Path + + +def _top_level(segment: str, directory: bool) -> str | None: + if not directory: + if segment.endswith((".py", ".pyc")): + segment = segment.rsplit(".", 1)[0] + elif segment.endswith((".so", ".pyd")): + segment = segment.split(".", 1)[0] + else: + return None + return segment if segment.isidentifier() else None + + +def _first_party_path(kind: str, short_path: str, workspace_prefix: str) -> str | None: + if short_path.startswith(("../", "/")): + return None + if kind in {"A", "B"}: + return short_path if short_path.startswith(workspace_prefix) else None + if kind in {"L", "Q"} and short_path.startswith(workspace_prefix): + return short_path + return workspace_prefix + short_path.removeprefix("./") + + +def generate( + *, records_path: str, workspace: str, escape: str, venv_escape: str +) -> tuple[str, str]: + workspace_prefix = workspace + "/" + import_roots = [] + wheel_root_coverage = {} + source_rows = [] + records = [] + wheel_imports = defaultdict(dict) + with Path(records_path).open(encoding="utf-8") as record_file: + for line in record_file: + row = line.rstrip("\r\n") + kind, separator, value = row.partition("\t") + if not separator: + raise ValueError(f"Invalid import index record: {row!r}") + if kind in {"S", "T", "L", "Q", "A", "B"}: + source = _first_party_path(kind, value, workspace_prefix) + if source is not None: + source_rows.append((source, kind in {"T", "Q", "B"})) + elif kind == "R": + import_roots.append(value) + elif kind in {"C", "H"}: + wheel_root_coverage[value] = kind == "C" + elif kind == "W": + entry, _, site_packages = value.partition("\t") + root = escape + "/" + site_packages + if "/" not in entry and entry.endswith((".dist-info", ".egg-info")): + records.append("D\t" + entry + "\t" + root) + continue + name = entry.split("/", 1)[0] + if name.endswith((".py", ".pyc")): + name = name.rsplit(".", 1)[0] + elif name.endswith((".so", ".pyd")): + name = name.split(".", 1)[0] + wheel_imports[name][root] = None + else: + raise ValueError(f"Invalid import index record: {row!r}") + + records = [ + "I\t" + name + "\t" + "\t".join(roots) for name, roots in wheel_imports.items() + ] + records + + roots = [("K", "")] + opaque_sources = {source for source, is_tree in source_rows if is_tree} + opaque_prefixes = tuple(path + "/" for path in opaque_sources) + trie = {} + for root in import_roots: + if wheel_root_coverage.get(root): + continue + segments = root.split("/") + if root.endswith("site-packages") and root not in wheel_root_coverage: + kind = "X" + elif root in opaque_sources or root.startswith(opaque_prefixes): + kind = "K" + elif root.startswith(workspace_prefix) and "site-packages" not in segments: + kind = "F" + node = trie + for segment in segments: + node = node.setdefault(segment, {}) + # Deduplicated import roots give each trie terminal one owner. + node[None] = len(roots) + else: + kind = "K" + roots.append((kind, root)) + + claims = defaultdict(set) + namespace_rows = defaultdict(list) + for source, is_tree in source_rows: + segments = source.split("/") + node = trie + for offset, segment in enumerate(segments): + position = node.get(None) + if position is not None: + name = _top_level(segment, is_tree or offset + 1 < len(segments)) + if name is not None: + claims[name].add(position) + namespace_rows[name].append((source, is_tree, position, offset)) + node = node.get(segment) + if node is None: + break + + claimed_positions = set().union(*claims.values()) + + del source_rows + namespace_claims = defaultdict(set) + regular_packages = set() + opaque_namespaces = set() + for top_level, rows in namespace_rows.items(): + if len(claims[top_level]) < 2: + continue + for source, is_tree, position, offset in rows: + segments = source.split("/") + package = top_level + for child_offset in range(offset + 1, len(segments)): + directory = is_tree or child_offset + 1 < len(segments) + child = _top_level(segments[child_offset], directory) + if child is None: + break + if child == "__init__" and not directory: + regular_packages.add(package) + break + package += "." + child + namespace_claims[package].add(position) + if is_tree: + opaque_namespaces.add(package) + + pth = [ + "import os, sys; _venv_bin = os.path.dirname(sys.executable); " + '_path = os.environ.get("PATH", ""); ' + 'os.environ["PATH"] = _path if _venv_bin in _path.split(os.pathsep) ' + "else _venv_bin + os.pathsep + _path; del _venv_bin, _path", + "import _aspect_rules_py_import_index", + ] + for position, (kind, root) in enumerate(roots): + # Unclaimed roots retain their physical sys.path entry. + if kind == "F" and position not in claimed_positions: + kind = "K" + relative_root = escape if not root else escape + "/" + root + index_kind = "K" if kind == "X" else kind + records.append("R\t" + index_kind + "\t" + relative_root) + if kind == "X": + # site supplies known_paths while executing .pth lines; reuse it to avoid rescans. + pth.append( + "import os, sys, site; " + "site.addsitedir(os.path.normpath(os.path.join(" + f'sys.prefix, "{venv_escape}", "{root}")), vars().get("known_paths"))' + ) + elif kind == "K": + pth.append(relative_root) + + for name, positions in sorted(claims.items()): + records.append("P\t" + name + "\t" + "\t".join(map(str, sorted(positions)))) + + opaque_namespace_prefixes = tuple(name + "." for name in opaque_namespaces) + for name, positions in sorted(namespace_claims.items()): + parent = name.rpartition(".")[0] + parent_positions = namespace_claims.get(parent) or claims.get(parent) + if ( + len(parent_positions) < 2 + or parent in regular_packages + or parent in opaque_namespaces + or parent.startswith(opaque_namespace_prefixes) + ): + continue + records.append("N\t" + name + "\t" + "\t".join(map(str, sorted(positions)))) + + return "\n".join(records) + "\n", "\n".join(pth) + "\n" + + +if __name__ == "__main__": + workspace, escape, venv_escape, index_file, pth_file = sys.argv[1:6] + helper_source, helper_output, records_path = sys.argv[6:] + index, pth = generate( + records_path=records_path, + workspace=workspace, + escape=escape, + venv_escape=venv_escape, + ) + Path(index_file).write_text(index, encoding="utf-8") + Path(pth_file).write_text(pth, encoding="utf-8") + Path(helper_output).write_bytes(Path(helper_source).read_bytes()) diff --git a/py/private/py_venv/py_venv.bzl b/py/private/py_venv/py_venv.bzl index 0ce2bebdf..3dd19476c 100644 --- a/py/private/py_venv/py_venv.bzl +++ b/py/private/py_venv/py_venv.bzl @@ -27,6 +27,7 @@ layout details. load("@bazel_lib//lib:expand_make_vars.bzl", "expand_locations", "expand_variables") load("@bazel_lib//lib:paths.bzl", "BASH_RLOCATION_FUNCTION", "to_rlocation_path") +load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") load("//py/private:py_library.bzl", _py_library = "py_library_utils") load("//py/private:py_semantics.bzl", _py_semantics = "semantics") load("//py/private:transitions.bzl", "python_transition") @@ -54,6 +55,21 @@ def _assemble_venv_target(ctx): ctx, extra_imports_depsets = virtual_resolution.imports, ) + srcs_depset = _py_library.make_srcs_depset( + ctx, + extra_depsets = virtual_resolution.srcs, + ) + + indexed_runfiles = None + if ( + ctx.attr.indexed_imports and + hasattr(ctx.attr, "_indexed_imports") and + ctx.attr._indexed_imports[BuildSettingInfo].value + ): + indexed_runfiles = _py_library.make_merged_runfiles( + ctx, + extra_depsets = [srcs_depset] + virtual_resolution.runfiles, + ) default_env = { "BAZEL_TARGET": str(ctx.label).lstrip("@"), @@ -79,12 +95,9 @@ def _assemble_venv_target(ctx): site_merge_script_py = ctx.file._site_merge_script, console_script_tmpl = ctx.file._console_script_tmpl, venv_name = ".{}".format(venv_stem), + indexed_runfiles = indexed_runfiles, ) - srcs_depset = _py_library.make_srcs_depset( - ctx, - extra_depsets = virtual_resolution.srcs, - ) runfiles = _py_library.make_merged_runfiles( ctx, extra_depsets = [py_toolchain.files] + virtual_resolution.runfiles, @@ -204,6 +217,10 @@ does not reinsert a wheel. default = False, doc = """`pyvenv.cfg` feature flag for the `include-system-site-packages` key.""", ), + "indexed_imports": attr.bool( + default = True, + doc = "Whether private virtual environments may use indexed imports.", + ), # Required for py_version attribute "_allowlist_function_transition": attr.label( default = "@bazel_tools//tools/allowlists/function_transition_allowlist", @@ -299,7 +316,19 @@ def _py_venv_lib_rule_impl(ctx): # `env`, `env_inherit`) aren't part of its rule contract. _py_venv_lib = rule( implementation = _py_venv_lib_rule_impl, - attrs = _lib_attrs, + attrs = _lib_attrs | { + "_indexed_imports": attr.label( + default = "//py:experimental_indexed_imports", + ), + "_import_index_shim": attr.label( + allow_single_file = True, + default = "//py/private/py_venv:templates/_aspect_rules_py_import_index.py", + ), + "_import_index_generator": attr.label( + allow_single_file = True, + default = "//py/private/py_venv:import_index.py", + ), + }, toolchains = _venv_toolchains, cfg = python_transition, ) @@ -325,6 +354,7 @@ _VENV_ONLY_ATTRS = [ "virtual_deps", "package_collisions", "include_system_site_packages", + "indexed_imports", "python_version", "dep_group", ] diff --git a/py/private/py_venv/templates/_aspect_rules_py_import_index.py b/py/private/py_venv/templates/_aspect_rules_py_import_index.py new file mode 100644 index 000000000..6be79dc01 --- /dev/null +++ b/py/private/py_venv/templates/_aspect_rules_py_import_index.py @@ -0,0 +1,432 @@ +"""Load wheel and first-party imports from a compact Bazel virtualenv. + +The generated TSV records wheel modules (I), wheel metadata (D), original +retained/virtual root order (R), virtual top-level modules (P), and virtual +namespace children (N). Virtual roots never enter sys.path. +""" + +from __future__ import annotations + +import os +import sys + +TYPE_CHECKING = False +if TYPE_CHECKING: + from collections.abc import Callable, Iterator, Sequence + from importlib.machinery import ModuleSpec + from importlib.metadata import Distribution, DistributionFinder + from pathlib import Path + from types import ModuleType + from typing import Any + + +def install_import_index() -> None: + """Resolve indexed imports without projecting packages or expanding sys.path.""" + site_packages = os.path.dirname(__file__) + index_path = os.path.join(site_packages, ".aspect_rules_py_import_index") + + for finder in sys.meta_path: + if getattr(finder, "_aspect_rules_py_import_index", None) == index_path: + return + + path_finder = next( + finder + for finder in sys.meta_path + if getattr(finder, "__name__", None) == "PathFinder" + ) + initial_path_hooks = tuple(sys.path_hooks) + indexed_roots = {} + indexed_distributions = {} + indexed_metadata_text = {} + ordered_import_roots = [] + retained_import_roots = {} + indexed_first_party = {} + indexed_namespace_portions = {} + + def normalize_distribution_name(name: str) -> str: + normalized = name.lower().replace("-", "_").replace(".", "_") + while "__" in normalized: + normalized = normalized.replace("__", "_") + return normalized + + with open(index_path, encoding="utf-8") as index_file: + for line in index_file: + kind, name, roots = line.rstrip("\r\n").split("\t", 2) + if kind == "I": + indexed_roots[name] = roots + elif kind == "D" and name.endswith((".dist-info", ".egg-info")): + stem = name.rsplit(".", 1)[0] + normalized_name = normalize_distribution_name(stem.partition("-")[0]) + record = name + "\t" + roots + indexed_distributions.setdefault(normalized_name, []).append(record) + elif kind == "R" and name in {"K", "F"} and "\t" not in roots: + if name == "K": + retained_import_roots[len(ordered_import_roots)] = os.path.normpath( + os.path.join(site_packages, roots) + ) + ordered_import_roots.append(roots) + elif kind in {"P", "N"} and ("." in name) == (kind == "N"): + indexed_first_party[name] = tuple(map(int, roots.split("\t"))) + else: + raise ValueError(f"Invalid Python import index record in {index_path}") + + stdlib_module_names = getattr(sys, "stdlib_module_names", ()) + + def indexed_path_distribution( + distribution_type: type[Distribution], + metadata_path: Path, + ref: Callable[[Distribution], Callable[[], Distribution | None]], + ) -> Distribution: + distribution = distribution_type(metadata_path) + original_read_text = distribution_type.read_text + canonical_path = os.path.normcase(os.path.abspath(os.fspath(metadata_path))) + distribution_reference = ref(distribution) + + def cached_read_text(filename: str) -> str | None: + instance = distribution_reference() + if instance is None: + raise ReferenceError("Indexed wheel distribution no longer exists") + if filename not in ( + "entry_points.txt", + "METADATA", + "PKG-INFO", + ): + return original_read_text(instance, filename) + + cache_key = (canonical_path, filename) + try: + return indexed_metadata_text[cache_key] + except KeyError: + # Indexed wheel runfiles are immutable, including absent files. + text = original_read_text(instance, filename) + indexed_metadata_text[cache_key] = text + return text + + # Preserve the concrete type without an instance/bound-method reference cycle. + distribution.read_text = cached_read_text + return distribution + + def distribution_records() -> Iterator[str]: + for records in indexed_distributions.values(): + yield from records + + def register_pkg_resources(module: ModuleType) -> None: + for entry in distribution_records(): + metadata_directory, _, relative_root = entry.partition("\t") + wheel_root = os.path.normpath(os.path.join(site_packages, relative_root)) + metadata_path = os.path.join(wheel_root, metadata_directory) + metadata = module.PathMetadata(wheel_root, metadata_path) + distribution = module.Distribution.from_location( + site_packages, + metadata_directory, + metadata, + ) + module.working_set.add(distribution, entry=site_packages, insert=False) + # Set the real location after activation to keep it off sys.path. + distribution.location = wheel_root + + def register_pkgutil(module: ModuleType) -> None: + original_extend_path = module.extend_path + + def extend_path(path: list[str], name: str) -> list[str]: + path = original_extend_path(path, name) + if not isinstance(path, list): + return path + package = name.replace(".", os.sep) + for position in indexed_first_party.get(name, ()): + root = os.path.join(site_packages, ordered_import_roots[position]) + portion = os.path.normpath(os.path.join(root, package)) + if portion not in path and os.path.isdir(portion): + path.append(portion) + return path + + module.extend_path = extend_path + + class _IndexedImportFinder: + _aspect_rules_py_import_index = index_path + + def _refresh_namespace( + self, namespace: str, parent_paths: Sequence[str] + ) -> ModuleSpec | None: + parent_name, separator, _ = namespace.rpartition(".") + if not separator: + return self.find_spec(namespace) + parent = sys.modules.get(parent_name) + current_paths = getattr(parent, "__path__", None) + if current_paths is not None: + spec = self.find_spec(namespace, current_paths) + if spec is not None: + return spec + return path_finder.find_spec(namespace, parent_paths) + + def _resolve_spec( + self, fullname: str, search_paths: Sequence[str], target: ModuleType | None + ) -> ModuleSpec | None: + spec = path_finder.find_spec(fullname, search_paths, target) + if spec is not None and spec.loader is None: + namespace_paths = spec.submodule_search_locations + if hasattr(namespace_paths, "_path_finder"): + namespace_paths._path_finder = self._refresh_namespace + return spec + + def find_spec( + self, + fullname: str, + path: Sequence[str] | None = None, + target: ModuleType | None = None, + ) -> ModuleSpec | None: + # Namespace pruning requires the original parent and import hooks. + if path is not None: + child_positions = indexed_first_party.get(fullname) + if child_positions is None or "." not in fullname: + return None + if tuple(sys.path_hooks) != initial_path_hooks: + return None + + try: + if sys.meta_path[sys.meta_path.index(self) + 1] is not path_finder: + return None + except (ValueError, IndexError): + return None + + parent_name, _, _ = fullname.rpartition(".") + parent = sys.modules.get(parent_name) + if getattr(parent, "__path__", None) is not path or not hasattr( + path, "_path_finder" + ): + return None + + parent_positions = indexed_first_party.get(parent_name) + known_portions = indexed_namespace_portions.get(parent_name) + if known_portions is None: + parent_directory = parent_name.replace(".", os.sep) + known_portions = { + os.path.normpath( + os.path.join( + site_packages, + ordered_import_roots[position], + parent_directory, + ) + ): position + for position in parent_positions + } + indexed_namespace_portions[parent_name] = known_portions + + child_owners = frozenset(child_positions) + search_paths = [] + removed_portion = False + # Prune only indexed portions; preserve physical and custom importers. + for portion in path: + owner = known_portions.get(portion) + if owner is None or owner in child_owners: + search_paths.append(portion) + continue + importer = sys.path_importer_cache.get(portion) + if importer is not None and type(importer).__name__ != "FileFinder": + search_paths.append(portion) + continue + removed_portion = True + + if not removed_portion: + return None + + return self._resolve_spec(fullname, search_paths, target) + + if "." in fullname or ( + fullname in stdlib_module_names and fullname != "pkgutil" + ): + return None + + roots = indexed_roots.get(fullname) + first_party_positions = indexed_first_party.get(fullname) + bridge = fullname if fullname == "pkgutil" else None + if indexed_distributions and fullname in { + "pkg_resources", + "importlib_metadata", + }: + bridge = fullname + if roots is None and first_party_positions is None: + if bridge is None: + return None + search_paths = sys.path + else: + try: + site_packages_position = sys.path.index(site_packages) + except ValueError: + return None + + search_paths = list(sys.path) + if first_party_positions is not None: + live_retained_roots = [] + for position, root in retained_import_roots.items(): + try: + path_position = search_paths.index(root) + except ValueError: + continue + live_retained_roots.append((position, path_position)) + + insertion_groups = {} + for position in first_party_positions: + relative_root = ordered_import_roots[position] + virtual_root = os.path.normpath( + os.path.join(site_packages, relative_root) + ) + if virtual_root in search_paths: + continue + + # Restore this import's root order without mutating sys.path. + insert_at = site_packages_position + 1 + for root_position, path_position in live_retained_roots: + if root_position > position: + insert_at = path_position + break + insert_at = path_position + 1 + insertion_groups.setdefault(insert_at, []).append(virtual_root) + + for insert_at in sorted(insertion_groups, reverse=True): + search_paths[insert_at:insert_at] = insertion_groups[insert_at] + + if roots is not None: + search_paths[ + site_packages_position + 1 : site_packages_position + 1 + ] = ( + os.path.normpath(os.path.join(site_packages, root)) + for root in roots.split("\t") + ) + spec = self._resolve_spec(fullname, search_paths, target) + if bridge is not None and spec is not None and spec.loader is not None: + original_exec_module = spec.loader.exec_module + + def exec_module(module: ModuleType) -> None: + original_exec_module(module) + # setuptools and the backport maintain separate metadata registries. + if bridge == "pkgutil": + register_pkgutil(module) + elif bridge == "pkg_resources": + register_pkg_resources(module) + else: + register_importlib_metadata(module) + + spec.loader.exec_module = exec_module + return spec + + def iter_modules(self, prefix: str = "") -> Iterator[tuple[str, bool]]: + for fullname in {**indexed_roots, **indexed_first_party}: + if "." in fullname: + continue + spec = self.find_spec(fullname) + if spec is not None: + yield prefix + fullname, spec.submodule_search_locations is not None + + sys.meta_path.insert(sys.meta_path.index(path_finder), _IndexedImportFinder()) + + pkgutil_module = sys.modules.get("pkgutil") + if pkgutil_module is not None: + register_pkgutil(pkgutil_module) + + if indexed_distributions: + + def install_distribution_resolver( + finder: Any, metadata_module: ModuleType | None = None + ) -> None: + original_find_distributions = finder.find_distributions + site_key = os.path.normcase(os.path.abspath(site_packages)) + + def find_distributions( + context: DistributionFinder.Context | None = None, + ) -> Iterator[Distribution]: + requested_name = getattr(context, "name", None) + if requested_name: + requested = indexed_distributions.get( + normalize_distribution_name(requested_name) + ) + if requested is None: + yield from original_find_distributions(context) + return + entries = requested + else: + entries = distribution_records() + + search_paths = list(getattr(context, "path", sys.path)) + site_packages_position = next( + ( + position + for position, path in enumerate(search_paths) + if os.path.normcase(os.path.abspath(os.fspath(path))) + == site_key + ), + None, + ) + if site_packages_position is None: + if context is None: + yield from original_find_distributions() + else: + yield from original_find_distributions(context) + return + + if metadata_module is None: + from importlib.metadata import DistributionFinder, PathDistribution + else: + DistributionFinder = metadata_module.DistributionFinder + PathDistribution = metadata_module.PathDistribution + from pathlib import Path + from weakref import ref + + context_values = {} if context is None else vars(context).copy() + # Insert indexed metadata at site-packages' original sys.path position. + context_values["path"] = search_paths[: site_packages_position + 1] + yield from original_find_distributions( + DistributionFinder.Context(**context_values) + ) + + for entry in entries: + metadata_directory, _, relative_root = entry.partition("\t") + wheel_root = os.path.normpath( + os.path.join(site_packages, relative_root) + ) + yield indexed_path_distribution( + PathDistribution, + Path(wheel_root) / metadata_directory, + ref, + ) + + trailing_paths = search_paths[site_packages_position + 1 :] + if trailing_paths: + context_values["path"] = trailing_paths + yield from original_find_distributions( + DistributionFinder.Context(**context_values) + ) + + if isinstance(finder, type): + finder.find_distributions = staticmethod(find_distributions) + else: + finder.find_distributions = find_distributions + + def register_importlib_metadata(module: ModuleType) -> None: + # The backport replaces PathFinder with its own metadata resolver. + metadata_finder_type = getattr(module, "MetadataPathFinder", None) + if metadata_finder_type is None: + return + metadata_finder = next( + ( + finder + for finder in sys.meta_path + if type(finder) is metadata_finder_type + ), + None, + ) + if metadata_finder is not None: + install_distribution_resolver(metadata_finder, module) + + importlib_metadata = sys.modules.get("importlib_metadata") + if hasattr(importlib_metadata, "MetadataPathFinder"): + register_importlib_metadata(importlib_metadata) + else: + install_distribution_resolver(path_finder) + + pkg_resources = sys.modules.get("pkg_resources") + if hasattr(pkg_resources, "working_set"): + register_pkg_resources(pkg_resources) + + +install_import_index() diff --git a/py/private/py_venv/tests/BUILD.bazel b/py/private/py_venv/tests/BUILD.bazel index a3d40cee7..cbe0ba160 100644 --- a/py/private/py_venv/tests/BUILD.bazel +++ b/py/private/py_venv/tests/BUILD.bazel @@ -73,4 +73,14 @@ py_test( main = "//py/private/py_venv/tests:test_link.py", ) +py_test( + name = "import_index_test", + srcs = [ + "import_index_test.py", + "//py/private/py_venv:import_index.py", + ], + imports = [".."], + main = "import_index_test.py", +) + virtuals_resolvers_test_suite(name = "virtuals_resolvers_tests") diff --git a/py/private/py_venv/tests/import_index_test.py b/py/private/py_venv/tests/import_index_test.py new file mode 100644 index 000000000..25c06feb3 --- /dev/null +++ b/py/private/py_venv/tests/import_index_test.py @@ -0,0 +1,81 @@ +"""Exercise wheel projection and first-party namespace index records.""" + +from pathlib import Path +import tempfile +import unittest + +from import_index import generate + + +class ImportIndexTest(unittest.TestCase): + def _generate(self, records: list[str]) -> tuple[list[str], list[str]]: + with tempfile.TemporaryDirectory(prefix="import-index-test-") as directory: + path = Path(directory) / "records" + path.write_text("\n".join(records) + "\n", encoding="utf-8") + index, pth = generate( + records_path=str(path), + workspace="_main", + escape="../../..", + venv_escape="../..", + ) + return index.splitlines(), pth.splitlines() + + def test_virtual_wheels_preserve_owner_and_metadata_order(self) -> None: + index, _ = self._generate( + [ + "R\t_main", + "W\tnamespace/first\texternal/shared", + "W\tfirst-1.dist-info\texternal/shared", + "W\todd.name.py\texternal/shared", + "W\tnamespace/second\texternal/shared-extra", + "W\tother.name.pyc\texternal/shared-extra", + "W\tnamespace/first_again\texternal/shared", + "W\tnative.cpython-312-x86_64-linux-gnu.so\texternal/shared-extra", + "W\tnamespace/nested.dist-info\texternal/shared-extra", + "W\todd name\texternal/shared", + "W\tsecond-1.egg-info\texternal/shared-extra", + ] + ) + + self.assertEqual( + [row for row in index if row.startswith(("I\t", "D\t"))], + [ + "I\tnamespace\t../../../external/shared\t../../../external/shared-extra", + "I\todd.name\t../../../external/shared", + "I\tother.name\t../../../external/shared-extra", + "I\tnative\t../../../external/shared-extra", + "I\todd name\t../../../external/shared", + "D\tfirst-1.dist-info\t../../../external/shared", + "D\tsecond-1.egg-info\t../../../external/shared-extra", + ], + ) + + def test_first_party_target_requires_no_wheel_records(self) -> None: + index, pth = self._generate( + [ + "R\t_main/project", + "S\tproject/service.py", + ] + ) + + self.assertIn("P\tservice\t1", index) + self.assertFalse(any(row.startswith(("I\t", "D\t")) for row in index)) + self.assertTrue(any("_aspect_rules_py_import_index" in row for row in pth)) + + def test_shared_first_party_namespaces_retain_each_owner(self) -> None: + index, _ = self._generate( + [ + "R\t_main/first", + "R\t_main/second", + "S\tfirst/shared/one.py", + "S\tsecond/shared/two.py", + ] + ) + + self.assertIn("P\tshared\t1\t2", index) + self.assertIn("N\tshared.one\t1", index) + self.assertIn("N\tshared.two\t2", index) + + +if __name__ == "__main__": + unittest.main() diff --git a/py/private/py_venv/venv.bzl b/py/private/py_venv/venv.bzl index e1a8e002f..96d7987a3 100644 --- a/py/private/py_venv/venv.bzl +++ b/py/private/py_venv/venv.bzl @@ -3,8 +3,8 @@ This module is the single place in rules_py that declares the files making up a Python venv. Both `py_binary` / `py_test` (each with its own internal venv, unless `expose_venv = True` routes them to a sibling py_venv) and -the standalone `py_venv` rule call `assemble_venv` to keep their layouts -bit-identical. +the standalone `py_venv` rule call `assemble_venv`. Exposed virtualenvs stay +physical; private virtualenvs can replace package projections with an index. The venv shape mirrors what CPython's `python -m venv` + pip install produces, so downstream tools (IDEs, `$VIRTUAL_ENV`-aware shells, @@ -67,6 +67,100 @@ _ADDSITEDIR_LINE = ( def _dict_to_exports(env): return ["export %s=\"%s\"" % (k, v) for (k, v) in env.items()] +def _import_name(entry, wheel): + """Return the importable top-level name, or None for metadata/data.""" + if "." not in entry: + if entry not in wheel.top_level_dirs and entry not in wheel.namespace_top_levels: + return None + name = entry + elif entry.endswith(".py"): + name = entry[:-3] + elif entry.endswith(".pyc"): + name = entry[:-4] + elif entry.endswith(".so") or entry.endswith(".pyd"): + name = entry.split(".", 1)[0] + else: + return None + return name if name and "-" not in name else None + +def _source_record(source): + path = source.short_path + if path.startswith("../"): + return None + if source.is_directory: + return "T\t" + path + if source.extension not in ("py", "pyc", "so", "pyd") and "/" in path: + path = path.rsplit("/", 1)[0] + "/" + return "S\t" + path + +def _symlink_record(symlink): + return ("Q\t" if symlink.target_file.is_directory else "L\t") + symlink.path + +def _root_symlink_record(symlink): + return ("B\t" if symlink.target_file.is_directory else "A\t") + symlink.path + +def _wheel_projection_record(projection): + return "W\t" + projection[0] + "\t" + projection[1] + +def _indexed_projection_plan(wheels, wheel_by_site_packages, fully_covered, projections, known_layout_site_pkgs, ordered_metadata = False): + """Select safe projections and retain existing collision/.pth semantics.""" + import_spellings = {} + projected_pth_sites = set() + candidate_entries = {} + for entry, site_packages in projections.items(): + wheel = wheel_by_site_packages.get(site_packages) + if wheel == None or entry in wheel.metadata_top_levels: + if ordered_metadata and wheel != None: + break + continue + spelling = entry if "/" not in entry else entry.split("/", 1)[0] + name = ( + spelling if "." not in spelling and "-" not in spelling and spelling in wheel.top_level_dirs else _import_name(spelling, wheel) + ) + if name == None: + if spelling.endswith(".pth"): + projected_pth_sites.add(site_packages) + continue + candidate_entries[entry] = name + if import_spellings.setdefault(name, spelling) != spelling: + import_spellings[name] = None + + if len(wheels) == len(fully_covered) + len(known_layout_site_pkgs): + wheels = [ + wheel_by_site_packages[site] + for site in list(known_layout_site_pkgs) + list(projected_pth_sites) + ] + for wheel in wheels: + site_packages = wheel.site_packages_rfpath + if site_packages in fully_covered: + if site_packages not in projected_pth_sites: + continue + if not any([entry.endswith(".pth") for entry, _ in wheel.tl_claims]): + continue + for entry, _ in wheel.tl_claims: + name = _import_name(entry, wheel) + if name != None: + import_spellings[name] = None + + eligible_wheels = {} + for entry, name in candidate_entries.items(): + site_packages = projections[entry] + if site_packages not in fully_covered or import_spellings[name] == None: + eligible_wheels[site_packages] = None + elif site_packages not in eligible_wheels: + eligible_wheels[site_packages] = True + + retained_projections = {} + for entry, site_packages in projections.items(): + if not eligible_wheels.get(site_packages) or ( + entry not in candidate_entries and + not entry.endswith(".dist-info") and + not entry.endswith(".egg-info") + ): + retained_projections[entry] = projections.pop(entry) + + return retained_projections, projections + def assemble_venv( ctx, *, @@ -81,7 +175,8 @@ def assemble_venv( venv_activate_tmpl, site_merge_script_py, console_script_tmpl, - venv_name): + venv_name, + indexed_runfiles = None): """Declare every file + symlink that makes up a venv for a target. Args: @@ -111,6 +206,7 @@ def assemble_venv( console_script_tmpl: File — the console-script wrapper template (usually `ctx.file._console_script_tmpl`). venv_name: str — the venv dir basename (e.g. "." + venv_stem). + indexed_runfiles: Optional runfiles whose paths seed indexed imports. Returns: struct with: @@ -120,7 +216,9 @@ def assemble_venv( / DefaultInfo aggregation. """ - top_level_to_site_pkgs, fully_covered_site_pkgs, console_scripts_map, merge_groups, data_file_to_site_pkgs, collisions = resolve_wheel_collisions(ctx, wheels) + wheel_by_site_packages = {} + known_layout_site_pkgs = set() + top_level_to_site_pkgs, fully_covered_site_pkgs, console_scripts_map, merge_groups, data_file_to_site_pkgs, collisions = resolve_wheel_collisions(ctx, wheels, wheel_by_site_packages, known_layout_site_pkgs) enforce_collision_policy(collisions, package_collisions) # All toolchain-derived path/flag math (runfiles escape arithmetic, @@ -137,20 +235,17 @@ def assemble_venv( wheel_py_ver = tc.wheel_py_ver site_packages_rel = tc.site_packages_rel - # site_packages_rfpath → install_tree, used only by the regular-package - # merge action below. The per-top-level symlinks and .pth lines locate - # each wheel by its runfiles path directly, not through this map. - tree_by_sp = {w.site_packages_rfpath: w.install_tree for w in wheels} - - # site_packages_rfpath → True for wheels whose top-level layout is known - # (they declare `top_levels`), so the per-top-level symlink loop projects - # their root entries — including any root `.pth` files — into the venv - # site-packages. Wheels that carry only `console_scripts` (e.g. source-built - # scripts) leave `top_levels` empty: nothing is projected for them, so their - # `.pth` line must use `site.addsitedir` (see `_format_imp`). - known_layout_site_pkgs = {w.site_packages_rfpath: True for w in wheels if w.top_levels} - declared = [] # accumulator for all outputs + wheel_projections = None + if indexed_runfiles != None: + top_level_to_site_pkgs, wheel_projections = _indexed_projection_plan( + wheels, + wheel_by_site_packages, + fully_covered_site_pkgs, + top_level_to_site_pkgs, + known_layout_site_pkgs, + ordered_metadata = True, + ) # Per-top-level site-packages symlink: a relative symlink escaping from # site-packages up to the runfiles root, then down into the owning @@ -222,7 +317,7 @@ def assemble_venv( # The merge runs as a build action under the exec-configuration # interpreter (same shape as WhlInstall's unpack action). Every # PyWheelsInfo record carries an install_tree (see providers.bzl), - # so each contributing wheel resolves in tree_by_sp. + # so each contributing wheel resolves in wheel_by_site_packages. for group in merge_groups: exec_toolchain = ctx.toolchains[EXEC_TOOLS_TOOLCHAIN] exec_runtime = exec_toolchain.exec_runtime if exec_toolchain else None @@ -245,7 +340,7 @@ def assemble_venv( arguments.add("--collision-policy", package_collisions) trees = [] for sp in group.site_packages_list: - tree = tree_by_sp[sp] + tree = wheel_by_site_packages[sp].install_tree trees.append(tree) arguments.add_all( [tree], @@ -289,34 +384,91 @@ def assemble_venv( ) return "{}/{}".format(escape, imp) - pth_lines = ctx.actions.args() - pth_lines.use_param_file("%s", use_always = True) - pth_lines.set_param_file_format("multiline") - pth_lines.add(escape) - - # Make wheel-declared console scripts reachable via `subprocess.run("name", ...)` - # without loading the distutils shim on every interpreter startup. - pth_lines.add( - "import os, sys; _venv_bin = os.path.dirname(sys.executable); " + - "_path = os.environ.get(\"PATH\", \"\"); " + - "os.environ[\"PATH\"] = _path if _venv_bin in _path.split(os.pathsep) " + - "else _venv_bin + os.pathsep + _path; del _venv_bin, _path", - ) - - # allow_closure lets _format_imp capture fully_covered_site_pkgs / - # known_layout_site_pkgs so we don't have to materialise imports_depset - # via .to_list(). - pth_lines.add_all(imports_depset, map_each = _format_imp, allow_closure = True) - site_packages_pth_file = ctx.actions.declare_file( "{}/{}.pth".format(site_packages_rel, venv_stem), ) - ctx.actions.write( - output = site_packages_pth_file, - content = pth_lines, - ) declared.append(site_packages_pth_file) + if indexed_runfiles != None: + index_file = ctx.actions.declare_file( + site_packages_prefix + ".aspect_rules_py_import_index", + ) + import_index_shim = ctx.actions.declare_file( + site_packages_prefix + "_aspect_rules_py_import_index.py", + ) + declared.extend([index_file, import_index_shim]) + + records = ctx.actions.args() + records.use_param_file("%s", use_always = True) + records.set_param_file_format("multiline") + records.add_all(imports_depset, format_each = "R\t%s") + records.add_all(list(fully_covered_site_pkgs), format_each = "C\t%s") + records.add_all(list(known_layout_site_pkgs), format_each = "H\t%s") + records.add_all( + indexed_runfiles.files, + map_each = _source_record, + expand_directories = False, + uniquify = True, + ) + records.add_all( + indexed_runfiles.symlinks, + map_each = _symlink_record, + ) + records.add_all( + indexed_runfiles.root_symlinks, + map_each = _root_symlink_record, + ) + records.add_all(wheel_projections.items(), map_each = _wheel_projection_record) + + arguments = ctx.actions.args() + arguments.add(ctx.file._import_index_generator) + arguments.add(ctx.workspace_name) + arguments.add(escape) + arguments.add(venv_to_runfiles_escape) + arguments.add(index_file) + arguments.add(site_packages_pth_file) + arguments.add(ctx.file._import_index_shim) + arguments.add(import_index_shim) + + exec_toolchain = ctx.toolchains[EXEC_TOOLS_TOOLCHAIN] + exec_runtime = exec_toolchain.exec_runtime if exec_toolchain else None + if exec_runtime == None: + fail("{}: indexed Python imports require an `{}` execution toolchain".format( + ctx.label, + EXEC_TOOLS_TOOLCHAIN, + )) + ctx.actions.run( + mnemonic = "PyImportIndex", + executable = exec_runtime.interpreter, + toolchain = EXEC_TOOLS_TOOLCHAIN, + arguments = [arguments, records], + inputs = depset( + direct = [ctx.file._import_index_generator, ctx.file._import_index_shim], + transitive = [exec_runtime.files], + ), + outputs = [index_file, site_packages_pth_file, import_index_shim], + execution_requirements = {"supports-path-mapping": "1"}, + ) + else: + pth_lines = ctx.actions.args() + pth_lines.use_param_file("%s", use_always = True) + pth_lines.set_param_file_format("multiline") + pth_lines.add(escape) + + # Make console scripts reachable without loading distutils at startup. + pth_lines.add( + "import os, sys; _venv_bin = os.path.dirname(sys.executable); " + + "_path = os.environ.get(\"PATH\", \"\"); " + + "os.environ[\"PATH\"] = _path if _venv_bin in _path.split(os.pathsep) " + + "else _venv_bin + os.pathsep + _path; del _venv_bin, _path", + ) + + pth_lines.add_all(imports_depset, map_each = _format_imp, allow_closure = True) + ctx.actions.write( + output = site_packages_pth_file, + content = pth_lines, + ) + pyvenv_cfg = ctx.actions.declare_file("{}/pyvenv.cfg".format(venv_name)) home_line = "home =\n" if tc.pyvenv_home == "" else "home = {}\n".format(tc.pyvenv_home) ctx.actions.write( diff --git a/py/private/py_venv/virtuals_resolvers.bzl b/py/private/py_venv/virtuals_resolvers.bzl index 2f33cced1..50ab2c165 100644 --- a/py/private/py_venv/virtuals_resolvers.bzl +++ b/py/private/py_venv/virtuals_resolvers.bzl @@ -72,7 +72,7 @@ def _distinct_ordered(keys): Collision precedence is "last distinct entry wins": the final element is the winner, everything before it is a loser. """ - return {k: True for k in keys}.keys() + return list(set(keys)) def _last_per_sp(claimants): """Last claim struct per distinct ``site_packages``, in first-claim order. @@ -88,24 +88,14 @@ def _new_state(): return struct( top_level_to_site_pkgs = {}, skipped_per_wheel = {}, - covered_per_wheel = {}, merge_groups = [], - conflicted_roots = {}, - ns_claimant_sps = {}, + conflicted_roots = set(), + ns_claimant_sps = set(), ) def _skip(state, sp, tl): """Route ``(sp, tl)`` to the ``.pth`` fallback.""" - state.skipped_per_wheel.setdefault(sp, {})[tl] = True - -def _cover(state, sp, tl): - """Mark ``(sp, tl)`` as projected, merged, or suppressed.""" - state.covered_per_wheel.setdefault(sp, {})[tl] = True - -def _cover_if_clean(state, sp, tl): - """Cover ``(sp, tl)`` only when it was not routed to ``.pth``.""" - if tl not in state.skipped_per_wheel.get(sp, {}): - _cover(state, sp, tl) + state.skipped_per_wheel.setdefault(sp, set()).add(tl) def _skip_entryless_and_split(unique_claimants, state, tl): """Route entryless claimants to ``.pth`` and return the rest. @@ -120,11 +110,6 @@ def _skip_entryless_and_split(unique_claimants, state, tl): _skip(state, c.site_packages, tl) return with_entries -def _cover_all_clean(claimants, state, tl): - """Cover every claimant that was not routed to ``.pth``.""" - for c in claimants: - _cover_if_clean(state, c.site_packages, tl) - def _make_collision_recorder(ctx, collisions): """Build a closure recording collisions for ``enforce_collision_policy``. @@ -161,15 +146,13 @@ def _resolve_entry_owners(claimants, tl, exclude_roots, state, complain): entry_owner = {} for c in claimants: for entry in c.ns_entries: - if _within_any(entry, exclude_roots): + if exclude_roots and _within_any(entry, exclude_roots): continue prior = entry_owner.get(entry) - if prior == None: - entry_owner[entry] = c - elif prior.site_packages != c.site_packages: + if prior != None and prior.site_packages != c.site_packages: complain("namespace entry", entry, prior.site_packages, c.site_packages) _skip(state, prior.site_packages, tl) - entry_owner[entry] = c + entry_owner[entry] = c return entry_owner def _dedupe_prefix_conflicts(entry_owner, tl, state, complain): @@ -186,7 +169,7 @@ def _dedupe_prefix_conflicts(entry_owner, tl, state, complain): not currently emitted. The conflict is surfaced via ``package_collisions`` rather than silently mis-merging. """ - for entry in list(entry_owner.keys()): + for entry in entry_owner.keys(): segments = entry.split("/") for depth in range(2, len(segments)): shallower = entry_owner.get("/".join(segments[:depth])) @@ -245,24 +228,14 @@ def _collapse_entry_projection(entry_owner, claimants, exclude_roots): owners, ) -def _build_wheel_lookup_sets(wheel_by_sp, sps): - """Pre-compute O(1) membership dicts for per-wheel root tuples. +def _wheel_lookup_sets(wheel_by_sp, sps, field): + """Index one wheel root field for constant-time membership.""" + return { + sp: set(getattr(wheel_by_sp[sp], field, ())) + for sp in sps + } - ``namespace_dirs``, ``regular_roots``, and ``native_roots`` are - tuples in the wheel record; converting them to dict keys once - avoids O(n) linear scans inside tight N x N loops. - """ - ns_dirs = {} - regular_roots = {} - native_roots = {} - for sp in sps: - w = wheel_by_sp[sp] - ns_dirs[sp] = {d: True for d in getattr(w, "namespace_dirs", ())} - regular_roots[sp] = {r: True for r in getattr(w, "regular_roots", ())} - native_roots[sp] = {r: True for r in getattr(w, "native_roots", ())} - return ns_dirs, regular_roots, native_roots - -def _scan_namespace_conflicts(tl, distinct_sps, wheel_by_sp, state): +def _scan_namespace_conflicts(tl, sps, wheel_by_sp, state): """Detect regular-package roots that span multiple namespace wheels. Cross-references each wheel's ``regular_roots`` against every other @@ -272,15 +245,15 @@ def _scan_namespace_conflicts(tl, distinct_sps, wheel_by_sp, state): ``__path__`` to the first directory found, so neither ``.pth`` nor per-entry symlinks can merge it. - Returns ``(conflicted_roots_dict, native_candidate_roots_dict)`` and - records every namespace claimant in ``state.ns_claimant_sps``. + Returns the conflicted roots and records every namespace claimant in + ``state.ns_claimant_sps``. """ - sps = list(distinct_sps) - ns_dirs, regular_roots, _native = _build_wheel_lookup_sets(wheel_by_sp, sps) + ns_dirs = _wheel_lookup_sets(wheel_by_sp, sps, "namespace_dirs") + regular_roots = _wheel_lookup_sets(wheel_by_sp, sps, "regular_roots") tl_prefix = tl + "/" - conflicted = {} + conflicted = set() for sp_a in sps: - state.ns_claimant_sps[sp_a] = True + state.ns_claimant_sps.add(sp_a) w_a = wheel_by_sp[sp_a] for root in getattr(w_a, "regular_roots", ()): if not root.startswith(tl_prefix): @@ -289,7 +262,7 @@ def _scan_namespace_conflicts(tl, distinct_sps, wheel_by_sp, state): if sp_b == sp_a: continue if root in ns_dirs[sp_b] or root in regular_roots[sp_b]: - conflicted[root] = True + conflicted.add(root) return conflicted def _classify_conflicted_roots(conflicted, unique_claimants, wheel_by_sp): @@ -300,23 +273,19 @@ def _classify_conflicted_roots(conflicted, unique_claimants, wheel_by_sp): Roots that cover a native candidate are promoted to avoid declaring both an ancestor and descendant as outputs. """ - sps = [c.site_packages for c in unique_claimants] - _, _, native_roots = _build_wheel_lookup_sets(wheel_by_sp, sps) - native_candidates = [ - root - for root in conflicted - if any([root in native_roots[c.site_packages] for c in unique_claimants]) - ] + native_roots = set() + for c in unique_claimants: + native_roots.update(getattr(wheel_by_sp[c.site_packages], "native_roots", ())) + native_candidates = [root for root in conflicted if root in native_roots] native_conflicted = _shallowest([ root for root in conflicted if _contains_any(root, native_candidates) ]) - mergeable = { - root: True - for root in conflicted - if not _within_any(root, native_conflicted) - } + mergeable = set() + for root in conflicted: + if not _within_any(root, native_conflicted): + mergeable.add(root) return native_conflicted, mergeable def _resolve_native_span( @@ -338,9 +307,10 @@ def _resolve_native_span( unless they carry duplicate metadata (their fallback would expose an unsuppressible duplicate entry). """ - _, regular_roots, _ = _build_wheel_lookup_sets( + regular_roots = _wheel_lookup_sets( wheel_by_sp, [c.site_packages for c in unique_claimants], + "regular_roots", ) native_winner_by_root = {} for root in native_roots: @@ -357,8 +327,8 @@ def _resolve_native_span( for c in unique_claimants: w = wheel_by_sp[c.site_packages] - ns_dirs = {d: True for d in getattr(w, "namespace_dirs", ())} - regs = {r: True for r in getattr(w, "regular_roots", ())} + ns_dirs = set(getattr(w, "namespace_dirs", ())) + regs = regular_roots[c.site_packages] for root, winner_sp in native_winner_by_root.items(): if (root in regs or root in ns_dirs or _contains_any(root, c.ns_entries)): if (c.site_packages != winner_sp and @@ -375,7 +345,6 @@ def _resolve_native_span( ) for entry, sp in projection.items(): state.top_level_to_site_pkgs[entry] = sp - _cover_all_clean(with_entries, state, tl) def _resolve_pure_namespace(unique_claimants, tl, state, complain): """Resolve a PEP 420 namespace top-level with no regular-span conflict. @@ -388,17 +357,15 @@ def _resolve_pure_namespace(unique_claimants, tl, state, complain): with_entries = _skip_entryless_and_split(unique_claimants, state, tl) if not with_entries: return - entry_owner = _resolve_entry_owners(with_entries, tl, [], state, complain) + entry_owner = _resolve_entry_owners(with_entries, tl, (), state, complain) _dedupe_prefix_conflicts(entry_owner, tl, state, complain) projection = _collapse_entry_projection(entry_owner, with_entries, []) for entry, sp in projection.items(): state.top_level_to_site_pkgs[entry] = sp - _cover_all_clean(with_entries, state, tl) def _resolve_directory_collision( tl, distinct_claimants, - any_namespace, state, duplicate_metadata_loser_sps): """Resolve a collision where all claimants are directories. @@ -419,14 +386,10 @@ def _resolve_directory_collision( winner = [c for c in distinct_claimants if not c.is_ns][-1] state.top_level_to_site_pkgs[tl] = winner.site_packages for c in distinct_claimants: - if (c.site_packages == winner.site_packages or - c.site_packages in duplicate_metadata_loser_sps): - _cover(state, c.site_packages, tl) - else: + if (c.site_packages != winner.site_packages and + c.site_packages not in duplicate_metadata_loser_sps): _skip(state, c.site_packages, tl) else: - for c in distinct_claimants: - _cover(state, c.site_packages, tl) state.merge_groups.append(struct( root = tl, site_packages_list = [c.site_packages for c in distinct_claimants], @@ -446,10 +409,7 @@ def _resolve_top_level( state.top_level_to_site_pkgs[tl] = claimants[0].site_packages return - all_namespace = all([c.is_ns for c in claimants]) - any_namespace = any([c.is_ns for c in claimants]) - - if all_namespace: + if all([c.is_ns for c in claimants]): unique_claimants = distinct_sp.values() tl_conflicted_roots = _scan_namespace_conflicts(tl, distinct_sp.keys(), wheel_by_sp, state) @@ -460,7 +420,7 @@ def _resolve_top_level( wheel_by_sp, ) for root in mergeable: - state.conflicted_roots[root] = True + state.conflicted_roots.add(root) _resolve_native_span( native_conflicted, @@ -477,6 +437,7 @@ def _resolve_top_level( _resolve_pure_namespace(unique_claimants, tl, state, complain) return + any_namespace = any([c.is_ns for c in claimants]) _complain_chain(complain, "top-level", tl, distinct_sp.keys()) distinct_claimants = distinct_sp.values() all_directories = any_namespace or all([c.is_dir for c in distinct_claimants]) @@ -485,7 +446,6 @@ def _resolve_top_level( _resolve_directory_collision( tl, distinct_claimants, - any_namespace, state, duplicate_metadata_loser_sps, ) @@ -502,21 +462,20 @@ def _resolve_top_level( _skip(state, c.site_packages, tl) state.top_level_to_site_pkgs[tl] = winner.site_packages -def _fold_merge_groups(wheels, wheel_by_sp, state): +def _fold_merge_groups(wheel_by_sp, state): """Fold conflicted roots into ``PySiteMerge`` merge groups. A conflicted root nested inside another is covered by the outer merge. Contributors are namespace-claimant wheels whose skeleton or regular roots include the path, in wheel traversal order. """ - ordered_sps = _distinct_ordered([w.site_packages_rfpath for w in wheels]) - for root in _shallowest(state.conflicted_roots.keys()): + for root in _shallowest(state.conflicted_roots): group_sps = [ sp - for sp in ordered_sps + for sp in wheel_by_sp if sp in state.ns_claimant_sps and ( - root in {d: True for d in getattr(wheel_by_sp[sp], "namespace_dirs", ())} or - root in {r: True for r in getattr(wheel_by_sp[sp], "regular_roots", ())} + root in getattr(wheel_by_sp[sp], "namespace_dirs", ()) or + root in getattr(wheel_by_sp[sp], "regular_roots", ()) ) ] if len(group_sps) >= 2: @@ -529,9 +488,11 @@ def _resolve_console_scripts(cs_claimants, complain): """Resolve console-script name collisions (last distinct wheel wins).""" console_scripts_map = {} for name, claimants in cs_claimants.items(): - distinct_sp = _last_per_sp(claimants) - _complain_chain(complain, "console script", name, distinct_sp.keys()) - winner = distinct_sp.values()[-1] + winner = claimants[0] + if len(claimants) > 1: + distinct_sp = _last_per_sp(claimants) + _complain_chain(complain, "console script", name, distinct_sp.keys()) + winner = distinct_sp.values()[-1] console_scripts_map[name] = struct(module = winner.module, func = winner.func) return console_scripts_map @@ -688,25 +649,19 @@ def _compute_fully_covered(wheels, state): Wheels without declared layout metadata (empty ``top_levels``) cannot be classified and are excluded. """ - fully_covered = {} + fully_covered = set() for w in wheels: if not w.top_levels: continue sp = w.site_packages_rfpath - skipped = state.skipped_per_wheel.get(sp, {}) - covered_roots = state.covered_per_wheel.get(sp, {}) - covered = True - for tl, _ in w.tl_claims: - if tl in skipped or ( - state.top_level_to_site_pkgs.get(tl) != sp and tl not in covered_roots - ): - covered = False - break - if covered: - fully_covered[sp] = True + if sp not in state.skipped_per_wheel or all([ + tl not in state.skipped_per_wheel[sp] + for tl, _ in w.tl_claims + ]): + fully_covered.add(sp) return fully_covered -def _resolve_metadata_collisions(metadata_claimants, state, fully_covered, complain, ctx): +def _resolve_metadata_collisions(metadata_claimants, metadata_duplicates, state, fully_covered, complain, ctx): """Resolve duplicate ``.dist-info`` / ``.egg-info`` entries. Python's metadata discovery scans every ``sys.path`` entry, so a @@ -716,23 +671,24 @@ def _resolve_metadata_collisions(metadata_claimants, state, fully_covered, compl ``package_collisions = "error"``. The winner is projected only when fully covered (fallback gone). """ - for tl, claimants in metadata_claimants.items(): - distinct = _distinct_ordered(claimants) - winner = distinct[-1] - for site_packages in distinct[:-1]: - if site_packages not in fully_covered: - fail(("{}: distribution metadata entry `{}` selects {}, but " + - "losing claimant {} remains on whole-wheel fallback.").format( - ctx.label, - tl, - winner, - site_packages, - )) - _complain_chain(complain, "distribution metadata entry", tl, distinct) + for tl, winner in metadata_claimants.items(): + if tl in metadata_duplicates: + distinct = _distinct_ordered(metadata_duplicates[tl]) + winner = distinct[-1] + for site_packages in distinct[:-1]: + if site_packages not in fully_covered: + fail(("{}: distribution metadata entry `{}` selects {}, but " + + "losing claimant {} remains on whole-wheel fallback.").format( + ctx.label, + tl, + winner, + site_packages, + )) + _complain_chain(complain, "distribution metadata entry", tl, distinct) if winner in fully_covered: state.top_level_to_site_pkgs[tl] = winner -def resolve_wheel_collisions(ctx, wheels): +def resolve_wheel_collisions(ctx, wheels, wheel_by_sp = None, known_layout_site_pkgs = None): """Walk ``PyWheelsInfo.wheels`` and produce merge plans for site-packages + bin/. Policy-agnostic: collisions are recorded, not reported. The caller @@ -747,28 +703,37 @@ def resolve_wheel_collisions(ctx, wheels): state = _new_state() tl_claimants = {} + tl_duplicates = {} metadata_claimants = {} + metadata_duplicates = {} cs_claimants = {} - wheel_by_sp = {} + wheel_by_sp = {} if wheel_by_sp == None else wheel_by_sp for w in wheels: wheel_by_sp[w.site_packages_rfpath] = w for tl in w.metadata_top_levels: - metadata_claimants.setdefault(tl, []).append(w.site_packages_rfpath) + if tl in metadata_claimants: + metadata_duplicates.setdefault(tl, [metadata_claimants[tl]]).append(w.site_packages_rfpath) + else: + metadata_claimants[tl] = w.site_packages_rfpath for tl, claim in w.tl_claims: - tl_claimants.setdefault(tl, []).append(claim) + if tl in tl_claimants: + tl_duplicates.setdefault(tl, [tl_claimants[tl]]).append(claim) + else: + tl_claimants[tl] = claim for name, claim in w.cs_claims: cs_claimants.setdefault(name, []).append(claim) - duplicate_metadata_loser_sps = { - loser: True - for claimants in metadata_claimants.values() - for loser in _distinct_ordered(claimants)[:-1] - } + duplicate_metadata_loser_sps = set() + for claimants in metadata_duplicates.values(): + duplicate_metadata_loser_sps.update(_distinct_ordered(claimants)[:-1]) - for tl, claimants in tl_claimants.items(): + for tl, claim in tl_claimants.items(): + if tl not in tl_duplicates: + state.top_level_to_site_pkgs[tl] = claim.site_packages + continue _resolve_top_level( tl, - claimants, + tl_duplicates[tl], wheel_by_sp, state, complain, @@ -776,11 +741,15 @@ def resolve_wheel_collisions(ctx, wheels): duplicate_metadata_loser_sps, ) - _fold_merge_groups(wheels, wheel_by_sp, state) + _fold_merge_groups(wheel_by_sp, state) console_scripts_map = _resolve_console_scripts(cs_claimants, complain) data_file_to_site_pkgs = _resolve_data_files(wheels, complain) fully_covered = _compute_fully_covered(wheels, state) - _resolve_metadata_collisions(metadata_claimants, state, fully_covered, complain, ctx) + if known_layout_site_pkgs != None: + for site_packages in state.skipped_per_wheel: + if site_packages not in fully_covered: + known_layout_site_pkgs.add(site_packages) + _resolve_metadata_collisions(metadata_claimants, metadata_duplicates, state, fully_covered, complain, ctx) return ( state.top_level_to_site_pkgs, diff --git a/py/tests/py-library-runfiles/BUILD.bazel b/py/tests/py-library-runfiles/BUILD.bazel index 70b10fff4..1f6489fd2 100644 --- a/py/tests/py-library-runfiles/BUILD.bazel +++ b/py/tests/py-library-runfiles/BUILD.bazel @@ -1,5 +1,5 @@ load("//py:defs.bzl", "py_binary", "py_library", "py_test", "py_venv") -load(":runfiles_test.bzl", "private_venv_provider_test", "py_library_runfiles_test", "py_venv_runfiles_test") +load(":runfiles_test.bzl", "indexed_imports_layout_test", "private_venv_provider_test", "py_library_runfiles_test", "py_venv_runfiles_test") package(default_testonly = True) @@ -52,6 +52,33 @@ private_venv_provider_test( target_under_test = ":_private_venv_bin.venv", ) +indexed_imports_layout_test( + name = "indexed_private_venv_test", + expect_index = True, + target_under_test = ":_private_venv_bin.venv", +) + +indexed_imports_layout_test( + name = "physical_public_venv_test", + expect_index = False, + target_under_test = ":public_venv", +) + +py_binary( + name = "physical_venv_bin", + srcs = ["main.py"], + imports = ["."], + indexed_imports = False, + main = "main.py", + deps = [":library"], +) + +indexed_imports_layout_test( + name = "physical_opt_out_venv_test", + expect_index = False, + target_under_test = ":_physical_venv_bin.venv", +) + py_test( name = "library_runtime_test", srcs = ["main.py"], diff --git a/py/tests/py-library-runfiles/runfiles_test.bzl b/py/tests/py-library-runfiles/runfiles_test.bzl index 6db807304..8058b706c 100644 --- a/py/tests/py-library-runfiles/runfiles_test.bzl +++ b/py/tests/py-library-runfiles/runfiles_test.bzl @@ -54,3 +54,30 @@ private_venv_provider_test = analysistest.make( "expected_sources": attr.string_list(mandatory = True), }, ) + +def _indexed_imports_layout_test_impl(ctx): + env = analysistest.begin(ctx) + target = analysistest.target_under_test(env) + outputs = [ + output.short_path + for action in analysistest.target_actions(env) + for output in action.outputs.to_list() + ] + runfiles = [file.short_path for file in target[DefaultInfo].default_runfiles.files.to_list()] + for suffix in [ + "/site-packages/.aspect_rules_py_import_index", + "/site-packages/_aspect_rules_py_import_index.py", + ]: + matches = [path for path in outputs if path.endswith(suffix)] + asserts.equals(env, 1 if ctx.attr.expect_index else 0, len(matches)) + if matches: + asserts.true(env, matches[0] in runfiles) + return analysistest.end(env) + +indexed_imports_layout_test = analysistest.make( + _indexed_imports_layout_test_impl, + attrs = {"expect_index": attr.bool(mandatory = True)}, + config_settings = { + str(Label("//py:experimental_indexed_imports")): True, + }, +) diff --git a/uv/private/extension/defs.bzl b/uv/private/extension/defs.bzl index a4129d368..b2e18e4eb 100644 --- a/uv/private/extension/defs.bzl +++ b/uv/private/extension/defs.bzl @@ -755,25 +755,29 @@ def _uv_impl(module_ctx): else: fail("Unsupported archive! {}".format(repr(sdist_cfg))) - # Wheel repos whose consuming package applies exclude_glob must carry their - # RECORD paths so whl_install can re-derive the layout after exclusion. All - # other wheels stay lean. Labels look like `@whl__pkg__hash//:whl`. - bdists_with_exclusions = {} + # Derive filtered layouts once when every consumer agrees on its exclusions. + # Shared wheels with conflicting exclusions retain their RECORD paths so + # each install can derive its own layout. + bdist_exclusions = {} for install_cfg in cfg.install_cfgs.values(): - if install_cfg.exclude_glob: - for whl_label in install_cfg.whls.values(): - if whl_label: - bdists_with_exclusions[whl_label.split("//", 1)[0].lstrip("@")] = True + for whl_label in install_cfg.whls.values(): + if not whl_label: + continue + bdist_name = whl_label.split("//", 1)[0].lstrip("@") + if bdist_exclusions.setdefault(bdist_name, install_cfg.exclude_glob) != install_cfg.exclude_glob: + bdist_exclusions[bdist_name] = None for bdist_name, bdist_cfg in cfg.bdist_cfgs.items(): # A per-wheel repo that downloads the wheel AND peeks its RECORD for the # install layout, so only the wheel a config selects is ever fetched. + exclusions = bdist_exclusions.get(bdist_name, []) whl_dist( name = bdist_name, url = bdist_cfg["url"], sha256 = _dist_sha256(bdist_cfg) or "", downloaded_file_path = url_basename(bdist_cfg["url"]), - carry_record_paths = bdist_name in bdists_with_exclusions, + exclude_glob = exclusions or [], + carry_record_paths = exclusions == None, ) # Resolve the sdist configure tool. The default is our bundled diff --git a/uv/private/py_entrypoint_binary/search.py b/uv/private/py_entrypoint_binary/search.py index 26dc306ba..880ab6eee 100644 --- a/uv/private/py_entrypoint_binary/search.py +++ b/uv/private/py_entrypoint_binary/search.py @@ -24,6 +24,13 @@ def optionxform(self, optionstr: str) -> str: opts = PARSER.parse_args() entrypoint = None +if "_aspect_rules_py_import_index" in sys.modules: + from importlib.metadata import entry_points + + match = next(iter(entry_points(group="console_scripts", name=opts.script)), None) + if match is not None: + entrypoint = match.value + for e in sys.path: if entrypoint: break diff --git a/uv/private/sdist_build/repository.bzl b/uv/private/sdist_build/repository.bzl index 5f35949b2..5a6827c22 100644 --- a/uv/private/sdist_build/repository.bzl +++ b/uv/private/sdist_build/repository.bzl @@ -265,6 +265,7 @@ py_binary( main = "@aspect_rules_py//uv/private/pep517_whl/tools:build_helper.py", srcs = ["@aspect_rules_py//uv/private/pep517_whl/tools:build_helper.py"], deps = {deps}, + indexed_imports = False, ) {rule}( diff --git a/uv/private/whl_install/dist_repository.bzl b/uv/private/whl_install/dist_repository.bzl index f26ad3c25..1bcc2a91d 100644 --- a/uv/private/whl_install/dist_repository.bzl +++ b/uv/private/whl_install/dist_repository.bzl @@ -37,7 +37,13 @@ def _whl_dist_impl(rctx): auth = get_auth(rctx, urls), ) - meta = extract_install_metadata(rctx, rctx.path(basename), basename) + meta = extract_install_metadata( + rctx, + rctx.path(basename), + basename, + rctx.attr.exclude_glob, + rctx.attr.carry_record_paths, + ) rctx.file("BUILD.bazel", content = """load("@aspect_rules_py//uv/private/whl_install:rule.bzl", "whl_dist") @@ -65,10 +71,8 @@ exports_files( # for every wheel (unlike record_paths): exclude_glob only removes # site-packages files, so the prefix data tree is never re-derived. data_files = _attr("data_files", meta.data_files), - # Only carried when a consuming package applies exclude_glob: whl_install - # re-derives the layout from these after exclusion. Kept off every other - # wheel so the common case doesn't pay for a full RECORD path list. - record_paths = _attr("record_paths", meta.record_paths) if rctx.attr.carry_record_paths else "", + # Conflicting or empty layouts retain their unfiltered RECORD paths. + record_paths = _attr("record_paths", meta.record_paths), )) # Hashless wheels record the discovered checksum so a re-fetch stays @@ -90,10 +94,11 @@ whl_dist = repository_rule( mandatory = True, doc = "The wheel's file name; also the on-disk output name.", ), + "exclude_glob": attr.string_list( + doc = "Exclusions shared by every consumer of this wheel.", + ), "carry_record_paths": attr.bool( - doc = "Emit the retained RECORD paths so whl_install can re-derive " + - "the layout after exclude_glob. Set only for wheels of packages " + - "that declare exclude_glob.", + doc = "Emit RECORD paths when consumers require different exclusions.", ), }, # Match http_file: the URL-derived canonical_id depends on this env var, so diff --git a/uv/private/whl_install/metadata.bzl b/uv/private/whl_install/metadata.bzl index cdc0930da..bc94dfe7b 100644 --- a/uv/private/whl_install/metadata.bzl +++ b/uv/private/whl_install/metadata.bzl @@ -32,6 +32,8 @@ def parse_record_path(line): containing an embedded newline (legal but vanishingly rare) is not handled -- the same limitation `importlib.metadata` has. """ + if not line.startswith("\""): + return line.partition(",")[0] path = [] # States mirror csv.reader's field parser: @@ -186,10 +188,8 @@ def native_roots_for_segments(segments, collision_roots = ()): # Keep parsing, matching, and cache-to-source matching in sync with # py/tools/unpack/{exclude_glob.py,unpack.py} and their shared test vectors. -# `whl_dist` extraction stays exclude-agnostic (the per-wheel repo never sees -# a per-package exclude_glob); `whl_install` applies these at analysis time to -# the selected wheel's layout so the advertised surface matches the install -# action's filtered tree. +# Wheel repositories apply exclusions shared by every consumer. Conflicting +# consumers instead filter the selected wheel's layout during analysis. def parse_exclude_glob(value): """Return the validated segments of a site-packages-relative glob.""" parts = value.split("/") @@ -224,23 +224,23 @@ def _exclude_glob_chunk_matches(value, pattern): def exclude_glob_matches(path, pattern): """Return whether a parsed glob excludes path or one of its parents.""" pattern = pattern + ["**"] - states = {0: True} + states = set([0]) for segment in path: for index in range(len(pattern)): if index in states and pattern[index] == "**": - states[index + 1] = True - next_states = {} + states.add(index + 1) + next_states = set() for index in range(len(pattern)): if index not in states: continue if pattern[index] == "**": - next_states[index] = True + next_states.add(index) elif _exclude_glob_chunk_matches(segment, pattern[index]): - next_states[index + 1] = True + next_states.add(index + 1) states = next_states for index in range(len(pattern)): if index in states and pattern[index] == "**": - states[index + 1] = True + states.add(index + 1) return len(pattern) in states def cache_source_path(path): @@ -319,7 +319,7 @@ def _namespace_dirs_and_roots(dirs_set, init_dirs, namespace_top_levels_set): """ namespace_dirs = [] regular_roots = [] - for d in sorted(dirs_set.keys()): + for d in sorted(dirs_set): segments = d.split("/") if segments[0] not in namespace_top_levels_set: continue @@ -490,48 +490,32 @@ def derive_layout(record_segments): """Derive the site-packages layout from filtered RECORD segment lists. `record_segments` are site-packages-relative paths (install-root escapes - already dropped). Run once at extraction, and again at analysis time (in - `whl_install`) over the segments that survive `exclude_glob` — so removing - an `__init__.py`, or the last file under a top-level, reclassifies - namespace/regular and drops stale entries instead of leaving the advertised - topology out of sync with the installed tree. + already dropped). Run once at extraction, and again at analysis time only + when wheel consumers disagree on exclusions. Removing an `__init__.py`, or + the last file under a top-level, reclassifies namespace/regular and drops + stale entries instead of leaving the advertised topology inconsistent. """ # First path segment = top-level name. Track which top-levels have a direct # `/__init__.py` (regular packages); the complement are PEP 420 # namespaces. Also record the directory skeleton, which dirs hold an # `__init__.py`, and which files are native. - top_levels_set = {} - regular_top_levels = {} - dirs_set = {} - init_dirs = {} + top_levels_set = set() + regular_top_levels = set() + dirs_set = set() + init_dirs = set() native_segments = [] for segments in record_segments: first_segment = segments[0] - top_levels_set[first_segment] = True - if len(segments) == 1 or (len(segments) >= 2 and segments[1] == "__init__.py"): - regular_top_levels[first_segment] = True + top_levels_set.add(first_segment) + if len(segments) == 1 or segments[1] == "__init__.py": + regular_top_levels.add(first_segment) if native_roots_for_segments(segments): native_segments.append(segments) for i in range(1, len(segments)): - dirs_set["/".join(segments[:i])] = True + dirs_set.add("/".join(segments[:i])) if len(segments) >= 2 and segments[-1] == "__init__.py": - init_dirs["/".join(segments[:-1])] = True - - # Namespace entries: for each path under a namespace top-level, descend to - # the shallowest concrete prefix — a dir with a direct `__init__.py`, or the - # file itself. Nested namespaces (`google/cloud/storage/…`) recurse. - namespace_entries_set = {} - for segments in record_segments: - if segments[0] in regular_top_levels or segments[0].endswith(".dist-info"): - continue - if len(segments) < 2: - continue - for depth in range(2, len(segments) + 1): - prefix = "/".join(segments[:depth]) - if depth == len(segments) or prefix in init_dirs: - namespace_entries_set[prefix] = True - break + init_dirs.add("/".join(segments[:-1])) top_level_dirs = sorted([ tl @@ -548,32 +532,41 @@ def derive_layout(record_segments): not tl.endswith(".dist-info") and not tl.endswith(".egg-info")) ]) - namespace_set = {tl: True for tl in namespaces} + namespace_set = set(namespaces) - namespace_entries = sorted([ - entry - for entry in namespace_entries_set - if entry.split("/")[0] in namespace_set - ]) + # Namespace entries: for each path under a namespace top-level, descend to + # the shallowest concrete prefix — a dir with a direct `__init__.py`, or the + # file itself. Nested namespaces (`google/cloud/storage/…`) recurse. + namespace_entries_set = set() + for segments in record_segments: + if segments[0] not in namespace_set: + continue + for depth in range(2, len(segments) + 1): + prefix = "/".join(segments[:depth]) + if depth == len(segments) or prefix in init_dirs: + namespace_entries_set.add(prefix) + break + + namespace_entries = sorted(namespace_entries_set) namespace_dirs, regular_roots = _namespace_dirs_and_roots(dirs_set, init_dirs, namespace_set) - native_roots = {} + native_roots = set() for segments in native_segments: for root in native_roots_for_segments(segments, namespace_dirs + regular_roots): - native_roots[root] = True + native_roots.add(root) return struct( - top_levels = sorted(top_levels_set.keys()), + top_levels = sorted(top_levels_set), top_level_dirs = top_level_dirs, namespace_top_levels = namespaces, namespace_entries = namespace_entries, namespace_dirs = namespace_dirs, regular_roots = regular_roots, - native_roots = sorted(native_roots.keys()), + native_roots = sorted(native_roots), ) -def extract_install_metadata(rctx, whl_path, basename): +def extract_install_metadata(rctx, whl_path, basename, exclude_glob, carry_record_paths): """Peek inside a wheel and derive the layout `PyWheelsInfo` consumes. Reads: @@ -588,6 +581,8 @@ def extract_install_metadata(rctx, whl_path, basename): whl_path: A resolved `rctx.path` to the wheel on disk. basename: The wheel's file name, which implies the `.dist-info` directory holding RECORD/entry_points.txt. + exclude_glob: Exclusions shared by every consumer of this wheel. + carry_record_paths: Whether conflicting consumers need unfiltered paths. Returns: A struct of sorted `list[str]` fields ready to pass straight through as @@ -603,6 +598,13 @@ def extract_install_metadata(rctx, whl_path, basename): # paths venv assembly projects. parsed = parse_record(record, data_directory) record_segments = parsed.record_segments + if exclude_glob: + patterns = [parse_exclude_glob(pattern) for pattern in exclude_glob] + record_segments = [ + segments + for segments in record_segments + if not record_path_excluded(segments, patterns) + ] # entry_points.txt: INI-style file. Only `[console_scripts]` interests # us — pip/uv synthesize executables under `bin/` from those at @@ -632,8 +634,9 @@ def extract_install_metadata(rctx, whl_path, basename): # A wheel's RECORD always lists at least its `.dist-info`, so a prebuilt # wheel's top_levels is never empty (empty stays reserved for source-built - # wheels of unknown layout). `record_paths` is preserved so whl_install can - # re-derive the layout after applying exclude_glob. + # wheels of unknown layout). Preserve RECORD paths for conflicting + # exclusions, or when exclusions empty a known layout; the latter keeps + # prebuilt wheels distinguishable from source-built wheels. layout = derive_layout(record_segments) return struct( top_levels = layout.top_levels, @@ -644,6 +647,9 @@ def extract_install_metadata(rctx, whl_path, basename): regular_roots = layout.regular_roots, native_roots = layout.native_roots, console_scripts = sorted(console_scripts.values()), - record_paths = ["/".join(segments) for segments in record_segments], + record_paths = [ + "/".join(segments) + for segments in parsed.record_segments + ] if carry_record_paths or not record_segments else [], data_files = parsed.data_files, ) diff --git a/uv/private/whl_install/rule.bzl b/uv/private/whl_install/rule.bzl index 2d585e28e..c68a04bac 100644 --- a/uv/private/whl_install/rule.bzl +++ b/uv/private/whl_install/rule.bzl @@ -9,10 +9,8 @@ load("//py/private/toolchain:types.bzl", "EXEC_TOOLS_TOOLCHAIN", "PY_TOOLCHAIN") # the built wheel; source_built_wheel consumes it below (unless overridden). load("//uv/private:source_built_wheel.bzl", "SourceBuiltWheelInfo") -# exclude_glob: whl_dist extraction is exclude-agnostic, so when a package -# declares exclusions the selected wheel's retained RECORD paths are filtered -# and the layout is RE-DERIVED here at analysis time (matching pre-derivation -# semantics — an excluded initializer reclassifies namespace/regular). +# Wheels shared by consumers with different exclusions require analysis-time +# layout derivation; all other layouts are derived by the wheel repository. load(":metadata.bzl", "derive_layout", "parse_exclude_glob", "record_path_excluded") PyWheelMetadataInfo = provider( @@ -38,7 +36,7 @@ PyWheelMetadataInfo = provider( "regular_roots": "Minimal `__init__.py`-carrying directories under the namespace top-levels.", "native_roots": "Collision roots containing native-library RECORD entries.", "console_scripts": "`[console_scripts]` entry points encoded as name=module:object.", - "record_paths": "Retained site-packages RECORD paths, for re-deriving the layout after exclude_glob. Empty unless a consuming package declares exclusions.", + "record_paths": "Site-packages RECORD paths for conflicting exclusions or empty filtered layouts.", "data_files": "PEP 427 `.data/data/` prefix-relative install paths (e.g. `share/...`), projected into the venv prefix.", }, ) @@ -168,39 +166,18 @@ def _whl_install(ctx): # in: it lives in a different repo that is never fetched or consulted here. meta = ctx.attr.src[PyWheelMetadataInfo] - # exclude_glob removes files from the install tree (via --exclude-glob on - # the action below). To keep the advertised layout consistent with that - # tree, filter the selected wheel's retained RECORD paths and RE-DERIVE the - # topology — matching the pre-derivation semantics: removing an initializer - # reclassifies a package regular→namespace, and removing the last file under - # a top-level drops it, so venv assembly never projects a dangling symlink - # or mis-merges. console_scripts live under bin/, so exclusions never touch - # them. record_paths is carried only for wheels of excluding packages; a - # source-built wheel has none, and its layout is already empty. + # Re-derive the layout only when this wheel's consumers have conflicting + # exclusions. Otherwise its repository already applied the shared policy. + # Filtering before derivation preserves regular/namespace classification. + layout = meta if ctx.attr.exclude_glob and meta.record_paths: patterns = [parse_exclude_glob(pattern) for pattern in ctx.attr.exclude_glob] - retained = [ - path.split("/") - for path in meta.record_paths - if not record_path_excluded(path.split("/"), patterns) - ] + retained = [] + for path in meta.record_paths: + segments = path.split("/") + if not record_path_excluded(segments, patterns): + retained.append(segments) layout = derive_layout(retained) - top_levels = layout.top_levels - top_level_dirs = layout.top_level_dirs - namespace_top_levels = layout.namespace_top_levels - namespace_entries = layout.namespace_entries - namespace_dirs = layout.namespace_dirs - regular_roots = layout.regular_roots - native_roots = layout.native_roots - else: - top_levels = meta.top_levels - top_level_dirs = meta.top_level_dirs - namespace_top_levels = meta.namespace_top_levels - namespace_entries = meta.namespace_entries - namespace_dirs = meta.namespace_dirs - regular_roots = meta.regular_roots - native_roots = meta.native_roots - console_scripts = meta.console_scripts # Prefix data files (`.data/data/`) are unaffected by exclude_glob (it only # removes site-packages files); the patch guard below keeps them consistent. @@ -229,11 +206,11 @@ def _whl_install(ctx): if patch_files: arguments.add("--patch-strip", str(ctx.attr.patch_strip)) arguments.add_all(patch_files, before_each = "--patch") - preserve_paths = {path: None for path in top_levels} - for path in namespace_entries + namespace_dirs + regular_roots: + preserve_paths = set(layout.top_levels) + for path in layout.namespace_entries + layout.namespace_dirs + layout.regular_roots: root = path.split("/")[0] if not root.endswith(".dist-info") and not root.endswith(".egg-info"): - preserve_paths[path] = None + preserve_paths.add(path) arguments.add_all( sorted(preserve_paths), before_each = "--preserve-path", @@ -335,15 +312,15 @@ def _whl_install(ctx): # venv assembly's per-top-level symlinks reference each wheel by # its natural runfiles path rather than through this File. wheels = depset(direct = [make_wheel_record( - top_levels = top_levels, - top_level_dirs = top_level_dirs, - namespace_top_levels = namespace_top_levels, - namespace_entries = namespace_entries, - namespace_dirs = namespace_dirs, - regular_roots = regular_roots, - native_roots = native_roots, + top_levels = layout.top_levels, + top_level_dirs = layout.top_level_dirs, + namespace_top_levels = layout.namespace_top_levels, + namespace_entries = layout.namespace_entries, + namespace_dirs = layout.namespace_dirs, + regular_roots = layout.regular_roots, + native_roots = layout.native_roots, site_packages_rfpath = site_packages_rfpath, - console_scripts = console_scripts, + console_scripts = meta.console_scripts, # unpack.py's data-file manifest guard (above) fails the build if a # patch alters the data set, so this list always matches the tree. data_files = data_files,