diff --git a/docs/api/py.md b/docs/api/py.md index f8fbde999..39fb1ad92 100644 --- a/docs/api/py.md +++ b/docs/api/py.md @@ -170,7 +170,7 @@ py_library(name, deps< | :------------- | :------------- | :------------- | :------------- | :------------- | | name | A unique name for this target. | Name | required | | | deps | Targets that produce Python code, commonly `py_library` rules. | List of labels | optional | `[]` | -| srcs | Python source files. | List of labels | optional | `[]` | +| srcs | Python source files.

`.pyi` type stubs listed here are carried as `PyInfo.transitive_pyi_files` rather than as runtime sources; both reach the runfiles of venvs and launchers that include this library. | List of labels | optional | `[]` | | data | Runtime dependencies of the program.

The transitive closure of the `data` dependencies will be available in the `.runfiles` folder for this binary/test. The program may optionally use the Runfiles lookup library to locate the data files, see https://pypi.org/project/bazel-runfiles/. Data is analyzed in the inherited caller configuration. Put artifacts that must match the terminal's Python environment in `deps`. | List of labels | optional | `[]` | | imports | List of import directories to be added to the PYTHONPATH. | List of strings | optional | `[]` | | resolutions | Satisfy a virtual_dep with a mapping from external package name to the label of an installed package that provides it. See virtual_deps. | Dictionary: String -> Label | optional | `{}` | @@ -288,7 +288,7 @@ from the already extracted whl file. 
 load("@aspect_rules_py//py:defs.bzl", "PyInfo")
 
-PyInfo(transitive_sources, imports, virtual_dependencies, virtual_resolutions)
+PyInfo(transitive_sources, transitive_pyi_files, imports, virtual_dependencies, virtual_resolutions)
Python source, import-path, and virtual-dependency information for a target's dependency closure. @@ -297,7 +297,8 @@ Python source, import-path, and virtual-dependency information for a target's de | Name | Description | | :------------- | :------------- | -| transitive_sources | depset[File] — postorder depset of first-party `.py` sources in the transitive closure. | +| transitive_sources | depset[File] — postorder depset of first-party runtime sources in the transitive closure; `.pyi` stubs are excluded. | +| transitive_pyi_files | depset[File] — postorder depset of `.pyi` type stubs in the transitive closure: stubs listed in `srcs` plus those carried by deps of either ruleset. | | imports | depset[str] — import roots to place on `sys.path` (rlocation-root-relative). | | virtual_dependencies | depset[str] — names of required virtual dependencies, independent of their resolution status. | | virtual_resolutions | depset[struct(virtual, target)] — virtual-dependency-name to concrete-target resolutions. | diff --git a/e2e/rules-python-interop/BUILD.bazel b/e2e/rules-python-interop/BUILD.bazel index 4fe9459d0..a0240ff10 100644 --- a/e2e/rules-python-interop/BUILD.bazel +++ b/e2e/rules-python-interop/BUILD.bazel @@ -8,6 +8,7 @@ load("@rules_python//python/zipapp:py_zipapp_binary.bzl", "py_zipapp_binary") load("@rules_shell//shell:sh_test.bzl", "sh_test") load(":exec_tools_facts.bzl", "exec_tools_facts", "with_python_version") load(":report_toolchain_version.bzl", "report_exec_version", "report_version") +load(":type_stubs_test.bzl", "foreign_stubs_test") # Both rulesets provision interpreters here, split by version so neither # shadows the other (see MODULE.bazel): 3.11 is rules_python's, everything else @@ -220,6 +221,32 @@ rules_py_test( deps = [":import_path_lib"], ) +# rules_python keeps `pyi_srcs` out of runfiles and in PyInfo only. Across the +# ruleset boundary rules_py must pick them up from the foreign provider and +# lay them down beside the modules in the venv. +rules_python_library( + name = "stubbed_lib", + srcs = ["stubbed/stubbed.py"], + imports = ["stubbed"], + pyi_srcs = ["stubbed/stubbed.pyi"], +) + +py_library( + name = "stubbed_wrapper", + deps = [":stubbed_lib"], +) + +foreign_stubs_test( + name = "foreign_stubs_test", + target_under_test = ":stubbed_wrapper", +) + +rules_py_test( + name = "rules_python_stubs_test", + srcs = ["rules_python_stubs_test.py"], + deps = [":stubbed_wrapper"], +) + # rules_python's console-script machinery reads the pip hub's dist-info to # generate a py_binary; unpinned, that binary lands on a rules_py-provisioned # toolchain. Its hub is parsed for 3.12 alone, so which wheel backs the script diff --git a/e2e/rules-python-interop/rules_python_stubs_test.py b/e2e/rules-python-interop/rules_python_stubs_test.py new file mode 100644 index 000000000..95d153656 --- /dev/null +++ b/e2e/rules-python-interop/rules_python_stubs_test.py @@ -0,0 +1,14 @@ +"""A rules_python library's `pyi_srcs` reach a rules_py consumer's runfiles. + +The stub is metadata in rules_python's PyInfo, never in its own runfiles; the +rules_py venv must carry it beside the module it annotates so a type checker +pointed at the venv resolves it. +""" + +import os + +import stubbed + +stub = os.path.splitext(stubbed.__file__)[0] + ".pyi" +assert os.path.exists(stub), "missing type stub next to " + stubbed.__file__ +assert stubbed.describe(1) == "stubbed 1" diff --git a/e2e/rules-python-interop/stubbed/stubbed.py b/e2e/rules-python-interop/stubbed/stubbed.py new file mode 100644 index 000000000..a4f57cd28 --- /dev/null +++ b/e2e/rules-python-interop/stubbed/stubbed.py @@ -0,0 +1,2 @@ +def describe(value: object) -> str: + return "stubbed " + str(value) diff --git a/e2e/rules-python-interop/stubbed/stubbed.pyi b/e2e/rules-python-interop/stubbed/stubbed.pyi new file mode 100644 index 000000000..9f5beb47f --- /dev/null +++ b/e2e/rules-python-interop/stubbed/stubbed.pyi @@ -0,0 +1 @@ +def describe(value: object) -> str: ... diff --git a/e2e/rules-python-interop/type_stubs_test.bzl b/e2e/rules-python-interop/type_stubs_test.bzl new file mode 100644 index 000000000..483b7c735 --- /dev/null +++ b/e2e/rules-python-interop/type_stubs_test.bzl @@ -0,0 +1,16 @@ +"""Analysis test: rules_python `pyi_srcs` surface in rules_py's `PyInfo`.""" + +load("@aspect_rules_py//py:defs.bzl", "PyInfo") +load("@bazel_skylib//lib:unittest.bzl", "analysistest", "asserts") + +def _basenames(files): + return sorted([file.basename for file in files.to_list()]) + +def _foreign_stubs_test_impl(ctx): + env = analysistest.begin(ctx) + info = analysistest.target_under_test(env)[PyInfo] + asserts.equals(env, ["stubbed.pyi"], _basenames(info.transitive_pyi_files)) + asserts.equals(env, ["stubbed.py"], _basenames(info.transitive_sources), "the stub is not a runtime source") + return analysistest.end(env) + +foreign_stubs_test = analysistest.make(_foreign_stubs_test_impl) diff --git a/e2e/rules-python-protobuf/BUILD.bazel b/e2e/rules-python-protobuf/BUILD.bazel index 2899a7924..6b9d279ff 100644 --- a/e2e/rules-python-protobuf/BUILD.bazel +++ b/e2e/rules-python-protobuf/BUILD.bazel @@ -1,6 +1,8 @@ -load("@aspect_rules_py//py:defs.bzl", "py_test") +load("@aspect_rules_py//py:defs.bzl", "py_library", "py_test") load("@protobuf//bazel:proto_library.bzl", "proto_library") +load("@protobuf//bazel:py_proto_library.bzl", "py_proto_library") load("@rules_proto_grpc_python//:defs.bzl", "python_proto_library") +load(":type_stubs_test.bzl", "generated_stubs_test") # A plain proto_library — the language-agnostic descriptor. Living at the # module root keeps the generated module a clean top-level `greeting_pb2`. @@ -28,3 +30,29 @@ py_test( main = "test.py", deps = [":greeting_py_proto"], ) + +# protobuf's own py_proto_library runs protoc with `--pyi_out`, so the +# generated bindings come with a `greeting_pb2.pyi` carried only in +# rules_python's `PyInfo.transitive_pyi_files`. +py_proto_library( + name = "greeting_py_proto_native", + deps = [":greeting_proto"], +) + +py_library( + name = "greeting_stubs_wrapper", + deps = [":greeting_py_proto_native"], +) + +generated_stubs_test( + name = "generated_stubs_test", + target_under_test = ":greeting_stubs_wrapper", +) + +py_test( + name = "stubs_test", + size = "small", + srcs = ["stubs_test.py"], + main = "stubs_test.py", + deps = [":greeting_py_proto_native"], +) diff --git a/e2e/rules-python-protobuf/MODULE.bazel b/e2e/rules-python-protobuf/MODULE.bazel index 38e0ca2f5..23ee4ae13 100644 --- a/e2e/rules-python-protobuf/MODULE.bazel +++ b/e2e/rules-python-protobuf/MODULE.bazel @@ -13,6 +13,7 @@ # own python_interpreters extension. bazel_dep(name = "aspect_rules_py") +bazel_dep(name = "bazel_skylib", version = "1.4.2") bazel_dep(name = "rules_proto_grpc_python", version = "5.8.0") # Declared directly so the `@protobuf//bazel:proto_library.bzl` load in diff --git a/e2e/rules-python-protobuf/stubs_test.py b/e2e/rules-python-protobuf/stubs_test.py new file mode 100644 index 000000000..76c2b3a66 --- /dev/null +++ b/e2e/rules-python-protobuf/stubs_test.py @@ -0,0 +1,14 @@ +"""protobuf's py_proto_library emits `greeting_pb2.pyi` only through +rules_python's `PyInfo.transitive_pyi_files`; the rules_py venv has to place it +beside `greeting_pb2.py` for type checkers to see the generated message types. +""" + +import os + +import greeting_pb2 + +stub = os.path.splitext(greeting_pb2.__file__)[0] + ".pyi" +assert os.path.exists(stub), "missing generated stub next to " + greeting_pb2.__file__ + +with open(stub) as handle: + assert "class Greeting" in handle.read(), stub diff --git a/e2e/rules-python-protobuf/type_stubs_test.bzl b/e2e/rules-python-protobuf/type_stubs_test.bzl new file mode 100644 index 000000000..31fa14335 --- /dev/null +++ b/e2e/rules-python-protobuf/type_stubs_test.bzl @@ -0,0 +1,16 @@ +"""Analysis test: generated `_pb2.pyi` stubs surface in rules_py's `PyInfo`.""" + +load("@aspect_rules_py//py:defs.bzl", "PyInfo") +load("@bazel_skylib//lib:unittest.bzl", "analysistest", "asserts") + +def _basenames(files): + return sorted([file.basename for file in files.to_list()]) + +def _generated_stubs_test_impl(ctx): + env = analysistest.begin(ctx) + info = analysistest.target_under_test(env)[PyInfo] + asserts.true(env, "greeting_pb2.pyi" in _basenames(info.transitive_pyi_files), "py_proto_library's stub propagates") + asserts.false(env, "greeting_pb2.pyi" in _basenames(info.transitive_sources), "the stub is not a runtime source") + return analysistest.end(env) + +generated_stubs_test = analysistest.make(_generated_stubs_test_impl) diff --git a/e2e/rules-python-provider-compat/BUILD.bazel b/e2e/rules-python-provider-compat/BUILD.bazel index f9e23dbdc..51472c9a3 100644 --- a/e2e/rules-python-provider-compat/BUILD.bazel +++ b/e2e/rules-python-provider-compat/BUILD.bazel @@ -1,5 +1,7 @@ load("@aspect_rules_py//py:defs.bzl", "py_binary", "py_library", rules_py_test = "py_test") +load("@rules_python//python:py_library.bzl", rules_python_library = "py_library") load("@rules_python//python:py_test.bzl", rules_python_test = "py_test") +load(":type_stubs_test.bzl", "merged_stubs_test") # One converted library, two consumers: the @rules_python test stands in # for a package the migration hasn't reached yet, the rules_py test for one it @@ -9,10 +11,27 @@ load("@rules_python//python:py_test.bzl", rules_python_test = "py_test") py_library( name = "lib", testonly = True, - srcs = ["lib.py"], + srcs = [ + "lib.py", + "lib.pyi", + ], imports = ["."], ) +# rules_python's PyInfo builder merges the emitted provider field by field, so +# the stub partition must arrive in the shape it expects: direct/transitive +# `pyi_files` populated, and no `.pyi` among `transitive_sources`. +rules_python_library( + name = "rules_python_wrapper", + testonly = True, + deps = [":lib"], +) + +merged_stubs_test( + name = "merged_stubs_test", + target_under_test = ":rules_python_wrapper", +) + rules_python_test( name = "rules_python_consumer_test", srcs = ["consumer_test.py"], diff --git a/e2e/rules-python-provider-compat/MODULE.bazel b/e2e/rules-python-provider-compat/MODULE.bazel index 4ae4a2407..1229edf65 100644 --- a/e2e/rules-python-provider-compat/MODULE.bazel +++ b/e2e/rules-python-provider-compat/MODULE.bazel @@ -11,6 +11,7 @@ requires. module(name = "rules_python_provider_compat") bazel_dep(name = "aspect_rules_py") +bazel_dep(name = "bazel_skylib", version = "1.4.2") bazel_dep(name = "rules_python", version = "1.9.0") local_path_override( diff --git a/e2e/rules-python-provider-compat/lib.pyi b/e2e/rules-python-provider-compat/lib.pyi new file mode 100644 index 000000000..bafa90087 --- /dev/null +++ b/e2e/rules-python-provider-compat/lib.pyi @@ -0,0 +1 @@ +GREETING: str diff --git a/e2e/rules-python-provider-compat/type_stubs_test.bzl b/e2e/rules-python-provider-compat/type_stubs_test.bzl new file mode 100644 index 000000000..e3fb7c2f6 --- /dev/null +++ b/e2e/rules-python-provider-compat/type_stubs_test.bzl @@ -0,0 +1,17 @@ +"""Analysis test: a rules_py stub survives merging into rules_python's PyInfo.""" + +load("@bazel_skylib//lib:unittest.bzl", "analysistest", "asserts") +load("@rules_python//python:py_info.bzl", "PyInfo") + +def _basenames(files): + return sorted([file.basename for file in files.to_list()]) + +def _merged_stubs_test_impl(ctx): + env = analysistest.begin(ctx) + info = analysistest.target_under_test(env)[PyInfo] + asserts.equals(env, [], _basenames(info.direct_pyi_files), "the wrapper declares no stubs of its own") + asserts.equals(env, ["lib.pyi"], _basenames(info.transitive_pyi_files)) + asserts.equals(env, ["lib.py"], _basenames(info.transitive_sources), "the stub is not a runtime source") + return analysistest.end(env) + +merged_stubs_test = analysistest.make(_merged_stubs_test_impl) diff --git a/py/private/py_info.bzl b/py/private/py_info.bzl index 57e174397..c4c6b61ee 100644 --- a/py/private/py_info.bzl +++ b/py/private/py_info.bzl @@ -11,7 +11,8 @@ deps to build the eventual venv or wheel. RulesPyInfo = provider( doc = "Python source, import-path, and virtual-dependency information for a target's dependency closure.", fields = { - "transitive_sources": "depset[File] — postorder depset of first-party `.py` sources in the transitive closure.", + "transitive_sources": "depset[File] — postorder depset of first-party runtime sources in the transitive closure; `.pyi` stubs are excluded.", + "transitive_pyi_files": "depset[File] — postorder depset of `.pyi` type stubs in the transitive closure: stubs listed in `srcs` plus those carried by deps of either ruleset.", "imports": "depset[str] — import roots to place on `sys.path` (rlocation-root-relative).", "virtual_dependencies": "depset[str] — names of required virtual dependencies, independent of their resolution status.", "virtual_resolutions": "depset[struct(virtual, target)] — virtual-dependency-name to concrete-target resolutions.", diff --git a/py/private/py_info_interop.bzl b/py/private/py_info_interop.bzl index 85855e0d6..7f3469fb4 100644 --- a/py/private/py_info_interop.bzl +++ b/py/private/py_info_interop.bzl @@ -3,8 +3,8 @@ The `deps` attribute on rules_py rules accepts targets built by either ruleset. rules_py always emits its own `PyInfo` (`//py/private:py_info.bzl`); native `@rules_python` targets (e.g. -a `py_proto_library`) carry `@rules_python`'s. Both expose `transitive_sources` -and `imports`, which is everything rules_py reads from a foreign dep. +a `py_proto_library`) carry `@rules_python`'s. Both expose the source, type +stub, and import information rules_py reads from a foreign dep. This module is the single place that knows about both providers. Rule code calls these accessors at the API edge instead of loading `@rules_python`'s @@ -37,3 +37,11 @@ def get_py_info(target): if RulesPythonPyInfo in target: return target[RulesPythonPyInfo] return None + +def get_transitive_pyi_files(target): + """Return the `.pyi` closure from either ruleset's `PyInfo`, or an empty depset for targets carrying neither.""" + if PyInfo in target: + return target[PyInfo].transitive_pyi_files + if RulesPythonPyInfo in target: + return target[RulesPythonPyInfo].transitive_pyi_files + return depset() diff --git a/py/private/py_library.bzl b/py/private/py_library.bzl index f56ec7f4c..1f5aef102 100644 --- a/py/private/py_library.bzl +++ b/py/private/py_library.bzl @@ -10,9 +10,18 @@ load("@rules_cc//cc/common:cc_info.bzl", "CcInfo") load("//py/private:providers.bzl", "PyWheelsInfo") load("//py/private:pth.bzl", "make_imports_depset") load("//py/private:py_info.bzl", "PyInfo") -load("//py/private:py_info_interop.bzl", "RulesPythonPyInfo", "get_py_info", "has_py_info") +load("//py/private:py_info_interop.bzl", "RulesPythonPyInfo", "get_py_info", "get_transitive_pyi_files", "has_py_info") load("//py/private:transitions.bzl", "reset_python_flags_transition") +def _is_type_stub(file): + return file.extension == "pyi" + +def _type_stubs(files): + return [file for file in files if _is_type_stub(file)] + +def _runtime_sources(files): + return [file for file in files if not _is_type_stub(file)] + def _make_instrumented_files_info(ctx): return coverage_common.instrumented_files_info( ctx, @@ -26,7 +35,7 @@ def _make_srcs_depset(ctx, extra_depsets = []): # `transitive_sources`. See py_info_interop.bzl. return depset( order = "postorder", - direct = ctx.files.srcs, + direct = _runtime_sources(ctx.files.srcs), transitive = [ get_py_info(target).transitive_sources for target in ctx.attr.deps @@ -34,6 +43,20 @@ def _make_srcs_depset(ctx, extra_depsets = []): ] + extra_depsets, ) +def _make_pyi_depset(ctx, extra_depsets = []): + # Stubs are partitioned out of `transitive_sources` to match rules_python's + # PyInfo shape, where `.pyi` files never count as runtime sources. Venvs + # and launchers put both depsets in runfiles, so a type checker pointed at + # the venv sees stubs next to the modules they annotate. + return depset( + order = "postorder", + direct = _type_stubs(ctx.files.srcs), + transitive = [ + get_transitive_pyi_files(target) + for target in ctx.attr.deps + ] + extra_depsets, + ) + def _make_virtual_depset(ctx): return depset( order = "postorder", @@ -46,13 +69,18 @@ def _make_virtual_depset(ctx): ) def _make_resolved_virtual_depset(target): - transitive = [target[DefaultInfo].files] + # A resolution target's default outputs stand in for its sources (a wheel + # install tree has no PyInfo of its own). Stubs among them belong to the + # pyi depset collected alongside, not to the runtime sources. + direct = _runtime_sources(target[DefaultInfo].files.to_list()) + transitive = [] info = get_py_info(target) if info: transitive.append(info.transitive_sources) return depset( order = "postorder", + direct = direct, transitive = transitive, ) @@ -77,6 +105,7 @@ def _resolve_virtuals(ctx): # Check for duplicate virtual dependency names. Those that map to the same resolution target would have been merged by the depset for us. seen = {} v_srcs = [] + v_pyi_files = [] v_runfiles = [] v_imports = [] @@ -88,6 +117,7 @@ def _resolve_virtuals(ctx): seen.update([[resolution.virtual, i]]) v_srcs.append(_make_resolved_virtual_depset(resolution.target)) + v_pyi_files.append(get_transitive_pyi_files(resolution.target)) v_runfiles.append(resolution.target[DefaultInfo].default_runfiles.files) info = get_py_info(resolution.target) @@ -100,6 +130,7 @@ def _resolve_virtuals(ctx): return struct( srcs = v_srcs, + pyi_files = v_pyi_files, runfiles = v_runfiles, imports = v_imports, ) @@ -153,6 +184,7 @@ def _make_merged_runfiles(ctx, extra_depsets = [], extra_runfiles = [], extra_ru def _py_library_impl(ctx): transitive_srcs = _make_srcs_depset(ctx) + transitive_pyi_files = _make_pyi_depset(ctx) imports = _make_imports_depset(ctx) virtuals = _make_virtual_depset(ctx) resolutions = _make_virtual_resolutions_depset(ctx) @@ -168,6 +200,7 @@ def _py_library_impl(ctx): PyInfo( imports = imports, transitive_sources = transitive_srcs, + transitive_pyi_files = transitive_pyi_files, virtual_dependencies = virtuals, virtual_resolutions = resolutions, ), @@ -180,18 +213,24 @@ def _py_library_impl(ctx): if getattr(ctx.attr, "_emit_rules_python_providers", None) and ctx.attr._emit_rules_python_providers[BuildSettingInfo].value: # Compatibility shim for trees mid-migration: keeps not-yet-converted # @rules_python py_* targets able to depend on this library. - # Only the two fields rules_py models are populated; virtual deps are + # Only the fields rules_py models are populated; virtual deps are # unrepresentable, so a @rules_python consumer never sees them. providers.append(RulesPythonPyInfo( imports = imports, transitive_sources = transitive_srcs, + direct_pyi_files = depset(_type_stubs(ctx.files.srcs)), + transitive_pyi_files = transitive_pyi_files, )) return providers _attrs = dict({ "srcs": attr.label_list( - doc = "Python source files.", + doc = """Python source files. + + `.pyi` type stubs listed here are carried as `PyInfo.transitive_pyi_files` + rather than as runtime sources; both reach the runfiles of venvs and + launchers that include this library.""", allow_files = True, ), "deps": attr.label_list( @@ -239,6 +278,7 @@ py_library_utils = struct( implementation = _py_library_impl, make_imports_depset = _make_imports_depset, make_merged_runfiles = _make_merged_runfiles, + make_pyi_depset = _make_pyi_depset, make_srcs_depset = _make_srcs_depset, make_wheels_depset = _make_wheels_depset, py_library_providers = _providers, diff --git a/py/private/py_unpacked_wheel.bzl b/py/private/py_unpacked_wheel.bzl index 895ae8a49..42b024fb3 100644 --- a/py/private/py_unpacked_wheel.bzl +++ b/py/private/py_unpacked_wheel.bzl @@ -75,6 +75,7 @@ def _py_unpacked_wheel_impl(ctx): ), PyInfo( imports = imports, + transitive_pyi_files = depset(), transitive_sources = depset([unpack_directory]), virtual_dependencies = depset(), virtual_resolutions = depset(), diff --git a/py/private/py_venv/py_venv.bzl b/py/private/py_venv/py_venv.bzl index 9b1738e7b..80acf927d 100644 --- a/py/private/py_venv/py_venv.bzl +++ b/py/private/py_venv/py_venv.bzl @@ -91,6 +91,10 @@ def _assemble_venv_target(ctx, executable, console_scripts): ctx, extra_depsets = virtual_resolution.srcs, ) + pyi_depset = _py_library.make_pyi_depset( + ctx, + extra_depsets = virtual_resolution.pyi_files, + ) runtime_files = depset( direct = assembled.declared_outputs, transitive = [ @@ -110,6 +114,7 @@ def _assemble_venv_target(ctx, executable, console_scripts): bin_python = assembled.bin_python, imports = imports_depset, runtime_runfiles = runfiles, + transitive_pyi_files = pyi_depset, transitive_sources = srcs_depset, runtime_files = runtime_files, console_scripts = depset(assembled.console_scripts), @@ -119,7 +124,9 @@ def _venv_providers(ctx, venv, venv_only, executable = None, include_sources = F """Providers emitted by both the executable and lib variants.""" runfiles = venv.runtime_runfiles.merge(ctx.runfiles(files = venv_only)) if include_sources: - runfiles = runfiles.merge(ctx.runfiles(transitive_files = venv.transitive_sources)) + runfiles = runfiles.merge(ctx.runfiles(transitive_files = depset( + transitive = [venv.transitive_sources, venv.transitive_pyi_files], + ))) return [ DefaultInfo( files = depset([executable]) if executable != None else None, diff --git a/py/private/py_venv/py_venv_exec.bzl b/py/private/py_venv/py_venv_exec.bzl index 2b165debb..d7c7f6a88 100644 --- a/py/private/py_venv/py_venv_exec.bzl +++ b/py/private/py_venv/py_venv_exec.bzl @@ -10,7 +10,7 @@ load("@bazel_lib//lib:expand_make_vars.bzl", "expand_locations", "expand_variabl load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") load("@hermetic_launcher//launcher:lib.bzl", "launcher") load("//py/private:py_info.bzl", "PyInfo") -load("//py/private:py_info_interop.bzl", "RulesPythonPyInfo", "get_py_info", "has_py_info") +load("//py/private:py_info_interop.bzl", "RulesPythonPyInfo", "get_py_info", "get_transitive_pyi_files", "has_py_info") load("//py/private:py_semantics.bzl", _py_semantics = "semantics") load("//py/private:transitions.bzl", "reset_python_flags_transition", "venv_python_transition") load(":types.bzl", "VirtualenvInfo", "venv_root") @@ -122,6 +122,9 @@ def _py_venv_exec_impl(ctx): get_py_info(target).transitive_sources for target in ctx.attr.data if has_py_info(target) + ] + [ + get_transitive_pyi_files(target) + for target in ctx.attr.data ] # First-party import sources attach explicitly; everything else the venv @@ -130,7 +133,7 @@ def _py_venv_exec_impl(ctx): # re-deriving the rest. runfiles = ctx.runfiles( files = ctx.files.data + [main], - transitive_files = depset(transitive = [vinfo.transitive_sources] + data_sources), + transitive_files = depset(transitive = [vinfo.transitive_sources, vinfo.transitive_pyi_files] + data_sources), ).merge(vinfo.runtime_runfiles).merge_all( [target[DefaultInfo].default_runfiles for target in ctx.attr.data], ) @@ -157,6 +160,7 @@ def _py_venv_exec_impl(ctx): # launcher will run with. `srcs` / `deps` live on the # sibling venv, not on this rule. imports = vinfo.imports, + transitive_pyi_files = vinfo.transitive_pyi_files, transitive_sources = vinfo.transitive_sources, virtual_dependencies = depset(), virtual_resolutions = depset(), @@ -171,6 +175,7 @@ def _py_venv_exec_impl(ctx): if ctx.attr._emit_rules_python_providers[BuildSettingInfo].value: providers.append(RulesPythonPyInfo( imports = vinfo.imports, + transitive_pyi_files = vinfo.transitive_pyi_files, transitive_sources = vinfo.transitive_sources, )) diff --git a/py/private/py_venv/types.bzl b/py/private/py_venv/types.bzl index 76676f145..a061fb54f 100644 --- a/py/private/py_venv/types.bzl +++ b/py/private/py_venv/types.bzl @@ -23,6 +23,7 @@ binary's launcher exec's the venv's `bin_python`. "bin_python": "File — the venv's bin/python symlink. Callers needing a launcher target point here.", "imports": "depset[str] — rlocation-root-relative import paths covered by this venv. Mirrors `PyInfo.imports` of the venv's dep closure.", "runtime_runfiles": "Runfiles — venv, wheels, and data without Python import sources.", + "transitive_pyi_files": "depset[File] — `.pyi` type stubs from `srcs`, `deps` and virtual-resolution targets. Kept apart from `transitive_sources` to mirror `PyInfo`; venvs that include sources put both in runfiles.", "transitive_sources": "depset[File] — source artifacts carried by this venv: its own `srcs`, sources from `deps` that emit `PyInfo`, and files contributed by virtual-resolution targets. Surfaced by py_binary as `PyInfo.transitive_sources` so downstream consumers see the same source closure they'd see if srcs/deps lived on the binary directly.", "runtime_files": "depset[File] — generated venv support files and the runfiles library; excludes dependency and interpreter runfiles.", "console_scripts": "depset[File] — `bin/` console-script wrappers. Not part of `runtime_runfiles`; launchers add them via `include_console_scripts`.", diff --git a/py/tests/py_venv_conflict/collision_order_test.bzl b/py/tests/py_venv_conflict/collision_order_test.bzl index beb7cb3d8..bba770399 100644 --- a/py/tests/py_venv_conflict/collision_order_test.bzl +++ b/py/tests/py_venv_conflict/collision_order_test.bzl @@ -68,6 +68,7 @@ printf 'VALUE = "namespace"\n' > "$site/mixed_top/from_namespace.py" ), PyInfo( imports = depset([site_packages]), + transitive_pyi_files = depset(), transitive_sources = depset([install_tree]), virtual_dependencies = depset(), virtual_resolutions = depset(), @@ -200,6 +201,7 @@ printf 'native' > "$site/collision_order/native_extension.so" ), PyInfo( imports = depset([site_packages]), + transitive_pyi_files = depset(), transitive_sources = depset([install_tree]), virtual_dependencies = depset(), virtual_resolutions = depset(), diff --git a/py/tests/py_venv_conflict/site_merge_order_test.bzl b/py/tests/py_venv_conflict/site_merge_order_test.bzl index 071819fae..13ea2a565 100644 --- a/py/tests/py_venv_conflict/site_merge_order_test.bzl +++ b/py/tests/py_venv_conflict/site_merge_order_test.bzl @@ -121,6 +121,7 @@ printf 'VALUE = %s\n' "$4" > "$site/other/from_final.py" ), PyInfo( imports = depset(site_packages_paths), + transitive_pyi_files = depset(), transitive_sources = depset(install_trees), virtual_dependencies = depset(), virtual_resolutions = depset(), @@ -230,6 +231,7 @@ printf 'VALUE = %s\n' "$4" > "$site/mixed/sibling.py" ), PyInfo( imports = depset(site_packages_paths), + transitive_pyi_files = depset(), transitive_sources = depset(install_trees), virtual_dependencies = depset(), virtual_resolutions = depset(), diff --git a/py/tests/type-stubs/BUILD.bazel b/py/tests/type-stubs/BUILD.bazel new file mode 100644 index 000000000..09f04cf5c --- /dev/null +++ b/py/tests/type-stubs/BUILD.bazel @@ -0,0 +1,138 @@ +load("//py:defs.bzl", "py_binary", "py_library", "py_test", "py_venv") +load( + ":type_stubs_test.bzl", + "first_party_stubs_test", + "launcher_stubs_test", + "merged_stubs_test", + "rules_python_provider_stubs_test", + "rules_python_pyi_fixture", + "venv_stubs_test", +) + +package(default_testonly = True) + +# A first-party library shipping its own stub next to the module. +py_library( + name = "annotated", + srcs = [ + "annotated.py", + "annotated.pyi", + ], + imports = ["."], +) + +first_party_stubs_test( + name = "first_party_stubs_test", + target_under_test = ":annotated", +) + +# Stubs that only exist in @rules_python's PyInfo (the py_proto_library shape). +rules_python_pyi_fixture( + name = "foreign_stubs", + srcs = ["foreign.pyi"], +) + +py_library( + name = "middle", + deps = [":foreign_stubs"], +) + +# Both origins merge one level further up. +py_library( + name = "consumer", + deps = [ + ":annotated", + ":middle", + ], +) + +merged_stubs_test( + name = "merged_stubs_test", + target_under_test = ":consumer", +) + +# Stubs reached only through a virtual dependency's resolution. +py_library( + name = "stubby_impl", + srcs = [ + "stubby.py", + "stubby.pyi", + ], + imports = ["."], +) + +py_library( + name = "needs_stubby", + virtual_deps = ["stubby"], +) + +py_binary( + name = "bin", + srcs = ["main.py"], + main = "main.py", + resolutions = {"stubby": ":stubby_impl"}, + deps = [ + ":consumer", + ":needs_stubby", + ], +) + +launcher_stubs_test( + name = "launcher_stubs_test", + target_under_test = ":bin", +) + +py_venv( + name = "venv", + srcs = ["main.py"], + resolutions = {"stubby": ":stubby_impl"}, + deps = [ + ":consumer", + ":needs_stubby", + ], +) + +venv_stubs_test( + name = "venv_stubs_test", + target_under_test = ":venv", +) + +rules_python_provider_stubs_test( + name = "rules_python_provider_library_test", + expected_direct = ["annotated.pyi"], + expected_transitive = ["annotated.pyi"], + target_under_test = ":annotated", +) + +rules_python_provider_stubs_test( + name = "rules_python_provider_consumer_test", + expected_direct = [], + expected_transitive = [ + "annotated.pyi", + "foreign.pyi", + ], + target_under_test = ":consumer", +) + +rules_python_provider_stubs_test( + name = "rules_python_provider_launcher_test", + expected_direct = [], + expected_transitive = [ + "annotated.pyi", + "foreign.pyi", + "stubby.pyi", + ], + target_under_test = ":bin", +) + +# Execution-time check: the stubs land in runfiles beside their modules. +py_test( + name = "stubs_in_runfiles_test", + srcs = ["main.py"], + main = "main.py", + resolutions = {"stubby": ":stubby_impl"}, + deps = [ + ":consumer", + ":needs_stubby", + ], +) diff --git a/py/tests/type-stubs/annotated.py b/py/tests/type-stubs/annotated.py new file mode 100644 index 000000000..52915e559 --- /dev/null +++ b/py/tests/type-stubs/annotated.py @@ -0,0 +1,2 @@ +def greeting(name: str) -> str: + return "hello, " + name diff --git a/py/tests/type-stubs/annotated.pyi b/py/tests/type-stubs/annotated.pyi new file mode 100644 index 000000000..bb97ee22c --- /dev/null +++ b/py/tests/type-stubs/annotated.pyi @@ -0,0 +1 @@ +def greeting(name: str) -> str: ... diff --git a/py/tests/type-stubs/foreign.pyi b/py/tests/type-stubs/foreign.pyi new file mode 100644 index 000000000..092e1bed5 --- /dev/null +++ b/py/tests/type-stubs/foreign.pyi @@ -0,0 +1 @@ +def foreign(value: int) -> int: ... diff --git a/py/tests/type-stubs/main.py b/py/tests/type-stubs/main.py new file mode 100644 index 000000000..15c1f3a9a --- /dev/null +++ b/py/tests/type-stubs/main.py @@ -0,0 +1,10 @@ +import os + +import annotated +import stubby + +for module in (annotated, stubby): + stub = os.path.splitext(module.__file__)[0] + ".pyi" + assert os.path.exists(stub), "missing type stub next to " + module.__file__ + +print(annotated.greeting(stubby.VALUE)) diff --git a/py/tests/type-stubs/stubby.py b/py/tests/type-stubs/stubby.py new file mode 100644 index 000000000..5d168ee25 --- /dev/null +++ b/py/tests/type-stubs/stubby.py @@ -0,0 +1 @@ +VALUE = "stubby" diff --git a/py/tests/type-stubs/stubby.pyi b/py/tests/type-stubs/stubby.pyi new file mode 100644 index 000000000..c8f2dc2fe --- /dev/null +++ b/py/tests/type-stubs/stubby.pyi @@ -0,0 +1 @@ +VALUE: str diff --git a/py/tests/type-stubs/type_stubs_test.bzl b/py/tests/type-stubs/type_stubs_test.bzl new file mode 100644 index 000000000..e67dcc935 --- /dev/null +++ b/py/tests/type-stubs/type_stubs_test.bzl @@ -0,0 +1,110 @@ +"""Analysis coverage for `.pyi` type-stub propagation. + +Stubs travel in `PyInfo.transitive_pyi_files`, partitioned away from +`transitive_sources`, whether they were listed in a rules_py `srcs`, reached +through a `@rules_python` dependency, or pulled in by a virtual-dependency +resolution. Venvs and launchers put them in runfiles next to the modules they +annotate. +""" + +load("@bazel_skylib//lib:unittest.bzl", "analysistest", "asserts") +load("//py/private:py_info.bzl", "PyInfo") +load("//py/private:py_info_interop.bzl", "RulesPythonPyInfo") +load("//py/private/py_venv:defs.bzl", "VirtualenvInfo") + +_ALL_STUBS = ["annotated.pyi", "foreign.pyi", "stubby.pyi"] + +# Resolved here so the label is canonical by the time skylib's transition, +# defined in another repo, receives it. +_EMIT_RULES_PYTHON_PROVIDERS = str(Label("//py/private:emit_rules_python_providers")) + +def _rules_python_pyi_fixture_impl(ctx): + return [RulesPythonPyInfo( + transitive_sources = depset(), + direct_pyi_files = depset(ctx.files.srcs), + transitive_pyi_files = depset(ctx.files.srcs), + )] + +# Stands in for a py_proto_library: a target whose only Python payload is +# stubs, advertised solely through @rules_python's provider. +rules_python_pyi_fixture = rule( + implementation = _rules_python_pyi_fixture_impl, + attrs = { + "srcs": attr.label_list(allow_files = [".pyi"]), + }, +) + +def _basenames(files): + return sorted([file.basename for file in files.to_list()]) + +def _has(paths, suffix): + return any([path.endswith(suffix) for path in paths]) + +def _runfile_paths(target): + return [file.short_path for file in target[DefaultInfo].default_runfiles.files.to_list()] + +def _first_party_stubs_test_impl(ctx): + env = analysistest.begin(ctx) + target = analysistest.target_under_test(env) + asserts.equals(env, ["annotated.pyi"], _basenames(target[PyInfo].transitive_pyi_files)) + asserts.equals(env, ["annotated.py"], _basenames(target[PyInfo].transitive_sources), "stubs are not runtime sources") + asserts.equals(env, ["annotated.py", "annotated.pyi"], _basenames(target[DefaultInfo].files), "default outputs keep every listed src") + return analysistest.end(env) + +first_party_stubs_test = analysistest.make(_first_party_stubs_test_impl) + +def _merged_stubs_test_impl(ctx): + env = analysistest.begin(ctx) + target = analysistest.target_under_test(env) + asserts.equals(env, ["annotated.pyi", "foreign.pyi"], _basenames(target[PyInfo].transitive_pyi_files)) + asserts.equals(env, ["annotated.py"], _basenames(target[PyInfo].transitive_sources), "a stubs-only foreign dep contributes no runtime sources") + return analysistest.end(env) + +merged_stubs_test = analysistest.make(_merged_stubs_test_impl) + +def _launcher_stubs_test_impl(ctx): + env = analysistest.begin(ctx) + target = analysistest.target_under_test(env) + asserts.equals(env, _ALL_STUBS, _basenames(target[PyInfo].transitive_pyi_files), "the launcher surfaces the venv's stub closure, resolutions included") + asserts.false(env, _has(_basenames(target[PyInfo].transitive_sources), ".pyi")) + paths = _runfile_paths(target) + for stub in _ALL_STUBS: + asserts.true(env, _has(paths, "/" + stub), "launcher runfiles carry " + stub) + asserts.true(env, _has(paths, "/stubby.py"), "resolved runtime sources still travel") + return analysistest.end(env) + +launcher_stubs_test = analysistest.make(_launcher_stubs_test_impl) + +def _venv_stubs_test_impl(ctx): + env = analysistest.begin(ctx) + target = analysistest.target_under_test(env) + asserts.equals(env, _ALL_STUBS, _basenames(target[VirtualenvInfo].transitive_pyi_files)) + paths = _runfile_paths(target) + for stub in _ALL_STUBS: + asserts.true(env, _has(paths, "/" + stub), "public venv runfiles carry " + stub) + return analysistest.end(env) + +venv_stubs_test = analysistest.make(_venv_stubs_test_impl) + +def _rules_python_provider_stubs_test_impl(ctx): + env = analysistest.begin(ctx) + target = analysistest.target_under_test(env) + info = target[RulesPythonPyInfo] + asserts.equals(env, ctx.attr.expected_direct, _basenames(info.direct_pyi_files)) + asserts.equals(env, ctx.attr.expected_transitive, _basenames(info.transitive_pyi_files)) + asserts.false(env, _has(_basenames(info.transitive_sources), ".pyi"), "@rules_python consumers never see stubs as sources") + return analysistest.end(env) + +# Under the migration flag the emitted @rules_python provider mirrors the +# stub partition, so a rules_python type-check aspect over a half-migrated +# tree sees the same closure on both sides of the boundary. +rules_python_provider_stubs_test = analysistest.make( + _rules_python_provider_stubs_test_impl, + attrs = { + "expected_direct": attr.string_list(mandatory = True), + "expected_transitive": attr.string_list(mandatory = True), + }, + config_settings = { + _EMIT_RULES_PYTHON_PROVIDERS: True, + }, +) diff --git a/uv/private/whl_install/rule.bzl b/uv/private/whl_install/rule.bzl index 919384a99..cdfd0c847 100644 --- a/uv/private/whl_install/rule.bzl +++ b/uv/private/whl_install/rule.bzl @@ -303,6 +303,7 @@ def _whl_install(ctx): transitive_sources = depset([ install_dir, ]), + transitive_pyi_files = depset(), imports = depset([site_packages_rfpath]), virtual_dependencies = depset(), virtual_resolutions = depset(),