From c7890559ef9c0406a118912e2cbc84f2ccd55221 Mon Sep 17 00:00:00 2001 From: Jason Bedard Date: Wed, 16 Sep 2026 16:33:03 -0700 Subject: [PATCH 1/5] docs: document minimum python versions --- README.md | 14 ++++++++++++++ docs/interpreter.md | 8 +++++--- pyproject.toml | 2 +- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index bd5d90d72..7aeab4097 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,20 @@ Built-in rules for creating optimized container images: bazel_dep(name = "aspect_rules_py", version = "1.11.2") ``` +### Requirements + +The minimum supported Python version is **3.10**. The launcher, test runners, and build +tools that run under your configured interpreter use 3.10 syntax, and CI only exercises +3.10 and newer. Older interpreters can still be fetched via `interpreters.configure()`, +but `py_binary` and `py_test` targets will fail at startup on them. + +Some `uv` features need newer versions: + +| Feature | Python | +| ----------------------------------------------------------------------------------------------------------------------- | ------ | +| Free-threaded interpreters (`freethreaded = True`), [first shipped in CPython 3.13](https://peps.python.org/pep-0703/) | 3.13+ | +| `pyproject.toml` parsing in sdist native-dependency detection (needs stdlib `tomllib`) | 3.11+ | + ### Quick Start Load rules from `aspect_rules_py` in your `BUILD` files: diff --git a/docs/interpreter.md b/docs/interpreter.md index c3c9e15e1..e3e9cc74b 100644 --- a/docs/interpreter.md +++ b/docs/interpreter.md @@ -14,8 +14,10 @@ are discovered automatically from PBS release artifacts and cached in your it — no repinning, no manifest regeneration. **No editorial decisions.** We don't decide which Python versions you can use. -Any version published in a PBS release is available. Need Python 3.8? Add an -older release date that includes it. +Any version published in a PBS release is available. Need a version that newer +releases dropped? Add an older release date that includes it. Note that the +rules themselves require Python 3.10 or newer at runtime (see +[Requirements](../README.md#requirements)). **Windows and cross-platform support.** 9 platforms are registered out of the box, including Windows (x86_64, aarch64, i686), Linux (glibc and musl), and @@ -65,7 +67,7 @@ interpreters.configure( ) interpreters.toolchain(python_version = "3.12") -interpreters.toolchain(python_version = "3.8") # Resolved from 20241002 +interpreters.toolchain(python_version = "3.10") # Resolved from 20241002 once newer releases drop it use_repo(interpreters, "python_interpreters") register_toolchains("@python_interpreters//:all") diff --git a/pyproject.toml b/pyproject.toml index eb142bf1f..c1e9870f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ dev-dependencies = [ ] [tool.ruff] -target-version = "py39" +target-version = "py310" extend-exclude = [ # Intentionally unparsable .py-as-data fixture. "py/tests/py-venv-multi-exec/broken_data.py", From ed8f20b53de206c29e0c66cd62412d32d0247ec2 Mon Sep 17 00:00:00 2001 From: Jason Bedard Date: Tue, 8 Sep 2026 21:50:58 -0700 Subject: [PATCH 2/5] feat: add pyc compilation of first-party python code via actions --- README.md | 74 +++- docs/api/py.md | 10 +- docs/interpreter.md | 5 +- docs/migrating.md | 8 + e2e/README.md | 6 +- e2e/cases/coverage-drivers/BUILD.bazel | 12 + e2e/cases/coverage-drivers/test.sh | 1 + e2e/cases/oci/py_image_layer/BUILD.bazel | 215 ++++++++++ .../oci/py_image_layer/assert_tar_paths.py | 5 + .../image_layer_analysis_tests.bzl | 44 ++ .../my_app_layers_fp_pyc_listing.yaml | 49 +++ .../my_app_layers_fp_pyc_only_listing.yaml | 44 ++ e2e/cases/oci/py_venv_image_layer/BUILD.bazel | 168 +++++++- .../my_app_pyc_amd64_layers_listing.yaml | 40 ++ .../my_app_pyc_only_amd64_layers_listing.yaml | 35 ++ e2e/cases/oci/test.sh | 16 + e2e/cases/pyc/BUILD.bazel | 138 +++++++ e2e/cases/pyc/entry_a.py | 4 + e2e/cases/pyc/entry_versioned.py | 16 + e2e/cases/pyc/noop.py | 1 + e2e/cases/pyc/pex_no_bytecode_test.py | 11 + e2e/cases/pyc/shared_lib.py | 1 + e2e/cases/pyc/test.sh | 91 +++++ e2e/cases/tools/asserts.bzl | 18 +- e2e/rules-python-interop/BUILD.bazel | 63 +++ .../cross-package/BUILD.bazel | 1 + .../cross-package/helper.py | 2 + e2e/rules-python-interop/precompiled_lib.py | 2 + .../reset-data-edges/BUILD.bazel | 47 ++- .../reset-data-edges/tests.bzl | 90 +++- .../rules_python_dep_test.py | 32 ++ e2e/rules-python-interop/test.sh | 76 +++- .../virtual-deps/django/BUILD.bazel | 10 + .../django/resolution_bytecode_test.py | 13 + e2e/rules-python-protobuf/BUILD.bazel | 46 +++ e2e/rules-python-protobuf/MODULE.bazel | 12 +- e2e/rules-python-protobuf/bytecode_test.py | 46 +++ .../native_greeting.proto | 6 + e2e/rules-python-protobuf/native_test.py | 4 + py/BUILD.bazel | 9 + py/defs.bzl | 23 ++ py/private/BUILD.bazel | 19 + py/private/interpreter/runtime.bzl | 26 ++ py/private/py_image_layer.bzl | 151 ++++++- py/private/py_info_interop.bzl | 13 + py/private/py_library.bzl | 23 +- py/private/py_pex_binary.bzl | 8 +- py/private/py_pytest_test.bzl | 4 + py/private/py_venv/BUILD.bazel | 2 + py/private/py_venv/py_venv.bzl | 18 +- py/private/py_venv/py_venv_exec.bzl | 153 +++++-- py/private/pyc.bzl | 385 ++++++++++++++++++ py/private/pyc_compile.py | 152 +++++++ py/private/toolchain/types.bzl | 5 +- py/private/transitions.bzl | 42 +- py/private/unittest_main.py | 6 +- py/tests/main-from-genrule/BUILD.bazel | 24 ++ py/tests/py-venv-multi-exec/BUILD.bazel | 373 ++++++++++++++++- py/tests/py-venv-multi-exec/entry_data.py | 2 + py/tests/py-venv-multi-exec/entry_sidecar.py | 6 + .../py-venv-multi-exec/entry_tree_source.py | 4 + .../py-venv-multi-exec/entry_versioned.py | 6 + .../py-venv-multi-exec/pyc_runfiles_test.bzl | 71 ++++ py/tests/py-venv-multi-exec/sidecar.txt | 1 + py/tests/py-venv-multi-exec/tree_module.py | 1 + py/tests/pyc-compile/BUILD.bazel | 78 ++++ py/tests/pyc-compile/assert_query_contains.sh | 7 + py/tests/pyc-compile/bytecode_compat_test.bzl | 65 +++ .../pyc-compile/pyc_layout_actions_test.bzl | 50 +++ .../snapshots/version_check_test.pyc | Bin 0 -> 12894 bytes py/tests/pyc-compile/version_check_test.py | 148 +++++++ py/tests/tmpdir-af-unix/BUILD.bazel | 50 +++ pyproject.toml | 3 +- 73 files changed, 3280 insertions(+), 110 deletions(-) create mode 100644 e2e/cases/oci/py_image_layer/snapshots/my_app_layers_fp_pyc_listing.yaml create mode 100644 e2e/cases/oci/py_image_layer/snapshots/my_app_layers_fp_pyc_only_listing.yaml create mode 100644 e2e/cases/oci/py_venv_image_layer/snapshots/my_app_pyc_amd64_layers_listing.yaml create mode 100644 e2e/cases/oci/py_venv_image_layer/snapshots/my_app_pyc_only_amd64_layers_listing.yaml create mode 100644 e2e/cases/pyc/BUILD.bazel create mode 100644 e2e/cases/pyc/entry_a.py create mode 100644 e2e/cases/pyc/entry_versioned.py create mode 100644 e2e/cases/pyc/noop.py create mode 100644 e2e/cases/pyc/pex_no_bytecode_test.py create mode 100644 e2e/cases/pyc/shared_lib.py create mode 100755 e2e/cases/pyc/test.sh create mode 100644 e2e/rules-python-interop/cross-package/BUILD.bazel create mode 100644 e2e/rules-python-interop/cross-package/helper.py create mode 100644 e2e/rules-python-interop/precompiled_lib.py create mode 100644 e2e/rules-python-interop/rules_python_dep_test.py create mode 100644 e2e/rules-python-interop/virtual-deps/django/resolution_bytecode_test.py create mode 100644 e2e/rules-python-protobuf/bytecode_test.py create mode 100644 e2e/rules-python-protobuf/native_greeting.proto create mode 100644 e2e/rules-python-protobuf/native_test.py create mode 100644 py/private/pyc.bzl create mode 100644 py/private/pyc_compile.py create mode 100644 py/tests/py-venv-multi-exec/entry_sidecar.py create mode 100644 py/tests/py-venv-multi-exec/entry_tree_source.py create mode 100644 py/tests/py-venv-multi-exec/pyc_runfiles_test.bzl create mode 100644 py/tests/py-venv-multi-exec/sidecar.txt create mode 100644 py/tests/py-venv-multi-exec/tree_module.py create mode 100644 py/tests/pyc-compile/BUILD.bazel create mode 100755 py/tests/pyc-compile/assert_query_contains.sh create mode 100644 py/tests/pyc-compile/bytecode_compat_test.bzl create mode 100644 py/tests/pyc-compile/pyc_layout_actions_test.bzl create mode 100644 py/tests/pyc-compile/snapshots/version_check_test.pyc create mode 100644 py/tests/pyc-compile/version_check_test.py diff --git a/README.md b/README.md index 7aeab4097..e57beb7fa 100644 --- a/README.md +++ b/README.md @@ -178,6 +178,75 @@ py_test( ) ``` +### First-party bytecode + +`py_binary` and `py_test` accept `pyc = "source" | "pyc" | "pyc_only"` +(a `select()` value is also accepted). The default is `source`, and may be +changed for inheriting targets with +`--@aspect_rules_py//py:pyc=source|pyc|pyc_only`; an explicit `pyc` +attribute pins the target's mode regardless of the flag. + +- `source` packages first-party `.py` sources. +- `pyc` packages sources and PEP 3147 `__pycache__` bytecode. +- `pyc_only` packages colocated first-party `.pyc` files without their source. + Tracebacks then carry no source lines, and every first-party source must be + directly owned by a rules_py `py_*` target. A `.py` file also declared through + `data` remains available as source because explicit runtime data takes + precedence over source stripping. + +Only `.py` files listed in a `py_*` target's `srcs` by their own file label +(checked-in or generated) are compiled. A `.py` file reached through a rule +target in `srcs` — a `filegroup`, a `genrule`, or another `py_library` — +stays in source form; `pyc_only` reports it as missing bytecode. + +Only sources directly owned by a `py_*` target's package are compiled. +Files a target lists from another package have no bytecode: `pyc` mode +ships them as plain source, and `pyc_only` fails analysis listing them. + +Dependencies built by rules_python rules (`py_proto_library`, pip hub +packages, unconverted `py_library` targets) are compiled by rules_py through +an aspect over `deps`. No rules_python `precompile` attribute or flag is +needed; with interpreters from the rules_py interpreter extension, +rules_python never precompiles. Bytecode rules_python does compile is reused. + +Limits: a `*_pb2.py` is compiled only when its `py_proto_library` shares the +`proto_library`'s package; the protobuf runtime is runfiles data and stays +source; a rules_python `py_library` carries its sources in its own runfiles, +so under `pyc_only` they ship beside the bytecode until the library is +converted to rules_py. + +Bytecode is compiled by an exec-platform interpreter of the target's +implementation/cache tag and feature version (major.minor; prereleases must +match exactly) when one is provisioned (the default with the +rules_py interpreter hub), so cross-platform builds work out of the box. With +toolchains not provisioned by rules_py (e.g. rules_python runtimes) the +target interpreter itself must be runnable on the build host, unless the +toolchain supplies a custom `pyc_compile_tool`. + +Compilation runs one `PyCompile` action per source, served by a Bazel +persistent worker so the interpreter starts once per worker rather than per +file; size the pool with `--worker_max_instances=PyCompile=N`. + +`bazel coverage` always runs `pyc_only` targets from sources so coverage.py +can instrument them. + +Bytecode is compiled at optimization level 0. Under `pyc` an optimized +interpreter (`-O`/`-OO`, `PYTHONOPTIMIZE`) ignores the cache and runs from +source. Under `pyc_only` there is no source, so `-O`/`-OO` +`interpreter_options` and `PYTHONOPTIMIZE` set or inherited by the launcher +or its venv fail analysis; with `isolated = False` a `PYTHONOPTIMIZE` from the +invoking shell still runs the level-0 bytecode unoptimized. + +`py_unittest_test` supports `pyc_only`. Because pytest collects `.py` source +files, `py_pytest_test` automatically falls back to `pyc` when `pyc_only` is +requested explicitly or through the global flag. + +`py_image_layer` accepts the same `pyc` attribute as a plain string (a +transition reads it, so no `select()`); unset, it inherits the +`--@aspect_rules_py//py:pyc` flag. Binaries with an unset `pyc` attribute +follow the image's mode automatically; a binary whose explicit `pyc` +attribute disagrees with the image fails analysis. + ## Dependency Resolution with `uv` `aspect_rules_py//uv` is our alternative to `rules_python`'s `pip.parse`: @@ -546,8 +615,9 @@ bazel run //:gazelle 1. **Swap the rules**: Load `py_binary`, `py_library`, `py_test` from `@aspect_rules_py//py:defs.bzl` instead of `@rules_python//python:defs.bzl` 2. **Migrate dependencies**: Replace `pip.parse` with `uv.declare_hub` and generate a `uv.lock` -3. **Optionally migrate toolchains**: Replace `rules_python` interpreter provisioning with - the `aspect_rules_py` interpreter extension for fully independent hermetic interpreters +3. **Migrate toolchains**: Replace `rules_python` interpreter provisioning with + the `aspect_rules_py` interpreter extension; rules_py then also compiles + bytecode for the rules_python-built targets that remain For detailed migration guidance, see [docs/migrating.md](docs/migrating.md). diff --git a/docs/api/py.md b/docs/api/py.md index f8fbde999..a08384541 100644 --- a/docs/api/py.md +++ b/docs/api/py.md @@ -464,7 +464,7 @@ workspace symlink in one step, set `expose_venv_link = True`. | name | Name of the rule. | none | | srcs | Python source files. | `[]` | | main | Entry point. Like rules_python, this is treated as a suffix of a file that should appear among the srcs. If absent, then `[name].py` is tried. As a final fallback, if the srcs has a single file, that is used as the main.

Note: the fallback runs at macro-evaluation time and operates on label strings, not resolved files — it cannot inspect a generated target's output basename. If `main` would resolve to a file produced by another rule (e.g. a `genrule` whose output happens to be `.py`), the macro can't see that and you must pass `main =` explicitly. | `None` | -| kwargs | additional named parameters forwarded to the underlying rule and the sibling py_venv. Three extras are handled by this macro:

* `include_console_scripts` (bool, default `False`) — when `True`, the binary's runfiles include the venv's wheel-declared `bin/` console-script wrappers so subprocesses can invoke them by name via `PATH`. Independent of `expose_venv`: the `.venv` target always carries wrappers for `bazel run`, the binary only with this flag. * `expose_venv` (bool, default `False`) — when `True`, emit a sibling `:{name}.venv` py_venv carrying all venv-shaping attrs (deps, imports, package_collisions, include_*_site_packages, interpreter_options). The `.venv` target is runnable (`bazel run :{name}.venv` drops into the hermetic interpreter). * `expose_venv_link` (bool, default `False`) — when `True`, additionally emit a `:{name}.venv_link` py_venv_link. `bazel run :{name}.venv_link` links the target's runfiles tree into the workspace and prints the nested venv path suitable for an IDE's interpreter setting. Implies `expose_venv = True`; passing `expose_venv = False, expose_venv_link = True` explicitly is rejected with a clear error. Equivalent to declaring an explicit `py_venv_link(name = "{name}.venv_link", venv = ":{name}.venv")` alongside the binary. | none | +| kwargs | additional named parameters forwarded to the underlying rule and the sibling py_venv. Three extras are handled by this macro:

* `include_console_scripts` (bool, default `False`) — when `True`, the binary's runfiles include the venv's wheel-declared `bin/` console-script wrappers so subprocesses can invoke them by name via `PATH`. Independent of `expose_venv`: the `.venv` target always carries wrappers for `bazel run`, the binary only with this flag. * `expose_venv` (bool, default `False`) — when `True`, emit a sibling `:{name}.venv` py_venv carrying all venv-shaping attrs (deps, imports, package_collisions, include_*_site_packages, interpreter_options). The `.venv` target is runnable (`bazel run :{name}.venv` drops into the hermetic interpreter). * `expose_venv_link` (bool, default `False`) — when `True`, additionally emit a `:{name}.venv_link` py_venv_link. `bazel run :{name}.venv_link` links the target's runfiles tree into the workspace and prints the nested venv path suitable for an IDE's interpreter setting. Implies `expose_venv = True`; passing `expose_venv = False, expose_venv_link = True` explicitly is rejected with a clear error. Equivalent to declaring an explicit `py_venv_link(name = "{name}.venv_link", venv = ":{name}.venv")` alongside the binary. * `pyc` (string) — first-party bytecode packaging. `"source"` ships only `.py` sources; `"pyc"` additionally ships PEP 3147 `__pycache__` bytecode; `"pyc_only"` ships colocated sourceless `.pyc` files (tracebacks then carry no source lines). Unset inherits the global `--@aspect_rules_py//py:pyc` flag; an explicit value pins the mode regardless of the flag. Configurable: a `select()` value is accepted. Bytecode compilation requires an executable, bytecode-compatible target interpreter, and `"pyc_only"` requires every first-party source to be directly owned by a rules_py `py_*` target; other first-party sources ship as source under `"pyc"`. A `.py` file also declared through `data` remains source because explicit runtime data takes precedence over source stripping. Only `.py` files listed in `srcs` by their own file label are compiled; files reached through a rule target in `srcs` (filegroup, genrule, py_library) stay source. `bazel coverage` always runs `"pyc_only"` targets from sources so coverage.py can instrument them. `"pyc_only"` rejects `-O`/`-OO` interpreter options and `PYTHONOPTIMIZE` settings because its level-0 bytecode has no source to fall back to; `"pyc"` runs from source instead. | none | @@ -476,8 +476,8 @@ load("@aspect_rules_py//py:defs.bzl", "py_image_layer") py_image_layer(name, binary, groups, group_execution_requirements, group_compress_levels, group_compression, group_compressors, allow_non_oci_layers, - warn_remote_cache_threshold_mb, warn_layer_count, platform, layer_tier, launcher_dir, - binaries, **kwargs) + warn_remote_cache_threshold_mb, warn_layer_count, platform, layer_tier, pyc, + launcher_dir, binaries, **kwargs) Create OCI-compatible tars from one or more py_binary targets. @@ -517,6 +517,7 @@ or pin a tier to a specific rule via the `py_layer_tier` attr below. | warn_layer_count | Warn when total layers exceed this. Default: 90. | `90` | | platform | Platform transition target. | `None` | | layer_tier | Optional py_layer_tier target pinned for this rule. Sets the `@aspect_rules_py//py:layer_tier` label_flag via the rule transition, overriding any command-line value for this rule's subgraph. | `None` | +| pyc | First-party bytecode mode for the image: "source", "pyc", or "pyc_only". Sets the `@aspect_rules_py//py:pyc` flag via the rule transition, so a `select()` is not accepted; empty (the default) inherits the flag's value. Binaries with an unset `pyc` attribute follow it automatically; a binary whose explicit `pyc` attribute disagrees fails analysis. | `""` | | launcher_dir | Absolute image directory for the binary launchers. Defaults to /app/bin with multiple binaries. Set RUNFILES_DIR=/app.runfiles in the image. | `""` | | binaries | Alternative to binary. A nonempty list of py_binary targets to include in the image. | `None` | | kwargs | Forwarded to inner rule. | none | @@ -568,6 +569,9 @@ Pytest is always the driver, so the entrypoint wiring is unambiguous. Include the `pytest` package (and `coverage`, if you want coverage) in `deps`. +Because pytest collects `.py` source paths, `pyc_only` requests fall back +to source-retaining `pyc` mode for these targets. + Every file in `srcs` is a test module that pytest collects (scoped to the target, not the whole runfiles tree). Put importable support code in `deps` and pytest's `conftest.py` in `data`; to select tests by name pattern, use diff --git a/docs/interpreter.md b/docs/interpreter.md index e3e9cc74b..2d5d67387 100644 --- a/docs/interpreter.md +++ b/docs/interpreter.md @@ -318,8 +318,9 @@ This interpreter provisioning is designed to coexist with `rules_python`: falls back to the hub's highest provisioned version — including the hub rules_py itself registers, so this resolves even in modules that provision interpreters only through `rules_python`'s `python.toolchain()`. rules_py - registers nothing under `rules_python`'s exec-tools type, leaving it — - including precompiling — entirely to `rules_python`. + registers nothing under `rules_python`'s exec-tools type; with interpreters + from `interpreters.toolchain()` alone, `rules_python` never precompiles and + rules_py compiles bytecode for `rules_python`-built dependencies itself. Note that runtimes provisioned by `interpreters.toolchain()` carry `rules_python`'s public `PyRuntimeInfo` (re-exported from diff --git a/docs/migrating.md b/docs/migrating.md index e9db90df7..368d5deee 100644 --- a/docs/migrating.md +++ b/docs/migrating.md @@ -59,6 +59,14 @@ providers. Temporary scaffolding: [virtual deps](/docs/virtual_deps.md) are not expressible in those providers (resolve them concretely in `deps`), and the flag belongs in `.bazelrc` only until the last rules_python target is gone. +## Bytecode for unconverted targets + +rules_py's `pyc` modes compile dependencies still built by rules_python rules +(`py_proto_library`, pip hub packages, unconverted `py_library` targets) +itself. Set no rules_python `precompile` attribute or flag. Under `pyc_only` +a rules_python `py_library` still ships its sources from its own runfiles; +converting it to rules_py's `py_library` is the fix. + ## Remaining notes Users are encouraged to send a Pull Request to add more documentation as they uncover issues during migrations. diff --git a/e2e/README.md b/e2e/README.md index b0a5b05d5..553670aa8 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -35,9 +35,9 @@ workspace above must not carry: its `.bazelrc` turns on the rules_python provide compatibility layer, so rules_python `py_*` targets can depend on a rules_py `py_library`. Its `test.sh` asserts the same dependency is rejected with the flag off. -`rules-python-protobuf` exercises rules_proto_grpc_python-generated bindings in -an isolated module so its rules_python/protobuf/grpc dependency graph does not -leak into the main test module. +`rules-python-protobuf` contains protobuf's native `py_proto_library` and +rules_proto_grpc_python consumer tests. Keeping both generators here prevents +their rules_python/protobuf dependency graph from leaking into the main test module. `crossbuild` covers `pep517_native_whl`'s cross-compilation path across the PEP 517 backends, each with more than one real package so no backend's cross diff --git a/e2e/cases/coverage-drivers/BUILD.bazel b/e2e/cases/coverage-drivers/BUILD.bazel index 7963b83c8..a02973bc4 100644 --- a/e2e/cases/coverage-drivers/BUILD.bazel +++ b/e2e/cases/coverage-drivers/BUILD.bazel @@ -52,6 +52,18 @@ py_pytest_test( ], ) +py_unittest_test( + name = "coverage_pyc_only_test", + srcs = ["cov_add_unittest_test.py"], + dep_group = "coverage-drivers", + imports = ["."], + pyc = "pyc_only", + deps = [ + ":lib", + "@pypi_coverage_drivers//coverage", + ], +) + # Baked pytest_args route py_pytest_test through its codegen branch, which # renders a per-test copy of pytest_main.py (via py_pytest_main) instead of the # shared main. Confirms the coverage teardown survives that template rendering. diff --git a/e2e/cases/coverage-drivers/test.sh b/e2e/cases/coverage-drivers/test.sh index 47ce3bb5e..54d11a5c8 100644 --- a/e2e/cases/coverage-drivers/test.sh +++ b/e2e/cases/coverage-drivers/test.sh @@ -61,5 +61,6 @@ check_coverage //coverage-drivers:coverage_pytest_test bazel-testlogs/coverage-d check_coverage //coverage-drivers:coverage_pytest_codegen_test bazel-testlogs/coverage-drivers/coverage_pytest_codegen_test/coverage.dat check_coverage //coverage-drivers:coverage_pytest_chdir_test bazel-testlogs/coverage-drivers/coverage_pytest_chdir_test/coverage.dat check_coverage //coverage-drivers:coverage_unittest_test bazel-testlogs/coverage-drivers/coverage_unittest_test/coverage.dat +check_coverage //coverage-drivers:coverage_pyc_only_test bazel-testlogs/coverage-drivers/coverage_pyc_only_test/coverage.dat echo "All coverage driver checks passed." diff --git a/e2e/cases/oci/py_image_layer/BUILD.bazel b/e2e/cases/oci/py_image_layer/BUILD.bazel index 908372495..b8a3fc7b7 100644 --- a/e2e/cases/oci/py_image_layer/BUILD.bazel +++ b/e2e/cases/oci/py_image_layer/BUILD.bazel @@ -82,6 +82,179 @@ assert_tar_listing( expected = "console_scripts_layers_listing.yaml", ) +py_binary( + name = "pyc_source_data_bin", + srcs = ["server.py"], + data = ["server.py"], + pyc = "pyc_only", +) + +py_image_layer( + name = "pyc_source_data_layers", + binary = ":pyc_source_data_bin", + pyc = "pyc_only", +) + +py_test( + name = "pyc_source_data_layers_test", + srcs = ["assert_tar_paths.py"], + args = [ + "--present=/oci/py_image_layer/server.py", + "--tar-contains=_default.tar.gz=/server.pyc", + "$(rootpaths :pyc_source_data_layers)", + ], + data = [":pyc_source_data_layers"], + main = "assert_tar_paths.py", +) + +# Source and data edges resolve this file in different configurations. +genrule( + name = "generated_source_data", + outs = ["generated_source_data.py"], + cmd = select({ + ":_python_3_11": "echo 'VALUE = \"source\"' > $@", + "//conditions:default": "echo 'VALUE = \"data\"' > $@", + }), +) + +py_binary( + name = "pyc_generated_source_data_bin", + srcs = [ + "server.py", + ":generated_source_data.py", + ], + data = [":generated_source_data.py"], + main = "server.py", + pyc = "pyc_only", + python_version = "3.11", +) + +py_image_layer( + name = "pyc_generated_source_data_layers", + binary = ":pyc_generated_source_data_bin", + launcher_dir = "/app/bin", + pyc = "pyc_only", +) + +filegroup( + name = "pyc_generated_source_data_validation", + srcs = [":pyc_generated_source_data_layers"], + output_group = "_validation", +) + +build_test( + name = "pyc_generated_source_data_validation_test", + targets = [":pyc_generated_source_data_validation"], +) + +py_library( + name = "pyc_rule_group_lib", + srcs = ["direct_source_helper.py"], +) + +py_binary( + name = "pyc_rule_group_bin", + srcs = ["server.py"], + pyc = "pyc_only", + deps = [":pyc_rule_group_lib"], +) + +py_image_layer( + name = "pyc_rule_group_layers", + binary = ":pyc_rule_group_bin", + groups = {":pyc_rule_group_lib": "fp_rule"}, + pyc = "pyc_only", +) + +py_test( + name = "pyc_rule_group_layers_test", + srcs = ["assert_tar_paths.py"], + args = [ + "--tar-contains=_fp_rule.tar.gz=/direct_source_helper.pyc", + "--tar-absent=_default.tar.gz=/direct_source_helper.pyc", + "--absent=/direct_source_helper.py", + "$(rootpaths :pyc_rule_group_layers)", + ], + data = [":pyc_rule_group_layers"], + main = "assert_tar_paths.py", +) + +py_layer_tier( + name = "pyc_tier_group_data_tier", + groups = { + "//oci/py_image_layer:pyc_rule_group_lib": "fp_tier", + }, +) + +py_binary( + name = "pyc_tier_group_data_bin", + srcs = ["server.py"], + data = [":pyc_rule_group_lib"], + pyc = "pyc_only", + deps = [":pyc_rule_group_lib"], +) + +py_image_layer( + name = "pyc_tier_group_data_layers", + binary = ":pyc_tier_group_data_bin", + layer_tier = ":pyc_tier_group_data_tier", + pyc = "pyc_only", +) + +# The library is also runtime data, so its source ships beside the bytecode. +py_test( + name = "pyc_tier_group_data_layers_test", + srcs = ["assert_tar_paths.py"], + args = [ + "--tar-contains=_fp_tier.tar.gz=/direct_source_helper.py", + "--tar-contains=_fp_tier.tar.gz=/direct_source_helper.pyc", + "--tar-absent=_default.tar.gz=/direct_source_helper.py", + "$(rootpaths :pyc_tier_group_data_layers)", + ], + data = [":pyc_tier_group_data_layers"], + main = "assert_tar_paths.py", +) + +# The group and venv edges resolve this file in different configurations. +genrule( + name = "generated_group_source", + outs = ["generated_group_source.py"], + cmd = "echo 'VALUE = 1' > $@", +) + +py_library( + name = "pyc_generated_group_lib", + srcs = [":generated_group_source.py"], +) + +py_binary( + name = "pyc_generated_group_bin", + srcs = ["server.py"], + pyc = "pyc_only", + python_version = "3.11", + deps = [":pyc_generated_group_lib"], +) + +py_image_layer( + name = "pyc_generated_group_layers", + binary = ":pyc_generated_group_bin", + groups = {":pyc_generated_group_lib": "fp_rule"}, + pyc = "pyc_only", +) + +py_test( + name = "pyc_generated_group_layers_test", + srcs = ["assert_tar_paths.py"], + args = [ + "--tar-contains=_fp_rule.tar.gz=/generated_group_source.pyc", + "--tar-absent=_default.tar.gz=/generated_group_source.pyc", + "--absent=/generated_group_source.py", + "$(rootpaths :pyc_generated_group_layers)", + ], + data = [":pyc_generated_group_layers"], + main = "assert_tar_paths.py", +) + # Docker-free counterpart to the container_structure_test: run the same # entrypoint (verify_all(imports=["colorama"])) on the host so the venv's # site-packages symlinks must resolve to real files and colorama must import. @@ -1194,6 +1367,48 @@ assert_tar_listing( expected = "my_app_layers_fp_listing.yaml", ) +py_image_layer( + name = "my_app_layers_fp_pyc", + binary = ":my_app_bin", + layer_tier = ":my_app_tier", + pyc = "pyc", +) + +platform_transition_filegroup( + name = "platform_layers_fp_pyc", + srcs = [":my_app_layers_fp_pyc"], + target_platform = ":x86_64_linux", +) + +assert_tar_listing( + name = "my_app_layers_fp_pyc_test", + actual = [":platform_layers_fp_pyc"], + exclude = ["python_interpreters"], + expected = "my_app_layers_fp_pyc_listing.yaml", + pyc = "pyc", +) + +py_image_layer( + name = "my_app_layers_fp_pyc_only", + binary = ":my_app_bin", + layer_tier = ":my_app_tier", + pyc = "pyc_only", +) + +platform_transition_filegroup( + name = "platform_layers_fp_pyc_only", + srcs = [":my_app_layers_fp_pyc_only"], + target_platform = ":x86_64_linux", +) + +assert_tar_listing( + name = "my_app_layers_fp_pyc_only_test", + actual = [":platform_layers_fp_pyc_only"], + exclude = ["python_interpreters"], + expected = "my_app_layers_fp_pyc_only_listing.yaml", + pyc = "pyc_only", +) + assert_tar_listing( name = "my_app_layers_multi_test", actual = [":platform_layers_multi"], diff --git a/e2e/cases/oci/py_image_layer/assert_tar_paths.py b/e2e/cases/oci/py_image_layer/assert_tar_paths.py index 50d998c01..4d597b1a9 100644 --- a/e2e/cases/oci/py_image_layer/assert_tar_paths.py +++ b/e2e/cases/oci/py_image_layer/assert_tar_paths.py @@ -7,6 +7,7 @@ def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--contains", action="append", default=[]) + parser.add_argument("--present", action="append", default=[]) parser.add_argument("--absent", action="append", default=[]) parser.add_argument("--count", action="append", default=[]) parser.add_argument("--tar-contains", action="append", default=[]) @@ -25,6 +26,10 @@ def main() -> int: if not any(expected in path for path in paths): parser.error("missing path containing {!r}".format(expected)) + for expected in args.present: + if not any(path.endswith(expected) for path in paths): + parser.error("missing path ending in {!r}".format(expected)) + for unexpected in args.absent: if any(path.endswith(unexpected) for path in paths): parser.error("unexpected path ending in {!r}".format(unexpected)) diff --git a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl index 1de14c4cf..04b9b0e93 100644 --- a/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl +++ b/e2e/cases/oci/py_image_layer/image_layer_analysis_tests.bzl @@ -3,6 +3,7 @@ load("@aspect_rules_py//py:defs.bzl", "py_binary", "py_image_layer", "py_layer_tier", "py_library") load("@bazel_features//:features.bzl", "bazel_features") load("@bazel_skylib//lib:unittest.bzl", "analysistest", "asserts") +load("@bazel_skylib//rules:build_test.bzl", "build_test") _PY_TOOLCHAIN = "@bazel_tools//tools/python:toolchain_type" @@ -227,6 +228,49 @@ def image_layer_analysis_test_suite(): name = "_wheel_scripts_tier", groups = {"@pip//build": "wheel_scripts"}, ) + + py_image_layer( + name = "_pyc_mixed_runtimes_layers", + binaries = [":_wheel_scripts_311", ":_wheel_scripts_312"], + launcher_dir = "/app/bin", + pyc = "pyc", + ) + build_test( + name = "pyc_mixed_runtimes_build_test", + targets = [":_pyc_mixed_runtimes_layers"], + ) + + _image_layer_failure( + name = "pyc_only_mixed_runtimes", + expected_error = "binaries compile conflicting bytecode for", + binaries = [":_wheel_scripts_311", ":_wheel_scripts_312"], + launcher_dir = "/app/bin", + pyc = "pyc_only", + ) + + for dep_group in ["images", "venv_images"]: + py_binary( + name = "_pyc_same_runtime_{}".format(dep_group), + srcs = ["server.py"], + dep_group = dep_group, + python_version = "3.11", + ) + for mode in ["pyc", "pyc_only"]: + layer_name = "_pyc_same_runtime_distinct_configs_{}_layers".format(mode) + py_image_layer( + name = layer_name, + binaries = [ + ":_pyc_same_runtime_images", + ":_pyc_same_runtime_venv_images", + ], + launcher_dir = "/app/bin", + pyc = mode, + ) + build_test( + name = "pyc_same_runtime_distinct_configs_{}_build_test".format(mode), + targets = [":" + layer_name], + ) + py_image_layer( name = "_configured_wheel_collision_layers", binaries = [":_wheel_scripts_311", ":_wheel_scripts_312"], diff --git a/e2e/cases/oci/py_image_layer/snapshots/my_app_layers_fp_pyc_listing.yaml b/e2e/cases/oci/py_image_layer/snapshots/my_app_layers_fp_pyc_listing.yaml new file mode 100644 index 000000000..85b510478 --- /dev/null +++ b/e2e/cases/oci/py_image_layer/snapshots/my_app_layers_fp_pyc_listing.yaml @@ -0,0 +1,49 @@ +--- +layer: 0 +files: + - -rwxr-xr-x 0 0 0 42 Jan 1 2023 ./app.runfiles/_main/oci/py_image_layer/branding/__init__.py + - -rwxr-xr-x 0 0 0 276 Jan 1 2023 ./app.runfiles/_main/oci/py_image_layer/branding/__pycache__/__init__.cpython-311.pyc + - -rwxr-xr-x 0 0 0 31 Jan 1 2023 ./app.runfiles/_main/oci/py_image_layer/branding/palette.txt +--- +layer: 1 +files: +--- +layer: 2 +files: + - -rwxr-xr-x 0 0 0 266 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/__init__.py + - -rwxr-xr-x 0 0 0 2522 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/ansi.py + - -rwxr-xr-x 0 0 0 11128 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/ansitowin32.py + - -rwxr-xr-x 0 0 0 3325 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/initialise.py + - -rwxr-xr-x 0 0 0 75 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/tests/__init__.py + - -rwxr-xr-x 0 0 0 2839 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/tests/ansi_test.py + - -rwxr-xr-x 0 0 0 10678 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/tests/ansitowin32_test.py + - -rwxr-xr-x 0 0 0 6741 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/tests/initialise_test.py + - -rwxr-xr-x 0 0 0 1866 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/tests/isatty_test.py + - -rwxr-xr-x 0 0 0 1079 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/tests/utils.py + - -rwxr-xr-x 0 0 0 3709 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/tests/winterm_test.py + - -rwxr-xr-x 0 0 0 6181 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/win32.py + - -rwxr-xr-x 0 0 0 7134 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/winterm.py +--- +layer: 3 +files: + - -rwxr-xr-x 0 0 0 17158 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama-0.4.6.dist-info/METADATA + - -rwxr-xr-x 0 0 0 1491 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama-0.4.6.dist-info/licenses/LICENSE.txt +--- +layer: 4 +files: + - -rwxr-xr-x 0 0 0 18856 Jan 1 2023 ./app + - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/_main/oci/py_image_layer/._my_app_bin.venv/bin/python3 -> python + - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/_main/oci/py_image_layer/._my_app_bin.venv/bin/python3.11 -> python + - -rwxr-xr-x 0 0 0 517 Jan 1 2023 ./app.runfiles/_main/oci/py_image_layer/._my_app_bin.venv/lib/python3.11/site-packages/_my_app_bin.venv.pth + - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/_main/oci/py_image_layer/._my_app_bin.venv/lib/python3.11/site-packages/colorama -> ../../../../../../../aspect_rules_py++uv+whl_install__images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama + - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/_main/oci/py_image_layer/._my_app_bin.venv/lib/python3.11/site-packages/colorama-0.4.6.dist-info -> ../../../../../../../aspect_rules_py++uv+whl_install__images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama-0.4.6.dist-info + - -rwxr-xr-x 0 0 0 111 Jan 1 2023 ./app.runfiles/_main/oci/py_image_layer/._my_app_bin.venv/pyvenv.cfg + - -rwxr-xr-x 0 0 0 276 Jan 1 2023 ./app.runfiles/_main/oci/py_image_layer/__main__.py + - -rwxr-xr-x 0 0 0 719 Jan 1 2023 ./app.runfiles/_main/oci/py_image_layer/__pycache__/__main__.cpython-311.pyc + - -rwxr-xr-x 0 0 0 10887 Jan 1 2023 ./app.runfiles/_main/tools/verify_venv/__pycache__/verify_venv.cpython-311.pyc + - -rwxr-xr-x 0 0 0 6709 Jan 1 2023 ./app.runfiles/_main/tools/verify_venv/verify_venv.py + - -rwxr-xr-x 0 0 0 * Jan 1 2023 ./app.runfiles/_repo_mapping + - -rwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/aspect_rules_py+/py/tests/internal-deps/adder/__init__.py + - -rwxr-xr-x 0 0 0 169 Jan 1 2023 ./app.runfiles/aspect_rules_py+/py/tests/internal-deps/adder/__pycache__/__init__.cpython-311.pyc + - -rwxr-xr-x 0 0 0 288 Jan 1 2023 ./app.runfiles/aspect_rules_py+/py/tests/internal-deps/adder/__pycache__/add.cpython-311.pyc + - -rwxr-xr-x 0 0 0 32 Jan 1 2023 ./app.runfiles/aspect_rules_py+/py/tests/internal-deps/adder/add.py diff --git a/e2e/cases/oci/py_image_layer/snapshots/my_app_layers_fp_pyc_only_listing.yaml b/e2e/cases/oci/py_image_layer/snapshots/my_app_layers_fp_pyc_only_listing.yaml new file mode 100644 index 000000000..a8895f5b7 --- /dev/null +++ b/e2e/cases/oci/py_image_layer/snapshots/my_app_layers_fp_pyc_only_listing.yaml @@ -0,0 +1,44 @@ +--- +layer: 0 +files: + - -rwxr-xr-x 0 0 0 276 Jan 1 2023 ./app.runfiles/_main/oci/py_image_layer/branding/__init__.pyc + - -rwxr-xr-x 0 0 0 31 Jan 1 2023 ./app.runfiles/_main/oci/py_image_layer/branding/palette.txt +--- +layer: 1 +files: +--- +layer: 2 +files: + - -rwxr-xr-x 0 0 0 266 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/__init__.py + - -rwxr-xr-x 0 0 0 2522 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/ansi.py + - -rwxr-xr-x 0 0 0 11128 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/ansitowin32.py + - -rwxr-xr-x 0 0 0 3325 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/initialise.py + - -rwxr-xr-x 0 0 0 75 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/tests/__init__.py + - -rwxr-xr-x 0 0 0 2839 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/tests/ansi_test.py + - -rwxr-xr-x 0 0 0 10678 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/tests/ansitowin32_test.py + - -rwxr-xr-x 0 0 0 6741 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/tests/initialise_test.py + - -rwxr-xr-x 0 0 0 1866 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/tests/isatty_test.py + - -rwxr-xr-x 0 0 0 1079 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/tests/utils.py + - -rwxr-xr-x 0 0 0 3709 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/tests/winterm_test.py + - -rwxr-xr-x 0 0 0 6181 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/win32.py + - -rwxr-xr-x 0 0 0 7134 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/winterm.py +--- +layer: 3 +files: + - -rwxr-xr-x 0 0 0 17158 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama-0.4.6.dist-info/METADATA + - -rwxr-xr-x 0 0 0 1491 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama-0.4.6.dist-info/licenses/LICENSE.txt +--- +layer: 4 +files: + - -rwxr-xr-x 0 0 0 18856 Jan 1 2023 ./app + - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/_main/oci/py_image_layer/._my_app_bin.venv/bin/python3 -> python + - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/_main/oci/py_image_layer/._my_app_bin.venv/bin/python3.11 -> python + - -rwxr-xr-x 0 0 0 517 Jan 1 2023 ./app.runfiles/_main/oci/py_image_layer/._my_app_bin.venv/lib/python3.11/site-packages/_my_app_bin.venv.pth + - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/_main/oci/py_image_layer/._my_app_bin.venv/lib/python3.11/site-packages/colorama -> ../../../../../../../aspect_rules_py++uv+whl_install__images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama + - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/_main/oci/py_image_layer/._my_app_bin.venv/lib/python3.11/site-packages/colorama-0.4.6.dist-info -> ../../../../../../../aspect_rules_py++uv+whl_install__images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama-0.4.6.dist-info + - -rwxr-xr-x 0 0 0 111 Jan 1 2023 ./app.runfiles/_main/oci/py_image_layer/._my_app_bin.venv/pyvenv.cfg + - -rwxr-xr-x 0 0 0 719 Jan 1 2023 ./app.runfiles/_main/oci/py_image_layer/__main__.pyc + - -rwxr-xr-x 0 0 0 10887 Jan 1 2023 ./app.runfiles/_main/tools/verify_venv/verify_venv.pyc + - -rwxr-xr-x 0 0 0 * Jan 1 2023 ./app.runfiles/_repo_mapping + - -rwxr-xr-x 0 0 0 169 Jan 1 2023 ./app.runfiles/aspect_rules_py+/py/tests/internal-deps/adder/__init__.pyc + - -rwxr-xr-x 0 0 0 288 Jan 1 2023 ./app.runfiles/aspect_rules_py+/py/tests/internal-deps/adder/add.pyc diff --git a/e2e/cases/oci/py_venv_image_layer/BUILD.bazel b/e2e/cases/oci/py_venv_image_layer/BUILD.bazel index 430d9fd37..6d836bc53 100644 --- a/e2e/cases/oci/py_venv_image_layer/BUILD.bazel +++ b/e2e/cases/oci/py_venv_image_layer/BUILD.bazel @@ -1,4 +1,4 @@ -load("@aspect_rules_py//py:defs.bzl", "py_binary", "py_image_layer", "py_test") +load("@aspect_rules_py//py:defs.bzl", "py_binary", "py_image_layer", "py_layer_tier", "py_test") load("@bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") load("@container_structure_test//:defs.bzl", "container_structure_test") load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load") @@ -183,3 +183,169 @@ container_structure_test( "@platforms//cpu:aarch64", ], ) + +py_image_layer( + name = "my_app_pyc_only_layers", + binary = ":my_app_bin", + pyc = "pyc_only", +) + +platform_transition_filegroup( + name = "pyc_only_amd64_layers", + srcs = [":my_app_pyc_only_layers"], + target_platform = ":amd64_linux", +) + +py_test( + name = "my_app_pyc_only_amd64_layers_test", + srcs = ["//oci/py_image_layer:assert_tar_paths.py"], + args = [ + "--contains=/oci/py_venv_image_layer/__main__.pyc", + "--absent=/oci/py_venv_image_layer/__main__.py", + "--count=/_wheels/=0", + "$(rootpaths :pyc_only_amd64_layers)", + ], + data = [":pyc_only_amd64_layers"], + main = "//oci/py_image_layer:assert_tar_paths.py", +) + +platform_transition_filegroup( + name = "pyc_only_arm64_layers", + srcs = [":my_app_pyc_only_layers"], + target_platform = ":arm64_linux", +) + +py_test( + name = "my_app_pyc_only_arm64_layers_test", + srcs = ["//oci/py_image_layer:assert_tar_paths.py"], + args = [ + "--contains=/oci/py_venv_image_layer/__main__.pyc", + "--absent=/oci/py_venv_image_layer/__main__.py", + "--count=/_wheels/=0", + "$(rootpaths :pyc_only_arm64_layers)", + ], + data = [":pyc_only_arm64_layers"], + main = "//oci/py_image_layer:assert_tar_paths.py", +) + +assert_tar_listing( + name = "my_app_pyc_only_amd64_layers_snapshot", + actual = [":pyc_only_amd64_layers"], + exclude = ["python_interpreters"], + expected = "my_app_pyc_only_amd64_layers_listing.yaml", + pyc = "pyc_only", +) + +py_image_layer( + name = "my_app_pyc_layers", + binary = ":my_app_bin", + pyc = "pyc", +) + +platform_transition_filegroup( + name = "pyc_amd64_layers", + srcs = [":my_app_pyc_layers"], + target_platform = ":amd64_linux", +) + +py_test( + name = "my_app_pyc_amd64_layers_test", + srcs = ["//oci/py_image_layer:assert_tar_paths.py"], + args = [ + "--contains=/oci/py_venv_image_layer/__main__.py", + "--contains=/oci/py_venv_image_layer/__pycache__/__main__.cpython-", + "--count=/_wheels/=0", + "$(rootpaths :pyc_amd64_layers)", + ], + data = [":pyc_amd64_layers"], + main = "//oci/py_image_layer:assert_tar_paths.py", +) + +assert_tar_listing( + name = "my_app_pyc_amd64_layers_snapshot", + actual = [":pyc_amd64_layers"], + exclude = ["python_interpreters"], + expected = "my_app_pyc_amd64_layers_listing.yaml", + pyc = "pyc", +) + +py_layer_tier( + name = "pyc_branding_tier", + groups = {"//oci/py_image_layer/branding": "branding"}, +) + +py_image_layer( + name = "my_app_pyc_grouped_layers", + binary = ":my_app_bin", + layer_tier = ":pyc_branding_tier", + pyc = "pyc_only", +) + +platform_transition_filegroup( + name = "pyc_grouped_amd64_layers", + srcs = [":my_app_pyc_grouped_layers"], + target_platform = ":amd64_linux", +) + +py_test( + name = "my_app_pyc_grouped_amd64_layers_test", + srcs = ["//oci/py_image_layer:assert_tar_paths.py"], + args = [ + "--tar-contains=_branding.tar.gz=/branding/__init__.pyc", + "--tar-absent=_default.tar.gz=/branding/__init__.pyc", + "--absent=/branding/__init__.py", + "--count=/_wheels/=0", + "$(rootpaths :pyc_grouped_amd64_layers)", + ], + data = [":pyc_grouped_amd64_layers"], + main = "//oci/py_image_layer:assert_tar_paths.py", +) + +oci_image( + name = "pyc_only_image", + base = "@ubuntu", + entrypoint = ["/app"], + tars = [":my_app_pyc_only_layers"], +) + +platform_transition_filegroup( + name = "pyc_only_amd64_image", + srcs = [":pyc_only_image"], + target_platform = ":amd64_linux", +) + +container_structure_test( + name = "py_pyc_only_amd64_image_command_test", + args = ["--verbosity=debug"], + configs = ["py_image_command_test.yaml"], + image = ":pyc_only_amd64_image", + platform = "linux/amd64", + tags = [ + "requires-docker", + "skip-on-bazel9", + ], + target_compatible_with = [ + "@platforms//cpu:x86_64", + ], +) + +py_binary( + name = "my_app_source_pinned_bin", + srcs = ["__main__.py"], + dep_group = "venv_images", + main = "__main__.py", + pyc = "source", + tags = ["manual"], + deps = [ + "//oci/py_image_layer/branding", + "//tools/verify_venv", + "@aspect_rules_py//py/tests/internal-deps/adder", + "@pypi_oci_py_venv_image_layer//colorama", + ], +) + +py_image_layer( + name = "my_app_pyc_mismatch_layers", + binary = ":my_app_source_pinned_bin", + pyc = "pyc_only", +) diff --git a/e2e/cases/oci/py_venv_image_layer/snapshots/my_app_pyc_amd64_layers_listing.yaml b/e2e/cases/oci/py_venv_image_layer/snapshots/my_app_pyc_amd64_layers_listing.yaml new file mode 100644 index 000000000..d02734c9b --- /dev/null +++ b/e2e/cases/oci/py_venv_image_layer/snapshots/my_app_pyc_amd64_layers_listing.yaml @@ -0,0 +1,40 @@ +--- +layer: 0 +files: + - -rwxr-xr-x 0 0 0 17158 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__venv_images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama-0.4.6.dist-info/METADATA + - -rwxr-xr-x 0 0 0 1491 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__venv_images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama-0.4.6.dist-info/licenses/LICENSE.txt + - -rwxr-xr-x 0 0 0 266 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__venv_images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/__init__.py + - -rwxr-xr-x 0 0 0 2522 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__venv_images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/ansi.py + - -rwxr-xr-x 0 0 0 11128 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__venv_images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/ansitowin32.py + - -rwxr-xr-x 0 0 0 3325 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__venv_images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/initialise.py + - -rwxr-xr-x 0 0 0 75 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__venv_images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/tests/__init__.py + - -rwxr-xr-x 0 0 0 2839 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__venv_images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/tests/ansi_test.py + - -rwxr-xr-x 0 0 0 10678 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__venv_images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/tests/ansitowin32_test.py + - -rwxr-xr-x 0 0 0 6741 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__venv_images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/tests/initialise_test.py + - -rwxr-xr-x 0 0 0 1866 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__venv_images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/tests/isatty_test.py + - -rwxr-xr-x 0 0 0 1079 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__venv_images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/tests/utils.py + - -rwxr-xr-x 0 0 0 3709 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__venv_images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/tests/winterm_test.py + - -rwxr-xr-x 0 0 0 6181 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__venv_images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/win32.py + - -rwxr-xr-x 0 0 0 7134 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__venv_images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/winterm.py +--- +layer: 1 +files: + - -rwxr-xr-x 0 0 0 18856 Jan 1 2023 ./app + - -rwxr-xr-x 0 0 0 42 Jan 1 2023 ./app.runfiles/_main/oci/py_image_layer/branding/__init__.py + - -rwxr-xr-x 0 0 0 276 Jan 1 2023 ./app.runfiles/_main/oci/py_image_layer/branding/__pycache__/__init__.cpython-311.pyc + - -rwxr-xr-x 0 0 0 31 Jan 1 2023 ./app.runfiles/_main/oci/py_image_layer/branding/palette.txt + - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/_main/oci/py_venv_image_layer/.my_app_bin.venv/bin/python3 -> python + - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/_main/oci/py_venv_image_layer/.my_app_bin.venv/bin/python3.11 -> python + - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/_main/oci/py_venv_image_layer/.my_app_bin.venv/lib/python3.11/site-packages/colorama -> ../../../../../../../aspect_rules_py++uv+whl_install__venv_images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama + - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/_main/oci/py_venv_image_layer/.my_app_bin.venv/lib/python3.11/site-packages/colorama-0.4.6.dist-info -> ../../../../../../../aspect_rules_py++uv+whl_install__venv_images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama-0.4.6.dist-info + - -rwxr-xr-x 0 0 0 522 Jan 1 2023 ./app.runfiles/_main/oci/py_venv_image_layer/.my_app_bin.venv/lib/python3.11/site-packages/my_app_bin.venv.pth + - -rwxr-xr-x 0 0 0 111 Jan 1 2023 ./app.runfiles/_main/oci/py_venv_image_layer/.my_app_bin.venv/pyvenv.cfg + - -rwxr-xr-x 0 0 0 463 Jan 1 2023 ./app.runfiles/_main/oci/py_venv_image_layer/__main__.py + - -rwxr-xr-x 0 0 0 1094 Jan 1 2023 ./app.runfiles/_main/oci/py_venv_image_layer/__pycache__/__main__.cpython-311.pyc + - -rwxr-xr-x 0 0 0 10887 Jan 1 2023 ./app.runfiles/_main/tools/verify_venv/__pycache__/verify_venv.cpython-311.pyc + - -rwxr-xr-x 0 0 0 6709 Jan 1 2023 ./app.runfiles/_main/tools/verify_venv/verify_venv.py + - -rwxr-xr-x 0 0 0 * Jan 1 2023 ./app.runfiles/_repo_mapping + - -rwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/aspect_rules_py+/py/tests/internal-deps/adder/__init__.py + - -rwxr-xr-x 0 0 0 169 Jan 1 2023 ./app.runfiles/aspect_rules_py+/py/tests/internal-deps/adder/__pycache__/__init__.cpython-311.pyc + - -rwxr-xr-x 0 0 0 288 Jan 1 2023 ./app.runfiles/aspect_rules_py+/py/tests/internal-deps/adder/__pycache__/add.cpython-311.pyc + - -rwxr-xr-x 0 0 0 32 Jan 1 2023 ./app.runfiles/aspect_rules_py+/py/tests/internal-deps/adder/add.py diff --git a/e2e/cases/oci/py_venv_image_layer/snapshots/my_app_pyc_only_amd64_layers_listing.yaml b/e2e/cases/oci/py_venv_image_layer/snapshots/my_app_pyc_only_amd64_layers_listing.yaml new file mode 100644 index 000000000..145999f9e --- /dev/null +++ b/e2e/cases/oci/py_venv_image_layer/snapshots/my_app_pyc_only_amd64_layers_listing.yaml @@ -0,0 +1,35 @@ +--- +layer: 0 +files: + - -rwxr-xr-x 0 0 0 17158 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__venv_images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama-0.4.6.dist-info/METADATA + - -rwxr-xr-x 0 0 0 1491 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__venv_images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama-0.4.6.dist-info/licenses/LICENSE.txt + - -rwxr-xr-x 0 0 0 266 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__venv_images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/__init__.py + - -rwxr-xr-x 0 0 0 2522 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__venv_images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/ansi.py + - -rwxr-xr-x 0 0 0 11128 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__venv_images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/ansitowin32.py + - -rwxr-xr-x 0 0 0 3325 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__venv_images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/initialise.py + - -rwxr-xr-x 0 0 0 75 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__venv_images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/tests/__init__.py + - -rwxr-xr-x 0 0 0 2839 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__venv_images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/tests/ansi_test.py + - -rwxr-xr-x 0 0 0 10678 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__venv_images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/tests/ansitowin32_test.py + - -rwxr-xr-x 0 0 0 6741 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__venv_images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/tests/initialise_test.py + - -rwxr-xr-x 0 0 0 1866 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__venv_images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/tests/isatty_test.py + - -rwxr-xr-x 0 0 0 1079 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__venv_images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/tests/utils.py + - -rwxr-xr-x 0 0 0 3709 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__venv_images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/tests/winterm_test.py + - -rwxr-xr-x 0 0 0 6181 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__venv_images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/win32.py + - -rwxr-xr-x 0 0 0 7134 Jan 1 2023 ./app.runfiles/aspect_rules_py++uv+whl_install__venv_images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama/winterm.py +--- +layer: 1 +files: + - -rwxr-xr-x 0 0 0 18856 Jan 1 2023 ./app + - -rwxr-xr-x 0 0 0 276 Jan 1 2023 ./app.runfiles/_main/oci/py_image_layer/branding/__init__.pyc + - -rwxr-xr-x 0 0 0 31 Jan 1 2023 ./app.runfiles/_main/oci/py_image_layer/branding/palette.txt + - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/_main/oci/py_venv_image_layer/.my_app_bin.venv/bin/python3 -> python + - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/_main/oci/py_venv_image_layer/.my_app_bin.venv/bin/python3.11 -> python + - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/_main/oci/py_venv_image_layer/.my_app_bin.venv/lib/python3.11/site-packages/colorama -> ../../../../../../../aspect_rules_py++uv+whl_install__venv_images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama + - lrwxr-xr-x 0 0 0 0 Jan 1 2023 ./app.runfiles/_main/oci/py_venv_image_layer/.my_app_bin.venv/lib/python3.11/site-packages/colorama-0.4.6.dist-info -> ../../../../../../../aspect_rules_py++uv+whl_install__venv_images__colorama__0_4_6/actual_install.install/lib/python3.11/site-packages/colorama-0.4.6.dist-info + - -rwxr-xr-x 0 0 0 522 Jan 1 2023 ./app.runfiles/_main/oci/py_venv_image_layer/.my_app_bin.venv/lib/python3.11/site-packages/my_app_bin.venv.pth + - -rwxr-xr-x 0 0 0 111 Jan 1 2023 ./app.runfiles/_main/oci/py_venv_image_layer/.my_app_bin.venv/pyvenv.cfg + - -rwxr-xr-x 0 0 0 1094 Jan 1 2023 ./app.runfiles/_main/oci/py_venv_image_layer/__main__.pyc + - -rwxr-xr-x 0 0 0 10887 Jan 1 2023 ./app.runfiles/_main/tools/verify_venv/verify_venv.pyc + - -rwxr-xr-x 0 0 0 * Jan 1 2023 ./app.runfiles/_repo_mapping + - -rwxr-xr-x 0 0 0 169 Jan 1 2023 ./app.runfiles/aspect_rules_py+/py/tests/internal-deps/adder/__init__.pyc + - -rwxr-xr-x 0 0 0 288 Jan 1 2023 ./app.runfiles/aspect_rules_py+/py/tests/internal-deps/adder/add.pyc diff --git a/e2e/cases/oci/test.sh b/e2e/cases/oci/test.sh index 9f944a1c0..5f583dd3e 100755 --- a/e2e/cases/oci/test.sh +++ b/e2e/cases/oci/test.sh @@ -146,3 +146,19 @@ if [[ "${USE_BAZEL_VERSION:-}" != 9* ]]; then fi echo "PASS: nested launcher prefixes share the same runfiles layout" fi + +echo "== a binary hard-wired to source fails under a pyc_only image ==" +if "$BAZEL" build //oci/py_venv_image_layer:my_app_pyc_mismatch_layers >"$output_log" 2>&1; then + fail "expected the pyc-mode mismatch to fail analysis" +fi +expect_diagnostic "has pyc=source but the image requires pyc=pyc_only" +echo "PASS: bytecode-mode mismatch is rejected" + +echo "== images inherit the global pyc flag when the attribute is unset ==" +if ! "$BAZEL" build --@aspect_rules_py//py:pyc=pyc_only \ + //oci/py_venv_image_layer:pyc_only_amd64_layers \ + //oci/py_venv_image_layer:my_app_layers >"$output_log" 2>&1; then + cat "$output_log" >&2 + fail "expected flag-inherited bytecode images to build" +fi +echo "PASS: global pyc flag flows into images" diff --git a/e2e/cases/pyc/BUILD.bazel b/e2e/cases/pyc/BUILD.bazel new file mode 100644 index 000000000..b26476a7c --- /dev/null +++ b/e2e/cases/pyc/BUILD.bazel @@ -0,0 +1,138 @@ +load("@aspect_rules_py//py:defs.bzl", "py_binary", "py_pex_binary", "py_test", "py_venv") +load("@aspect_rules_py//py/private/py_venv:defs.bzl", "py_venv_exec_test") + +# Fixtures for test.sh, which flips the global flag and inspects the +# action/configuration graph. The launchers mirror py/tests/py-venv-multi-exec, +# where their runfiles assertions live. + +py_venv( + name = "shared_venv", + srcs = [ + "entry_a.py", + "shared_lib.py", + ], + imports = ["."], +) + +py_venv_exec_test( + name = "test_pyc_default", + main = "entry_a.py", + venv = ":shared_venv", +) + +py_venv_exec_test( + name = "test_pyc_cache", + main = "entry_a.py", + pyc = "pyc", + venv = ":shared_venv", +) + +py_venv_exec_test( + name = "test_pyc_only", + main = "entry_a.py", + pyc = "pyc_only", + venv = ":shared_venv", +) + +py_venv( + name = "shared_versioned_venv", + srcs = ["entry_versioned.py"], + imports = ["."], +) + +[ + py_venv_exec_test( + name = "{}_python_{}".format( + prefix, + version.replace(".", ""), + ), + main = "entry_versioned.py", + pyc = mode, + python_version = version, + venv = ":shared_versioned_venv", + ) + for mode, prefix in { + "source": "test_pyc_source", + "pyc": "test_pyc", + "pyc_only": "test_pyc_only", + }.items() + for version in ("3.12", "3.13") +] + +genrule( + name = "gen_main", + outs = ["main_generated.py"], + cmd = "echo 'print(\"ok\")' > $@", +) + +# A rule target in `srcs` is opaque to bytecode compilation. +py_binary( + name = "main_from_genrule_bin", + srcs = [":gen_main"], + main = ":gen_main", +) + +# Bytecode compilation requires the generated file's own label in `srcs`. +py_binary( + name = "main_from_genrule_pyc_bin", + srcs = ["main_generated.py"], + main = ":gen_main", + pyc = "pyc_only", +) + +platform( + name = "linux_amd64", + constraint_values = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], + # Pin libc so the PBS linux-gnu toolchain matches regardless of host. + flags = ["--@aspect_rules_py//uv/private/constraints/platform:platform_libc=glibc"], +) + +platform( + name = "linux_arm64", + constraint_values = [ + "@platforms//os:linux", + "@platforms//cpu:aarch64", + ], + flags = ["--@aspect_rules_py//uv/private/constraints/platform:platform_libc=glibc"], +) + +py_binary( + name = "noop_bin", + srcs = ["noop.py"], +) + +# The emitted .venv_link runs rules_py's own link script; bytecode modes never apply to it. +py_binary( + name = "link_bin", + srcs = ["noop.py"], + expose_venv_link = True, +) + +py_pex_binary( + name = "noop_pex", + binary = ":noop_bin", + python_interpreter_constraints = [], +) + +py_test( + name = "pex_no_bytecode_test", + srcs = ["pex_no_bytecode_test.py"], + data = [":noop_pex"], + main = "pex_no_bytecode_test.py", +) + +py_binary( + name = "pyc_only_noop_bin", + srcs = ["noop.py"], + pyc = "pyc_only", + tags = ["manual"], +) + +py_pex_binary( + name = "pyc_only_noop_pex", + binary = ":pyc_only_noop_bin", + tags = ["manual"], +) diff --git a/e2e/cases/pyc/entry_a.py b/e2e/cases/pyc/entry_a.py new file mode 100644 index 000000000..95c564b56 --- /dev/null +++ b/e2e/cases/pyc/entry_a.py @@ -0,0 +1,4 @@ +import shared_lib + +assert shared_lib.GREETING == "hello from the shared venv" +print("entry_a ok") diff --git a/e2e/cases/pyc/entry_versioned.py b/e2e/cases/pyc/entry_versioned.py new file mode 100644 index 000000000..e140259e3 --- /dev/null +++ b/e2e/cases/pyc/entry_versioned.py @@ -0,0 +1,16 @@ +"""Checks that a shared py_venv is configured for the launcher's Python.""" + +import os +import sys + +EXPECTED = { + "test_pyc_source_python_312": (3, 12), + "test_pyc_source_python_313": (3, 13), + "test_pyc_python_312": (3, 12), + "test_pyc_python_313": (3, 13), + "test_pyc_only_python_312": (3, 12), + "test_pyc_only_python_313": (3, 13), +} + +assert sys.version_info[:2] == EXPECTED[os.environ["BAZEL_TARGET_NAME"]] +print("versioned venv ok") diff --git a/e2e/cases/pyc/noop.py b/e2e/cases/pyc/noop.py new file mode 100644 index 000000000..1198e767c --- /dev/null +++ b/e2e/cases/pyc/noop.py @@ -0,0 +1 @@ +print("noop") diff --git a/e2e/cases/pyc/pex_no_bytecode_test.py b/e2e/cases/pyc/pex_no_bytecode_test.py new file mode 100644 index 000000000..f2bf80ebf --- /dev/null +++ b/e2e/cases/pyc/pex_no_bytecode_test.py @@ -0,0 +1,11 @@ +"""The PEX edge resets the pyc flag, so a bytecode-mode build ships no .pyc.""" + +import os +import zipfile +from pathlib import Path + +pex = Path(os.environ["RUNFILES_DIR"]) / "_main/pyc/noop_pex.pex" +with zipfile.ZipFile(pex) as zf: + names = zf.namelist() +assert any(name.endswith("noop.py") for name in names), names[:10] +assert not any(name.endswith(".pyc") for name in names), names[:10] diff --git a/e2e/cases/pyc/shared_lib.py b/e2e/cases/pyc/shared_lib.py new file mode 100644 index 000000000..db7e4194a --- /dev/null +++ b/e2e/cases/pyc/shared_lib.py @@ -0,0 +1 @@ +GREETING = "hello from the shared venv" diff --git a/e2e/cases/pyc/test.sh b/e2e/cases/pyc/test.sh new file mode 100755 index 000000000..050aad902 --- /dev/null +++ b/e2e/cases/pyc/test.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# +# Bytecode-mode checks that need a top-level build setting or a look at +# Bazel's action/configuration graph. The cases/test.sh aggregator runs it. +set -euo pipefail + +cd "$(dirname "$0")/.." # e2e/cases workspace root + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +count_compile_actions() { + bazel aquery "$@" 2>/dev/null | grep -c '^action ' || true +} + +echo "== PEX ignores first-party bytecode mode ==" +bazel test --@aspect_rules_py//py:pyc=pyc //pyc:pex_no_bytecode_test + +echo "== PEX rejects an explicitly bytecode-configured binary ==" +pex_error="$(mktemp)" +cross_dir="$(mktemp -d)" +trap 'rm -rf "$pex_error" "$cross_dir"' EXIT +if bazel build //pyc:pyc_only_noop_pex >"$pex_error" 2>&1; then + fail "expected explicitly bytecode-configured PEX input to fail" +fi +grep -Fq "to use pyc=source" "$pex_error" || fail "expected PEX bytecode-mode diagnostic" + +echo "== venv_link launchers ignore the global flag ==" +bazel build --@aspect_rules_py//py:pyc=pyc_only //pyc:link_bin.venv_link + +echo "== direct venv consumers follow the global flag; explicit pyc pins ==" +bazel test --@aspect_rules_py//py:pyc=pyc //pyc:test_pyc_default +bazel build --@aspect_rules_py//py:pyc=pyc //pyc:test_pyc_source_python_313 +pinned_manifest="bazel-bin/pyc/test_pyc_source_python_313.runfiles_manifest" +test -f "$pinned_manifest" || fail "missing runfiles manifest $pinned_manifest" +if grep -q '\.pyc' "$pinned_manifest"; then + fail "explicit pyc=source did not pin source mode under a pyc flag" +fi + +echo "== source mode declares but never executes or ships bytecode ==" +source_actions="$(count_compile_actions --@aspect_rules_py//py:pyc=source "mnemonic('PyCompile', deps(//pyc:test_pyc_default))")" +test "$source_actions" -gt 0 || fail "expected auto-declared bytecode compile actions in source mode" +bazel build --@aspect_rules_py//py:pyc=source //pyc:test_pyc_default +manifest="bazel-bin/pyc/test_pyc_default.runfiles_manifest" +test -f "$manifest" || fail "missing runfiles manifest $manifest" +if grep -q '\.pyc' "$manifest"; then + fail "source mode unexpectedly shipped .pyc runfiles" +fi + +echo "== rule targets in srcs are opaque to bytecode ==" +genrule_actions="$(count_compile_actions "mnemonic('PyCompile', deps(//pyc:main_from_genrule_bin))")" +test "$genrule_actions" = 0 || fail "genrule-in-srcs unexpectedly declared bytecode compile actions" + +echo "== all bytecode modes share one configured venv and its compilation actions ==" +venv_configs="$(bazel cquery "deps(//pyc:test_pyc_default, 1) union deps(//pyc:test_pyc_cache, 1) union deps(//pyc:test_pyc_only, 1)" 2>/dev/null | grep -c ':shared_venv ' || true)" +test "$venv_configs" = 1 || fail "expected one configured shared_venv across all pyc modes, got $venv_configs" +single_count="$(count_compile_actions "mnemonic('PyCompile', deps(//pyc:test_pyc_cache))")" +combined_count="$(count_compile_actions "mnemonic('PyCompile', deps(set(//pyc:test_pyc_default //pyc:test_pyc_cache //pyc:test_pyc_only)))")" +test "$single_count" -gt 0 || fail "expected shared venv to compile first-party bytecode" +test "$single_count" = "$combined_count" || fail "shared venv bytecode compilation was duplicated ($single_count vs $combined_count)" + +echo "== bytecode forks per python version, not per pyc mode ==" +v313_count="$(count_compile_actions "mnemonic('PyCompile', deps(//pyc:test_pyc_python_313))")" +v312_count="$(count_compile_actions "mnemonic('PyCompile', deps(//pyc:test_pyc_python_312))")" +matrix_count="$(count_compile_actions "mnemonic('PyCompile', deps(set(//pyc:test_pyc_source_python_313 //pyc:test_pyc_source_python_312 //pyc:test_pyc_python_313 //pyc:test_pyc_python_312 //pyc:test_pyc_only_python_313 //pyc:test_pyc_only_python_312)))")" +test "$v313_count" -gt 0 || fail "expected 3.13 bytecode compile actions" +test "$v312_count" -gt 0 || fail "expected 3.12 bytecode compile actions" +test "$matrix_count" = "$((v313_count + v312_count))" || + fail "pyc/version matrix duplicated compile actions ($matrix_count vs $v313_count + $v312_count)" +versioned_configs="$(bazel cquery "deps(//pyc:test_pyc_source_python_313, 1) union deps(//pyc:test_pyc_source_python_312, 1) union deps(//pyc:test_pyc_python_313, 1) union deps(//pyc:test_pyc_python_312, 1) union deps(//pyc:test_pyc_only_python_313, 1) union deps(//pyc:test_pyc_only_python_312, 1)" 2>/dev/null | grep -c ':shared_versioned_venv ' || true)" +test "$versioned_configs" = 2 || fail "expected one configured shared_versioned_venv per python version, got $versioned_configs" + +echo "== bytecode cross-compiles via the exec interpreter ==" +for plat in linux_amd64 linux_arm64; do + bazel build "--platforms=//pyc:${plat}" //pyc:main_from_genrule_pyc_bin + pyc_path="$(bazel cquery "--platforms=//pyc:${plat}" --output=files //pyc:main_from_genrule_pyc_bin 2>/dev/null | grep '\.pyc$')" + test -n "$pyc_path" || fail "no .pyc output for platform ${plat}" + cp "$pyc_path" "$cross_dir/${plat}.pyc" +done +cmp "$cross_dir/linux_amd64.pyc" "$cross_dir/linux_arm64.pyc" || + fail "cross-compiled bytecode differs between target platforms" + +echo "== launchers remain in the top-level configuration ==" +launcher_path="$(bazel cquery --output=starlark --starlark:expr='target.files.to_list()[0].path' //pyc:main_from_genrule_bin 2>/dev/null)" +case "$launcher_path" in + *-ST-*) fail "launcher was transitioned away from the top-level configuration: $launcher_path" ;; + bazel-out/*/bin/*) ;; + *) fail "unexpected launcher output path: $launcher_path" ;; +esac diff --git a/e2e/cases/tools/asserts.bzl b/e2e/cases/tools/asserts.bzl index f7835537c..68ee5efd0 100644 --- a/e2e/cases/tools/asserts.bzl +++ b/e2e/cases/tools/asserts.bzl @@ -7,13 +7,12 @@ load("@bazel_lib//lib:write_source_files.bzl", "write_source_file") # the venv/runfiles source layer only if something bypasses the pip-package # layer's `_should_skip_pkg_path` filter — e.g. a reintroduced # `_wheels/` intermediate tree, which also re-duplicates wheel files. -# Asserted by a Docker-free py_test (see assert_tar_listing); intentionally an -# invariant, not exact bytes, so it survives snapshot regeneration. -_FORBIDDEN_LAYER_PATHS = [ - "__pycache__", - "[.]pyc", - "/_wheels/", -] +# Keyed by mode because first-party bytecode is allowed in bytecode modes. +_FORBIDDEN_LAYER_PATHS = { + "source": ["__pycache__", "[.]pyc", "/_wheels/"], + "pyc": ["site-packages/.*[.]pyc", "/_wheels/"], + "pyc_only": ["__pycache__", "site-packages/.*[.]pyc", "/_wheels/"], +} # Paths whose byte size varies across Bazel releases or builds. We keep # the rows in the listing (so a missing/renamed file would still be @@ -34,7 +33,7 @@ _FILTERED_PATHS = [ "/bazel_tools/tools/bash/runfiles/runfiles.bash", ] -def assert_tar_listing(name, actual, expected, exclude = [], disjoint = True, **kwargs): +def assert_tar_listing(name, actual, expected, exclude = [], disjoint = True, pyc = "source", **kwargs): """Snapshot and invariant tests over the tar listings of image layers. Renders `bsdtar -tv` rows for every tar in `actual` into one multi-layer @@ -59,6 +58,7 @@ def assert_tar_listing(name, actual, expected, exclude = [], disjoint = True, ** rows they keep. The disjointness test always sees every row. disjoint: set False to skip the disjointness test for layouts with intentional cross-layer overlap. + pyc: the image's `pyc` mode; selects which bytecode paths are forbidden. **kwargs: forwarded to the `write_source_file` snapshot target. """ actual_listing = "{}_listing".format(name) @@ -163,7 +163,7 @@ done > $@ name = "{}_no_forbidden_paths".format(name), srcs = ["//tools:assert_absent.py"], main = "//tools:assert_absent.py", - args = ["$(rootpath :{})".format(actual_listing)] + _FORBIDDEN_LAYER_PATHS, + args = ["$(rootpath :{})".format(actual_listing)] + _FORBIDDEN_LAYER_PATHS[pyc], data = [":{}".format(actual_listing)], testonly = True, **test_kwargs diff --git a/e2e/rules-python-interop/BUILD.bazel b/e2e/rules-python-interop/BUILD.bazel index 4fe9459d0..fd1232d07 100644 --- a/e2e/rules-python-interop/BUILD.bazel +++ b/e2e/rules-python-interop/BUILD.bazel @@ -208,6 +208,11 @@ rules_py_test( ], ) +rules_python_library( + name = "precompiled_lib", + srcs = ["precompiled_lib.py"], +) + rules_python_library( name = "import_path_lib", srcs = ["rules_python_import/rp_import.py"], @@ -220,6 +225,64 @@ rules_py_test( deps = [":import_path_lib"], ) +rules_py_test( + name = "rules_python_dep_test", + srcs = ["rules_python_dep_test.py"], + deps = [":precompiled_lib"], +) + +rules_py_test( + name = "rules_python_dep_pyc_test", + srcs = ["rules_python_dep_test.py"], + env = {"EXPECT_PYC": "pyc"}, + main = "rules_python_dep_test.py", + pyc = "pyc", + deps = [":precompiled_lib"], +) + +# Same dependency on the foreign runtime: rules_py's hub has no 3.11, so the +# aspect must read rules_python's PyRuntimeInfo for the cache tag and compile +# with the target interpreter rather than an exec-tools fallback. +rules_py_test( + name = "rules_python_dep_pyc_311_test", + srcs = ["rules_python_dep_test.py"], + env = {"EXPECT_PYC": "pyc"}, + main = "rules_python_dep_test.py", + pyc = "pyc", + python_version = "3.11", + deps = [":precompiled_lib"], +) + +rules_py_test( + name = "rules_python_dep_pyc_only_311_test", + srcs = ["rules_python_dep_test.py"], + env = { + "EXPECT_PYC": "pyc_only", + "EXPECT_SOURCE_IN_RUNFILES": "1", + }, + main = "rules_python_dep_test.py", + pyc = "pyc_only", + python_version = "3.11", + deps = [":precompiled_lib"], +) + +# The aspect only compiles sources owned by the library's own package, so a +# source borrowed from another package leaves pyc_only incomplete. test.sh +# asserts the analysis failure names the file. +rules_python_library( + name = "cross_package_lib", + srcs = ["//cross-package:helper.py"], +) + +rules_py_test( + name = "cross_package_pyc_only_test", + srcs = ["rules_python_dep_test.py"], + main = "rules_python_dep_test.py", + pyc = "pyc_only", + tags = ["manual"], + deps = [":cross_package_lib"], +) + # rules_python's console-script machinery reads the pip hub's dist-info to # generate a py_binary; unpinned, that binary lands on a rules_py-provisioned # toolchain. Its hub is parsed for 3.12 alone, so which wheel backs the script diff --git a/e2e/rules-python-interop/cross-package/BUILD.bazel b/e2e/rules-python-interop/cross-package/BUILD.bazel new file mode 100644 index 000000000..0bcb5b025 --- /dev/null +++ b/e2e/rules-python-interop/cross-package/BUILD.bazel @@ -0,0 +1 @@ +exports_files(["helper.py"]) diff --git a/e2e/rules-python-interop/cross-package/helper.py b/e2e/rules-python-interop/cross-package/helper.py new file mode 100644 index 000000000..4860f406c --- /dev/null +++ b/e2e/rules-python-interop/cross-package/helper.py @@ -0,0 +1,2 @@ +def answer() -> int: + return 42 diff --git a/e2e/rules-python-interop/precompiled_lib.py b/e2e/rules-python-interop/precompiled_lib.py new file mode 100644 index 000000000..4860f406c --- /dev/null +++ b/e2e/rules-python-interop/precompiled_lib.py @@ -0,0 +1,2 @@ +def answer() -> int: + return 42 diff --git a/e2e/rules-python-interop/reset-data-edges/BUILD.bazel b/e2e/rules-python-interop/reset-data-edges/BUILD.bazel index 300fcabc5..6f13ed774 100644 --- a/e2e/rules-python-interop/reset-data-edges/BUILD.bazel +++ b/e2e/rules-python-interop/reset-data-edges/BUILD.bazel @@ -1,5 +1,5 @@ load("@aspect_rules_py//py:defs.bzl", "py_binary", "py_library", "py_venv") -load(":tests.bzl", "probe", "reset_data_edges_test_suite", "root", "terminal", "versioned_terminal") +load(":tests.bzl", "probe", "pyc_fanout_root", "reset_data_edges_test_suite", "root", "terminal", "versioned_terminal") package(default_testonly = True) @@ -170,4 +170,49 @@ root( ], ) +probe(name = "pyc_binary_data_probe") + +probe(name = "pyc_data_probe") + +py_library( + name = "pyc_shared", + srcs = ["main.py"], + tags = ["manual"], +) + +py_library( + name = "pyc_data_library", + srcs = ["main.py"], + data = [":pyc_data_probe"], + tags = ["manual"], +) + +py_venv( + name = "canonical_venv", + srcs = ["main.py"], + tags = ["manual"], + deps = [":pyc_shared"], +) + +py_binary( + name = "pyc_bin", + srcs = ["main.py"], + data = [":pyc_binary_data_probe"], + main = "main.py", + tags = ["manual"], + deps = [":pyc_shared"], +) + +# Analyze the same consumers under source and pyc_only. Binaries retain both +# modes while venvs, their dependencies, and runtime data canonicalize. +pyc_fanout_root( + name = "pyc_reset_root", + tags = ["manual"], + deps = [ + ":canonical_venv", + ":pyc_bin", + ":pyc_data_library", + ], +) + reset_data_edges_test_suite() diff --git a/e2e/rules-python-interop/reset-data-edges/tests.bzl b/e2e/rules-python-interop/reset-data-edges/tests.bzl index 8e33a6ce4..eaf3466cf 100644 --- a/e2e/rules-python-interop/reset-data-edges/tests.bzl +++ b/e2e/rules-python-interop/reset-data-edges/tests.bzl @@ -10,9 +10,10 @@ _PYTHON_VERSION_FLAG = "@aspect_rules_py//py/private/interpreter:python_version" _RPY_VERSION_FLAG = "@rules_python//python/config_settings:python_version" _FREETHREADED_FLAG = "@aspect_rules_py//py/private/interpreter:freethreaded" _RPY_FREETHREADED_FLAG = "@rules_python//python/config_settings:py_freethreaded" +_PYC_FLAG = "@aspect_rules_py//py:pyc" _ProbeInfo = provider(fields = ["file"]) -_ProbeFilesInfo = provider(fields = ["files", "modes", "bin_dirs"]) +_ProbeFilesInfo = provider(fields = ["files", "modes", "pyc_modes", "bin_dirs"]) def _probe_impl(ctx): out = ctx.actions.declare_file(ctx.label.name + ".txt") @@ -34,6 +35,7 @@ def _probe_aspect_impl(target, ctx): transitive = [] transitive_modes = [] + transitive_pyc_modes = [] transitive_bin_dirs = [] deps = [] for attr_name in ["data", "deps"]: @@ -48,6 +50,7 @@ def _probe_aspect_impl(target, ctx): if _ProbeFilesInfo in dep: transitive.append(dep[_ProbeFilesInfo].files) transitive_modes.append(dep[_ProbeFilesInfo].modes) + transitive_pyc_modes.append(dep[_ProbeFilesInfo].pyc_modes) transitive_bin_dirs.append(dep[_ProbeFilesInfo].bin_dirs) # Record every visited target; the test impl filters to the names it @@ -64,6 +67,10 @@ def _probe_aspect_impl(target, ctx): return [_ProbeFilesInfo( files = depset(direct = direct, transitive = transitive), modes = depset(direct = modes, transitive = transitive_modes), + pyc_modes = depset( + direct = [(ctx.label.name, ctx.attr._pyc[BuildSettingInfo].value)], + transitive = transitive_pyc_modes, + ), bin_dirs = depset(direct = bin_dirs, transitive = transitive_bin_dirs), )] @@ -72,6 +79,7 @@ _probe_aspect = aspect( attr_aspects = ["data", "deps", "venv"], attrs = { "_freethreaded": attr.label(default = _FREETHREADED_FLAG), + "_pyc": attr.label(default = _PYC_FLAG), "_rpy_freethreaded": attr.label(default = _RPY_FREETHREADED_FLAG), }, ) @@ -112,6 +120,7 @@ def _root_impl(ctx): return [_ProbeFilesInfo( files = depset(transitive = transitive), modes = depset(transitive = [dep[_ProbeFilesInfo].modes for dep in ctx.attr.deps]), + pyc_modes = depset(transitive = [dep[_ProbeFilesInfo].pyc_modes for dep in ctx.attr.deps]), bin_dirs = depset(transitive = [dep[_ProbeFilesInfo].bin_dirs for dep in ctx.attr.deps]), )] @@ -169,6 +178,32 @@ root = rule( }, ) +def _pyc_split_transition_impl(_settings, _attr): + return { + "source": {_PYC_FLAG: "source"}, + "pyc_only": {_PYC_FLAG: "pyc_only"}, + } + +_pyc_split_transition = transition( + implementation = _pyc_split_transition_impl, + inputs = [], + outputs = [_PYC_FLAG], +) + +pyc_fanout_root = rule( + implementation = _root_impl, + attrs = { + "deps": attr.label_list( + allow_empty = False, + aspects = [_probe_aspect], + cfg = _pyc_split_transition, + ), + "_allowlist_function_transition": attr.label( + default = "@bazel_tools//tools/allowlists/function_transition_allowlist", + ), + }, +) + def _reset_data_edges_test_impl(ctx): env = analysistest.begin(ctx) files = analysistest.target_under_test(env)[_ProbeFilesInfo].files.to_list() @@ -234,6 +269,54 @@ def _shared_binaries_test_impl(ctx): _shared_binaries_test = analysistest.make(_shared_binaries_test_impl) +def _pyc_resets_test_impl(ctx): + env = analysistest.begin(ctx) + under_test = analysistest.target_under_test(env)[_ProbeFilesInfo] + actual = {} + for name, mode in under_test.pyc_modes.to_list(): + actual.setdefault(name, {})[mode] = True + + expected = { + # The runnable and a directly configured library remain mode-specific. + "pyc_bin": {"pyc_only": True, "source": True}, + "pyc_data_library": {"pyc_only": True, "source": True}, + + # Venvs, their Python graph, and runtime data are mode-independent. + "_pyc_bin.venv": {"source": True}, + "canonical_venv": {"source": True}, + "pyc_binary_data_probe": {"source": True}, + "pyc_data_probe": {"source": True}, + "pyc_shared": {"source": True}, + } + for name, modes in expected.items(): + asserts.equals(env, modes, actual.get(name), name + " pyc configurations") + + bin_dirs = {} + for name, path in under_test.bin_dirs.to_list(): + bin_dirs.setdefault(name, {})[path] = True + for name in [ + "_pyc_bin.venv", + "canonical_venv", + "pyc_binary_data_probe", + "pyc_data_probe", + "pyc_shared", + ]: + asserts.equals(env, 1, len(bin_dirs[name]), name + " should have one configured target") + for name in ["pyc_bin", "pyc_data_library"]: + asserts.equals(env, 2, len(bin_dirs[name]), name + " should retain both terminal configurations") + + files = under_test.files.to_list() + for basename in ["pyc_binary_data_probe.txt", "pyc_data_probe.txt"]: + asserts.equals( + env, + 1, + len([f for f in files if f.basename == basename]), + basename + " should be one shared artifact", + ) + return analysistest.end(env) + +_pyc_resets_test = analysistest.make(_pyc_resets_test_impl) + def reset_data_edges_test_suite(): _reset_data_edges_test( name = "reset_data_edges_test", @@ -250,3 +333,8 @@ def reset_data_edges_test_suite(): tags = ["manual"], target_under_test = ":binaries_root", ) + _pyc_resets_test( + name = "pyc_resets_test", + tags = ["manual"], + target_under_test = ":pyc_reset_root", + ) diff --git a/e2e/rules-python-interop/rules_python_dep_test.py b/e2e/rules-python-interop/rules_python_dep_test.py new file mode 100644 index 000000000..a49f17c47 --- /dev/null +++ b/e2e/rules-python-interop/rules_python_dep_test.py @@ -0,0 +1,32 @@ +"""Asserts a rules_py binary ships a @rules_python py_library dependency's +sources or bytecode as the bytecode mode dictates. Files are checked before the +import so the interpreter cannot have written the cache itself. + +rules_python keeps the library's srcs in its own runfiles, so pyc_only only +drops the source when rules_python's omit_source retention drops it too. +""" + +import os +import sys + +mode = os.environ.get("EXPECT_PYC", "source") +source_in_runfiles = os.environ.get("EXPECT_SOURCE_IN_RUNFILES") == "1" +root = os.path.join(os.environ["TEST_SRCDIR"], os.environ["TEST_WORKSPACE"]) +source = os.path.join(root, "precompiled_lib.py") +pycache = os.path.join( + root, "__pycache__", "precompiled_lib.{}.pyc".format(sys.implementation.cache_tag) +) +legacy = os.path.join(root, "precompiled_lib.pyc") + +present = {p for p in (source, pycache, legacy) if os.path.exists(p)} +expected = { + "source": {source}, + "pyc": {source, pycache}, + "pyc_only": {legacy} | ({source} if source_in_runfiles else set()), +}[mode] +assert present == expected, (mode, present) + +import precompiled_lib # noqa: E402 + +assert precompiled_lib.answer() == 42 +print("OK") diff --git a/e2e/rules-python-interop/test.sh b/e2e/rules-python-interop/test.sh index 3fa204bee..f40b2ba7f 100755 --- a/e2e/rules-python-interop/test.sh +++ b/e2e/rules-python-interop/test.sh @@ -9,8 +9,6 @@ set -euo pipefail cd "$(dirname "$0")" # e2e/rules-python-interop -BAZEL="${BAZEL:-bazel}" - # The repo check is what makes the rules_python-flag runs meaningful at 3.12: # rules_python's own default toolchain reports that version too, so a broken # fallback in rules_py's hub would be invisible if only the version were @@ -21,7 +19,7 @@ check_resolved_runtime() { local version="$3" local repo="$4" local got - got="$("$BAZEL" run --lockfile_mode=off "--${flag}=${version}" \ + got="$(bazel run --lockfile_mode=off "--${flag}=${version}" \ -- "${target}" 2>/dev/null)" if [[ "${got}" != "${version} "* || "${got}" != *"${repo}"* ]]; then echo "FAIL: set ${flag}=${version}, expected ${target} to report ${version} from ${repo}, got ${got}" >&2 @@ -34,12 +32,12 @@ check_resolved_runtime() { # transition normalizes both flags, so either entry point reaches every # version — including 3.11, which only rules_python provisions. for version in 3.10 3.11 3.12 3.13 3.14; do - "$BAZEL" run \ + bazel run \ --lockfile_mode=off \ "--@aspect_rules_py//py:python_version=${version}" \ -- //:version_check "${version}" - "$BAZEL" run \ + bazel run \ --lockfile_mode=off \ "--@rules_python//python/config_settings:python_version=${version}" \ -- //:version_check "${version}" @@ -62,13 +60,13 @@ done # declares no 3.11, so nothing of ours shadows it. check_resolved_runtime @rules_python//python/config_settings:python_version //:report_version 3.11 rules_python -"$BAZEL" build \ +bazel build \ --lockfile_mode=off \ --@aspect_rules_py//py:python_version=3.13 \ -- \ //:python_launcher -launcher_version="$(<"$("$BAZEL" info --lockfile_mode=off bazel-bin)/python_launcher.txt")" +launcher_version="$(<"$(bazel info --lockfile_mode=off bazel-bin)/python_launcher.txt")" if [[ "${launcher_version}" != "3.13" ]]; then echo "FAIL: rules_py Python version selected launcher ${launcher_version}, expected 3.13" >&2 exit 1 @@ -76,9 +74,71 @@ fi echo "PASS: rules_py Python version selected the 3.13 launcher" -"$BAZEL" test --lockfile_mode=off \ +bazel test --lockfile_mode=off \ --@aspect_rules_py//py:python_version=3.13 \ -- \ //reset-data-edges:passthrough_terminals_test \ + //reset-data-edges:pyc_resets_test \ //reset-data-edges:reset_data_edges_test \ //reset-data-edges:shared_binaries_test + +# Exercise rules_python's source-retention modes. +retention=@rules_python//python/config_settings:precompile_source_retention +bazel test --lockfile_mode=off \ + --@aspect_rules_py//py:pyc=pyc_only --test_env=EXPECT_PYC=pyc_only \ + --test_env=EXPECT_SOURCE_IN_RUNFILES=1 \ + -- //:rules_python_dep_test +bazel test --lockfile_mode=off "--${retention}=omit_source" \ + -- //:rules_python_dep_test +bazel test --lockfile_mode=off "--${retention}=omit_source" \ + --@aspect_rules_py//py:pyc=pyc --test_env=EXPECT_PYC=pyc \ + -- //:rules_python_dep_test +bazel test --lockfile_mode=off "--${retention}=omit_source" \ + --@aspect_rules_py//py:pyc=pyc_only --test_env=EXPECT_PYC=pyc_only \ + -- //:rules_python_dep_test + +# Exercise rules_python with precompilation disabled. +precompile=@rules_python//python/config_settings:precompile +bazel test --lockfile_mode=off "--${precompile}=force_disabled" \ + --@aspect_rules_py//py:pyc=pyc_only --test_env=EXPECT_PYC=pyc_only \ + --test_env=EXPECT_SOURCE_IN_RUNFILES=1 \ + -- //:rules_python_dep_test + +# A source owned by another package is outside the aspect's reach. +cross_package_log="$(mktemp)" +if bazel build --lockfile_mode=off -- //:cross_package_pyc_only_test >"$cross_package_log" 2>&1; then + echo "FAIL: pyc_only accepted a rules_python library with a cross-package source" >&2 + exit 1 +fi +if ! grep -Fq "pyc_only could not compile all first-party sources: cross-package/helper.py" "$cross_package_log"; then + cat "$cross_package_log" >&2 + echo "FAIL: pyc_only failure did not name the uncompiled cross-package source" >&2 + exit 1 +fi +rm -f "$cross_package_log" + +# Verify which compiler produces each layout. +actions="$(mktemp)" +query="$(mktemp)" +trap 'rm -f "$actions" "$query"' EXIT +echo 'mnemonic("PyCompile", deps(//:rules_python_dep_test))' >"$query" +check_compiler() { + local flag="$1" output="$2" tool="$3" count + bazel aquery --lockfile_mode=off --output=text "$flag" \ + --query_file="$query" >"$actions" 2>/dev/null + count="$(awk -v RS='' -v out="$output" 'match($0, /Outputs: \[[^]]*\]/) && index(substr($0, RSTART, RLENGTH), out) { c++ } END { print c + 0 }' "$actions")" + if [[ "$count" != 1 ]]; then + echo "FAIL: ${flag}: expected one action producing ${output}, got ${count}" >&2 + exit 1 + fi + if ! awk -v RS='' -v out="$output" 'match($0, /Outputs: \[[^]]*\]/) && index(substr($0, RSTART, RLENGTH), out)' "$actions" | grep -Fq "$tool"; then + echo "FAIL: ${flag}: expected ${output} to be compiled by ${tool}" >&2 + exit 1 + fi +} +check_compiler "--${retention}=keep_source" "__pycache__/precompiled_lib.cpython-312.pyc" precompiler +check_compiler "--${retention}=keep_source" "/precompiled_lib.pyc" pyc_compile.py +check_compiler "--${retention}=omit_source" "__pycache__/precompiled_lib.cpython-312.pyc" pyc_compile.py +check_compiler "--${retention}=omit_source" "/precompiled_lib.pyc" precompiler +check_compiler "--${precompile}=force_disabled" "__pycache__/precompiled_lib.cpython-312.pyc" pyc_compile.py +check_compiler "--${precompile}=force_disabled" "/precompiled_lib.pyc" pyc_compile.py diff --git a/e2e/rules-python-interop/virtual-deps/django/BUILD.bazel b/e2e/rules-python-interop/virtual-deps/django/BUILD.bazel index bc213f921..829316ed1 100644 --- a/e2e/rules-python-interop/virtual-deps/django/BUILD.bazel +++ b/e2e/rules-python-interop/virtual-deps/django/BUILD.bazel @@ -117,3 +117,13 @@ py_test( }, deps = [":proj"], ) + +py_test( + name = "resolution_bytecode_test", + srcs = ["resolution_bytecode_test.py"], + main = "resolution_bytecode_test.py", + package_collisions = "warning", + pyc = "pyc_only", + resolutions = django_resolutions, + deps = [":proj"], +) diff --git a/e2e/rules-python-interop/virtual-deps/django/resolution_bytecode_test.py b/e2e/rules-python-interop/virtual-deps/django/resolution_bytecode_test.py new file mode 100644 index 000000000..85e808d91 --- /dev/null +++ b/e2e/rules-python-interop/virtual-deps/django/resolution_bytecode_test.py @@ -0,0 +1,13 @@ +"""A virtual dep resolved to a rules_python pip hub package gets bytecode from +rules_py's aspect over `resolutions`; pyc_only would otherwise fail analysis +listing the hub's sources. The hub's own runfiles still carry the source.""" + +import os + +import django + +assert "+pip+django_312_django" in django.__file__, django.__file__ +assert django.__file__.endswith("__init__.py"), django.__file__ +legacy = django.__file__[: -len(".py")] + ".pyc" +assert os.path.exists(legacy), os.listdir(os.path.dirname(django.__file__)) +print("OK") diff --git a/e2e/rules-python-protobuf/BUILD.bazel b/e2e/rules-python-protobuf/BUILD.bazel index 2899a7924..e6baa20db 100644 --- a/e2e/rules-python-protobuf/BUILD.bazel +++ b/e2e/rules-python-protobuf/BUILD.bazel @@ -1,5 +1,6 @@ load("@aspect_rules_py//py:defs.bzl", "py_test") load("@protobuf//bazel:proto_library.bzl", "proto_library") +load("@protobuf//bazel:py_proto_library.bzl", "py_proto_library") load("@rules_proto_grpc_python//:defs.bzl", "python_proto_library") # A plain proto_library — the language-agnostic descriptor. Living at the @@ -28,3 +29,48 @@ py_test( main = "test.py", deps = [":greeting_py_proto"], ) + +proto_library( + name = "native_greeting_proto", + srcs = ["native_greeting.proto"], +) + +py_proto_library( + name = "native_greeting_py_proto", + deps = [":native_greeting_proto"], +) + +py_test( + name = "native_test", + size = "small", + srcs = ["native_test.py"], + deps = [":native_greeting_py_proto"], +) + +# Bytecode for each generator under both modes. py_proto_library reaches the +# protobuf runtime through runfiles alone, so only rules_proto_grpc's PyInfo +# edge to @protobuf//:protobuf_python gets the runtime compiled. +[ + py_test( + name = "{}_{}_test".format(generator, mode), + size = "small", + srcs = ["bytecode_test.py"], + env = { + "EXPECT_PYC": mode, + "EXPECT_RUNTIME_PYC": runtime_pyc, + "MODULE": module, + "MODULE_DIR": module_dir, + }, + main = "bytecode_test.py", + pyc = mode, + deps = [dep], + ) + for generator, dep, module, module_dir, runtime_pyc in [ + ("native", ":native_greeting_py_proto", "native_greeting_pb2", "", "0"), + ("grpc", ":greeting_py_proto", "greeting_pb2", "greeting_py_proto_pb", "1"), + ] + for mode in [ + "pyc", + "pyc_only", + ] +] diff --git a/e2e/rules-python-protobuf/MODULE.bazel b/e2e/rules-python-protobuf/MODULE.bazel index 38e0ca2f5..4a5fb0dfb 100644 --- a/e2e/rules-python-protobuf/MODULE.bazel +++ b/e2e/rules-python-protobuf/MODULE.bazel @@ -1,12 +1,8 @@ -"Standalone e2e: rules_py consuming rules_proto_grpc_python-generated code" +"Standalone e2e: rules_py consuming protobuf-generated rules_python libraries" -# rules_proto_grpc_python generates Python bindings as a rules_python -# py_library (which advertises the upstream PyInfo provider) and pulls in the -# protobuf runtime via @protobuf//:protobuf_python. This lives in its own -# module — rather than as a case in the shared e2e MODULE — so the -# rules_proto_grpc/protobuf/grpc dependency surface never leaks into the -# standard cases. It complements //examples/protobuf, which exercises the same -# consumer seam against protobuf's native py_proto_library. +# protobuf's py_proto_library and rules_proto_grpc_python both generate Python +# bindings as rules_python py_library targets. This lives in its own module so +# the protobuf/grpc dependency surface never leaks into the main workspace. # # rules_python is named nowhere here or in BUILD.bazel — it is only pulled in # transitively, and the interpreter toolchain is provisioned through rules_py's diff --git a/e2e/rules-python-protobuf/bytecode_test.py b/e2e/rules-python-protobuf/bytecode_test.py new file mode 100644 index 000000000..5d47802fa --- /dev/null +++ b/e2e/rules-python-protobuf/bytecode_test.py @@ -0,0 +1,46 @@ +"""Bytecode for a rules_python-generated proto module comes from rules_py's +deps aspect; rules_python's own runfiles retain the generated source in every +mode. The protobuf runtime is compiled only when it reaches the test through +PyInfo deps (rules_proto_grpc) rather than runfiles alone (py_proto_library). +Files are checked before the import so the interpreter cannot have written +them itself. +""" + +import importlib +import os +import sys + +mode = os.environ["EXPECT_PYC"] +module = os.environ["MODULE"] +runtime_compiled = os.environ["EXPECT_RUNTIME_PYC"] == "1" +root = os.path.join( + os.environ["TEST_SRCDIR"], os.environ["TEST_WORKSPACE"], os.environ["MODULE_DIR"] +) +runtime = os.path.join( + os.environ["TEST_SRCDIR"], "protobuf+", "python", "google", "protobuf" +) +tag = sys.implementation.cache_tag + + +def layout(directory: str, stem: str) -> tuple[str, str]: + pycache = os.path.join(directory, "__pycache__", "{}.{}.pyc".format(stem, tag)) + legacy = os.path.join(directory, stem + ".pyc") + return {"pyc": (pycache, legacy), "pyc_only": (legacy, pycache)}[mode] + + +expected, unexpected = layout(root, module) +assert os.path.exists(os.path.join(root, module + ".py")), os.listdir(root) +assert os.path.exists(expected), os.listdir(root) +assert not os.path.exists(unexpected), os.listdir(root) + +runtime_expected, runtime_unexpected = layout(runtime, "message") +assert os.path.exists(os.path.join(runtime, "message.py")), os.listdir(runtime) +assert os.path.exists(runtime_expected) == runtime_compiled, os.listdir(runtime) +assert not os.path.exists(runtime_unexpected), os.listdir(runtime) + +generated = importlib.import_module(module) +descriptor = next(iter(generated.DESCRIPTOR.message_types_by_name.values())) +message_class = getattr(generated, descriptor.name) +message = message_class(**{descriptor.fields[0].name: "pyc"}) +assert message_class.FromString(message.SerializeToString()) == message +print("OK") diff --git a/e2e/rules-python-protobuf/native_greeting.proto b/e2e/rules-python-protobuf/native_greeting.proto new file mode 100644 index 000000000..500c90505 --- /dev/null +++ b/e2e/rules-python-protobuf/native_greeting.proto @@ -0,0 +1,6 @@ +syntax = "proto3"; + +message NativeGreeting { + string recipient = 1; + int32 times = 2; +} diff --git a/e2e/rules-python-protobuf/native_test.py b/e2e/rules-python-protobuf/native_test.py new file mode 100644 index 000000000..7538c7b6f --- /dev/null +++ b/e2e/rules-python-protobuf/native_test.py @@ -0,0 +1,4 @@ +from native_greeting_pb2 import NativeGreeting + +greeting = NativeGreeting(recipient="rules_py", times=2) +assert NativeGreeting.FromString(greeting.SerializeToString()) == greeting diff --git a/py/BUILD.bazel b/py/BUILD.bazel index ead55c05b..2f5ad44e1 100644 --- a/py/BUILD.bazel +++ b/py/BUILD.bazel @@ -1,4 +1,6 @@ load("@bazel_lib//:bzl_library.bzl", "bzl_library") +load("@bazel_skylib//rules:common_settings.bzl", "string_flag") +load("//py/private:pyc.bzl", "PYC_MODES") load("//py/private/interpreter:launcher.bzl", "py_interpreter_launcher") # Consumed by //docs:py_api (stardoc). exports_files gives the implicit .bzl file @@ -71,6 +73,13 @@ label_flag( visibility = ["//visibility:public"], ) +string_flag( + name = "pyc", + build_setting_default = "source", + values = PYC_MODES, + visibility = ["//visibility:public"], +) + bzl_library( name = "current_py_toolchain", srcs = ["current_py_toolchain.bzl"], diff --git a/py/defs.bzl b/py/defs.bzl index 3dca5a2b0..76ae2790b 100644 --- a/py/defs.bzl +++ b/py/defs.bzl @@ -159,6 +159,29 @@ def py_binary(name, srcs = [], main = None, **kwargs): explicit `py_venv_link(name = "{name}.venv_link", venv = ":{name}.venv")` alongside the binary. + * `pyc` (string) — first-party bytecode packaging. + `"source"` ships only `.py` sources; `"pyc"` additionally + ships PEP 3147 `__pycache__` bytecode; `"pyc_only"` ships + colocated sourceless `.pyc` files (tracebacks then carry + no source lines). Unset inherits the global + `--@aspect_rules_py//py:pyc` flag; an explicit value pins + the mode regardless of the flag. Configurable: a + `select()` value is accepted. Bytecode compilation + requires an executable, bytecode-compatible target + interpreter, and `"pyc_only"` requires every first-party + source to be directly owned by a rules_py `py_*` target; + other first-party sources ship as source under `"pyc"`. + A `.py` file also declared through `data` remains source + because explicit runtime data takes precedence over source + stripping. + Only `.py` files listed in `srcs` by their own file label + are compiled; files reached through a rule target in + `srcs` (filegroup, genrule, py_library) stay source. + `bazel coverage` always runs `"pyc_only"` targets from + sources so coverage.py can instrument them. + `"pyc_only"` rejects `-O`/`-OO` interpreter options and + `PYTHONOPTIMIZE` settings because its level-0 bytecode has no + source to fall back to; `"pyc"` runs from source instead. """ _py_binary_with_venv( diff --git a/py/private/BUILD.bazel b/py/private/BUILD.bazel index 5d0b2d588..62b043799 100644 --- a/py/private/BUILD.bazel +++ b/py/private/BUILD.bazel @@ -19,6 +19,7 @@ exports_files( "unittest_main.py", "merge_repo_mappings.awk", "modify_mtree.awk", + "pyc_compile.py", ], visibility = ["//visibility:public"], ) @@ -56,6 +57,19 @@ py_layer_tier( visibility = ["//visibility:public"], ) +bzl_library( + name = "pyc", + srcs = ["pyc.bzl"], + deps = [ + ":providers", + ":py_info_interop", + ":transitions", + "//py/private/toolchain:types", + "@bazel_skylib//lib:paths", + "@bazel_skylib//rules:common_settings", + ], +) + bzl_library( name = "py_image_layer", srcs = ["py_image_layer.bzl"], @@ -64,9 +78,11 @@ bzl_library( ":providers", ":py_info", ":py_info_interop", + ":pyc", "//py/private/py_venv:types", "//py/private/toolchain:types", "@bazel_lib//lib:transitions", + "@bazel_skylib//rules:common_settings", "@tar.bzl//tar:mtree", "@tar.bzl//tar:tar", ], @@ -91,6 +107,7 @@ bzl_library( ":py_info_interop", ":py_semantics", ":py_wheel", + ":pyc", ":transitions", "@bazel_skylib//lib:new_sets", "@bazel_skylib//lib:types", @@ -160,6 +177,8 @@ bzl_library( deps = [ ":providers", ":py_info", + ":pyc", + ":transitions", "//py/private/py_venv:types", "//py/private/toolchain:types", "@bazel_skylib//rules:common_settings", diff --git a/py/private/interpreter/runtime.bzl b/py/private/interpreter/runtime.bzl index 9c0d89fc2..f58a47169 100644 --- a/py/private/interpreter/runtime.bzl +++ b/py/private/interpreter/runtime.bzl @@ -41,6 +41,14 @@ def _py_runtime_toolchain_impl(ctx): zip_main_template = ctx.file._zip_main_template, ) + pyc_compile_tool = None + if ctx.attr.pyc_compile_tool: + pyc_compile_tool = struct( + executable = ctx.attr.pyc_compile_tool[DefaultInfo].files_to_run, + arguments = [], + inputs = depset(), + ) + return [ runtime, platform_common.ToolchainInfo( @@ -50,6 +58,7 @@ def _py_runtime_toolchain_impl(ctx): py3_runtime = runtime, # The //py/private/toolchain:exec_tools_toolchain_type contract. exec_runtime = runtime, + pyc_compile_tool = pyc_compile_tool, ), DefaultInfo(files = depset([ctx.file.interpreter], transitive = [runtime.files])), ] @@ -79,6 +88,23 @@ build host regardless of the target platform being built for).""", "abi_flags": attr.string( doc = "CPython ABI flag suffix, e.g. \"t\" for freethreaded.", ), + "pyc_compile_tool": attr.label( + doc = """Self-contained executable replacing the default bytecode +compiler script, e.g. a prebuilt binary. Takes `SRC OUT DFILE` triples +(source input, bytecode output, logical traceback path), one per action today, +`--legacy` to also write the colocated sourceless `.pyc` beside a PEP 3147 +`__pycache__` `OUT` (identical bytes), and `--expect-version VERSION` (the +target runtime's Python version, to verify or ignore), all passed through one +`@ARGFILE`; pyc_compile.py is the reference implementation. Runs once per +action; only the reference script is additionally driven as a Bazel persistent +worker. +Built for and runs on the exec platform, but emitted bytecode must exactly +match this runtime's format — the `--expect-version` contract. Must not +resolve a Python toolchain: this toolchain's own resolution would cycle +through it.""", + executable = True, + cfg = "exec", + ), "_bootstrap_template": attr.label( allow_single_file = True, default = "@rules_python//python/private:bootstrap_template", diff --git a/py/private/py_image_layer.bzl b/py/private/py_image_layer.bzl index f1ab787b5..cce554ca2 100644 --- a/py/private/py_image_layer.bzl +++ b/py/private/py_image_layer.bzl @@ -30,6 +30,7 @@ Sharing model: - Ungrouped pip packages: squashed by the rule into one per-rule tar. """ +load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") load( "//py/private:compression.bzl", "DEFAULT_ALGORITHM", @@ -45,6 +46,8 @@ load( load("//py/private:providers.bzl", "PyWheelsInfo") load("//py/private:py_info.bzl", "PyInfo") load("//py/private:py_info_interop.bzl", "has_py_info") +load("//py/private:pyc.bzl", "PYC_MODES", "PYC_MODE_ATTRS", "PycInfo", "PycModeInfo") +load("//py/private:transitions.bzl", "PYC_FLAG") load("//py/private/py_venv:types.bzl", "VirtualenvInfo") load("//py/private/toolchain:types.bzl", "PY_TOOLCHAIN", "interpreter_files_and_version") @@ -890,13 +893,14 @@ def _platform_cfg_impl(settings, attr): result = { "//command_line_option:platforms": [attr.platform] if attr.platform else settings["//command_line_option:platforms"], "@aspect_rules_py//py:layer_tier": str(attr.layer_tier) if attr.layer_tier else settings["@aspect_rules_py//py:layer_tier"], + PYC_FLAG: attr.pyc if attr.pyc else settings[PYC_FLAG], } return result _platform_cfg = transition( implementation = _platform_cfg_impl, - inputs = ["//command_line_option:platforms", "@aspect_rules_py//py:layer_tier"], - outputs = ["//command_line_option:platforms", "@aspect_rules_py//py:layer_tier"], + inputs = ["//command_line_option:platforms", "@aspect_rules_py//py:layer_tier", PYC_FLAG], + outputs = ["//command_line_option:platforms", "@aspect_rules_py//py:layer_tier", PYC_FLAG], ) def _skip_path(f): @@ -1074,12 +1078,47 @@ def _declare_group_tar(ctx, rule_codecs, plan, bsdtar, bsdtar_files, out_basenam ) return tar_out +def _fp_files_for_pyc(files, mode, pyc_by_source_path, retained_source_short_paths): + """Rewrite a first-party group's files for the image's bytecode mode. + + Matched by runfiles path: group files and the launcher's PycInfo are + analyzed in different configurations. Unmapped or data-retained .py ship as-is. + """ + if mode == "source" or not pyc_by_source_path: + return files + out = [] + for f in files.to_list(): + entries = pyc_by_source_path.get(f.short_path) if f.extension == "py" else None + if entries == None: + out.append(f) + elif mode == "pyc": + out.append(f) + + out.extend([entry.pycache for entry in entries]) + else: + if f.short_path in retained_source_short_paths: + out.append(f) + out.extend([entry.pyc for entry in entries]) + return depset(out) + def _py_image_layer_impl(ctx): binaries = ctx.attr.binaries if not binaries: fail("py_image_layer requires at least one binary") single_binary = len(binaries) == 1 infos = [binary[_LayerInfo] for binary in binaries] + + effective_pyc = ctx.attr._pyc_flag[BuildSettingInfo].value + for binary in binaries: + binary_mode = binary[PycModeInfo].mode if PycModeInfo in binary else "source" + if binary_mode != effective_pyc: + fail("{}: binary {} has pyc={} but the image requires pyc={}; drop the binary's explicit pyc attribute or align it with the image".format( + ctx.label, + binary.label, + binary_mode, + effective_pyc, + )) + bsdtar, bsdtar_files = _tar_toolchain(ctx) # Normalized labels can collide across lock universes, and one wheel target @@ -1189,17 +1228,66 @@ def _py_image_layer_impl(ctx): interpreter_map = lambda f, d: _interpreter_file_to_mtree(f, d, owner, group) rule_group_names = {gname: True for gname in ctx.attr.groups.values()} - rule_group_files = [] - rule_groups = [] + rule_group_specs = [] for dep, group_name in ctx.attr.groups.items(): dep_label = normalize_label(str(dep.label)) if dep_label in pip_labels: continue files = dep[DefaultInfo].files - rule_group_files.append(files) - rule_groups.append((group_name, files)) + rule_group_specs.append((group_name, files)) + + fp_layer_entries = [] + for info in infos: + fp_layer_entries.extend(info.first_party_layers.to_list()) + pyc_by_source_path = {} + pyc_files = [] + pyc_only_sources = [] + retained_source_paths = {} + retained_source_short_paths = {} + if effective_pyc == "pyc_only": + for binary in binaries: + for f in binary[DefaultInfo].default_runfiles.files.to_list(): + retained_source_paths[f.path] = True + retained_source_short_paths[f.short_path] = True + if effective_pyc != "source": + # Configured artifacts for one source share a runfiles path. + pyc_by_dest = {} + for binary in binaries: + if PycInfo not in binary: + continue + pyc_info = binary[PycInfo] + pyc_files.append(pyc_info.pycache_files if effective_pyc == "pyc" else pyc_info.legacy_files) + for entry in pyc_info.entries.to_list(): + dest = entry.source.short_path + prev = pyc_by_dest.get(dest) + if prev != None: + if prev.pycache == entry.pycache: + continue + if effective_pyc == "pyc_only" and prev.pycache.basename != entry.pycache.basename: + fail("{}: binaries compile conflicting bytecode for {} (different Python runtimes or configurations); align the binaries' python_version or use pyc = \"source\"".format( + ctx.label, + dest, + )) + else: + pyc_by_dest[dest] = entry + pyc_by_source_path.setdefault(entry.source.short_path, []).append(entry) + if effective_pyc == "pyc_only" and entry.source.path not in retained_source_paths: + pyc_only_sources.append(entry.source) + fp_entries = [ + struct( + label = entry.label, + group = entry.group, + files = _fp_files_for_pyc(entry.files, effective_pyc, pyc_by_source_path, retained_source_short_paths), + ) + for entry in fp_layer_entries + ] + rule_groups = [ + (group_name, _fp_files_for_pyc(files, effective_pyc, pyc_by_source_path, retained_source_short_paths)) + for group_name, files in rule_group_specs + ] + rule_group_files = [files for _, files in rule_groups] - source_files = depset(transitive = [info.source_files for info in infos]) + source_files = depset(transitive = [info.source_files for info in infos] + pyc_files) if repo_mapping != None: source_files = depset(direct = [repo_mapping], transitive = [source_files]) rule_group_map = lambda f, d: ( @@ -1209,14 +1297,13 @@ def _py_image_layer_impl(ctx): first_party_reference_files = [] fp_by_group = {} seen_fp_labels = {} - for info in infos: - for entry in info.first_party_layers.to_list(): - first_party_reference_files.append(entry.files) - if single_binary: - if entry.label in seen_fp_labels: - continue - seen_fp_labels[entry.label] = True - fp_by_group.setdefault(entry.group, []).append(entry.files) + for entry in fp_entries: + first_party_reference_files.append(entry.files) + if single_binary: + if entry.label in seen_fp_labels: + continue + seen_fp_labels[entry.label] = True + fp_by_group.setdefault(entry.group, []).append(entry.files) # Interpreter tars are declared at the configured toolchain, so identical # runtimes action-share while distinct interpreter artifacts are retained. @@ -1348,6 +1435,11 @@ def _py_image_layer_impl(ctx): # snapshot here to avoid double-bookkeeping during construction. dep_tars = list(all_tars) + source_skip_files = depset( + direct = pyc_only_sources, + transitive = source_exclusion_files, + ) if pyc_only_sources or source_exclusion_files else None + source_tar = _declare_group_tar( ctx, rule_codecs, @@ -1360,7 +1452,7 @@ def _py_image_layer_impl(ctx): source_map, "Creating source layer for %s" % ctx.label, symlink_mappings, - skip_files = depset(transitive = source_exclusion_files) if source_exclusion_files else None, + skip_files = source_skip_files, ) all_tars.append(source_tar) @@ -1393,11 +1485,13 @@ def _py_image_layer_impl(ctx): validation_args.add("--mtree") validation_arguments.append(mtree_args) - validation_skip_args = _path_set_args(ctx, source_exclusion_files) - validation_flag_args = ctx.actions.args() - validation_flag_args.add("--skip") - validation_arguments.extend([validation_flag_args, validation_skip_args]) - validation_inputs.extend([source_files] + source_exclusion_files) + validation_inputs.append(source_files) + if source_skip_files: + validation_skip_args = _path_set_args(ctx, [source_skip_files]) + validation_flag_args = ctx.actions.args() + validation_flag_args.add("--skip") + validation_arguments.extend([validation_flag_args, validation_skip_args]) + validation_inputs.append(source_skip_files) ctx.actions.run( executable = ctx.executable._validator, @@ -1442,6 +1536,11 @@ _py_image_layer = rule( "warn_layer_count": attr.int(default = 90), "platform": attr.label(default = None, providers = [platform_common.PlatformInfo]), "layer_tier": attr.label(default = None, providers = [PyLayerTierInfo]), + "pyc": attr.string( + default = "", + values = [""] + PYC_MODES, + doc = "First-party bytecode mode for the image; empty inherits the `//py:pyc` flag. Read by a transition, so not configurable.", + ), "_layer_tier": attr.label( default = "//py:layer_tier", providers = [PyLayerTierInfo], @@ -1465,7 +1564,7 @@ _py_image_layer = rule( cfg = "exec", executable = True, ), - }, + } | PYC_MODE_ATTRS, cfg = _platform_cfg, toolchains = [_TAR_TOOLCHAIN], ) @@ -1483,6 +1582,7 @@ def py_image_layer( warn_layer_count = 90, platform = None, layer_tier = None, + pyc = "", launcher_dir = "", binaries = None, **kwargs): @@ -1537,6 +1637,12 @@ def py_image_layer( layer_tier: Optional py_layer_tier target pinned for this rule. Sets the `@aspect_rules_py//py:layer_tier` label_flag via the rule transition, overriding any command-line value for this rule's subgraph. + pyc: First-party bytecode mode for the image: "source", "pyc", or + "pyc_only". Sets the `@aspect_rules_py//py:pyc` flag via the rule + transition, so a `select()` is not accepted; empty (the default) + inherits the flag's value. Binaries with an unset `pyc` attribute + follow it automatically; a binary whose explicit `pyc` attribute + disagrees fails analysis. launcher_dir: Absolute image directory for the binary launchers. Defaults to /app/bin with multiple binaries. Set RUNFILES_DIR=/app.runfiles in the image. @@ -1575,6 +1681,7 @@ def py_image_layer( warn_layer_count = warn_layer_count, platform = platform, layer_tier = layer_tier, + pyc = pyc, tags = tags, **kwargs ) diff --git a/py/private/py_info_interop.bzl b/py/private/py_info_interop.bzl index 85855e0d6..b1b700532 100644 --- a/py/private/py_info_interop.bzl +++ b/py/private/py_info_interop.bzl @@ -37,3 +37,16 @@ def get_py_info(target): if RulesPythonPyInfo in target: return target[RulesPythonPyInfo] return None + +def get_transitive_sources(target): + """The target's transitive first-party sources, from either `PyInfo`. + + `@rules_python` drops precompiled sources from `transitive_sources` under + `precompile_source_retention = omit_source` and carries them only in + `transitive_implicit_pyc_source_files`. + """ + info = get_py_info(target) + return depset(transitive = [ + info.transitive_sources, + getattr(info, "transitive_implicit_pyc_source_files", depset()), + ]) diff --git a/py/private/py_library.bzl b/py/private/py_library.bzl index f56ec7f4c..8dad0b494 100644 --- a/py/private/py_library.bzl +++ b/py/private/py_library.bzl @@ -10,7 +10,8 @@ load("@rules_cc//cc/common:cc_info.bzl", "CcInfo") load("//py/private:providers.bzl", "PyWheelsInfo") load("//py/private:pth.bzl", "make_imports_depset") load("//py/private:py_info.bzl", "PyInfo") -load("//py/private:py_info_interop.bzl", "RulesPythonPyInfo", "get_py_info", "has_py_info") +load("//py/private:py_info_interop.bzl", "RulesPythonPyInfo", "get_py_info", "get_transitive_sources", "has_py_info") +load("//py/private:pyc.bzl", "PYC_ATTRS", "PYC_TOOLCHAINS", "compile_pycs", "make_pyc_info", "own_compile_sources", "pyc_aspect") load("//py/private:transitions.bzl", "reset_python_flags_transition") def _make_instrumented_files_info(ctx): @@ -28,7 +29,7 @@ def _make_srcs_depset(ctx, extra_depsets = []): order = "postorder", direct = ctx.files.srcs, transitive = [ - get_py_info(target).transitive_sources + get_transitive_sources(target) for target in ctx.attr.deps if has_py_info(target) ] + extra_depsets, @@ -47,9 +48,8 @@ def _make_virtual_depset(ctx): def _make_resolved_virtual_depset(target): transitive = [target[DefaultInfo].files] - info = get_py_info(target) - if info: - transitive.append(info.transitive_sources) + if has_py_info(target): + transitive.append(get_transitive_sources(target)) return depset( order = "postorder", @@ -177,6 +177,14 @@ def _py_library_impl(ctx): instrumented_files_info, ] + compiled = compile_pycs(ctx, own_compile_sources(ctx.attr.srcs)) + providers.append(make_pyc_info( + compiled, + sources = ctx.files.srcs, + deps = ctx.attr.deps, + resolutions = getattr(ctx.attr, "resolutions", {}).values(), + )) + if getattr(ctx.attr, "_emit_rules_python_providers", None) and ctx.attr._emit_rules_python_providers[BuildSettingInfo].value: # Compatibility shim for trees mid-migration: keeps not-yet-converted # @rules_python py_* targets able to depend on this library. @@ -203,6 +211,7 @@ _attrs = dict({ # rules_py emits @rules_python providers only under the # migration-only //py:emit_rules_python_providers flag. providers = [[PyInfo], [RulesPythonPyInfo], [CcInfo]], + aspects = [pyc_aspect], ), "data": attr.label_list( doc = """Runtime dependencies of the program. @@ -225,6 +234,7 @@ _attrs = dict({ See virtual_deps. """, providers = [[PyInfo], [RulesPythonPyInfo]], + aspects = [pyc_aspect], ), }) @@ -250,6 +260,7 @@ py_library = rule( attrs = dict({ "virtual_deps": attr.string_list(allow_empty = True, default = []), "_emit_rules_python_providers": attr.label(default = "//py/private:emit_rules_python_providers"), - }, **py_library_utils.attrs), + }, **py_library_utils.attrs) | PYC_ATTRS, provides = py_library_utils.py_library_providers, + toolchains = PYC_TOOLCHAINS, ) diff --git a/py/private/py_pex_binary.bzl b/py/private/py_pex_binary.bzl index 4827b21cd..99d870b0d 100644 --- a/py/private/py_pex_binary.bzl +++ b/py/private/py_pex_binary.bzl @@ -22,6 +22,8 @@ load("@bazel_lib//lib:paths.bzl", "to_rlocation_path") load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") load("//py/private:providers.bzl", "PyWheelsInfo") load("//py/private:py_info.bzl", "PyInfo") +load("//py/private:pyc.bzl", "PycModeInfo") +load("//py/private:transitions.bzl", "reset_pyc_transition") load("//py/private/py_venv:types.bzl", "PY_VENV_KINDS", "VirtualenvInfo", "venv_root") load("//py/private/py_venv:virtuals_resolvers.bzl", "VENV_OWNED_ROOTS") load("//py/private/toolchain:types.bzl", "PY_TOOLCHAIN", "interpreter_files_and_version") @@ -185,7 +187,9 @@ def _dep_arg(wheel): return "--dependency={}/{}".format(wheel.install_tree.path, suffix) def _py_python_pex_impl(ctx): - binary = ctx.attr.binary + binary = _single_target(ctx.attr.binary) + if PycModeInfo in binary and binary[PycModeInfo].mode != "source": + fail("py_pex_binary {} requires binary {} to use pyc=source".format(ctx.label, binary.label)) binary_default = binary[DefaultInfo] # py_venv_exec emits depset([launcher, main]) — the non-executable file is @@ -312,7 +316,7 @@ def _py_python_pex_impl(ctx): _attrs = dict({ "binary": attr.label( executable = True, - cfg = "target", + cfg = reset_pyc_transition, mandatory = True, doc = "The py_binary target to package.", aspects = [_closure_aspect], diff --git a/py/private/py_pytest_test.bzl b/py/private/py_pytest_test.bzl index cede410b3..d6c92f297 100644 --- a/py/private/py_pytest_test.bzl +++ b/py/private/py_pytest_test.bzl @@ -123,6 +123,9 @@ def py_pytest_test( Include the `pytest` package (and `coverage`, if you want coverage) in `deps`. + Because pytest collects `.py` source paths, `pyc_only` requests fall back + to source-retaining `pyc` mode for these targets. + Every file in `srcs` is a test module that pytest collects (scoped to the target, not the whole runfiles tree). Put importable support code in `deps` and pytest's `conftest.py` in `data`; to select tests by name pattern, use @@ -151,6 +154,7 @@ def py_pytest_test( fail("py_pytest_test provides its own entrypoint; `main` is not supported. Use py_pytest_main + py_test for a custom main.") kwargs["testonly"] = True + kwargs["source_retention_required"] = True deps = list(deps) main = pytest_driver_wiring( diff --git a/py/private/py_venv/BUILD.bazel b/py/private/py_venv/BUILD.bazel index 8df0198f8..852ab3bfa 100644 --- a/py/private/py_venv/BUILD.bazel +++ b/py/private/py_venv/BUILD.bazel @@ -63,6 +63,7 @@ bzl_library( "//py/private:py_info", "//py/private:py_library", "//py/private:py_semantics", + "//py/private:pyc", "//py/private:transitions", "@bazel_lib//lib:expand_make_vars", "@bazel_lib//lib:paths", @@ -79,6 +80,7 @@ bzl_library( ":venv", "//py/private:py_library", "//py/private:py_semantics", + "//py/private:pyc", "//py/private:transitions", "//py/private/toolchain:types", "@bazel_lib//lib:expand_make_vars", diff --git a/py/private/py_venv/py_venv.bzl b/py/private/py_venv/py_venv.bzl index 9b1738e7b..f083d1c68 100644 --- a/py/private/py_venv/py_venv.bzl +++ b/py/private/py_venv/py_venv.bzl @@ -29,7 +29,8 @@ load("@bazel_lib//lib:expand_make_vars.bzl", "expand_locations", "expand_variabl load("@bazel_lib//lib:paths.bzl", "BASH_RLOCATION_FUNCTION", "to_rlocation_path") load("//py/private:py_library.bzl", _py_library = "py_library_utils") load("//py/private:py_semantics.bzl", _py_semantics = "semantics") -load("//py/private:transitions.bzl", "python_transition") +load("//py/private:pyc.bzl", "PYC_ATTRS", "compile_pycs", "make_pyc_info", "own_compile_sources") +load("//py/private:transitions.bzl", "py_venv_transition") load("//py/private/toolchain:types.bzl", "EXEC_TOOLS_TOOLCHAIN", "PY_TOOLCHAIN") load(":py_venv_exec.bzl", _py_venv_exec = "py_venv_exec") load(":types.bzl", "VirtualenvInfo", "venv_root") @@ -120,6 +121,7 @@ def _venv_providers(ctx, venv, venv_only, executable = None, include_sources = F runfiles = venv.runtime_runfiles.merge(ctx.runfiles(files = venv_only)) if include_sources: runfiles = runfiles.merge(ctx.runfiles(transitive_files = venv.transitive_sources)) + return [ DefaultInfo( files = depset([executable]) if executable != None else None, @@ -135,6 +137,12 @@ def _venv_providers(ctx, venv, venv_only, executable = None, include_sources = F dependency_attributes = ["deps"], extensions = ["py"], ), + make_pyc_info( + compile_pycs(ctx, own_compile_sources(ctx.attr.srcs)), + sources = ctx.files.srcs, + deps = ctx.attr.deps, + resolutions = getattr(ctx.attr, "resolutions", {}).values(), + ), ] def _py_venv_rule_impl(ctx): @@ -257,6 +265,7 @@ does not reinsert a wheel. }) _lib_attrs.update(**_py_library.attrs) +_lib_attrs.update(**PYC_ATTRS) # Attrs only the executable variant reads — launcher template, REPL # flags, env vars forwarded via RunEnvironmentInfo. @@ -305,7 +314,7 @@ _py_venv = rule( attrs = _attrs, toolchains = _venv_toolchains, executable = True, - cfg = python_transition, + cfg = py_venv_transition, ) def _py_venv_lib_rule_impl(ctx): @@ -325,7 +334,7 @@ _py_venv_lib = rule( "include_console_scripts": attr.bool(default = False), }, toolchains = _venv_toolchains, - cfg = python_transition, + cfg = py_venv_transition, ) def _wrap_with_debug(rule): @@ -503,6 +512,9 @@ def py_venv_link(name, venv, link_name = None, **kwargs): **kwargs: Forwarded to the underlying `py_binary`. """ link_script = str(Label("//py/private/py_venv:templates/link.py")) + + # The link script is not part of the venv, so bytecode modes cannot apply. + kwargs["pyc"] = "source" _py_venv_exec( name = name, main = link_script, diff --git a/py/private/py_venv/py_venv_exec.bzl b/py/private/py_venv/py_venv_exec.bzl index 2b165debb..1326966cd 100644 --- a/py/private/py_venv/py_venv_exec.bzl +++ b/py/private/py_venv/py_venv_exec.bzl @@ -10,8 +10,9 @@ load("@bazel_lib//lib:expand_make_vars.bzl", "expand_locations", "expand_variabl load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") load("@hermetic_launcher//launcher:lib.bzl", "launcher") load("//py/private:py_info.bzl", "PyInfo") -load("//py/private:py_info_interop.bzl", "RulesPythonPyInfo", "get_py_info", "has_py_info") +load("//py/private:py_info_interop.bzl", "RulesPythonPyInfo", "get_transitive_sources", "has_py_info") load("//py/private:py_semantics.bzl", _py_semantics = "semantics") +load("//py/private:pyc.bzl", "PYC_MODES", "PYC_MODE_ATTRS", "PycInfo", "PycModeInfo") load("//py/private:transitions.bzl", "reset_python_flags_transition", "venv_python_transition") load(":types.bzl", "VirtualenvInfo", "venv_root") @@ -29,6 +30,99 @@ def _single_venv(value): return value[0] return value +def _pyc_mode(ctx): + mode = ctx.attr.pyc + if mode == "": + mode = ctx.attr._pyc_flag[BuildSettingInfo].value + if mode == "pyc_only": + if ctx.configuration.coverage_enabled: + return "source" + if ctx.attr.source_retention_required: + return "pyc" + return mode + +def _requests_optimization(opt): + """Whether a single-dash interpreter option bundles `-O` (`-OO`, `-BO`). + + `-W`, `-X`, `-c`, `-m` end the bundle and take the rest as their argument + (`-Wignore::foo.OldWarning`), as does the first non-letter. + """ + if not opt.startswith("-") or opt.startswith("--"): + return False + for letter in opt[1:].elems(): + if not letter.isalpha() or letter in "WXcm": + return False + if letter == "O": + return True + return False + +def _resolve_pyc(ctx, venv, main, passed_env, inherited_env): + mode = _pyc_mode(ctx) + vinfo = venv[VirtualenvInfo] + if mode == "source": + return struct( + entrypoint = main, + info = None, + mode = mode, + venv_files = vinfo.transitive_sources, + ) + + if PycInfo not in venv: + fail("{}: bytecode mode requires a rules_py py_venv, which always carries first-party bytecode mappings".format(ctx.label)) + + info = venv[PycInfo] + if mode == "pyc": + return struct( + entrypoint = main, + info = info, + mode = mode, + venv_files = depset(transitive = [vinfo.transitive_sources, info.pycache_files]), + ) + + # Sourceless bytecode is level 0 and loads regardless of the optimization requested. + for opt in ctx.attr.interpreter_options: + if _requests_optimization(opt): + fail("{}: pyc_only ships level-0 bytecode without sources; interpreter_options {} is incompatible. Use pyc = \"pyc\" or \"source\" for optimized interpreters.".format(ctx.label, opt)) + if ( + "PYTHONOPTIMIZE" in ctx.attr.env or + "PYTHONOPTIMIZE" in ctx.attr.env_inherit or + "PYTHONOPTIMIZE" in passed_env or + "PYTHONOPTIMIZE" in inherited_env + ): + fail("{}: pyc_only does not support PYTHONOPTIMIZE in env or env_inherit, even when set to 0. Remove it or use pyc = \"pyc\" or \"source\".".format(ctx.label)) + + if not info.complete: + missing = sorted([src.short_path for src in info.missing_sources.to_list()]) + fail("{}: pyc_only could not compile all first-party sources{}".format( + ctx.label, + ": " + ", ".join(missing) if missing else "", + )) + + # The generated venv compiles the launcher's main as one of its direct + # srcs. Match by runfiles path because generated sources in the venv's + # transitioned configuration have different exec paths. + main_entry = None + for entry in info.direct_entries: + if entry.source.short_path == main.short_path: + main_entry = entry + break + if main_entry == None: + for entry in info.entries.to_list(): + if entry.source.short_path == main.short_path: + main_entry = entry + break + if main_entry == None: + fail(("{}: pyc_only requested but no bytecode was produced for main {}. " + + "The source must be directly owned by a rules_py py_* target and the exec " + + "Python must exactly match the target Python version.").format(ctx.label, main)) + + return struct( + entrypoint = main_entry.pyc, + info = info, + mode = mode, + venv_files = info.sourceless_files, + ) + def _py_venv_exec_impl(ctx): # The launcher itself doesn't need a python toolchain — it just # exec's the sibling venv's `bin/python`, whose path was already @@ -48,14 +142,6 @@ def _py_venv_exec_impl(ctx): venv = _single_venv(ctx.attr.venv) vinfo = venv[VirtualenvInfo] - # Merge env vars: start from the venv's `env` (if any), then - # overlay the binary's own — binary wins on key conflicts. Same - # merge for inherited env-var names. Bazel-contextual identifiers - # (BAZEL_TARGET, etc.) overlay last and are stripped from - # `inherited_env` so a stray `env_inherit` entry can't let the - # caller's shell shadow the contextual label — per - # https://bazel.build/rules/lib/providers/RunEnvironmentInfo, an - # inherited value wins over `environment` when both are present. passed_env = {} inherited_env = [] if RunEnvironmentInfo in venv: @@ -63,14 +149,15 @@ def _py_venv_exec_impl(ctx): passed_env = dict(venv_env.environment) inherited_env = list(venv_env.inherited_environment) - # Owned by the rule. The lib venv variant carries no `env` to guard, - # so guard here to match the executable variant's check. + pyc = _resolve_pyc(ctx, venv, main, passed_env, inherited_env) + + # Library venvs have no RunEnvironmentInfo, so the launcher owns VIRTUAL_ENV. if "VIRTUAL_ENV" in ctx.attr.env: fail("py_binary/py_test {}: `VIRTUAL_ENV` is set by the rule and cannot be overridden via `env`.".format(ctx.label)) - # Set here so it's present even for the lib venv variant, which has - # no RunEnvironmentInfo to carry it. passed_env["VIRTUAL_ENV"] = venv_root(vinfo.bin_python) + + # Overlay binary values, then protect contextual values from env_inherit. for k, v in ctx.attr.env.items(): passed_env[k] = expand_variables( ctx, @@ -83,6 +170,7 @@ def _py_venv_exec_impl(ctx): passed_env["BAZEL_TARGET"] = str(ctx.label).lstrip("@") passed_env["BAZEL_WORKSPACE"] = ctx.workspace_name passed_env["BAZEL_TARGET_NAME"] = ctx.attr.name + inherited_env = [n for n in inherited_env if n not in _CONTEXTUAL_ENV_KEYS] # When `isolated = False`, drop Python's `-I` flag so PYTHONPATH is @@ -106,7 +194,7 @@ def _py_venv_exec_impl(ctx): transformed_args = transformed_args, ) embedded_args, transformed_args = launcher.append_runfile( - file = main, + file = pyc.entrypoint, embedded_args = embedded_args, transformed_args = transformed_args, ) @@ -119,18 +207,15 @@ def _py_venv_exec_impl(ctx): # Merge runfiles, supporting `py_venv_exec(main)` not being in the `py_venv` runfiles. data_sources = [ - get_py_info(target).transitive_sources + get_transitive_sources(target) for target in ctx.attr.data if has_py_info(target) ] - # First-party import sources attach explicitly; everything else the venv - # needs at runtime (venv files, wheels, data) comes from its - # runtime_runfiles, so a terminal can substitute the source set without - # re-deriving the rest. + # rules_python may retain its sources in the venv's runtime runfiles. runfiles = ctx.runfiles( - files = ctx.files.data + [main], - transitive_files = depset(transitive = [vinfo.transitive_sources] + data_sources), + files = ctx.files.data + ([] if pyc.mode == "pyc_only" else [main]), + transitive_files = depset(transitive = [pyc.venv_files] + data_sources), ).merge(vinfo.runtime_runfiles).merge_all( [target[DefaultInfo].default_runfiles for target in ctx.attr.data], ) @@ -146,7 +231,7 @@ def _py_venv_exec_impl(ctx): providers = [ DefaultInfo( - files = depset([executable_launcher, main]), + files = depset([executable_launcher, pyc.entrypoint]), executable = executable_launcher, runfiles = runfiles, ), @@ -166,7 +251,10 @@ def _py_venv_exec_impl(ctx): environment = passed_env, inherited_environment = inherited_env, ), + PycModeInfo(mode = pyc.mode), ] + if pyc.info != None: + providers.append(pyc.info) if ctx.attr._emit_rules_python_providers[BuildSettingInfo].value: providers.append(RulesPythonPyInfo( @@ -211,14 +299,29 @@ the macro layer in `//py:defs.bzl`. The binary's launcher exec's the referenced venv's `bin/python`; its runfiles inherit the venv's runtime runfiles for wheels and runtime data, -and add first-party sources from `VirtualenvInfo.transitive_sources` at +and add first-party sources (or their compiled bytecode, per `pyc`) at their usual rlocation paths. The edge transition forwards this launcher's `python_version` / `freethreaded` choices to the venv's configuration, so several launchers can resolve one venv label under different interpreter versions or GIL modes; unset, the inherited configuration passes through -untouched. +untouched. `pyc` is launcher-only — the venv always declares bytecode +actions, so every mode shares one configured venv. """, ), + "pyc": attr.string( + default = "", + values = [""] + PYC_MODES, + doc = """First-party bytecode packaging: `source` ships only `.py` +sources; `pyc` additionally ships PEP 3147 `__pycache__` bytecode; +`pyc_only` ships colocated sourceless `.pyc` files. Empty (the default) +follows the global `--@aspect_rules_py//py:pyc` flag; an explicit value +pins the mode regardless of the flag. Configurable: `select()` values +are accepted.""", + ), + "source_retention_required": attr.bool( + default = False, + doc = "Internal: downgrade pyc_only to pyc for source-collecting test drivers.", + ), "python_version": attr.string( default = "", doc = "Python version for this direct py_venv_exec consumer. Usually set on py_binary/py_test instead.", @@ -276,7 +379,7 @@ that must match the terminal's Python environment in `deps`. "_emit_rules_python_providers": attr.label( default = "//py/private:emit_rules_python_providers", ), -}) +}) | PYC_MODE_ATTRS _test_attrs = dict({ # Magic attribute to make coverage --combined_report flag work. diff --git a/py/private/pyc.bzl b/py/private/pyc.bzl new file mode 100644 index 000000000..b3c6af3d1 --- /dev/null +++ b/py/private/pyc.bzl @@ -0,0 +1,385 @@ +"""First-party Python bytecode compilation. + +`PycInfo` is declared unconditionally by `py_library` / `py_venv`; the compile +actions only run when a terminal's `pyc` attribute or the `//py:pyc` flag +requests bytecode, so launchers sharing a configured library share them. +""" + +load("@bazel_skylib//lib:paths.bzl", "paths") +load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") +load("//py/private:providers.bzl", "PyWheelsInfo") +load("//py/private:py_info_interop.bzl", "RulesPythonPyInfo", "get_py_info", "has_py_info") +load("//py/private:transitions.bzl", "PYC_FLAG") +load("//py/private/toolchain:types.bzl", "EXEC_TOOLS_TOOLCHAIN", "PY_TOOLCHAIN") + +PYC_MODES = ["source", "pyc", "pyc_only"] + +PycInfo = provider( + doc = "Private: first-party Python bytecode artifacts.", + fields = { + "complete": "bool — whether every transitive Python source has a bytecode entry.", + "direct_entries": "list[struct(source, pyc, pycache)] — bytecode mappings declared directly by this target.", + "entries": "depset[struct(source, pyc, pycache)] — transitive bytecode mappings; pyc is the colocated sourceless layout, byte-identical to pycache.", + "legacy_files": "depset[File] — colocated sourceless .pyc files.", + "missing_sources": "depset[File] — Python sources without bytecode entries.", + "pycache_files": "depset[File] — PEP 3147 __pycache__ files for source-retaining mode.", + "sourceless_files": "depset[File] — colocated .pyc files plus non-Python source artifacts retained by pyc_only.", + }, +) + +PycModeInfo = provider( + doc = "Private: effective first-party bytecode mode of a runnable target.", + fields = {"mode": "One of source, pyc, or pyc_only."}, +) + +PYC_ATTRS = { + "_pyc_compiler": attr.label( + default = "//py/private:pyc_compile.py", + allow_single_file = True, + ), +} + +# Terminals read the flag as the default for an unset `pyc` attribute. +PYC_MODE_ATTRS = { + "_pyc_flag": attr.label( + default = PYC_FLAG, + providers = [BuildSettingInfo], + ), +} + +# Compilation is optional until a pyc consumer requests complete bytecode. +PYC_TOOLCHAINS = [ + config_common.toolchain_type(PY_TOOLCHAIN, mandatory = False), + config_common.toolchain_type(EXEC_TOOLS_TOOLCHAIN, mandatory = False), +] + +_PRERELEASE_ABBREVS = {"alpha": "a", "beta": "b", "candidate": "rc"} + +def own_compile_sources(srcs_targets): + """Files this target compiles: `srcs` entries that are file labels. + + Files reached through a rule target in `srcs` (filegroup, genrule, + py_library) stay in source form. + + Args: + srcs_targets: Targets of the rule's `srcs` attribute. + + Returns: + list[File] — the compilable sources. + """ + files = [] + for target in srcs_targets: + if has_py_info(target): + continue + fs = target[DefaultInfo].files.to_list() + if len(fs) == 1 and _is_file_target(target.label, fs[0]): + files.append(fs[0]) + return files + +def _is_file_target(label, f): + """Whether `label` names the file itself rather than a rule providing it.""" + owner = f.owner + if owner == None: + return False + if f.is_source: + return owner == label + + if owner == label: + return False + if owner.package != label.package or owner.workspace_name != label.workspace_name: + return False + path = f.short_path + if path.startswith("../"): + path = path.split("/", 2)[2] + if label.package: + path = path[len(label.package) + 1:] + return path == label.name + +def _expected_version(runtime): + """Version string the compiler must match, or None when unknown.""" + version_info = getattr(runtime, "interpreter_version_info", None) + if version_info == None: + return None + expected = "{}.{}.{}".format(version_info.major, version_info.minor, getattr(version_info, "micro", None) or 0) + releaselevel = getattr(version_info, "releaselevel", None) + if releaselevel and releaselevel != "final": + expected += _PRERELEASE_ABBREVS.get(releaselevel, releaselevel) + str(getattr(version_info, "serial", None) or 0) + return expected + +def _bytecode_key(runtime): + version_info = getattr(runtime, "interpreter_version_info", None) + if version_info == None: + return None + + pyc_tag = getattr(runtime, "pyc_tag", None) + implementation_name = getattr(runtime, "implementation_name", None) + if pyc_tag: + runtime_identity = ("pyc_tag", str(pyc_tag)) + elif implementation_name: + runtime_identity = ("implementation", str(implementation_name)) + else: + return None + + key = [ + runtime_identity, + str(getattr(version_info, "major", None)), + str(getattr(version_info, "minor", None)), + ] + releaselevel = getattr(version_info, "releaselevel", None) or "final" + if releaselevel != "final": + key += [ + str(getattr(version_info, "micro", None) or 0), + releaselevel, + str(getattr(version_info, "serial", None) or 0), + ] + return tuple(key) + +def bytecode_compatible(exec_runtime, target_runtime): + """Whether `exec_runtime` emits bytecode loadable by `target_runtime`. + + Final releases match on implementation/cache tag and major.minor; + prereleases require an exact version match. + """ + target_key = _bytecode_key(target_runtime) + return target_key != None and target_key == _bytecode_key(exec_runtime) + +def pycache_tag(runtime): + """Return the runtime's PEP 3147 cache tag. + + Args: + runtime: PyRuntimeInfo of the target toolchain. + + Returns: + The `pyc_tag` field, else `-`, else None. + """ + tag = getattr(runtime, "pyc_tag", None) + if tag: + return tag + implementation_name = getattr(runtime, "implementation_name", None) + version = getattr(runtime, "interpreter_version_info", None) + if not implementation_name or version == None: + return None + return "{}-{}{}".format(implementation_name, version.major, version.minor) + +def compile_pycs(ctx, srcs, existing = {}): + """Compile this rule's own first-party sources to bytecode. + + Only sources directly owned by this target's package are compiled — a + ``sibling=`` declaration (which keeps the bytecode's natural runfiles + location next to its source) is only permitted for files of the declaring + package. Foreign sources are expected to be compiled by their own owning + target; the pyc_only terminal validates that none remain. + + Args: + ctx: rule or aspect ctx carrying PYC_ATTRS and PYC_TOOLCHAINS. + srcs: list[File] — the rule's direct sources. + existing: dict[short_path, File] — bytecode the owning target already + declares at the natural paths; taken instead of compiled. + + Returns: + struct(entries, legacy_files, pycache_files) of lists; empty lists + when no bytecode-compatible compiler is available. + """ + entries = [] + legacy_files = [] + pycache_files = [] + + target_toolchain = ctx.toolchains[PY_TOOLCHAIN] + target_runtime = target_toolchain.py3_runtime if target_toolchain != None else None + + # Prefer a custom tool, then a compatible exec runtime, then the target runtime. + pyc_compile_tool = getattr(target_toolchain, "pyc_compile_tool", None) if target_toolchain != None else None + tool_toolchain = PY_TOOLCHAIN + if pyc_compile_tool == None and target_runtime != None: + exec_toolchain = ctx.toolchains[EXEC_TOOLS_TOOLCHAIN] + exec_runtime = getattr(exec_toolchain, "exec_runtime", None) if exec_toolchain != None else None + compile_runtime = None + if exec_runtime != None and getattr(exec_runtime, "interpreter", None) != None and bytecode_compatible(exec_runtime, target_runtime): + compile_runtime = exec_runtime + tool_toolchain = EXEC_TOOLS_TOOLCHAIN + elif target_runtime.interpreter != None: + compile_runtime = target_runtime + if compile_runtime != None: + pyc_compile_tool = struct( + executable = compile_runtime.interpreter, + arguments = ["-S", "-s", "-B", ctx.file._pyc_compiler], + inputs = depset( + [compile_runtime.interpreter, ctx.file._pyc_compiler], + transitive = [compile_runtime.files], + ), + supports_workers = True, + ) + + pyc_tag = pycache_tag(target_runtime) + if target_runtime == None or pyc_compile_tool == None or pyc_tag == None: + return struct(entries = entries, legacy_files = legacy_files, pycache_files = pycache_files) + + expected_version = _expected_version(target_runtime) + + # Startup arguments stay on the command line; per-source arguments go + # through a flagfile so a persistent worker receives them per request. + tool_args = ctx.actions.args() + tool_args.add_all(pyc_compile_tool.arguments) + execution_requirements = {} + if getattr(pyc_compile_tool, "supports_workers", False): + execution_requirements = {"supports-workers": "1", "requires-worker-protocol": "json"} + + # Per-source actions remain identical when multiple targets share a source. + for src in srcs: + if src.extension != "py": + continue + if src.owner.package != ctx.label.package or src.owner.workspace_name != ctx.label.workspace_name: + continue + stem = src.basename[:-3] + directory = src.short_path.rpartition("/")[0] + pycache_basename = "{}.{}.pyc".format(stem, pyc_tag) + pyc = existing.get(paths.join(directory, stem + ".pyc")) + pycache = existing.get(paths.join(directory, "__pycache__", pycache_basename)) + + outputs = [] + if pycache == None: + pycache = ctx.actions.declare_file("__pycache__/{}".format(pycache_basename), sibling = src) + outputs.append(pycache) + if pyc == None: + pyc = ctx.actions.declare_file(stem + ".pyc", sibling = src) + outputs.append(pyc) + if outputs: + compile_args = ctx.actions.args() + compile_args.set_param_file_format("multiline") + compile_args.use_param_file("@%s", use_always = True) + if expected_version: + compile_args.add("--expect-version", expected_version) + if len(outputs) == 2: + compile_args.add("--legacy") + compile_args.add(src) + compile_args.add(outputs[0]) + compile_args.add(src.short_path) + ctx.actions.run( + executable = pyc_compile_tool.executable, + toolchain = tool_toolchain, + arguments = [tool_args, compile_args], + execution_requirements = execution_requirements, + inputs = depset( + direct = [src], + transitive = [pyc_compile_tool.inputs], + ), + outputs = outputs, + mnemonic = "PyCompile", + progress_message = "Python precompiling {} into {}".format( + src.short_path, + ", ".join([out.short_path for out in outputs]), + ), + env = { + "PYTHONHASHSEED": "0", + "PYTHONNOUSERSITE": "1", + "PYTHONSAFEPATH": "1", + }, + ) + entries.append(struct(source = src, pyc = pyc, pycache = pycache)) + legacy_files.append(pyc) + pycache_files.append(pycache) + + return struct(entries = entries, legacy_files = legacy_files, pycache_files = pycache_files) + +def make_pyc_info(compiled, sources = [], deps = [], resolutions = []): + """Merge this rule's own compiled bytecode with its dependencies'. + + Args: + compiled: the struct returned by `compile_pycs` (or None). + sources: list[File] — this rule's direct contribution to PyInfo. + deps: Targets whose PycInfo (when present) is inherited. + resolutions: additional Targets (virtual-dep resolutions) to inherit. + + Returns: + PycInfo + """ + transitive_entries = [] + transitive_legacy_files = [] + transitive_missing_sources = [] + transitive_pycache_files = [] + transitive_sourceless_files = [] + complete = True + for dep in list(deps) + list(resolutions): + if PycInfo in dep: + dep_pyc = dep[PycInfo] + transitive_entries.append(dep_pyc.entries) + transitive_legacy_files.append(dep_pyc.legacy_files) + transitive_missing_sources.append(dep_pyc.missing_sources) + transitive_pycache_files.append(dep_pyc.pycache_files) + transitive_sourceless_files.append(dep_pyc.sourceless_files) + complete = complete and dep_pyc.complete + elif has_py_info(dep) and PyWheelsInfo not in dep: + complete = False + + direct_entries = compiled.entries if compiled else [] + compiled_sources = {entry.source.short_path: True for entry in direct_entries} + missing_sources = [] + retained_sources = [] + for src in sources: + if src.extension != "py": + retained_sources.append(src) + elif src.short_path not in compiled_sources: + missing_sources.append(src) + complete = complete and not missing_sources + + return PycInfo( + complete = complete, + direct_entries = direct_entries, + entries = depset( + direct = direct_entries, + transitive = transitive_entries, + ), + legacy_files = depset( + direct = compiled.legacy_files if compiled else [], + transitive = transitive_legacy_files, + ), + missing_sources = depset( + direct = missing_sources, + transitive = transitive_missing_sources, + ), + pycache_files = depset( + direct = compiled.pycache_files if compiled else [], + transitive = transitive_pycache_files, + ), + sourceless_files = depset( + direct = retained_sources + (compiled.legacy_files if compiled else []), + transitive = transitive_sourceless_files, + ), + ) + +def _pyc_aspect_impl(target, ctx): + # Srcs-less producers such as py_proto_library expose only transitive sources. + if hasattr(ctx.rule.files, "srcs"): + srcs = ctx.rule.files.srcs + else: + srcs = [ + src + for src in get_py_info(target).transitive_sources.to_list() + if src.owner.package == ctx.label.package and src.owner.workspace_name == ctx.label.workspace_name + ] + + # Reuse bytecode outputs to avoid conflicting actions. + existing = {} + for action in target.actions: + for out in action.outputs.to_list(): + if out.extension == "pyc": + existing[out.short_path] = out + return [make_pyc_info( + compile_pycs(ctx, srcs, existing = existing), + sources = srcs, + deps = getattr(ctx.rule.attr, "deps", []), + )] + +pyc_aspect = aspect( + doc = """Compiles bytecode for @rules_python targets reached through `deps`. + +Applies only to rules advertising @rules_python's PyInfo, so it never visits +or propagates through rules_py targets. Bytecode rules_python already +compiled for a source is reused; the missing layout is compiled here.""", + implementation = _pyc_aspect_impl, + attr_aspects = ["deps"], + attrs = PYC_ATTRS, + required_providers = [[RulesPythonPyInfo]], + toolchains = PYC_TOOLCHAINS, + provides = [PycInfo], +) diff --git a/py/private/pyc_compile.py b/py/private/pyc_compile.py new file mode 100644 index 000000000..0f2f5affc --- /dev/null +++ b/py/private/pyc_compile.py @@ -0,0 +1,152 @@ +"""Compile Python sources into PEP 552 unchecked-hash bytecode. + +Usage: pyc_compile.py [--expect-version VERSION] [--legacy] [@ARGFILE] SRC OUT DFILE... + pyc_compile.py --persistent_worker + +Each ``SRC OUT DFILE`` triple compiles ``SRC`` to ``OUT`` (a PEP 3147 +``__pycache__`` file or a colocated sourceless ``.pyc``) with ``DFILE`` stored +as its logical source path. ``--legacy`` also writes the same bytes to the +colocated ``.pyc`` beside a ``__pycache__`` ``OUT``. As a Bazel persistent +worker the same arguments arrive per request on stdin. Imports stay minimal: +without a worker the interpreter is spawned once per source, so module loading +dominates. +""" + +import importlib.util +import marshal +import sys + +_PRERELEASE_ABBREVS = {"alpha": "a", "beta": "b", "candidate": "rc"} + + +class CompileError(Exception): + pass + + +def parse_args(argv: list[str]) -> tuple[str | None, bool, list[str]]: + expect_version = None + legacy = False + files = [] + args = iter(argv) + for arg in args: + if arg.startswith("@"): + with open(arg[1:]) as f: + nested_version, nested_legacy, nested_files = parse_args( + f.read().splitlines() + ) + expect_version = nested_version or expect_version + legacy = legacy or nested_legacy + files += nested_files + elif arg == "--expect-version": + expect_version = next(args, None) + elif arg.startswith("--expect-version="): + expect_version = arg.partition("=")[2] + elif arg == "--legacy": + legacy = True + else: + files.append(arg) + return expect_version, legacy, files + + +def legacy_path(src: str, out: str) -> str: + """``pkg/__pycache__/mod..pyc`` beside ``mod.py`` -> ``pkg/mod.pyc``.""" + cache_dir = out.rpartition("/")[0] + if cache_dir.rpartition("/")[2] != "__pycache__": + raise CompileError("--legacy requires a __pycache__ output, got {}".format(out)) + return ( + cache_dir[: -len("__pycache__")] + + src.rpartition("/")[2][: -len(".py")] + + ".pyc" + ) + + +def check_version(expected: str) -> None: + actual = "{}.{}.{}".format(*sys.version_info[:3]) + if sys.version_info.releaselevel != "final": + actual += _PRERELEASE_ABBREVS.get( + sys.version_info.releaselevel, sys.version_info.releaselevel + ) + str(sys.version_info.serial) + # Bytecode magic is stable within a stable feature release but may change + # between prereleases: full equality is required when either side is a + # prerelease, otherwise major.minor must match. + prerelease = ( + sys.version_info.releaselevel != "final" + or not expected.replace(".", "").isdigit() + ) + if actual.split(".")[:2] != expected.split(".")[:2] or ( + prerelease and actual != expected + ): + raise CompileError( + "pyc compiler is Python {}, expected {}: emitted bytecode would " + "not match the target runtime".format(actual, expected) + ) + + +def compile_all(argv: list[str]) -> None: + expect_version, legacy, files = parse_args(argv) + if not files or len(files) % 3: + raise CompileError("expected SRC OUT DFILE triples") + if expect_version: + check_version(expect_version) + for src, out, dfile in zip(*[iter(files)] * 3): + data = compile_source(src, dfile) + write(out, data) + if legacy: + write(legacy_path(src, out), data) + + +def write(path: str, data: bytes) -> None: + with open(path, "wb") as f: + f.write(data) + + +def compile_source(src: str, dfile: str) -> bytes: + with open(src, "rb") as f: + source = f.read() + try: + code = compile(source, dfile, "exec", dont_inherit=True, optimize=0) + except SyntaxError as exc: + raise CompileError("{}: {}".format(dfile, exc)) from exc + # PEP 552 unchecked hash-based pyc: magic, flags=1, source hash, code. + return b"".join( + [ + importlib.util.MAGIC_NUMBER, + (1).to_bytes(4, "little"), + importlib.util.source_hash(source), + marshal.dumps(code), + ] + ) + + +def worker_loop() -> None: + """Serve Bazel JSON work requests, one per line on stdin, sequentially.""" + import json + + for line in sys.stdin: + request = json.loads(line) + if request.get("cancel"): + continue + response = { + "exitCode": 0, + "output": "", + "requestId": request.get("requestId", 0), + } + try: + compile_all(request["arguments"]) + except CompileError as exc: + response.update(exitCode=1, output=str(exc)) + sys.stdout.write(json.dumps(response) + "\n") + sys.stdout.flush() + + +def main() -> None: + if sys.argv[1:] == ["--persistent_worker"]: + worker_loop() + return + try: + compile_all(sys.argv[1:]) + except CompileError as exc: + sys.exit(str(exc)) + + +main() diff --git a/py/private/toolchain/types.bzl b/py/private/toolchain/types.bzl index 3cfdb5ddc..c41fe596b 100644 --- a/py/private/toolchain/types.bzl +++ b/py/private/toolchain/types.bzl @@ -1,8 +1,9 @@ """Constants for toolchain types""" PY_TOOLCHAIN = "@bazel_tools//tools/python:toolchain_type" -EXEC_TOOLS_TOOLCHAIN = "@aspect_rules_py//py/private/toolchain:exec_tools_toolchain_type" -NATIVE_BUILD_TOOLCHAIN = "@aspect_rules_py//py/private/toolchain:native_build_toolchain_type" + +EXEC_TOOLS_TOOLCHAIN = Label("@aspect_rules_py//py/private/toolchain:exec_tools_toolchain_type") +NATIVE_BUILD_TOOLCHAIN = Label("@aspect_rules_py//py/private/toolchain:native_build_toolchain_type") def interpreter_files_and_version(toolchain): """Interpreter files and version from a resolved PY_TOOLCHAIN target. diff --git a/py/private/transitions.bzl b/py/private/transitions.bzl index 27269764f..e88e7d7bb 100644 --- a/py/private/transitions.bzl +++ b/py/private/transitions.bzl @@ -3,6 +3,11 @@ _DEP_GROUP_FLAG = "@aspect_rules_py//uv/private/constraints/dep_group:dep_group" _DEP_GROUP_BASELINE_FLAG = "@aspect_rules_py//uv/private/constraints/dep_group:baseline" +# Only terminal rules read this flag (as the default for an unset `pyc` +# attribute). Venv and runtime-data transitions reset it before entering +# mode-independent subgraphs; it is not part of the general python_transition. +PYC_FLAG = "@aspect_rules_py//py:pyc" + # Our own python_version flag, replacing the rules_python one. _PYTHON_VERSION_FLAG = "@aspect_rules_py//py/private/interpreter:python_version" _PYTHON_VERSION_BASELINE_FLAG = "@aspect_rules_py//py/private/interpreter:baseline_python_version" @@ -95,16 +100,32 @@ python_transition = transition( outputs = _ALL_FLAGS, ) +def _py_venv_transition_impl(settings, attr): + acc = _python_transition_base(settings, attr, validate = True) + acc[PYC_FLAG] = "source" + return acc + +# A venv declares bytecode actions and PycInfo independently of the terminal's +# packaging mode. Canonicalize that mode on the venv itself so direct builds and +# every kind of incoming edge share one configured venv and dependency graph. +py_venv_transition = transition( + implementation = _py_venv_transition_impl, + inputs = _ALL_FLAGS + [PYC_FLAG], + outputs = _ALL_FLAGS + [PYC_FLAG], +) + # The launcher -> venv edge. Validation never runs here: the venv's own rule # transition always applies next, may override either half of a version/GIL # combination, and is the sole authority for rejecting the final configuration. def _venv_python_transition_impl(settings, attr): - return _python_transition_base(settings, attr, validate = False) + acc = _python_transition_base(settings, attr, validate = False) + acc[PYC_FLAG] = "source" + return acc venv_python_transition = transition( implementation = _venv_python_transition_impl, - inputs = _ALL_FLAGS, - outputs = _ALL_FLAGS, + inputs = _ALL_FLAGS + [PYC_FLAG], + outputs = _ALL_FLAGS + [PYC_FLAG], ) # Runtime data is outside the Python environment selected by terminal attrs. @@ -112,7 +133,7 @@ venv_python_transition = transition( # clear the scratch state so data targets share the caller's canonical # configuration. def _reset_python_flags_transition_impl(settings, _attr): - acc = {} + acc = {PYC_FLAG: "source"} for flag, baseline_flag in _FLAG_BASELINE_PAIRS: baseline = settings[baseline_flag] if baseline == _BASELINE_UNSET: @@ -126,6 +147,15 @@ def _reset_python_flags_transition_impl(settings, _attr): reset_python_flags_transition = transition( implementation = _reset_python_flags_transition_impl, - inputs = _ALL_FLAGS, - outputs = _ALL_FLAGS, + inputs = _ALL_FLAGS + [PYC_FLAG], + outputs = _ALL_FLAGS + [PYC_FLAG], +) + +def _reset_pyc_transition_impl(_settings, _attr): + return {PYC_FLAG: "source"} + +reset_pyc_transition = transition( + implementation = _reset_pyc_transition_impl, + inputs = [PYC_FLAG], + outputs = [PYC_FLAG], ) diff --git a/py/private/unittest_main.py b/py/private/unittest_main.py index 8a1a59173..b33d42b43 100644 --- a/py/private/unittest_main.py +++ b/py/private/unittest_main.py @@ -57,7 +57,11 @@ def _import_test_modules(test_files: list[str]) -> list[ModuleType]: while rel.startswith("../"): rel = rel[len("../"):] mod_name = rel[:-len(".py")].replace("/", ".") - loader = importlib.machinery.SourceFileLoader(mod_name, path) + if os.path.exists(path): + loader = importlib.machinery.SourceFileLoader(mod_name, path) + else: + # pyc_only runfiles replace the source with a colocated .pyc. + loader = importlib.machinery.SourcelessFileLoader(mod_name, path[:-len(".py")] + ".pyc") spec = importlib.util.spec_from_loader(mod_name, loader) if spec is None: raise ImportError("cannot load test module from %r" % path) diff --git a/py/tests/main-from-genrule/BUILD.bazel b/py/tests/main-from-genrule/BUILD.bazel index 2d0d6143d..6c5f50394 100644 --- a/py/tests/main-from-genrule/BUILD.bazel +++ b/py/tests/main-from-genrule/BUILD.bazel @@ -1,3 +1,4 @@ +load("@bazel_skylib//rules:build_test.bzl", "build_test") load("//py:defs.bzl", "py_binary", "py_test") package(default_testonly = True) @@ -16,6 +17,29 @@ py_binary( main = ":gen_main", ) +# Bytecode compilation requires the generated file's label in `srcs`. +py_binary( + name = "main_from_genrule_pyc_bin", + srcs = ["main_generated.py"], + main = ":gen_main", + pyc = "pyc_only", +) + +py_binary( + name = "main_from_genrule_pyc_and_source_bin", + srcs = ["main_generated.py"], + main = ":gen_main", + pyc = "pyc", +) + +build_test( + name = "pyc_binaries_build", + targets = [ + ":main_from_genrule_pyc_bin", + ":main_from_genrule_pyc_and_source_bin", + ], +) + genrule( name = "gen_test_main", outs = ["test_main_generated.py"], diff --git a/py/tests/py-venv-multi-exec/BUILD.bazel b/py/tests/py-venv-multi-exec/BUILD.bazel index e8339da4b..6af216f96 100644 --- a/py/tests/py-venv-multi-exec/BUILD.bazel +++ b/py/tests/py-venv-multi-exec/BUILD.bazel @@ -1,12 +1,40 @@ -load("@rules_python//python:defs.bzl", "py_library") +load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") load("//py:defs.bzl", "py_library", "py_venv") load("//py/private/py_venv:defs.bzl", "py_venv_exec_test") load(":env_inherit_test.bzl", "contextual_keys_not_inherited_test", "env_inherit_test") load(":freethreaded_flag_test.bzl", "freethreaded_flag_test") +load(":pyc_runfiles_test.bzl", "pyc_failure_test", "pyc_runfiles_test") load(":runfiles_mapping.bzl", "runfiles_mapping") package(default_testonly = True) +# A TreeArtifact in srcs is opaque to bytecode: it ships as source under pyc_only. +copy_to_directory( + name = "tree_sources", + srcs = ["tree_module.py"], + out = "tree_sources_tree", + root_paths = ["py/tests/py-venv-multi-exec"], +) + +py_venv( + name = "tree_source_venv", + srcs = [ + "entry_tree_source.py", + ":tree_sources", + ], + imports = [ + ".", + "tree_sources_tree", + ], +) + +py_venv_exec_test( + name = "test_pyc_only_tree_source", + main = "entry_tree_source.py", + pyc = "pyc_only", + venv = ":tree_source_venv", +) + # A single py_venv exec'd through multiple py_venv_exec consumers, # each differing in `main`, `env`, `args`, `data`, or other launcher- # level config. Demonstrates that the venv-shaping attrs (srcs, deps, @@ -47,6 +75,27 @@ py_venv( ], ) +py_library( + name = "dependency_main_library", + srcs = [ + "entry_a.py", + "shared_lib.py", + ], +) + +py_venv( + name = "dependency_main_venv", + imports = ["."], + deps = [":dependency_main_library"], +) + +py_venv_exec_test( + name = "test_pyc_only_dependency_main", + main = "entry_a.py", + pyc = "pyc_only", + venv = ":dependency_main_venv", +) + # A venv may list a source file exported by another package; it ships at its # owning package's runfiles path. py_venv( @@ -112,11 +161,193 @@ py_venv_exec_test( venv = ":shared_venv", ) +py_venv_exec_test( + name = "test_pyc_only_transitive_python_data", + main = "entry_runtime_data.py", + pyc = "pyc_only", + venv = ":runtime_data_venv", +) + +pyc_runfiles_test( + name = "test_pyc_only_transitive_python_data_runfiles", + expect_legacy = False, + mode = "pyc_only", + module = "runtime_data", + protect_source = True, + target_under_test = ":test_pyc_only_transitive_python_data", +) + +py_venv( + name = "sidecar_venv", + srcs = [ + "entry_sidecar.py", + "sidecar.txt", + ], + imports = ["."], +) + +py_venv_exec_test( + name = "test_pyc_only_sidecar_srcs", + main = "entry_sidecar.py", + pyc = "pyc_only", + venv = ":sidecar_venv", +) + +pyc_runfiles_test( + name = "test_pyc_only_sidecar_srcs_runfiles", + expect_suffixes = ["/sidecar.txt"], + mode = "pyc_only", + module = "entry_sidecar", + target_under_test = ":test_pyc_only_sidecar_srcs", +) + +py_venv_exec_test( + name = "test_foreign_source_pyc", + main = "entry_foreign.py", + pyc = "pyc", + venv = ":foreign_source_venv", +) + +pyc_runfiles_test( + name = "test_foreign_source_pyc_runfiles", + expect_pyc = False, + mode = "pyc", + module = "foreign_source", + target_under_test = ":test_foreign_source_pyc", +) + +py_venv_exec_test( + name = "_test_foreign_source_pyc_only", + main = "entry_foreign.py", + pyc = "pyc_only", + tags = ["manual"], + venv = ":foreign_source_venv", +) + +# Source-retaining pyc tolerates optimization: CPython skips level-0 caches under -O. +py_venv_exec_test( + name = "test_pyc_optimized", + interpreter_options = ["-O"], + main = "entry_a.py", + pyc = "pyc", + venv = ":shared_venv", +) + +# A capital O inside another option's argument is not `-O`. +py_venv_exec_test( + name = "test_pyc_warning_option", + interpreter_options = ["-Wignore:Overwriting:UserWarning"], + main = "entry_a.py", + pyc = "pyc", + venv = ":shared_venv", +) + +py_venv_exec_test( + name = "test_pyc_bundled_optimize", + interpreter_options = ["-BO"], + main = "entry_a.py", + pyc = "pyc", + venv = ":shared_venv", +) + +py_venv_exec_test( + name = "test_pyc_inherit_optimize", + env_inherit = ["PYTHONOPTIMIZE"], + main = "entry_a.py", + pyc = "pyc", + venv = ":shared_venv", +) + +py_venv( + name = "pyc_optimize_env_venv", + srcs = [ + "entry_a.py", + "shared_lib.py", + ], + env = {"PYTHONOPTIMIZE": "0"}, + imports = ["."], +) + +py_venv_exec_test( + name = "test_pyc_venv_optimize_env", + main = "entry_a.py", + pyc = "pyc", + venv = ":pyc_optimize_env_venv", +) + +# Sourceless bytecode cannot honour an optimization request. +py_venv_exec_test( + name = "_test_pyc_repeated_optimize", + interpreter_options = ["-OO"], + main = "entry_a.py", + pyc = "pyc_only", + tags = ["manual"], + venv = ":shared_venv", +) + +pyc_failure_test( + name = "test_pyc_repeated_optimize_failure", + expected_error = "pyc_only ships level-0 bytecode", + target_under_test = ":_test_pyc_repeated_optimize", +) + +py_venv_exec_test( + name = "_test_pyc_only_pythonoptimize", + env = {"PYTHONOPTIMIZE": "0"}, + main = "entry_a.py", + pyc = "pyc_only", + tags = ["manual"], + venv = ":shared_venv", +) + +pyc_failure_test( + name = "test_pyc_only_pythonoptimize_failure", + expected_error = "pyc_only does not support PYTHONOPTIMIZE", + target_under_test = ":_test_pyc_only_pythonoptimize", +) + +py_venv( + name = "_pyc_optimize_inherit_venv", + srcs = ["entry_a.py"], + env_inherit = ["PYTHONOPTIMIZE"], + tags = ["manual"], +) + +py_venv_exec_test( + name = "_test_pyc_venv_optimize_inherit", + isolated = False, + main = "entry_a.py", + pyc = "pyc_only", + tags = ["manual"], + venv = ":_pyc_optimize_inherit_venv", +) + +pyc_failure_test( + name = "test_pyc_venv_optimize_inherit_failure", + expected_error = "pyc_only does not support PYTHONOPTIMIZE", + target_under_test = ":_test_pyc_venv_optimize_inherit", +) + +pyc_failure_test( + name = "test_foreign_source_pyc_only_failure", + expected_error = "pyc_only could not compile all first-party sources", + target_under_test = ":_test_foreign_source_pyc_only", +) + # Same shared venv, two different `main` entry points. Both reach # `shared_lib` via the venv's sys.path. py_venv_exec_test( name = "test_entry_a", main = "entry_a.py", + pyc = "pyc", + venv = ":shared_venv", +) + +py_venv_exec_test( + name = "test_pyc_only_broken_python_data", + data = [":broken_python_data"], + main = "entry_a.py", + pyc = "pyc_only", venv = ":shared_venv", ) @@ -126,9 +357,66 @@ py_venv_exec_test( venv = ":shared_venv", ) -# One py_venv label, resolved in two consumer configurations. A materialized -# venv cannot be shared across interpreter versions, but Bazel configures this -# label independently for each launcher's `python_version`. +py_venv_exec_test( + name = "test_pyc_source", + main = "entry_a.py", + venv = ":shared_venv", +) + +py_venv_exec_test( + name = "test_pyc_cache", + main = "entry_a.py", + pyc = "pyc", + venv = ":shared_venv", +) + +py_venv_exec_test( + name = "test_pyc_only", + data = [":entry_a_mapping"], + main = "entry_a.py", + pyc = "pyc_only", + venv = ":shared_venv", +) + +pyc_runfiles_test( + name = "test_pyc_source_runfiles", + mode = "source", + target_under_test = ":test_pyc_source", +) + +pyc_runfiles_test( + name = "test_pyc_cache_runfiles", + forbid_entry_substring = "site-packages", + mode = "pyc", + target_under_test = ":test_pyc_cache", +) + +pyc_runfiles_test( + name = "test_pyc_only_runfiles", + check_mappings = True, + mode = "pyc_only", + target_under_test = ":test_pyc_only", +) + +py_venv_exec_test( + name = "test_pyc_data", + data = ["data_alpha.txt"], + env = {"DATA_PATH": "$(rootpath :data_alpha.txt)"}, + main = "entry_data.py", + pyc = "pyc", + venv = ":shared_venv", +) + +py_venv_exec_test( + name = "test_pyc_only_data", + data = ["data_beta.txt"], + env = {"DATA_PATH": "$(rootpath :data_beta.txt)"}, + main = "entry_data.py", + pyc = "pyc_only", + venv = ":shared_venv", +) + +# One py_venv label, resolved independently for each launcher's Python version. py_venv( name = "shared_versioned_venv", srcs = ["entry_versioned.py"], @@ -149,6 +437,14 @@ py_venv_exec_test( venv = ":shared_versioned_venv", ) +py_venv_exec_test( + name = "test_pyc_python_313", + main = "entry_versioned.py", + pyc = "pyc", + python_version = "3.13", + venv = ":shared_versioned_venv", +) + py_venv_exec_test( name = "test_python_313_freethreaded", freethreaded = "true", @@ -157,6 +453,75 @@ py_venv_exec_test( venv = ":shared_versioned_venv", ) +py_venv_exec_test( + name = "test_pyc_python_312", + main = "entry_versioned.py", + pyc = "pyc", + python_version = "3.12", + venv = ":shared_versioned_venv", +) + +py_venv_exec_test( + name = "test_pyc_select", + main = "entry_a.py", + pyc = select({"//conditions:default": "pyc"}), + venv = ":shared_venv", +) + +pyc_runfiles_test( + name = "test_pyc_select_runfiles", + mode = "pyc", + target_under_test = ":test_pyc_select", +) + +py_venv_exec_test( + name = "test_pyc_source_python_313", + main = "entry_versioned.py", + pyc = "source", + python_version = "3.13", + venv = ":shared_versioned_venv", +) + +py_venv_exec_test( + name = "test_pyc_source_python_312", + main = "entry_versioned.py", + pyc = "source", + python_version = "3.12", + venv = ":shared_versioned_venv", +) + +py_venv_exec_test( + name = "test_pyc_only_python_313", + main = "entry_versioned.py", + pyc = "pyc_only", + python_version = "3.13", + venv = ":shared_versioned_venv", +) + +py_venv_exec_test( + name = "test_pyc_only_python_312", + main = "entry_versioned.py", + pyc = "pyc_only", + python_version = "3.12", + venv = ":shared_versioned_venv", +) + +pyc_runfiles_test( + name = "test_pyc_python_313_runfiles", + mode = "pyc", + module = "entry_versioned", + pyc_tag = "cpython-313", + target_under_test = ":test_pyc_python_313", +) + +pyc_runfiles_test( + name = "test_pyc_python_312_runfiles", + mode = "pyc", + module = "entry_versioned", + pyc_tag = "cpython-312", + target_under_test = ":test_pyc_python_312", +) + # Regression: the launcher edge must not validate intermediate flag combos. # Under a global freethreaded flag this venv pins GIL mode back off; a # launcher choosing only python_version passes through an intermediate diff --git a/py/tests/py-venv-multi-exec/entry_data.py b/py/tests/py-venv-multi-exec/entry_data.py index 405c1863f..422260ea9 100644 --- a/py/tests/py-venv-multi-exec/entry_data.py +++ b/py/tests/py-venv-multi-exec/entry_data.py @@ -9,6 +9,8 @@ EXPECTATIONS = { "test_data_alpha": "alpha-payload\n", "test_data_beta": "beta-payload\n", + "test_pyc_data": "alpha-payload\n", + "test_pyc_only_data": "beta-payload\n", } expected = EXPECTATIONS.get(target) diff --git a/py/tests/py-venv-multi-exec/entry_sidecar.py b/py/tests/py-venv-multi-exec/entry_sidecar.py new file mode 100644 index 000000000..923108127 --- /dev/null +++ b/py/tests/py-venv-multi-exec/entry_sidecar.py @@ -0,0 +1,6 @@ +import os + +here = os.path.dirname(__file__) +content = open(os.path.join(here, "sidecar.txt")).read() +assert content == "sidecar-content\n", repr(content) +print("sidecar ok") diff --git a/py/tests/py-venv-multi-exec/entry_tree_source.py b/py/tests/py-venv-multi-exec/entry_tree_source.py new file mode 100644 index 000000000..89d8715c2 --- /dev/null +++ b/py/tests/py-venv-multi-exec/entry_tree_source.py @@ -0,0 +1,4 @@ +import tree_module + +assert tree_module.VALUE == 42 +assert tree_module.__file__.endswith(".py"), tree_module.__file__ diff --git a/py/tests/py-venv-multi-exec/entry_versioned.py b/py/tests/py-venv-multi-exec/entry_versioned.py index 8db5b3144..97411e66e 100644 --- a/py/tests/py-venv-multi-exec/entry_versioned.py +++ b/py/tests/py-venv-multi-exec/entry_versioned.py @@ -10,6 +10,12 @@ "test_python_313": ((3, 13), False), "test_python_313_freethreaded": ((3, 13), True), "test_python_312_gil_pinned": ((3, 12), False), + "test_pyc_python_312": ((3, 12), False), + "test_pyc_python_313": ((3, 13), False), + "test_pyc_source_python_312": ((3, 12), False), + "test_pyc_source_python_313": ((3, 13), False), + "test_pyc_only_python_312": ((3, 12), False), + "test_pyc_only_python_313": ((3, 13), False), } version, freethreaded = EXPECTED[os.environ["BAZEL_TARGET_NAME"]] diff --git a/py/tests/py-venv-multi-exec/pyc_runfiles_test.bzl b/py/tests/py-venv-multi-exec/pyc_runfiles_test.bzl new file mode 100644 index 000000000..b8ad519c6 --- /dev/null +++ b/py/tests/py-venv-multi-exec/pyc_runfiles_test.bzl @@ -0,0 +1,71 @@ +"""Runfiles assertions for all bytecode modes on one shared py_venv.""" + +load("@bazel_skylib//lib:unittest.bzl", "analysistest", "asserts") +load("//py/private:pyc.bzl", "PycInfo") + +def _has(paths, suffix): + return any([path.endswith(suffix) for path in paths]) + +def _mapping_has(entries, path, suffix): + return any([entry.path == path and entry.target_file.short_path.endswith(suffix) for entry in entries.to_list()]) + +def _pyc_runfiles_test_impl(ctx): + env = analysistest.begin(ctx) + target = analysistest.target_under_test(env) + paths = [f.short_path for f in target[DefaultInfo].default_runfiles.files.to_list()] + mode = ctx.attr.mode + module = ctx.attr.module + + has_source = _has(paths, "/{}.py".format(module)) + has_legacy = _has(paths, "/{}.pyc".format(module)) + has_pycache = any(["/__pycache__/{}.".format(module) in path and path.endswith(".pyc") for path in paths]) + + asserts.equals(env, mode != "pyc_only" or ctx.attr.protect_source, has_source, "source retention") + asserts.equals(env, mode == "pyc_only" and ctx.attr.expect_legacy, has_legacy, "colocated sourceless bytecode") + asserts.equals(env, mode == "pyc" and ctx.attr.expect_pyc, has_pycache, "PEP 3147 cache bytecode") + asserts.equals(env, mode != "source", PycInfo in target, "bytecode provider is opt-in") + if ctx.attr.pyc_tag: + asserts.true( + env, + any(["/__pycache__/{}.{}.pyc".format(module, ctx.attr.pyc_tag) in path for path in paths]), + "PEP 3147 cache tag", + ) + if ctx.attr.check_mappings: + runfiles = target[DefaultInfo].default_runfiles + asserts.true(env, _mapping_has(runfiles.symlinks, "mapped.py", "/entry_a.py"), "data symlink preserved") + asserts.true(env, _mapping_has(runfiles.root_symlinks, "root-mapped.py", "/entry_a.py"), "data root symlink preserved") + for suffix in ctx.attr.expect_suffixes: + asserts.true(env, _has(paths, suffix), "expected runfile " + suffix) + if ctx.attr.forbid_entry_substring: + asserts.false( + env, + any([ctx.attr.forbid_entry_substring in entry.source.short_path for entry in target[PycInfo].entries.to_list()]), + "third-party sources are not first-party bytecode entries", + ) + return analysistest.end(env) + +pyc_runfiles_test = analysistest.make( + _pyc_runfiles_test_impl, + attrs = { + "mode": attr.string(mandatory = True), + "module": attr.string(default = "entry_a"), + "pyc_tag": attr.string(default = ""), + "check_mappings": attr.bool(default = False), + "expect_pyc": attr.bool(default = True), + "expect_legacy": attr.bool(default = True), + "protect_source": attr.bool(default = False), + "forbid_entry_substring": attr.string(default = ""), + "expect_suffixes": attr.string_list(default = []), + }, +) + +def _pyc_failure_test_impl(ctx): + env = analysistest.begin(ctx) + asserts.expect_failure(env, ctx.attr.expected_error) + return analysistest.end(env) + +pyc_failure_test = analysistest.make( + _pyc_failure_test_impl, + attrs = {"expected_error": attr.string(mandatory = True)}, + expect_failure = True, +) diff --git a/py/tests/py-venv-multi-exec/sidecar.txt b/py/tests/py-venv-multi-exec/sidecar.txt new file mode 100644 index 000000000..ede0126b8 --- /dev/null +++ b/py/tests/py-venv-multi-exec/sidecar.txt @@ -0,0 +1 @@ +sidecar-content diff --git a/py/tests/py-venv-multi-exec/tree_module.py b/py/tests/py-venv-multi-exec/tree_module.py new file mode 100644 index 000000000..b11ef08a3 --- /dev/null +++ b/py/tests/py-venv-multi-exec/tree_module.py @@ -0,0 +1 @@ +VALUE = 42 diff --git a/py/tests/pyc-compile/BUILD.bazel b/py/tests/pyc-compile/BUILD.bazel new file mode 100644 index 000000000..f86f09436 --- /dev/null +++ b/py/tests/pyc-compile/BUILD.bazel @@ -0,0 +1,78 @@ +load("@bazel_lib//lib:write_source_files.bzl", "write_source_files") +load("@rules_shell//shell:sh_test.bzl", "sh_test") +load("//py:defs.bzl", "py_library", "py_test", "py_venv") +load("//py/private/py_venv:defs.bzl", "py_venv_exec") +load(":bytecode_compat_test.bzl", "bytecode_compat_test_suite") +load(":pyc_layout_actions_test.bzl", "pyc_layout_actions_test") + +package(default_testonly = True) + +genquery( + name = "pyc_bzl_library_deps", + expression = "deps(//py/private:pyc)", + scope = ["//py/private:pyc"], +) + +sh_test( + name = "pyc_bzl_library_providers_dep_repro_test", + srcs = ["assert_query_contains.sh"], + args = [ + "$(location :pyc_bzl_library_deps)", + "//py/private:providers", + ], + data = [":pyc_bzl_library_deps"], +) + +py_test( + name = "version_check_test", + srcs = ["version_check_test.py"], + data = ["//py/private:pyc_compile.py"], + env = {"PYC_COMPILE": "$(rootpath //py/private:pyc_compile.py)"}, +) + +# Refresh after changing the source or Python 3.13 interpreter: +# bazel run //py/tests/pyc-compile:snapshots +py_venv( + name = "snapshot_venv", + srcs = ["version_check_test.py"], + python_version = "3.13", +) + +py_venv_exec( + name = "snapshot_bin", + main = "version_check_test.py", + pyc = "pyc_only", + python_version = "3.13", + venv = ":snapshot_venv", +) + +genrule( + name = "snapshot_pyc", + srcs = [":snapshot_bin"], + outs = ["snapshot_extracted.pyc"], + cmd = "for f in $(SRCS); do case $$f in *.pyc) cp $$f $@;; esac; done", +) + +write_source_files( + name = "snapshots", + files = { + "snapshots/version_check_test.pyc": ":snapshot_pyc", + }, +) + +bytecode_compat_test_suite(name = "bytecode_compat_test") + +py_library( + name = "layout_lib", + srcs = ["version_check_test.py"], +) + +pyc_layout_actions_test( + name = "py_library_layout_actions_test", + target_under_test = ":layout_lib", +) + +pyc_layout_actions_test( + name = "py_venv_layout_actions_test", + target_under_test = ":snapshot_venv", +) diff --git a/py/tests/pyc-compile/assert_query_contains.sh b/py/tests/pyc-compile/assert_query_contains.sh new file mode 100755 index 000000000..71db444a2 --- /dev/null +++ b/py/tests/pyc-compile/assert_query_contains.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail + +query_output="$1" +expected="$2" + +grep -Fx -- "$expected" "$query_output" diff --git a/py/tests/pyc-compile/bytecode_compat_test.bzl b/py/tests/pyc-compile/bytecode_compat_test.bzl new file mode 100644 index 000000000..96029c249 --- /dev/null +++ b/py/tests/pyc-compile/bytecode_compat_test.bzl @@ -0,0 +1,65 @@ +"""Unit tests for the exec-interpreter bytecode compatibility gate.""" + +load("@bazel_skylib//lib:unittest.bzl", "asserts", "unittest") +load("//py/private:pyc.bzl", "bytecode_compatible", "pycache_tag") + +def _runtime(major = 3, minor = 12, micro = 4, releaselevel = "final", serial = 0, abi_flags = "", implementation_name = "cpython", pyc_tag = None): + return struct( + interpreter_version_info = struct( + major = major, + minor = minor, + micro = micro, + releaselevel = releaselevel, + serial = serial, + ), + abi_flags = abi_flags, + implementation_name = implementation_name, + pyc_tag = pyc_tag, + ) + +def _bytecode_compat_test_impl(ctx): + env = unittest.begin(ctx) + + asserts.true(env, bytecode_compatible(_runtime(), _runtime()), "identical") + asserts.true(env, bytecode_compatible(_runtime(micro = 3), _runtime(micro = 9)), "final micro differs") + asserts.false(env, bytecode_compatible(_runtime(minor = 11), _runtime(minor = 12)), "minor differs") + asserts.false(env, bytecode_compatible(_runtime(major = 2), _runtime()), "major differs") + asserts.true(env, bytecode_compatible(_runtime(abi_flags = "t"), _runtime()), "ABI does not affect bytecode") + asserts.false(env, bytecode_compatible(_runtime(), _runtime(implementation_name = "pypy")), "implementation differs") + asserts.true(env, bytecode_compatible(_runtime(implementation_name = "pypy"), _runtime(implementation_name = "pypy")), "implementation matches") + asserts.false(env, bytecode_compatible(_runtime(pyc_tag = "cpython-312"), _runtime(pyc_tag = "vendor-312")), "pyc tag differs") + asserts.true(env, bytecode_compatible( + _runtime(implementation_name = None, pyc_tag = "vendor-312"), + _runtime(implementation_name = None, pyc_tag = "vendor-312"), + ), "pyc tag matches without implementation") + asserts.false(env, bytecode_compatible( + _runtime(implementation_name = None), + _runtime(implementation_name = None), + ), "runtime identity unknown") + + rc1 = _runtime(minor = 14, micro = 0, releaselevel = "candidate", serial = 1) + rc2 = _runtime(minor = 14, micro = 0, releaselevel = "candidate", serial = 2) + final = _runtime(minor = 14, micro = 0) + asserts.true(env, bytecode_compatible(rc1, rc1), "same prerelease") + asserts.false(env, bytecode_compatible(rc1, rc2), "prerelease serial differs") + asserts.false(env, bytecode_compatible(rc1, final), "prerelease exec vs final target") + asserts.false(env, bytecode_compatible(final, rc1), "final exec vs prerelease target") + asserts.true(env, bytecode_compatible(rc1, _runtime(minor = 14, micro = None, releaselevel = "candidate", serial = 1)), "prerelease without micro") + + asserts.true(env, bytecode_compatible(struct( + implementation_name = "cpython", + interpreter_version_info = struct(major = 3, minor = 12), + ), _runtime()), "sparse version_info") + asserts.false(env, bytecode_compatible(struct(), _runtime()), "exec lacks version_info") + asserts.false(env, bytecode_compatible(_runtime(), struct()), "target lacks version_info") + + asserts.equals(env, "vendor-312", pycache_tag(_runtime(pyc_tag = "vendor-312")), "explicit cache tag") + asserts.equals(env, "pypy-312", pycache_tag(_runtime(implementation_name = "pypy")), "derived non-CPython cache tag") + asserts.equals(env, None, pycache_tag(_runtime(implementation_name = None)), "unknown runtime cache tag") + + return unittest.end(env) + +bytecode_compat_test = unittest.make(_bytecode_compat_test_impl) + +def bytecode_compat_test_suite(name): + unittest.suite(name, bytecode_compat_test) diff --git a/py/tests/pyc-compile/pyc_layout_actions_test.bzl b/py/tests/pyc-compile/pyc_layout_actions_test.bzl new file mode 100644 index 000000000..7f06ba5d2 --- /dev/null +++ b/py/tests/pyc-compile/pyc_layout_actions_test.bzl @@ -0,0 +1,50 @@ +"""One compile action emits both bytecode layouts of a source. + +A `py_library` declares its bytecode actions in every mode, so a second spawn +or a copy toolchain for the colocated `.pyc` would be paid by every consumer, +including `source` mode. +""" + +load("@bazel_skylib//lib:unittest.bzl", "analysistest", "asserts") +load("//py/private:pyc.bzl", "PycInfo") + +def _pyc_layout_actions_test_impl(ctx): + env = analysistest.begin(ctx) + target = analysistest.target_under_test(env) + + producers = {} + for action in analysistest.target_actions(env): + for out in action.outputs.to_list(): + producers[out.short_path] = action + + asserts.false( + env, + "CopyFile" in [action.mnemonic for action in producers.values()], + "colocated bytecode must not be copied through the coreutils toolchain", + ) + + entries = target[PycInfo].direct_entries + asserts.true(env, len(entries) > 0, "target declares bytecode entries") + for entry in entries: + pycache_action = producers.get(entry.pycache.short_path) + pyc_action = producers.get(entry.pyc.short_path) + asserts.equals( + env, + "PyCompile", + pycache_action.mnemonic if pycache_action else None, + "__pycache__ producer for " + entry.source.short_path, + ) + asserts.equals( + env, + "PyCompile", + pyc_action.mnemonic if pyc_action else None, + "colocated .pyc producer for " + entry.source.short_path, + ) + asserts.true( + env, + pycache_action != None and pycache_action == pyc_action, + "both layouts come from one action for " + entry.source.short_path, + ) + return analysistest.end(env) + +pyc_layout_actions_test = analysistest.make(_pyc_layout_actions_test_impl) diff --git a/py/tests/pyc-compile/snapshots/version_check_test.pyc b/py/tests/pyc-compile/snapshots/version_check_test.pyc new file mode 100644 index 0000000000000000000000000000000000000000..29da805ab2d0b194d9c9778be7b6a7f4591fd2a9 GIT binary patch literal 12894 zcmb_jYit|Go!{kh$>m!T^|mcjvSNvPzih{`tW=g2*|jCxT+vM<|pQrEA;N|ld`=>R{Jp4rXosnp-L|>CMg3rlFtT!}y-79IY1)^Tb z;DcU+es44&>4ZRZN(lS11BMK)22SzWsQlq**Vd<*d<`|CQHot!C6~MM6tB@z$Eol> z^<27K8zsjxuiS=3ouDW;HNq*SE|ZeWUH2Y?THi0;;#g*$BNpYo`}Cm#*<;8hkecxa z{e)p=wUkpk^ql+tTP@_@m$>8-I~ELkLrzZ80X`eERbJ$~9`N1t;Bx?`WD};s;b3^&lX)yuKonEY3t!t>){XW z>FV}mb^A)qN_(RESi*Yz-oR76F|994>dVsl+N8cVu21P(*E!po{b<_WkhC{EHQUnW zgGuwjwAq<7J0EtY8&4%0Po*2rB^%E@V>HFaKbmdtGU)_P6T{efI>OKjDR{!Ks|zWNpfwirIaBH!x{@j0tB3 zDXiVnk5Mk$N`Yy!x2p37XyzxkRv9&~XT-NH_=tm_YDJ^q8d2yiUYV>%b(G<8xh92u_9qLL=NDs>Qc~+@qcv&G+kO>+Uzr zHqD8#V@nrPM(4f0rv}T6Z?5z2sutVVIm3PZtbWe0##MgJP{#5@&4v~` zW0rU{aeN@zI*_uS0~XNgethBW3-4+c^z-@!+q^BNPZiX!bggP%UN_qoYhxu#O$k$7 zLR0lIpp>bZ!*?orX1qs%8qjc2;_KDf;$1$WQ>8;aheMk&y!6lzn@q?NpuXfe=& zqb#rCbzNFsPfB!E6n}u{pp~G$zrkm3$$|S)r_mP3=dvDd)@_5W&oE?oy0{06O&`0>LvZeO+VfU z)i6Lw9|+$H3X!l-40i;%%e15g@jfO?Y8j`j7>K$jrPBOGXd&fCps9$x0dnmJ()J^( z_9G89kBrO4N7iNQ%3G=W{-pi28SOiqP>Ju~t;Esjc z^7JS+tpWneXREmXgP;81{jm?D@w2~}PFA&lT>IH@(&kEVE_v;&vN5LU!fjdxp~S%N z*)$MXqbw}ll$yNdX+;BsHEvpwqF(5Qhx=B|CAt;038iLW)ljXJo4G=}HCN~py5p#W zh+W*JA%=y-Oho+83-V3JFL6XphKM^^O7>!bR$jtB$fF4o3W)M@;(T(lQlGIBr1Bk$ znR()b`eHwjd(@_mDlD6eF5H>F6Dvs-)XlK}b)r8}P!k(U+B!e$n9;7=>}lKKRomf& zttD0rB+jndT2|NV1j zfnRd$_;PtQ+Mv0JtaOh8v&vvebLB~{e6ciL(U7cYi1)^CJ$h^Ttu^lSI%mFbo3*{K zd7xj?Kd>%YW4BY~txvc%^3}FU4qRx$6+Mt49}2GdsRoyP>*vi>Q2eLk6fWE{Jp}yD z!!zML5`4NIp|Y+nH3V+c9tTiYQ6MQXpW|a^^I5aFsi_=zkDe3jI91$~Ett zTh^|;42@oNZX;R6q0CYNi9B23Ds&3mnwu}0C_5%yMCKB>1OB2Sm82u;6W&&dNKCYn zc1-r+tEtBBRMqKKu4m_z(X=I>^RFTEqEqHo`3|%&E+k2=tjDL6fH=%}liQaE)W$R` zJ5Pj}*ewfXgT?~7C_*8Ue4JE}B^wKPiN3idTNa*3ICNKPP+vK)1~ z>iOrM-*u)>T}Ym~@Y$Kqt|U%fN;F_*{Hl402oHN+A>6vN28Ya0 zPIgGZmrQp~3W1Cp?FfdW0RirTiVh+0y{VuOa1cguWPsrSPuzE1zzGQhK%B-c(83s$ z#ECb9lfxi;1O%SMPIzxb1h@pl^2H}aa^TEZERrb_y&e!e6G3>o&2p(E==2B2#$b@) zs9eKCLIYBxIy_Y?zY1d!-vR=5jmere9Y~rEq)qinQ+<3OW$Iiv7u+A59ZXbSS~Fi> zFLk6#>yo8)Tm12wq`Bt9zI1I@vbHN-`%1F*mEZNHyDuiYFQ&U)$!=G|eKpyA6@2tE zk%XegfH7S@oc4sR;~#%GO!ijE-o|14-_{;%n)H&B=q!@$ap1;4z!;TV^eZ(xY+R zBg?X7jXO>%qJ8mx(Sx$3vIkX5RS+JkXnVr7?^HWk&{Po-35o<>o}a#PSdz|pC5rBK z(5@(Mi^7hgBB24GVT%b0(t{^vRHAlZ{Z`@-=GK^bW|Pj`Ssk;t2hS)^u4usZg}DK`PUbxQr4GdnEdTdDGvH4Z}h`Tn5t@Cnzt6Z?oX;za!E#j%fKgQRa7 zumcW22O%%zj0D0XDrkm%K&F*;1D9Zgg>Z-p;(!yfsLs6tqVtRz#&aApl0`el@e$*^ zPLblUi=f@|9OAfJp0K;<6|Jiz_kJap)o$!wPnF)mImp|Y$iE+__K5hG$r1nEYm8Hw zJpuiY&<{g=u7@xc#{8di&CQ{}xYu_#$7Qg0dCh6s6053)7P4`31arMe@))lc$|S82 z@cNxPIW%$(OLUpOiRNL+#sh4FMnVxE?1K75l*Firl(=x~b<$Q_pj_d19%4F5-%QVzl(tGoO^T_LZ@p>Nv=MT4wBf z$?$2l9`p6azT^6B7Cx#B!rrp6wy(O%212Kx)npN2HROoAgkjOWBL%8vnL7J33h3R; zv5ZIwYjY1#Kg?p1P6p^Q2d}C;TJu_m06(I@Z^$H9!d&Vldh1aTu zYI1%?COiUj!?aU+!-&vc&r49M4pvPw)Kn}i&`LUli!w@dTX&c2L&vvEcCyDL3b?Bx z(ME~>mTWi~wg5U0%P44smynQnXD{Rwn~BJ9J#1PLAd_7NhtvU^ zc{~9v#UBDu5yM5r3w85#3r+J)8N`;RirQxMC{^ngJJ&3SW9{o^%l(1bfw{8_=jYEa z22#bfv1qcman0NWGSuq$f`pBe5H=o8A8Jk>YL5F>29_sMhq@mJKJNX^^(WlG*DTa~ zVmbU=#9+d|S!&NzGoMr&&$Morp;_pD47GNTvyPygB?&$&u1a3YZ89lQ)G$WT)J%QV zh)52myTwHCAc700l9#pu2P1F&@LWJ8NCcdu&seuky{yS|l1@#kWp?>4K=W;}GCAT8 zo@{0VGh;LF7V+Ov5#2%YA`tKi6=nP3@r)SiPaS#XamVA{#F3sv;j8;Oe8E`hFAsfi zNY&oWD#ZVQ&-H?Gh=$RT1fOjmI~K!QytYMyXW)2!s|I%m$-R6IPQ#n_RmEoIvthed zZJ60x4no8Q=SC85X(bT~-m0dC@vMNGv1d=jDRKr*=o$EeliC+lzJ+e>-u5*-$JlIM z+RF;IQ&(A0(+U8yQ|BPC5oSk9mF6xyE|QlZ?93{&xU3mGKv5>jL4|eU-7It+yg{FR z1F&G^HeZA?7Ft8Z30sSru)dYz?N_oU(QcluTgMk-TfUgkLgl;dQD62lzIlDJqnznK ztF1`?FxULVm%xbR7Nt0%ZR0tVUz?&vP&bg|a%3`FF0Lb6ZZcWGHtAnkbLC1#^wQ4y zf596vyXZ>K$`y5=s3Bj69aD1pG0Is6rx3;?5x>M-^ZI2Kr;!Y36G15Lyk9ba(jiAW zMOpH_2BQ>$NKo6usR!X_aFuj$B-0lNeTniL4r`8kgW+j=bMqu_OhHHv_5dLKa5ErC z)HX94t9Qch*)WhJfF4v6fL&PGFunI?3t%@JVCTuf#9P#Rtb-bK$T%FQPcU$HOC$UQ zx`s_xIC~{V;MA3WYGP7^TTM%wL<`$Qv7viB5QZS1&?CK~u7P*Vr==U6s*u>zl5P7p z@IDdY@+EbTN>udzmRo6mzxF}HKQzRy{Jc3{`}@k4-*SH@ag(=?*3)>nH5i8DBEl(X zBjA>tY(L|=!rtis77(Sg1UYi9EZPVA>%tAZ)yXEV)4s(lIgKrFt6ISQ69I9U=Vg+; zd1whIQzk>fXebzl0~7xU^&~bF@%lwM_NfvB643T=aV3Lg0)~*SWX`{Eeg1k(m$KE( zX!FBGUsN85*Cx%a>y-x*ji(>qPBaXDW(Sfq4?z(aK;|a6K0LZo{Oj_M${(LdHS{IT zXEzLNv2ljmG*d+f7pi_*_0(LHC~ZuXG+}kw)Jj(L{)S6c^rnh>*AE;94g8gXB{si# z&%*~DFaNsvqvmwSx04;;UhBA+DjrN<8rug%pY*fh*{b#A7n^=pV+8hH1 zwEs?72OOM}CILv`QH)&;$)^R%B+lc(V~`$?Wb}9-yfOuWM2}?lc<_ix=84S%+lhim zpbMuZJi-a+LBQ}4vBe}dI)$eq&tn0x#=nEPH<0`lk})Jb9ua~~O z!Qi#;HFEvJVA*8he$z~|Et~L$SCR1arcTG6+$_+t9h+tydwR1#$F?CMxvtFumOV#r zS{b(OYhxKJ?@^Ia6BvcwHT<32b8E-&1#uuO{s5dIbG z;2A$zL~z|Q0D6SqK^d?*CB}ekFf>j7CpGj3%Jey9_?#;IoHBk++2G$72Gjk@*~+=W zq@m`XZry0P-!j_*CsegDfBdzCq4S>Z3&)W-|2Uc~J)5BH8#*0rU$k#haM>u*(e-o8 z2IT7(JIN*16=yy;iBIb{i^4SBK6fWwR-Y`Z-vs?cw{KiyD6M&BI>l71Yt26%e0%V3 MFKs|kC7Hqh0SzCad;kCd literal 0 HcmV?d00001 diff --git a/py/tests/pyc-compile/version_check_test.py b/py/tests/pyc-compile/version_check_test.py new file mode 100644 index 000000000..104fc3d95 --- /dev/null +++ b/py/tests/pyc-compile/version_check_test.py @@ -0,0 +1,148 @@ +"""Tests for pyc_compile.py's argument contract and --expect-version guard.""" + +import json +import os +import subprocess +import sys +import tempfile +import unittest + +SCRIPT = os.environ["PYC_COMPILE"] +PYTHON = [sys.executable, "-S", "-s", "-B", SCRIPT] + +_PRERELEASE_ABBREVS = {"alpha": "a", "beta": "b", "candidate": "rc"} + + +def running_version() -> str: + version = "{}.{}.{}".format(*sys.version_info[:3]) + if sys.version_info.releaselevel != "final": + version += _PRERELEASE_ABBREVS.get( + sys.version_info.releaselevel, sys.version_info.releaselevel + ) + str(sys.version_info.serial) + return version + + +def triple(tmp: str, name: str) -> list[str]: + src = os.path.join(tmp, name + ".py") + with open(src, "w") as f: + f.write("x = 1\n") + pycache = os.path.join(tmp, "__pycache__", name + ".cpython-00.pyc") + os.makedirs(os.path.dirname(pycache), exist_ok=True) + return [src, pycache, name + ".py"] + + +class VersionCheckTest(unittest.TestCase): + def setUp(self) -> None: + self.tmp = tempfile.mkdtemp(dir=os.environ.get("TEST_TMPDIR")) + + def run_compile(self, *argv: str) -> "subprocess.CompletedProcess[str]": + return subprocess.run(PYTHON + list(argv), capture_output=True, text=True) + + def compile( + self, expect_version: str | None = None + ) -> tuple["subprocess.CompletedProcess[str]", str]: + files = triple(self.tmp, "mod") + argv = ["--expect-version", expect_version] if expect_version else [] + return self.run_compile(*argv, *files), files[1] + + def assert_compiled( + self, result: "subprocess.CompletedProcess[str]", pycache: str + ) -> None: + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(os.path.exists(pycache)) + + def test_no_expect_version(self) -> None: + self.assert_compiled(*self.compile()) + + def test_exact_version(self) -> None: + self.assert_compiled(*self.compile(running_version())) + + def test_feature_version_only(self) -> None: + self.assert_compiled(*self.compile("{}.{}".format(*sys.version_info[:2]))) + + def test_different_micro_final(self) -> None: + if sys.version_info.releaselevel != "final": + self.skipTest("prerelease interpreters require exact version match") + other_micro = "{}.{}.{}".format( + sys.version_info.major, sys.version_info.minor, sys.version_info.micro + 1 + ) + self.assert_compiled(*self.compile(other_micro)) + + def test_wrong_feature_version(self) -> None: + result, pycache = self.compile("2.0.0") + self.assertNotEqual(result.returncode, 0) + self.assertIn("expected 2.0.0", result.stderr) + self.assertFalse(os.path.exists(pycache)) + + def test_prerelease_expected_requires_exact(self) -> None: + expected = "{}.{}.{}rc9".format(*sys.version_info[:3]) + if running_version() == expected: + self.skipTest("interpreter is coincidentally the tested prerelease") + result, _ = self.compile(expected) + self.assertNotEqual(result.returncode, 0) + self.assertIn("expected " + expected, result.stderr) + + def test_multiple_triples(self) -> None: + one = triple(self.tmp, "one") + two = triple(self.tmp, "two") + result = self.run_compile(*one, *two) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(os.path.exists(one[1])) + self.assertTrue(os.path.exists(two[1])) + + def test_legacy_writes_colocated_copy(self) -> None: + files = triple(self.tmp, "mod") + legacy = os.path.join(self.tmp, "mod.pyc") + result = self.run_compile("--legacy", *files) + self.assertEqual(result.returncode, 0, result.stderr) + with open(files[1], "rb") as cache, open(legacy, "rb") as colocated: + self.assertEqual(cache.read(), colocated.read()) + + def test_legacy_requires_pycache_output(self) -> None: + src, _, dfile = triple(self.tmp, "mod") + result = self.run_compile( + "--legacy", src, os.path.join(self.tmp, "mod.pyc"), dfile + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("__pycache__", result.stderr) + + def test_argfile(self) -> None: + files = triple(self.tmp, "mod") + argfile = os.path.join(self.tmp, "args") + with open(argfile, "w") as f: + f.write("\n".join(files) + "\n") + self.assert_compiled(self.run_compile("@" + argfile), files[1]) + + def test_incomplete_triple(self) -> None: + result = self.run_compile(*triple(self.tmp, "mod")[:2]) + self.assertNotEqual(result.returncode, 0) + self.assertIn("triples", result.stderr) + + def test_persistent_worker(self) -> None: + good = triple(self.tmp, "good") + bad = triple(self.tmp, "bad") + requests = [ + {"requestId": 1, "arguments": good}, + {"requestId": 2, "arguments": ["--expect-version", "2.0.0", *bad]}, + {"requestId": 3, "cancel": True}, + {"requestId": 4, "arguments": triple(self.tmp, "again")}, + ] + result = subprocess.run( + PYTHON + ["--persistent_worker"], + input="".join(json.dumps(r) + "\n" for r in requests), + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 0, result.stderr) + responses = [json.loads(line) for line in result.stdout.splitlines()] + self.assertEqual([r["requestId"] for r in responses], [1, 2, 4]) + self.assertEqual(responses[0]["exitCode"], 0) + self.assertEqual(responses[2]["exitCode"], 0) + self.assertEqual(responses[1]["exitCode"], 1) + self.assertIn("expected 2.0.0", responses[1]["output"]) + self.assertTrue(os.path.exists(good[1])) + self.assertFalse(os.path.exists(bad[1])) + + +if __name__ == "__main__": + unittest.main() diff --git a/py/tests/tmpdir-af-unix/BUILD.bazel b/py/tests/tmpdir-af-unix/BUILD.bazel index 6572e3054..93f5162db 100644 --- a/py/tests/tmpdir-af-unix/BUILD.bazel +++ b/py/tests/tmpdir-af-unix/BUILD.bazel @@ -1,4 +1,5 @@ load("//py:defs.bzl", "py_library", "py_pytest_test", "py_test", "py_unittest_test") +load("//py/tests/py-venv-multi-exec:pyc_runfiles_test.bzl", "pyc_runfiles_test") package(default_testonly = True) @@ -18,12 +19,61 @@ py_pytest_test( ], ) +py_pytest_test( + name = "pytest_pyc_af_unix_test", + srcs = ["pytest_af_unix_test.py"], + # Exercise configurable pyc forwarding through the public wrapper. + pyc = select({"//conditions:default": "pyc"}), + deps = [ + ":af_unix_probe", + "@pypi//pytest", + ], +) + +pyc_runfiles_test( + name = "pytest_pyc_runfiles_test", + mode = "pyc", + module = "pytest_af_unix_test", + target_under_test = ":pytest_pyc_af_unix_test", +) + +py_pytest_test( + name = "pytest_pyc_only_af_unix_test", + srcs = ["pytest_af_unix_test.py"], + pyc = "pyc_only", + deps = [ + ":af_unix_probe", + "@pypi//pytest", + ], +) + +pyc_runfiles_test( + name = "pytest_pyc_only_runfiles_test", + mode = "pyc", + module = "pytest_af_unix_test", + target_under_test = ":pytest_pyc_only_af_unix_test", +) + py_unittest_test( name = "unittest_af_unix_test", srcs = ["unittest_af_unix_test.py"], deps = [":af_unix_probe"], ) +py_unittest_test( + name = "unittest_pyc_af_unix_test", + srcs = ["unittest_af_unix_test.py"], + pyc = "pyc", + deps = [":af_unix_probe"], +) + +pyc_runfiles_test( + name = "unittest_pyc_runfiles_test", + mode = "pyc", + module = "unittest_af_unix_test", + target_under_test = ":unittest_pyc_af_unix_test", +) + py_test( name = "raw_af_unix_test", srcs = ["raw_af_unix_test.py"], diff --git a/pyproject.toml b/pyproject.toml index c1e9870f1..679059455 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,8 @@ dev-dependencies = [ [tool.ruff] target-version = "py310" extend-exclude = [ - # Intentionally unparsable .py-as-data fixture. + # Intentionally unparsable .py-as-data fixture; must stay broken so a + # bytecode compile of it would fail loudly. "py/tests/py-venv-multi-exec/broken_data.py", # OCI layer snapshots pin the byte sizes of these fixtures. "e2e/cases/oci/py_image_layer/branding/**", From 0561046ab2a57dbe5abacd0ec665680950af5514 Mon Sep 17 00:00:00 2001 From: Jason Bedard Date: Wed, 16 Sep 2026 16:55:47 -0700 Subject: [PATCH 3/5] fixup! feat: add pyc compilation of first-party python code via actions --- py/private/pyc_compile.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/py/private/pyc_compile.py b/py/private/pyc_compile.py index 0f2f5affc..78dc539a9 100644 --- a/py/private/pyc_compile.py +++ b/py/private/pyc_compile.py @@ -12,6 +12,8 @@ dominates. """ +from __future__ import annotations + import importlib.util import marshal import sys From b2b5c54a946f23e16546a83a9adf0ae71d467daa Mon Sep 17 00:00:00 2001 From: Jason Bedard Date: Wed, 16 Sep 2026 16:58:22 -0700 Subject: [PATCH 4/5] fixup! feat: add pyc compilation of first-party python code via actions --- py/private/interpreter/runtime.bzl | 2 +- py/private/pyc.bzl | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/py/private/interpreter/runtime.bzl b/py/private/interpreter/runtime.bzl index f58a47169..f70f805e8 100644 --- a/py/private/interpreter/runtime.bzl +++ b/py/private/interpreter/runtime.bzl @@ -46,7 +46,7 @@ def _py_runtime_toolchain_impl(ctx): pyc_compile_tool = struct( executable = ctx.attr.pyc_compile_tool[DefaultInfo].files_to_run, arguments = [], - inputs = depset(), + tools = depset(), ) return [ diff --git a/py/private/pyc.bzl b/py/private/pyc.bzl index b3c6af3d1..8704fb0dd 100644 --- a/py/private/pyc.bzl +++ b/py/private/pyc.bzl @@ -203,7 +203,7 @@ def compile_pycs(ctx, srcs, existing = {}): pyc_compile_tool = struct( executable = compile_runtime.interpreter, arguments = ["-S", "-s", "-B", ctx.file._pyc_compiler], - inputs = depset( + tools = depset( [compile_runtime.interpreter, ctx.file._pyc_compiler], transitive = [compile_runtime.files], ), @@ -259,10 +259,8 @@ def compile_pycs(ctx, srcs, existing = {}): toolchain = tool_toolchain, arguments = [tool_args, compile_args], execution_requirements = execution_requirements, - inputs = depset( - direct = [src], - transitive = [pyc_compile_tool.inputs], - ), + inputs = [src], + tools = [pyc_compile_tool.tools], outputs = outputs, mnemonic = "PyCompile", progress_message = "Python precompiling {} into {}".format( From e3128093fea4c11cb1a3332b0c14630d6d56ea97 Mon Sep 17 00:00:00 2001 From: Jason Bedard Date: Wed, 16 Sep 2026 17:08:25 -0700 Subject: [PATCH 5/5] fixup! feat: add pyc compilation of first-party python code via actions --- e2e/cases/oci/test.sh | 14 ++++++++++++++ py/private/py_image_layer.bzl | 5 ++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/e2e/cases/oci/test.sh b/e2e/cases/oci/test.sh index 5f583dd3e..7f7e24a7c 100755 --- a/e2e/cases/oci/test.sh +++ b/e2e/cases/oci/test.sh @@ -162,3 +162,17 @@ if ! "$BAZEL" build --@aspect_rules_py//py:pyc=pyc_only \ fail "expected flag-inherited bytecode images to build" fi echo "PASS: global pyc flag flows into images" + +echo "== sourceless images build unchanged under coverage ==" +if ! "$BAZEL" build --collect_code_coverage \ + //oci/py_venv_image_layer:pyc_only_amd64_layers >"$output_log" 2>&1; then + cat "$output_log" >&2 + fail "expected the pyc_only image to build under coverage" +fi +coverage_layers="$("$BAZEL" cquery --collect_code_coverage --output=files //oci/py_venv_image_layer:pyc_only_amd64_layers 2>/dev/null)" +for layer in $coverage_layers; do + if tar tzf "$layer" | grep -q '/_main/oci/py_venv_image_layer/__main__\.py$'; then + fail "coverage build shipped the source entrypoint in a pyc_only image: $layer" + fi +done +echo "PASS: coverage does not alter image contents" diff --git a/py/private/py_image_layer.bzl b/py/private/py_image_layer.bzl index cce554ca2..858bc4dd4 100644 --- a/py/private/py_image_layer.bzl +++ b/py/private/py_image_layer.bzl @@ -889,9 +889,12 @@ def _parse_exec_requirements(entries): reqs[k] = v return reqs +# Images package runtime artifacts: coverage instrumentation of the wrapped +# binaries is dropped so `bazel coverage` builds the same layers as `bazel build`. def _platform_cfg_impl(settings, attr): result = { "//command_line_option:platforms": [attr.platform] if attr.platform else settings["//command_line_option:platforms"], + "//command_line_option:collect_code_coverage": False, "@aspect_rules_py//py:layer_tier": str(attr.layer_tier) if attr.layer_tier else settings["@aspect_rules_py//py:layer_tier"], PYC_FLAG: attr.pyc if attr.pyc else settings[PYC_FLAG], } @@ -900,7 +903,7 @@ def _platform_cfg_impl(settings, attr): _platform_cfg = transition( implementation = _platform_cfg_impl, inputs = ["//command_line_option:platforms", "@aspect_rules_py//py:layer_tier", PYC_FLAG], - outputs = ["//command_line_option:platforms", "@aspect_rules_py//py:layer_tier", PYC_FLAG], + outputs = ["//command_line_option:platforms", "//command_line_option:collect_code_coverage", "@aspect_rules_py//py:layer_tier", PYC_FLAG], ) def _skip_path(f):