From 61b93e4b75e02ef8edef78b3bc197799601839e1 Mon Sep 17 00:00:00 2001 From: Alex Gaynor Date: Sun, 7 Jun 2026 09:06:39 -0700 Subject: [PATCH] Set PYO3_BASE_PYTHON to the stable base interpreter path When a project is built by a PEP 517 frontend with build isolation, the frontend creates a randomly-named temporary virtualenv and maturin points PYO3_PYTHON inside it. Because pyo3-build-config registers rerun-if-env-changed=PYO3_PYTHON, the ephemeral path defeats cargo's build cache and forces recompilation of the pyo3 crates on every build, even when the underlying interpreter is identical. maturin now additionally exports PYO3_BASE_PYTHON pointing at the stable base interpreter path (sys._base_executable). pyo3 versions that support it prefer it over PYO3_PYTHON and don't trigger rebuilds when only PYO3_PYTHON changes, keeping the build cache warm. Older pyo3 versions simply ignore the variable, and PYO3_PYTHON is still set to the venv path for build scripts that rely on it. See https://github.com/PyO3/pyo3/issues/6113 Co-Authored-By: Claude Opus 4.8 (1M context) --- Changelog.md | 1 + src/compile.rs | 16 ++++++++++++++++ src/python_interpreter/discovery.rs | 11 +++++++++++ .../get_interpreter_metadata.py | 4 ++++ src/python_interpreter/mod.rs | 10 ++++++++++ src/python_interpreter/resolver.rs | 2 ++ 6 files changed, 44 insertions(+) diff --git a/Changelog.md b/Changelog.md index 7a75985bd..0952b9ec7 100644 --- a/Changelog.md +++ b/Changelog.md @@ -4,6 +4,7 @@ * `--find-interpreter` now discovers free-threaded CPython interpreters (`python3.14t` and newer) on all platforms. The experimental 3.13t is no longer built by default; build it explicitly with `-i python3.13t`. * `cffi` is no longer automatically added as a build dependency of maturin on PyPy, which has `cffi` pre-installed as part of the PyPy distribution +* Set `PYO3_BASE_PYTHON` to the stable base interpreter path (`sys._base_executable`) alongside `PYO3_PYTHON`. With pyo3 versions that support it, this keeps cargo's build cache warm across PEP 517 builds in ephemeral virtualenvs ([pyo3#6113](https://github.com/PyO3/pyo3/issues/6113)) ## 1.13.3 diff --git a/src/compile.rs b/src/compile.rs index 3105f7183..ca8ec0c1e 100644 --- a/src/compile.rs +++ b/src/compile.rs @@ -892,6 +892,22 @@ fn configure_pyo3_env( "PYO3_ENVIRONMENT_SIGNATURE", interpreter.environment_signature(), ); + + // Also point pyo3 at the stable base interpreter path (PEP 405 + // `sys._base_executable`). pyo3 versions that support it prefer + // `PYO3_BASE_PYTHON` over `PYO3_PYTHON` and don't trigger rebuilds + // when only `PYO3_PYTHON` changes, which keeps cargo's build cache + // warm across PEP 517 builds in ephemeral (randomly-named) + // virtualenvs. Older pyo3 versions simply ignore the variable. + // See https://github.com/PyO3/pyo3/issues/6113 + if let Some(base_executable) = interpreter + .base_executable + .as_deref() + .filter(|base| base.is_file()) + { + debug!("Setting PYO3_BASE_PYTHON to {}", base_executable.display()); + build_command.env("PYO3_BASE_PYTHON", base_executable); + } } // and legacy pyo3 versions diff --git a/src/python_interpreter/discovery.rs b/src/python_interpreter/discovery.rs index 55e3a18a2..ee3e35941 100644 --- a/src/python_interpreter/discovery.rs +++ b/src/python_interpreter/discovery.rs @@ -32,6 +32,10 @@ const GET_INTERPRETER_METADATA: &str = include_str!("get_interpreter_metadata.py pub(super) struct InterpreterMetadataMessage { pub implementation_name: String, pub executable: Option, + // `sys._base_executable`: the base interpreter path when running inside a + // venv, otherwise (usually) the same as `executable`. Absent on + // interpreters that don't define `sys._base_executable`. + pub base_executable: Option, pub major: usize, pub minor: usize, pub abiflags: Option, @@ -623,6 +627,7 @@ fn from_metadata_message( .executable .map(PathBuf::from) .unwrap_or_else(|| executable.as_ref().to_path_buf()); + let base_executable = message.base_executable.map(PathBuf::from); if target.is_windows() { 'windows_arch_check: { @@ -676,6 +681,7 @@ fn from_metadata_message( gil_disabled: message.gil_disabled, }, executable, + base_executable, platform, runnable: true, implementation_name: message.implementation_name, @@ -883,6 +889,7 @@ mod tests { ext_suffix: Some(".pyd".to_string()), platform: platform.to_string(), executable: None, + base_executable: None, soabi: None, gil_disabled: false, system: "windows".to_string(), @@ -935,6 +942,7 @@ mod tests { gil_disabled: false, }, executable: PathBuf::from("python3.10"), + base_executable: None, platform: Some(platform.replace("-", "_")), runnable: true, implementation_name: "CPython".to_string(), @@ -980,6 +988,7 @@ mod tests { gil_disabled: false, }, executable: PathBuf::from("python3.10"), + base_executable: None, platform: Some("unknown_platform".to_string()), runnable: true, implementation_name: "CPython".to_string(), @@ -1010,6 +1019,7 @@ mod tests { ext_suffix: Some(".cp314-win_amd64.pyd".to_string()), platform: "win-amd64".to_string(), executable: None, + base_executable: None, soabi: None, gil_disabled: false, system: "windows".to_string(), @@ -1031,6 +1041,7 @@ mod tests { ext_suffix: Some(".cp314t-win_amd64.pyd".to_string()), platform: "win-amd64".to_string(), executable: None, + base_executable: None, soabi: None, gil_disabled: true, system: "windows".to_string(), diff --git a/src/python_interpreter/get_interpreter_metadata.py b/src/python_interpreter/get_interpreter_metadata.py index e2fd63d8d..729d6ebb2 100644 --- a/src/python_interpreter/get_interpreter_metadata.py +++ b/src/python_interpreter/get_interpreter_metadata.py @@ -24,6 +24,10 @@ # Pyston has sys.implementation.name == "pyston" while platform.python_implementation() == cpython "implementation_name": sys.implementation.name, "executable": sys.executable or None, + # The base interpreter path when running inside a venv (PEP 405). Used to + # set `PYO3_BASE_PYTHON` to a stable path so that ephemeral virtualenv + # paths don't invalidate cargo's build cache. + "base_executable": getattr(sys, "_base_executable", None) or None, "major": sys.version_info.major, "minor": sys.version_info.minor, "abiflags": sysconfig.get_config_var("ABIFLAGS"), diff --git a/src/python_interpreter/mod.rs b/src/python_interpreter/mod.rs index 75823a95b..a194a6b8c 100644 --- a/src/python_interpreter/mod.rs +++ b/src/python_interpreter/mod.rs @@ -88,6 +88,14 @@ pub struct PythonInterpreter { /// /// Just the name of the binary in PATH does also work, e.g. `python3.5` pub executable: PathBuf, + /// Comes from `sys._base_executable`: the base interpreter path when + /// `executable` is inside a venv (PEP 405), otherwise usually the same as + /// `executable`. + /// + /// Used to set `PYO3_BASE_PYTHON` to a stable interpreter path so that + /// ephemeral virtualenv paths (as created by PEP 517 build frontends with + /// build isolation) don't invalidate cargo's build cache. + pub base_executable: Option, /// Comes from `sysconfig.get_platform()` /// /// Note that this can be `None` when cross compiling @@ -244,6 +252,7 @@ impl PythonInterpreter { PythonInterpreter { config, executable: PathBuf::new(), + base_executable: None, platform: None, runnable: false, implementation_name, @@ -272,6 +281,7 @@ impl PythonInterpreter { gil_disabled: false, }, executable: PathBuf::new(), + base_executable: None, platform: None, runnable: false, implementation_name: "cpython".to_string(), diff --git a/src/python_interpreter/resolver.rs b/src/python_interpreter/resolver.rs index 539e65aa8..19b715af7 100644 --- a/src/python_interpreter/resolver.rs +++ b/src/python_interpreter/resolver.rs @@ -330,6 +330,7 @@ impl<'a> InterpreterResolver<'a> { let interp = PythonInterpreter { config, executable: PathBuf::new(), + base_executable: None, platform: None, runnable: false, implementation_name, @@ -858,6 +859,7 @@ impl<'a> InterpreterResolver<'a> { gil_disabled, }, executable: PathBuf::new(), + base_executable: None, platform: None, runnable: false, implementation_name: interpreter_kind.to_string().to_ascii_lowercase(),