From 5ab6a478cb27ccd0e978a6cd43582843da6b9bc2 Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Fri, 18 Sep 2026 00:34:32 +0800 Subject: [PATCH] fix(python): isolate the torch.compile cache per xdist worker `test_torch_index_with_nans` fails intermittently on the Windows python job with `PermissionError: [Errno 13]` on a path under `torchinductor_runneradmin`. It has now reproduced on two consecutive runs of the same branch, once per parametrized case, so it is not a one-off. `lance.torch.distance` decorates six kernels with `@torch.compile`, the suite runs with `PYTEST_WORKERS=auto`, and nothing sets `TORCHINDUCTOR_CACHE_DIR`. Every worker that reaches the torch accelerator therefore compiles the same six functions into torch's single default cache directory. Inductor writes a generated module by replacing it, which POSIX permits while another process holds it open and Windows refuses -- which is why only this job is affected. Point each worker at its own subdirectory. Serial runs keep torch's default so the cache stays warm, and the per-worker directories share one temp root for the same reason. `TORCH_COMPILE_DISABLE` would also stop the race but would drop coverage of those six kernels, which the Windows job sets up MSVC specifically to exercise. ## Testing `uv run make lint` from `python/`. Verified the mechanism with a throwaway test asserting that each worker's `TORCHINDUCTOR_CACHE_DIR` ends in its own worker id: under `-n 2` the two workers get `.../lance_torchinductor/gw0` and `.../gw1`, and with the helper disabled the variable is unset for both. I cannot reproduce the Windows failure itself on macOS, so that the race is what fails there remains inferred from the log and the configuration above. --- python/python/tests/conftest.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/python/python/tests/conftest.py b/python/python/tests/conftest.py index c9d8911fd2b..5064c6b791f 100644 --- a/python/python/tests/conftest.py +++ b/python/python/tests/conftest.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright The Lance Authors +import os import sys +import tempfile from typing import Optional import pytest @@ -86,6 +88,28 @@ def pytest_addoption(parser): ) +def _isolate_torch_inductor_cache(workerinput) -> None: + """Give each xdist worker its own torch.compile cache directory. + + `lance.torch.distance` decorates six kernels with `@torch.compile`, so every + worker that reaches the torch accelerator compiles the same functions into + torch's single default cache directory. Inductor writes each generated module + by replacing it, which POSIX allows while another process holds it open and + Windows refuses, so the shared directory surfaces as an intermittent + `PermissionError` out of the codecache rather than a test failure. + + Only under xdist: a single process cannot race itself, and leaving torch's + default alone there keeps the cache warm across runs. The per-worker + directories live under the same temp root for the same reason. + """ + if workerinput is None: + return + root = os.environ.get("TORCHINDUCTOR_CACHE_DIR") or os.path.join( + tempfile.gettempdir(), "lance_torchinductor" + ) + os.environ["TORCHINDUCTOR_CACHE_DIR"] = os.path.join(root, workerinput["workerid"]) + + def pytest_configure(config): config.addinivalue_line( "markers", @@ -103,6 +127,7 @@ def pytest_configure(config): ) workerinput = getattr(config, "workerinput", None) + _isolate_torch_inductor_cache(workerinput) if workerinput is not None: from compat.compat_decorator import use_version_snapshot