From d25936cd5a74a2e5d7ca366d552249999f11f214 Mon Sep 17 00:00:00 2001 From: magaonka-amd Date: Thu, 4 Jun 2026 17:37:46 -0500 Subject: [PATCH] [ROCm] xdist: select GPU from runner-allocated ROCR slice CI runners isolate each job to a subset of the host's physical GPUs by injecting ROCR_VISIBLE_DEVICES (e.g. "0,3,4,5"); these are absolute physical indices and may be non-contiguous. Re-setting ROCR_VISIBLE_DEVICES does not compose across processes, so the previous `xdist_worker_number % num_rocm_devices` selection could land a worker on a GPU owned by another job running on the same host. Pick the device from the inherited ROCR_VISIBLE_DEVICES slice instead, and fall back to the old behaviour when no slice is injected (local dev). --- conftest.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/conftest.py b/conftest.py index cd8414369b89..3abbeef26136 100644 --- a/conftest.py +++ b/conftest.py @@ -79,9 +79,17 @@ def pytest_collection() -> None: if not xdist_worker_name.startswith("gw"): return xdist_worker_number = int(xdist_worker_name[len("gw") :]) - os.environ["ROCR_VISIBLE_DEVICES"] = str( - xdist_worker_number % num_rocm_devices + allocated = os.environ.get("ROCR_VISIBLE_DEVICES") + allocated_tokens = ( + [t.strip() for t in allocated.split(",") if t.strip()] + if allocated + else [] ) + if allocated_tokens: + selected = allocated_tokens[xdist_worker_number % len(allocated_tokens)] + else: + selected = str(xdist_worker_number % num_rocm_devices) + os.environ["ROCR_VISIBLE_DEVICES"] = selected # ROCR_VISIBLE_DEVICES filters HSA to a single physical device, which # becomes HIP index 0. The container env-file may preset # HIP_VISIBLE_DEVICES to all GPUs; override to "0" so HIP doesn't try to