Skip to content

Fix setup_ppcie to prepare gpus and switches. #403

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Evrard-Nil wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Fix setup_ppcie to prepare gpus and switches. #403

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
acaa1af
Fix setup_ppcie to prepare gpus and switches.
Evrard-Nil Aug 22, 2025
3fdd0ef
Enhance GPU argument handling in run_td: support 'all' to detect avai…
Evrard-Nil Oct 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 22 additions & 9 deletions guest-tools/run_td
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ def add_gpus(cmd, gpus):
index = index + 1

def pci_devices(vendor, product) -> list[str]:
output = subprocess.check_output(["lspci", "-d", f"{vendor}:{product}"], stderr=subprocess.STDOUT)
output = subprocess.check_output(["lspci", "-D", "-d", f"{vendor}:{product}"], stderr=subprocess.STDOUT)
devices = []
for line in output.splitlines():
address = line.decode().strip().split(" ")[0]
Expand All @@ -100,7 +100,8 @@ def setup_ppcie(cmd):
print("PPCIe mode requires 4 NVSwitches")
exit(1)

# prepare_gpus(gpus)
all_devices = gpus + nvswitches
prepare_gpus(all_devices)

cmd.extend([
"-object", "iommufd,id=iommufd0",
Expand Down Expand Up @@ -138,13 +139,13 @@ def setup_ppcie(cmd):
"-fw_cfg", "name=opt/ovmf/X-PciMmio64Mb8,string=262144",

"-device", "pcie-root-port,port=24,chassis=9,id=pci.9,bus=pcie.0,multifunction=on,addr=0x4",
"-device", f"vfio-pci,host=0000:{nvswitches[0]},bus=pci.9,addr=0x0,iommufd=iommufd0",
"-device", f"vfio-pci,host={nvswitches[0]},bus=pci.9,addr=0x0,iommufd=iommufd0",
"-device", "pcie-root-port,port=25,chassis=10,id=pci.10,bus=pcie.0,addr=0x4.0x1",
"-device", f"vfio-pci,host=0000:{nvswitches[1]},bus=pci.10,addr=0x0,iommufd=iommufd0",
"-device", f"vfio-pci,host={nvswitches[1]},bus=pci.10,addr=0x0,iommufd=iommufd0",
"-device", "pcie-root-port,port=26,chassis=11,id=pci.11,bus=pcie.0,addr=0x4.0x2",
"-device", f"vfio-pci,host=0000:{nvswitches[2]},bus=pci.11,addr=0x0,iommufd=iommufd0",
"-device", f"vfio-pci,host={nvswitches[2]},bus=pci.11,addr=0x0,iommufd=iommufd0",
"-device", "pcie-root-port,port=27,chassis=12,id=pci.12,bus=pcie.0,addr=0x4.0x3",
"-device", f"vfio-pci,host=0000:{nvswitches[3]},bus=pci.12,addr=0x0,iommufd=iommufd0",
"-device", f"vfio-pci,host={nvswitches[3]},bus=pci.12,addr=0x0,iommufd=iommufd0",
])

def do_run(img_path, gpus):
Expand Down Expand Up @@ -187,7 +188,7 @@ def do_run(img_path, gpus):

add_vsock(qemu_cmds)

if args.gpus and len(args.gpus) == 8:
if args.gpus and len(gpus) == 8:
setup_ppcie(qemu_cmds)
else:
add_gpus(qemu_cmds, gpus)
Expand All @@ -205,12 +206,24 @@ def run_td(args):
return
if args.image:
td_img=args.image
do_run(td_img, args.gpus.split(',') if args.gpus else [])

# Handle GPU arguments
gpus = []
if args.gpus:
if args.gpus.lower() == 'all':
# Detect all available GPUs (NVIDIA H100)
gpus = pci_devices("10de", "2335")
if gpus:
print(f"Detected {len(gpus)} GPU(s): {gpus}")
else:
gpus = args.gpus.split(',')

do_run(td_img, gpus)

if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument("--image", type=str, help="Guest image")
parser.add_argument("--gpus", type=str, help="GPUs to pass-through")
parser.add_argument("--gpus", type=str, help="GPUs to pass-through (comma-separated list of PCI addresses or 'all' to pass all GPUs)")
parser.add_argument("--clean", action='store_true', help="Clean the current VM")
parser.add_argument("--foreground", action='store_true', help="Run in foreground")
args = parser.parse_args()
Expand Down