diff --git a/ddp-qwen-broadcast.py b/ddp-qwen-broadcast.py new file mode 100644 index 0000000..4178a89 --- /dev/null +++ b/ddp-qwen-broadcast.py @@ -0,0 +1,331 @@ +from __future__ import annotations +import os +import sys +import time +import math +import logging +import socket +import functools +import itertools +from datetime import datetime, timedelta +import torch +import torch.nn as nn +import torch.distributed as dist +from torch.utils.data import DataLoader +from torch.utils.data.distributed import DistributedSampler +from transformers import AutoModelForCausalLM, AutoTokenizer +from datasets import load_dataset + +# --- DDP Import --- +from torch.nn.parallel import DistributedDataParallel as DDP + +# --- SUPPRESS WARNINGS --- +import warnings +warnings.filterwarnings("ignore") +os.environ["TORCH_CPP_LOG_LEVEL"] = "ERROR" +os.environ["TORCH_DISTRIBUTED_DEBUG"] = "OFF" + +# ----------------------------------------------------------- +# Configuration +# ----------------------------------------------------------- +TRAIN_NODES = 2 +GPUS_PER_NODE = 8 +TRAIN_WORLD_SIZE = TRAIN_NODES * GPUS_PER_NODE +INFERENCE_MASTER_RANK = TRAIN_WORLD_SIZE + +# ----------------------------------------------------------- +# Logging Setup (Fixed Flushing) +# ----------------------------------------------------------- +def setup_logging(rank: int) -> str: + log_dir = "logs" + os.makedirs(log_dir, exist_ok=True) + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + log_file = os.path.join(log_dir, f"rank_{rank}_{timestamp}.log") + + logger = logging.getLogger() + logger.setLevel(logging.INFO) + + # Clear existing handlers + for h in list(logger.handlers): + logger.removeHandler(h) + + # File Handler + fh = logging.FileHandler(log_file) + fh.setLevel(logging.INFO) + + # Console Handler with Force Flush + ch = logging.StreamHandler(sys.stdout) + ch.setLevel(logging.INFO) + + # Custom Formatter + formatter = logging.Formatter(fmt="%(asctime)s | Rank %(rank)d | %(levelname)s | %(message)s", datefmt="%H:%M:%S") + fh.setFormatter(formatter) + ch.setFormatter(formatter) + + # Rank Filter + class RankFilter(logging.Filter): + def filter(self, record): + record.rank = rank + return True + + fh.addFilter(RankFilter()) + ch.addFilter(RankFilter()) + + logger.addHandler(fh) + logger.addHandler(ch) + return log_file + +# ----------------------------------------------------------- +# Dataset +# ----------------------------------------------------------- +def get_wikitext_dataset(tokenizer: AutoTokenizer, seq_len: int = 128): + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + # Only log on rank 0 to avoid spam, unless error + if local_rank != 0: + import datasets + datasets.logging.set_verbosity_error() + + logging.info("Loading WikiText-2 dataset...") + dataset = load_dataset("wikitext", "wikitext-2-raw-v1", split="train") + + def tokenize_function(examples): + return tokenizer(examples["text"]) + + tokenized_datasets = dataset.map(tokenize_function, batched=True, num_proc=4, remove_columns=["text"]) + all_input_ids = list(itertools.chain(*tokenized_datasets["input_ids"])) + data_tensor = torch.tensor(all_input_ids, dtype=torch.long) + + class WikiTextDataset(torch.utils.data.Dataset): + def __init__(self, data, seq_len): + self.data = data + self.seq_len = seq_len + self.num_samples = (len(self.data) - 1) // self.seq_len + def __len__(self): return self.num_samples + def __getitem__(self, idx): + start = idx * self.seq_len + end = start + self.seq_len + return self.data[start : end] + + logging.info(f"Dataset ready. Samples: {(len(data_tensor)-1)//seq_len}") + return WikiTextDataset(data_tensor, seq_len) + +# ----------------------------------------------------------- +# Broadcast Helper (Standard PyTorch TCP/NCCL) +# ----------------------------------------------------------- +def broadcast_model_to_inference(model, rank, bridge_group, is_sender=False): + """ + Sends model weights using standard torch.distributed.broadcast. + This works over whatever backend the bridge_group uses (likely Gloo or NCCL). + """ + if bridge_group is None: return + + try: + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + device = torch.device("cuda", local_rank) + except: + device = torch.device("cuda") + + if is_sender: + logging.info("Broadcasting weights to Inference Node...") + else: + logging.info("Waiting for weights from Training Node...") + # Force flush to ensure log appears before blocking operation + sys.stdout.flush() + + t0 = time.time() + + # We iterate over parameters to ensure robust transfer + state_dict = model.state_dict() if not isinstance(model, DDP) else model.module.state_dict() + items = list(state_dict.items()) + total_bytes = sum(t.numel() * t.element_size() for _, t in items) + + with torch.no_grad(): + for param in model.parameters(): + # Ensure tensor is on GPU for NCCL broadcast (if using NCCL backend) + # Or ensure consistent device for Gloo. + if param.device.type == "cpu": + gpu_param = param.data.to(device) + dist.broadcast(gpu_param, src=0, group=bridge_group) + # If receiver, move back to CPU if model requires it (rare) + else: + dist.broadcast(param.data, src=0, group=bridge_group) + + duration = time.time() - t0 + bw = (total_bytes / 1e9) / duration if duration > 0 else 0 + + if is_sender: + logging.info(f"Broadcast complete. Time: {duration:.4f}s | BW: {bw:.2f} GB/s") + else: + logging.info(f"Weights received. Time: {duration:.4f}s") + sys.stdout.flush() + +# ----------------------------------------------------------- +# Reward Function +# ----------------------------------------------------------- +def compute_reward(token_ids, tokenizer): + # Dynamic "the" finding for Qwen + # Note: encoded[0] is usually the token ID. Qwen might encode " the" vs "the". + # For simplicity/speed we calc this once or here. + # Assuming "the" (common) is what we want. + target_id = tokenizer.encode("the", add_special_tokens=False)[0] + + matches = (token_ids == target_id).float() + rewards = matches.sum(dim=1) + rewards = (rewards * 0.5) - 0.5 + return rewards + +# ----------------------------------------------------------- +# Trainer Loop (DDP) +# ----------------------------------------------------------- +def run_trainer(rank, world_size, train_group, bridge_group): + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + torch.cuda.set_device(local_rank) + device = torch.device("cuda", local_rank) + + logging.info("Initializing DDP Actor Model (Qwen2.5-0.5B)...") + tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-0.5B") + # Qwen doesn't set pad_token by default usually + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + + model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen2.5-0.5B").to(device) + + # --- DDP WRAPPER --- + model = DDP(model, device_ids=[local_rank], output_device=local_rank, process_group=train_group) + + dataset = get_wikitext_dataset(tokenizer, seq_len=128) + sampler = DistributedSampler(dataset, num_replicas=dist.get_world_size(group=train_group), rank=dist.get_rank(group=train_group), shuffle=True) + + BATCH_SIZE = 4 + dataloader = DataLoader(dataset, batch_size=BATCH_SIZE, sampler=sampler, num_workers=2) + optimizer = torch.optim.AdamW(model.parameters(), lr=1e-5) + + num_epochs = 2 + + for epoch in range(1, num_epochs + 1): + model.train() + sampler.set_epoch(epoch) + epoch_start = time.time() + total_reward = 0.0 + + for i, batch_ids in enumerate(dataloader): + # Debug limit + if i > 50: break + + batch_ids = batch_ids.to(device) + optimizer.zero_grad() + + outputs = model(batch_ids, labels=batch_ids) + logits = outputs.logits + + shift_logits = logits[..., :-1, :].contiguous() + shift_labels = batch_ids[..., 1:].contiguous() + loss_fct = nn.CrossEntropyLoss(reduction='none') + neg_log_probs = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1)) + neg_log_probs = neg_log_probs.view(shift_labels.size()) + + rewards = compute_reward(batch_ids, tokenizer) + rewards_expanded = rewards.unsqueeze(1).expand_as(neg_log_probs) + rl_loss = (neg_log_probs * rewards_expanded).mean() + + rl_loss.backward() + optimizer.step() + + total_reward += rewards.mean().item() + + if i % 20 == 0 and rank == 0: + logging.info(f"Batch {i}/{len(dataloader)} | RL Loss: {rl_loss.item():.4f} | Avg Reward: {rewards.mean().item():.2f}") + + epoch_duration = time.time() - epoch_start + if rank == 0: + avg_reward = total_reward / (i+1) + logging.info(f"--- RL EPOCH {epoch} STATS ---") + logging.info(f"Duration: {epoch_duration:.2f} seconds") + logging.info(f"Avg Reward: {avg_reward:.4f}") + logging.info(f"-------------------------") + sys.stdout.flush() + + # --- BRIDGE SYNC --- + if rank == 0: + # DDP wraps model in .module + broadcast_model_to_inference(model.module, rank, bridge_group, is_sender=True) + +# ----------------------------------------------------------- +# Inference Loop +# ----------------------------------------------------------- +def run_inference(rank, bridge_group): + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + torch.cuda.set_device(local_rank) + device = torch.device("cuda", local_rank) + + logging.info("Initializing Inference Model (Qwen2.5-0.5B)...") + tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-0.5B") + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + + model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen2.5-0.5B").to(device) + model.eval() + + num_epochs = 2 + for epoch in range(1, num_epochs + 1): + logging.info(f"Waiting for RL model update (Epoch {epoch})...") + sys.stdout.flush() + + # This will block until Rank 0 calls broadcast + broadcast_model_to_inference(model, rank, bridge_group, is_sender=False) + + logging.info("Running inference test...") + test_input = "The AI scientist discovered" + inputs = tokenizer(test_input, return_tensors="pt").to(device) + + with torch.no_grad(): + outputs = model.generate( + **inputs, + max_new_tokens=30, + do_sample=True, + temperature=0.8, + repetition_penalty=1.2 + ) + + generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True) + logging.info(f"--- [INFERENCE RESULT EPOCH {epoch}] ---") + logging.info(f"Output: {generated_text}") + logging.info("---------------------------------------") + sys.stdout.flush() + +# ----------------------------------------------------------- +# Main +# ----------------------------------------------------------- +def main(): + # Use NCCL everywhere for simplicity if hardware supports it, + # but GLOO is safer for the bridge group if network config is tricky. + # Here we use NCCL globally for performance. + dist.init_process_group(backend="nccl", init_method="env://", timeout=timedelta(minutes=60)) + rank = dist.get_rank() + log_file = setup_logging(rank) + + if rank == 0: + logging.info(f"NCCL_SOCKET_IFNAME: {os.environ.get('NCCL_SOCKET_IFNAME', 'Not Set')}") + + train_ranks = list(range(0, TRAIN_WORLD_SIZE)) + train_group = dist.new_group(ranks=train_ranks) + + bridge_ranks = [0, INFERENCE_MASTER_RANK] + # Use NCCL for bridge too so we use RDMA/Broadcoms + bridge_group = dist.new_group(ranks=bridge_ranks) + + if rank in train_ranks: + my_bridge = bridge_group if rank == 0 else None + run_trainer(rank, TRAIN_WORLD_SIZE, train_group, my_bridge) + elif rank == INFERENCE_MASTER_RANK: + run_inference(rank, bridge_group) + else: + logging.info("Idling...") + pass + + logging.info("Waiting for all ranks to complete...") + dist.barrier() + dist.destroy_process_group() + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/ddp-qwen-uccl.py b/ddp-qwen-uccl.py new file mode 100644 index 0000000..d442f82 --- /dev/null +++ b/ddp-qwen-uccl.py @@ -0,0 +1,357 @@ +from __future__ import annotations +import os +import sys +import time +import math +import logging +import socket +import functools +import itertools +from datetime import datetime, timedelta +import torch +import torch.nn as nn +import torch.distributed as dist +from torch.utils.data import DataLoader +from torch.utils.data.distributed import DistributedSampler +from transformers import AutoModelForCausalLM, AutoTokenizer +from datasets import load_dataset + +# --- DDP Import --- +from torch.nn.parallel import DistributedDataParallel as DDP + +# UCCL Import +try: + from uccl import collective +except ImportError: + print("Error: 'uccl' library not found. Please ensure it is installed.") + sys.exit(1) + +# --- SUPPRESS WARNINGS --- +import warnings +warnings.filterwarnings("ignore") +os.environ["TORCH_CPP_LOG_LEVEL"] = "ERROR" +os.environ["TORCH_DISTRIBUTED_DEBUG"] = "OFF" + +# ----------------------------------------------------------- +# Configuration +# ----------------------------------------------------------- +TRAIN_NODES = 2 +GPUS_PER_NODE = 8 +TRAIN_WORLD_SIZE = TRAIN_NODES * GPUS_PER_NODE +INFERENCE_MASTER_RANK = TRAIN_WORLD_SIZE + +# ----------------------------------------------------------- +# Logging Setup +# ----------------------------------------------------------- +def setup_logging(rank: int) -> str: + log_dir = "logs" + os.makedirs(log_dir, exist_ok=True) + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + log_file = os.path.join(log_dir, f"rank_{rank}_{timestamp}.log") + + logger = logging.getLogger() + logger.setLevel(logging.INFO) + + for h in list(logger.handlers): + logger.removeHandler(h) + + fh = logging.FileHandler(log_file) + fh.setLevel(logging.INFO) + + ch = logging.StreamHandler(sys.stdout) + ch.setLevel(logging.INFO) + + formatter = logging.Formatter(fmt="%(asctime)s | Rank %(rank)d | %(levelname)s | %(message)s", datefmt="%H:%M:%S") + fh.setFormatter(formatter) + ch.setFormatter(formatter) + + class RankFilter(logging.Filter): + def filter(self, record): + record.rank = rank + return True + + fh.addFilter(RankFilter()) + ch.addFilter(RankFilter()) + + logger.addHandler(fh) + logger.addHandler(ch) + return log_file + +# ----------------------------------------------------------- +# Dataset +# ----------------------------------------------------------- +def get_wikitext_dataset(tokenizer: AutoTokenizer, seq_len: int = 128): + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + if local_rank != 0: + import datasets + datasets.logging.set_verbosity_error() + + logging.info("Loading WikiText-2 dataset...") + dataset = load_dataset("wikitext", "wikitext-2-raw-v1", split="train") + + def tokenize_function(examples): + return tokenizer(examples["text"]) + + tokenized_datasets = dataset.map(tokenize_function, batched=True, num_proc=4, remove_columns=["text"]) + all_input_ids = list(itertools.chain(*tokenized_datasets["input_ids"])) + data_tensor = torch.tensor(all_input_ids, dtype=torch.long) + + class WikiTextDataset(torch.utils.data.Dataset): + def __init__(self, data, seq_len): + self.data = data + self.seq_len = seq_len + self.num_samples = (len(self.data) - 1) // self.seq_len + def __len__(self): return self.num_samples + def __getitem__(self, idx): + start = idx * self.seq_len + end = start + self.seq_len + return self.data[start : end] + + logging.info(f"Dataset ready. Samples: {(len(data_tensor)-1)//seq_len}") + return WikiTextDataset(data_tensor, seq_len) + +# ----------------------------------------------------------- +# UCCL Collective Broadcast Helper (RDMA) +# ----------------------------------------------------------- +def broadcast_model_uccl(model, rank, is_sender=False): + """ + Uses uccl.collective to send/recv model weights via RDMA. + Rank 0 sends to Rank 16. + """ + SRC_RANK = 0 + DST_RANK = INFERENCE_MASTER_RANK + + # Only participate if sender or receiver + if rank != SRC_RANK and rank != DST_RANK: + return + + try: + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + device = torch.device("cuda", local_rank) + except: + device = torch.device("cuda") + + if is_sender: + logging.info("UCCL: Starting Broadcast (Sender)...") + else: + logging.info("UCCL: Waiting for Broadcast (Receiver)...") + sys.stdout.flush() + + # DDP Handling: Access .module state dict if wrapped + if isinstance(model, DDP): + state_dict = model.module.state_dict() + else: + state_dict = model.state_dict() + + items = list(state_dict.items()) + total_bytes = sum(t.numel() * t.element_size() for _, t in items) + + t0 = time.perf_counter() + + for name, tensor in items: + # 1. Move to GPU if needed (UCCL RDMA requirement) + if not tensor.is_cuda: + tensor_gpu = tensor.cuda(device, non_blocking=True) + else: + tensor_gpu = tensor + + # 2. Ensure contiguous memory layout + if not tensor_gpu.is_contiguous(): + tensor_gpu = tensor_gpu.contiguous() + + # 3. Register (Pin) memory for zero-copy RDMA + collective.register_tensor(tensor_gpu) + + if is_sender: + collective.send(tensor_gpu, dst=DST_RANK) + else: + collective.recv(tensor_gpu, src=SRC_RANK) + + # Copy back logic for receiver + if not tensor.is_cuda: + tensor.copy_(tensor_gpu.cpu()) + elif tensor.data_ptr() != tensor_gpu.data_ptr(): + tensor.copy_(tensor_gpu) + + duration = time.perf_counter() - t0 + bw = (total_bytes / 1e9) / duration if duration > 0 else 0 + + if is_sender: + logging.info(f"UCCL Broadcast Complete. Time: {duration:.4f}s | BW: {bw:.2f} GB/s") + else: + logging.info(f"UCCL Receive Complete. Updated Model. Time: {duration:.4f}s") + sys.stdout.flush() + +# ----------------------------------------------------------- +# Reward Function +# ----------------------------------------------------------- +def compute_reward(token_ids, tokenizer): + # Dynamic "the" finding for Qwen + target_id = tokenizer.encode("the", add_special_tokens=False)[0] + + matches = (token_ids == target_id).float() + rewards = matches.sum(dim=1) + rewards = (rewards * 0.5) - 0.5 + return rewards + +# ----------------------------------------------------------- +# Trainer Loop (DDP) +# ----------------------------------------------------------- +def run_trainer(rank, world_size, train_group): + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + torch.cuda.set_device(local_rank) + device = torch.device("cuda", local_rank) + + logging.info("Initializing DDP Actor Model (Qwen2.5-0.5B)...") + tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-0.5B") + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + + model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen2.5-0.5B").to(device) + + # --- DDP WRAPPER --- + # We pass the NCCL train_group to ensure fast gradient sync + model = DDP(model, device_ids=[local_rank], output_device=local_rank, process_group=train_group) + + dataset = get_wikitext_dataset(tokenizer, seq_len=128) + sampler = DistributedSampler(dataset, num_replicas=dist.get_world_size(group=train_group), rank=dist.get_rank(group=train_group), shuffle=True) + + BATCH_SIZE = 4 + dataloader = DataLoader(dataset, batch_size=BATCH_SIZE, sampler=sampler, num_workers=2) + optimizer = torch.optim.AdamW(model.parameters(), lr=1e-5) + + num_epochs = 2 + + for epoch in range(1, num_epochs + 1): + model.train() + sampler.set_epoch(epoch) + epoch_start = time.time() + total_reward = 0.0 + + for i, batch_ids in enumerate(dataloader): + # Debug limit + if i > 50: break + + batch_ids = batch_ids.to(device) + optimizer.zero_grad() + + outputs = model(batch_ids, labels=batch_ids) + logits = outputs.logits + + shift_logits = logits[..., :-1, :].contiguous() + shift_labels = batch_ids[..., 1:].contiguous() + loss_fct = nn.CrossEntropyLoss(reduction='none') + neg_log_probs = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1)) + neg_log_probs = neg_log_probs.view(shift_labels.size()) + + rewards = compute_reward(batch_ids, tokenizer) + rewards_expanded = rewards.unsqueeze(1).expand_as(neg_log_probs) + rl_loss = (neg_log_probs * rewards_expanded).mean() + + rl_loss.backward() + optimizer.step() + + total_reward += rewards.mean().item() + + if i % 20 == 0 and rank == 0: + logging.info(f"Batch {i}/{len(dataloader)} | RL Loss: {rl_loss.item():.4f} | Avg Reward: {rewards.mean().item():.2f}") + + epoch_duration = time.time() - epoch_start + if rank == 0: + avg_reward = total_reward / (i+1) + logging.info(f"--- RL EPOCH {epoch} STATS ---") + logging.info(f"Duration: {epoch_duration:.2f} seconds") + logging.info(f"Avg Reward: {avg_reward:.4f}") + logging.info(f"-------------------------") + sys.stdout.flush() + + # --- BRIDGE SYNC (UCCL) --- + if rank == 0: + logging.info("Broadcasting weights via UCCL RDMA...") + # DDP wraps model in .module + broadcast_model_uccl(model, rank, is_sender=True) + +# ----------------------------------------------------------- +# Inference Loop +# ----------------------------------------------------------- +def run_inference(rank): + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + torch.cuda.set_device(local_rank) + device = torch.device("cuda", local_rank) + + logging.info("Initializing Inference Model (Qwen2.5-0.5B)...") + tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-0.5B") + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + + model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen2.5-0.5B").to(device) + model.eval() + + num_epochs = 2 + for epoch in range(1, num_epochs + 1): + logging.info(f"Waiting for UCCL model update (Epoch {epoch})...") + sys.stdout.flush() + + # Receive via UCCL Collective + broadcast_model_uccl(model, rank, is_sender=False) + + logging.info("Running inference test...") + test_input = "The AI scientist discovered" + inputs = tokenizer(test_input, return_tensors="pt").to(device) + + with torch.no_grad(): + outputs = model.generate( + **inputs, + max_new_tokens=30, + do_sample=True, + temperature=0.8, + repetition_penalty=1.2 + ) + + generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True) + logging.info(f"--- [INFERENCE RESULT EPOCH {epoch}] ---") + logging.info(f"Output: {generated_text}") + logging.info("---------------------------------------") + sys.stdout.flush() + +# ----------------------------------------------------------- +# Main +# ----------------------------------------------------------- +def main(): + # Sync Interfaces: Copy NCCL to GLOO if needed + if "NCCL_SOCKET_IFNAME" in os.environ and "GLOO_SOCKET_IFNAME" not in os.environ: + os.environ["GLOO_SOCKET_IFNAME"] = os.environ["NCCL_SOCKET_IFNAME"] + + # 1. Init GLOO (Required for UCCL Control Plane) + dist.init_process_group(backend="gloo", init_method="env://", timeout=timedelta(minutes=60)) + rank = dist.get_rank() + log_file = setup_logging(rank) + + if rank == 0: + logging.info(f"NCCL_SOCKET_IFNAME: {os.environ.get('NCCL_SOCKET_IFNAME', 'Not Set')}") + + # 2. Init UCCL (RDMA Data Plane) + logging.info("Initializing UCCL Collective...") + collective.init_collective(num_cpus=4) + + # 3. Setup NCCL Group (Fast Training Plane) + # We must create a separate NCCL group for DDP to avoid using the slow GLOO backend for gradients + train_ranks = list(range(0, TRAIN_WORLD_SIZE)) + train_group = dist.new_group(ranks=train_ranks, backend="nccl") + + if rank in train_ranks: + run_trainer(rank, TRAIN_WORLD_SIZE, train_group) + elif rank == INFERENCE_MASTER_RANK: + run_inference(rank) + else: + logging.info("Idling...") + pass + + logging.info("Waiting for all ranks to complete...") + dist.barrier() + + collective.finalize_collective() + dist.destroy_process_group() + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/ddp_rl_broadcast.py b/ddp_rl_broadcast.py new file mode 100644 index 0000000..6050739 --- /dev/null +++ b/ddp_rl_broadcast.py @@ -0,0 +1,318 @@ +from __future__ import annotations +import os +import sys +import time +import math +import logging +import socket +import functools +import itertools +from datetime import datetime, timedelta +import torch +import torch.nn as nn +import torch.distributed as dist +from torch.utils.data import DataLoader +from torch.utils.data.distributed import DistributedSampler +from transformers import GPT2LMHeadModel, GPT2Tokenizer +from transformers.models.gpt2.modeling_gpt2 import GPT2Block +from datasets import load_dataset + +# --- DDP Import --- +from torch.nn.parallel import DistributedDataParallel as DDP + +# --- SUPPRESS WARNINGS --- +import warnings +warnings.filterwarnings("ignore") +os.environ["TORCH_CPP_LOG_LEVEL"] = "ERROR" +os.environ["TORCH_DISTRIBUTED_DEBUG"] = "OFF" + +# ----------------------------------------------------------- +# Configuration +# ----------------------------------------------------------- +TRAIN_NODES = 2 +GPUS_PER_NODE = 8 +TRAIN_WORLD_SIZE = TRAIN_NODES * GPUS_PER_NODE +INFERENCE_MASTER_RANK = TRAIN_WORLD_SIZE + +# ----------------------------------------------------------- +# Logging Setup (Fixed Flushing) +# ----------------------------------------------------------- +def setup_logging(rank: int) -> str: + log_dir = "logs" + os.makedirs(log_dir, exist_ok=True) + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + log_file = os.path.join(log_dir, f"rank_{rank}_{timestamp}.log") + + logger = logging.getLogger() + logger.setLevel(logging.INFO) + + # Clear existing handlers + for h in list(logger.handlers): + logger.removeHandler(h) + + # File Handler + fh = logging.FileHandler(log_file) + fh.setLevel(logging.INFO) + + # Console Handler with Force Flush + ch = logging.StreamHandler(sys.stdout) + ch.setLevel(logging.INFO) + + # Custom Formatter + formatter = logging.Formatter(fmt="%(asctime)s | Rank %(rank)d | %(levelname)s | %(message)s", datefmt="%H:%M:%S") + fh.setFormatter(formatter) + ch.setFormatter(formatter) + + # Rank Filter + class RankFilter(logging.Filter): + def filter(self, record): + record.rank = rank + return True + + fh.addFilter(RankFilter()) + ch.addFilter(RankFilter()) + + logger.addHandler(fh) + logger.addHandler(ch) + return log_file + +# ----------------------------------------------------------- +# Dataset +# ----------------------------------------------------------- +def get_wikitext_dataset(tokenizer: GPT2Tokenizer, seq_len: int = 128): + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + # Only log on rank 0 to avoid spam, unless error + if local_rank != 0: + import datasets + datasets.logging.set_verbosity_error() + + logging.info("Loading WikiText-2 dataset...") + dataset = load_dataset("wikitext", "wikitext-2-raw-v1", split="train") + + def tokenize_function(examples): + return tokenizer(examples["text"]) + + tokenized_datasets = dataset.map(tokenize_function, batched=True, num_proc=4, remove_columns=["text"]) + all_input_ids = list(itertools.chain(*tokenized_datasets["input_ids"])) + data_tensor = torch.tensor(all_input_ids, dtype=torch.long) + + class WikiTextDataset(torch.utils.data.Dataset): + def __init__(self, data, seq_len): + self.data = data + self.seq_len = seq_len + self.num_samples = (len(self.data) - 1) // self.seq_len + def __len__(self): return self.num_samples + def __getitem__(self, idx): + start = idx * self.seq_len + end = start + self.seq_len + return self.data[start : end] + + logging.info(f"Dataset ready. Samples: {(len(data_tensor)-1)//seq_len}") + return WikiTextDataset(data_tensor, seq_len) + +# ----------------------------------------------------------- +# Broadcast Helper (Standard PyTorch TCP/NCCL) +# ----------------------------------------------------------- +def broadcast_model_to_inference(model, rank, bridge_group, is_sender=False): + """ + Sends model weights using standard torch.distributed.broadcast. + This works over whatever backend the bridge_group uses (likely Gloo or NCCL). + """ + if bridge_group is None: return + + try: + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + device = torch.device("cuda", local_rank) + except: + device = torch.device("cuda") + + if is_sender: + logging.info("Broadcasting weights to Inference Node...") + else: + logging.info("Waiting for weights from Training Node...") + # Force flush to ensure log appears before blocking operation + sys.stdout.flush() + + t0 = time.time() + + # We iterate over parameters to ensure robust transfer + with torch.no_grad(): + for param in model.parameters(): + # Ensure tensor is on GPU for NCCL broadcast (if using NCCL backend) + # Or ensure consistent device for Gloo. + if param.device.type == "cpu": + gpu_param = param.data.to(device) + dist.broadcast(gpu_param, src=0, group=bridge_group) + # If receiver, move back to CPU if model requires it (rare) + else: + dist.broadcast(param.data, src=0, group=bridge_group) + + if is_sender: + duration = time.time() - t0 + logging.info(f"Broadcast complete. Time: {duration:.2f}s") + else: + logging.info("Weights received.") + sys.stdout.flush() + +# ----------------------------------------------------------- +# Reward Function +# ----------------------------------------------------------- +def compute_reward(token_ids, tokenizer): + target_id = 262 # "the" + matches = (token_ids == target_id).float() + rewards = matches.sum(dim=1) + rewards = (rewards * 0.5) - 0.5 + return rewards + +# ----------------------------------------------------------- +# Trainer Loop (DDP) +# ----------------------------------------------------------- +def run_trainer(rank, world_size, train_group, bridge_group): + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + torch.cuda.set_device(local_rank) + device = torch.device("cuda", local_rank) + + tokenizer = GPT2Tokenizer.from_pretrained("gpt2") + tokenizer.pad_token = tokenizer.eos_token + + logging.info("Initializing DDP Actor Model...") + model = GPT2LMHeadModel.from_pretrained("gpt2").to(device) + + # --- DDP WRAPPER --- + model = DDP(model, device_ids=[local_rank], output_device=local_rank, process_group=train_group) + + dataset = get_wikitext_dataset(tokenizer, seq_len=128) + sampler = DistributedSampler(dataset, num_replicas=dist.get_world_size(group=train_group), rank=dist.get_rank(group=train_group), shuffle=True) + + BATCH_SIZE = 4 + dataloader = DataLoader(dataset, batch_size=BATCH_SIZE, sampler=sampler, num_workers=2) + optimizer = torch.optim.AdamW(model.parameters(), lr=1e-5) + + num_epochs = 2 + + for epoch in range(1, num_epochs + 1): + model.train() + sampler.set_epoch(epoch) + epoch_start = time.time() + total_reward = 0.0 + + for i, batch_ids in enumerate(dataloader): + # Debug limit + # if i > 50: break + + batch_ids = batch_ids.to(device) + optimizer.zero_grad() + + outputs = model(batch_ids, labels=batch_ids) + logits = outputs.logits + + shift_logits = logits[..., :-1, :].contiguous() + shift_labels = batch_ids[..., 1:].contiguous() + loss_fct = nn.CrossEntropyLoss(reduction='none') + neg_log_probs = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1)) + neg_log_probs = neg_log_probs.view(shift_labels.size()) + + rewards = compute_reward(batch_ids, tokenizer) + rewards_expanded = rewards.unsqueeze(1).expand_as(neg_log_probs) + rl_loss = (neg_log_probs * rewards_expanded).mean() + + rl_loss.backward() + optimizer.step() + + total_reward += rewards.mean().item() + + if i % 20 == 0 and rank == 0: + logging.info(f"Batch {i}/{len(dataloader)} | RL Loss: {rl_loss.item():.4f} | Avg Reward: {rewards.mean().item():.2f}") + + epoch_duration = time.time() - epoch_start + if rank == 0: + avg_reward = total_reward / (i+1) + logging.info(f"--- RL EPOCH {epoch} STATS ---") + logging.info(f"Duration: {epoch_duration:.2f} seconds") + logging.info(f"Avg Reward: {avg_reward:.4f}") + logging.info(f"-------------------------") + sys.stdout.flush() + + # --- BRIDGE SYNC --- + if rank == 0: + # DDP wraps model in .module + broadcast_model_to_inference(model.module, rank, bridge_group, is_sender=True) + +# ----------------------------------------------------------- +# Inference Loop +# ----------------------------------------------------------- +def run_inference(rank, bridge_group): + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + torch.cuda.set_device(local_rank) + device = torch.device("cuda", local_rank) + + tokenizer = GPT2Tokenizer.from_pretrained("gpt2") + tokenizer.pad_token = tokenizer.eos_token + + logging.info("Initializing Inference Model...") + model = GPT2LMHeadModel.from_pretrained("gpt2").to(device) + model.eval() + + num_epochs = 2 + for epoch in range(1, num_epochs + 1): + logging.info(f"Waiting for RL model update (Epoch {epoch})...") + sys.stdout.flush() + + # This will block until Rank 0 calls broadcast + broadcast_model_to_inference(model, rank, bridge_group, is_sender=False) + + logging.info("Running inference test...") + test_input = "The AI scientist discovered" + inputs = tokenizer(test_input, return_tensors="pt").to(device) + + with torch.no_grad(): + outputs = model.generate( + **inputs, + max_new_tokens=30, + do_sample=True, + temperature=0.8, + repetition_penalty=1.2 + ) + + generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True) + logging.info(f"--- [INFERENCE RESULT EPOCH {epoch}] ---") + logging.info(f"Output: {generated_text}") + logging.info("---------------------------------------") + sys.stdout.flush() + +# ----------------------------------------------------------- +# Main +# ----------------------------------------------------------- +def main(): + # Use NCCL everywhere for simplicity if hardware supports it, + # but GLOO is safer for the bridge group if network config is tricky. + # Here we use NCCL globally for performance. + dist.init_process_group(backend="nccl", init_method="env://", timeout=timedelta(minutes=60)) + rank = dist.get_rank() + log_file = setup_logging(rank) + + if rank == 0: + logging.info(f"NCCL_SOCKET_IFNAME: {os.environ.get('NCCL_SOCKET_IFNAME', 'Not Set')}") + + train_ranks = list(range(0, TRAIN_WORLD_SIZE)) + train_group = dist.new_group(ranks=train_ranks) + + bridge_ranks = [0, INFERENCE_MASTER_RANK] + # Use NCCL for bridge too so we use RDMA/Broadcoms + bridge_group = dist.new_group(ranks=bridge_ranks) + + if rank in train_ranks: + my_bridge = bridge_group if rank == 0 else None + run_trainer(rank, TRAIN_WORLD_SIZE, train_group, my_bridge) + elif rank == INFERENCE_MASTER_RANK: + run_inference(rank, bridge_group) + else: + logging.info("Idling...") + pass + + logging.info("Waiting for all ranks to complete...") + dist.barrier() + dist.destroy_process_group() + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/ddp_rl_uccl.py b/ddp_rl_uccl.py new file mode 100644 index 0000000..09792f1 --- /dev/null +++ b/ddp_rl_uccl.py @@ -0,0 +1,344 @@ +from __future__ import annotations +import os +import sys +import time +import logging +import functools +import itertools +from datetime import datetime, timedelta +import torch +import torch.nn as nn +import torch.distributed as dist +from torch.utils.data import DataLoader +from torch.utils.data.distributed import DistributedSampler +from transformers import GPT2LMHeadModel, GPT2Tokenizer +from transformers.models.gpt2.modeling_gpt2 import GPT2Block +from datasets import load_dataset + +# UCCL Import +try: + from uccl import collective +except ImportError: + print("Error: 'uccl' library not found. Please ensure it is installed.") + sys.exit(1) + +# --- SUPPRESS WARNINGS --- +import warnings +warnings.filterwarnings("ignore") +os.environ["TORCH_CPP_LOG_LEVEL"] = "ERROR" +os.environ["TORCH_DISTRIBUTED_DEBUG"] = "OFF" + +# DDP Import +from torch.nn.parallel import DistributedDataParallel as DDP + +# ----------------------------------------------------------- +# Configuration +# ----------------------------------------------------------- +TRAIN_NODES = 2 +GPUS_PER_NODE = 8 +TRAIN_WORLD_SIZE = TRAIN_NODES * GPUS_PER_NODE +INFERENCE_MASTER_RANK = TRAIN_WORLD_SIZE + +# ----------------------------------------------------------- +# Logging Setup +# ----------------------------------------------------------- +def setup_logging(rank: int) -> str: + log_dir = "logs" + os.makedirs(log_dir, exist_ok=True) + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + log_file = os.path.join(log_dir, f"rank_{rank}_{timestamp}.log") + + logger = logging.getLogger() + logger.setLevel(logging.INFO) + for h in list(logger.handlers): + logger.removeHandler(h) + + fh = logging.FileHandler(log_file) + fh.setLevel(logging.INFO) + ch = logging.StreamHandler(sys.stdout) + ch.setLevel(logging.INFO) + formatter = logging.Formatter(fmt="%(asctime)s | Rank %(rank)d | %(levelname)s | %(message)s", datefmt="%H:%M:%S") + + class RankFilter(logging.Filter): + def filter(self, record): + record.rank = rank + return True + + fh.addFilter(RankFilter()) + ch.addFilter(RankFilter()) + fh.setFormatter(formatter) + ch.setFormatter(formatter) + logger.addHandler(fh) + logger.addHandler(ch) + return log_file + +# ----------------------------------------------------------- +# Dataset +# ----------------------------------------------------------- +def get_wikitext_dataset(tokenizer: GPT2Tokenizer, seq_len: int = 128): + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + if local_rank != 0: + import datasets + datasets.logging.set_verbosity_error() + + logging.info("Loading WikiText-2 dataset...") + dataset = load_dataset("wikitext", "wikitext-2-raw-v1", split="train") + + def tokenize_function(examples): + return tokenizer(examples["text"]) + + tokenized_datasets = dataset.map(tokenize_function, batched=True, num_proc=4, remove_columns=["text"]) + all_input_ids = list(itertools.chain(*tokenized_datasets["input_ids"])) + data_tensor = torch.tensor(all_input_ids, dtype=torch.long) + + class WikiTextDataset(torch.utils.data.Dataset): + def __init__(self, data, seq_len): + self.data = data + self.seq_len = seq_len + self.num_samples = (len(self.data) - 1) // self.seq_len + def __len__(self): return self.num_samples + def __getitem__(self, idx): + start = idx * self.seq_len + end = start + self.seq_len + return self.data[start : end] + + logging.info(f"Dataset ready. Samples: {(len(data_tensor)-1)//seq_len}") + return WikiTextDataset(data_tensor, seq_len) + +# ----------------------------------------------------------- +# UCCL Collective Broadcast Helper +# ----------------------------------------------------------- +def broadcast_model_uccl(model, rank, is_sender=False): + """ + Uses uccl.collective to send/recv model weights. + Rank 0 sends to Rank 16. + """ + SRC_RANK = 0 + DST_RANK = INFERENCE_MASTER_RANK + + if rank != SRC_RANK and rank != DST_RANK: + return + + try: + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + device = torch.device("cuda", local_rank) + except: + device = torch.device("cuda") + + if is_sender: + logging.info("UCCL: Starting Broadcast (Sender)...") + else: + logging.info("UCCL: Waiting for Broadcast (Receiver)...") + + # Access state dict depending on if it's DDP wrapped or standard + if isinstance(model, DDP): + state_dict = model.module.state_dict() + else: + state_dict = model.state_dict() + + items = list(state_dict.items()) + total_bytes = sum(t.numel() * t.element_size() for _, t in items) + + t0 = time.perf_counter() + + for name, tensor in items: + # 1. Move to GPU if needed + if not tensor.is_cuda: + tensor_gpu = tensor.cuda(device, non_blocking=True) + else: + tensor_gpu = tensor + + # 2. Ensure contiguous + if not tensor_gpu.is_contiguous(): + tensor_gpu = tensor_gpu.contiguous() + + # 3. Register (Pin) + collective.register_tensor(tensor_gpu) + + if is_sender: + collective.send(tensor_gpu, dst=DST_RANK) + else: + collective.recv(tensor_gpu, src=SRC_RANK) + + # Copy back logic + if not tensor.is_cuda: + tensor.copy_(tensor_gpu.cpu()) + elif tensor.data_ptr() != tensor_gpu.data_ptr(): + tensor.copy_(tensor_gpu) + + duration = time.perf_counter() - t0 + bw = (total_bytes / 1e9) / duration + + if is_sender: + logging.info(f"UCCL Broadcast Complete. Time: {duration:.3f}s | BW: {bw:.2f} GB/s") + else: + logging.info(f"UCCL Receive Complete. Updated Model.") + +# ----------------------------------------------------------- +# RL Trainer Loop +# ----------------------------------------------------------- +def compute_reward(token_ids, tokenizer): + target_id = 262 # "the" + matches = (token_ids == target_id).float() + rewards = matches.sum(dim=1) + rewards = (rewards * 0.5) - 0.5 + return rewards + +def run_trainer(rank, world_size, train_group): + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + torch.cuda.set_device(local_rank) + device = torch.device("cuda", local_rank) + + tokenizer = GPT2Tokenizer.from_pretrained("gpt2") + tokenizer.pad_token = tokenizer.eos_token + + logging.info("Initializing DDP Actor Model...") + model = GPT2LMHeadModel.from_pretrained("gpt2").to(device) + + # --- DDP WRAP --- + # DDP replicates the model on every GPU. + model = DDP(model, device_ids=[local_rank], output_device=local_rank, process_group=train_group) + + dataset = get_wikitext_dataset(tokenizer, seq_len=128) + sampler = DistributedSampler(dataset, num_replicas=dist.get_world_size(group=train_group), rank=dist.get_rank(group=train_group), shuffle=True) + + BATCH_SIZE = 4 + dataloader = DataLoader(dataset, batch_size=BATCH_SIZE, sampler=sampler, num_workers=2) + optimizer = torch.optim.AdamW(model.parameters(), lr=1e-5) + + num_epochs = 2 + total_training_start = time.time() + + for epoch in range(1, num_epochs + 1): + model.train() + sampler.set_epoch(epoch) + total_reward = 0.0 + epoch_start = time.time() + + for i, batch_ids in enumerate(dataloader): + if i > 40: break + batch_ids = batch_ids.to(device) + optimizer.zero_grad() + + outputs = model(batch_ids, labels=batch_ids) + logits = outputs.logits + + shift_logits = logits[..., :-1, :].contiguous() + shift_labels = batch_ids[..., 1:].contiguous() + loss_fct = nn.CrossEntropyLoss(reduction='none') + neg_log_probs = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1)) + neg_log_probs = neg_log_probs.view(shift_labels.size()) + + rewards = compute_reward(batch_ids, tokenizer) + rewards_expanded = rewards.unsqueeze(1).expand_as(neg_log_probs) + rl_loss = (neg_log_probs * rewards_expanded).mean() + + rl_loss.backward() + optimizer.step() + + total_reward += rewards.mean().item() + + if i % 20 == 0 and rank == 0: + logging.info(f"Batch {i}/{len(dataloader)} | RL Loss: {rl_loss.item():.4f} | Avg Reward: {rewards.mean().item():.2f}") + + epoch_duration = time.time() - epoch_start + total_tokens_processed = len(dataloader) * BATCH_SIZE * 128 * dist.get_world_size(group=train_group) + throughput = total_tokens_processed / epoch_duration + + if rank == 0: + avg_reward = total_reward / len(dataloader) + logging.info(f"--- RL EPOCH {epoch} STATS ---") + logging.info(f"Duration: {epoch_duration:.2f} seconds") + logging.info(f"Throughput: {throughput:.2f} tokens/sec") + logging.info(f"Avg Reward: {avg_reward:.4f}") + logging.info(f"-------------------------") + + # --- BRIDGE SYNC (UCCL Collective) --- + if rank == 0: + logging.info("Preparing weights for UCCL broadcast from DDP Master...") + # In DDP, Rank 0 already has the full model in model.module + # No gathering required! + broadcast_model_uccl(model, rank, is_sender=True) + + total_training_time = time.time() - total_training_start + if rank == 0: + logging.info(f"Total Session Time: {total_training_time:.2f} seconds") + +# ----------------------------------------------------------- +# Inference Loop +# ----------------------------------------------------------- +def run_inference(rank): + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + torch.cuda.set_device(local_rank) + device = torch.device("cuda", local_rank) + + tokenizer = GPT2Tokenizer.from_pretrained("gpt2") + tokenizer.pad_token = tokenizer.eos_token + + logging.info("Initializing Inference Model...") + model = GPT2LMHeadModel.from_pretrained("gpt2").to(device) + model.eval() + + num_epochs = 2 + for epoch in range(1, num_epochs + 1): + logging.info(f"Waiting for UCCL model update (Epoch {epoch})...") + + # Receive via UCCL Collective + broadcast_model_uccl(model, rank, is_sender=False) + + logging.info("Running inference test...") + test_input = "The AI scientist discovered" + inputs = tokenizer(test_input, return_tensors="pt").to(device) + + with torch.no_grad(): + outputs = model.generate( + **inputs, + max_new_tokens=30, + do_sample=True, + temperature=0.8, + repetition_penalty=1.2 + ) + + generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True) + logging.info(f"--- [INFERENCE RESULT EPOCH {epoch}] ---") + logging.info(f"Output: {generated_text}") + +# ----------------------------------------------------------- +# Main +# ----------------------------------------------------------- +def main(): + if "NCCL_SOCKET_IFNAME" in os.environ and "GLOO_SOCKET_IFNAME" not in os.environ: + os.environ["GLOO_SOCKET_IFNAME"] = os.environ["NCCL_SOCKET_IFNAME"] + + # 1. Init GLOO for UCCL + dist.init_process_group(backend="gloo", init_method="env://", timeout=timedelta(minutes=60)) + rank = dist.get_rank() + log_file = setup_logging(rank) + + if rank == 0: + logging.info(f"NCCL_SOCKET_IFNAME: {os.environ.get('NCCL_SOCKET_IFNAME', 'Not Set')}") + + # 2. Init UCCL + logging.info("Initializing UCCL Collective...") + collective.init_collective(num_cpus=4) + + # 3. Setup NCCL Group for DDP + train_ranks = list(range(0, TRAIN_WORLD_SIZE)) + train_group = dist.new_group(ranks=train_ranks, backend="nccl") + + if rank in train_ranks: + run_trainer(rank, TRAIN_WORLD_SIZE, train_group) + elif rank == INFERENCE_MASTER_RANK: + run_inference(rank) + else: + logging.info("Idling...") + pass + + logging.info("Waiting for all ranks to complete...") + dist.barrier() + + collective.finalize_collective() + dist.destroy_process_group() + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/fsdp-qwen-broadcast.py b/fsdp-qwen-broadcast.py new file mode 100644 index 0000000..90bf269 --- /dev/null +++ b/fsdp-qwen-broadcast.py @@ -0,0 +1,350 @@ +from __future__ import annotations +import os +import sys +import time +import math +import logging +import socket +import functools +import itertools +from datetime import datetime, timedelta +import torch +import torch.nn as nn +import torch.distributed as dist +from torch.utils.data import DataLoader +from torch.utils.data.distributed import DistributedSampler +from transformers import AutoModelForCausalLM, AutoTokenizer +from datasets import load_dataset + +# --- FSDP Imports --- +from torch.distributed.fsdp import ( + FullyShardedDataParallel as FSDP, + StateDictType, + FullStateDictConfig, +) +from torch.distributed.fsdp.wrap import ( + transformer_auto_wrap_policy, +) + +# Import Qwen Block for wrapping policy +try: + from transformers.models.qwen2.modeling_qwen2 import Qwen2DecoderLayer +except ImportError: + # Fallback if transformers version is old, though Qwen2 usually requires new version + Qwen2DecoderLayer = None + +# --- SUPPRESS WARNINGS --- +import warnings +warnings.filterwarnings("ignore") +os.environ["TORCH_CPP_LOG_LEVEL"] = "ERROR" +os.environ["TORCH_DISTRIBUTED_DEBUG"] = "OFF" + +# ----------------------------------------------------------- +# Configuration +# ----------------------------------------------------------- +TRAIN_NODES = 2 +GPUS_PER_NODE = 8 +TRAIN_WORLD_SIZE = TRAIN_NODES * GPUS_PER_NODE +INFERENCE_MASTER_RANK = TRAIN_WORLD_SIZE + +# ----------------------------------------------------------- +# Logging Setup +# ----------------------------------------------------------- +def setup_logging(rank: int) -> str: + log_dir = "logs" + os.makedirs(log_dir, exist_ok=True) + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + log_file = os.path.join(log_dir, f"rank_{rank}_{timestamp}.log") + + logger = logging.getLogger() + logger.setLevel(logging.INFO) + + for h in list(logger.handlers): + logger.removeHandler(h) + + fh = logging.FileHandler(log_file) + fh.setLevel(logging.INFO) + + ch = logging.StreamHandler(sys.stdout) + ch.setLevel(logging.INFO) + + formatter = logging.Formatter(fmt="%(asctime)s | Rank %(rank)d | %(levelname)s | %(message)s", datefmt="%H:%M:%S") + fh.setFormatter(formatter) + ch.setFormatter(formatter) + + class RankFilter(logging.Filter): + def filter(self, record): + record.rank = rank + return True + + fh.addFilter(RankFilter()) + ch.addFilter(RankFilter()) + + logger.addHandler(fh) + logger.addHandler(ch) + return log_file + +# ----------------------------------------------------------- +# Dataset +# ----------------------------------------------------------- +def get_wikitext_dataset(tokenizer: AutoTokenizer, seq_len: int = 128): + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + if local_rank != 0: + import datasets + datasets.logging.set_verbosity_error() + + logging.info("Loading WikiText-2 dataset...") + dataset = load_dataset("wikitext", "wikitext-2-raw-v1", split="train") + + def tokenize_function(examples): + return tokenizer(examples["text"]) + + tokenized_datasets = dataset.map(tokenize_function, batched=True, num_proc=4, remove_columns=["text"]) + all_input_ids = list(itertools.chain(*tokenized_datasets["input_ids"])) + data_tensor = torch.tensor(all_input_ids, dtype=torch.long) + + class WikiTextDataset(torch.utils.data.Dataset): + def __init__(self, data, seq_len): + self.data = data + self.seq_len = seq_len + self.num_samples = (len(self.data) - 1) // self.seq_len + def __len__(self): return self.num_samples + def __getitem__(self, idx): + start = idx * self.seq_len + end = start + self.seq_len + return self.data[start : end] + + logging.info(f"Dataset ready. Samples: {(len(data_tensor)-1)//seq_len}") + return WikiTextDataset(data_tensor, seq_len) + +# ----------------------------------------------------------- +# Broadcast Helper (Standard PyTorch TCP/NCCL) +# ----------------------------------------------------------- +def broadcast_model_to_inference(model, rank, bridge_group, is_sender=False): + """ + Sends model weights using standard torch.distributed.broadcast. + """ + if bridge_group is None: return + + try: + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + device = torch.device("cuda", local_rank) + except: + device = torch.device("cuda") + + if is_sender: + logging.info("Broadcasting weights to Inference Node...") + else: + logging.info("Waiting for weights from Training Node...") + sys.stdout.flush() + + t0 = time.time() + + # Iterate parameters. If model is FSDP wrapped, this usually iterates flattened params + # BUT we pass a CPU copy of the full model to this function from the Trainer, so it's a standard model. + # The Receiver also passes a standard model. + with torch.no_grad(): + for param in model.parameters(): + # If tensor is on CPU (sender gather result), move to GPU for NCCL broadcast + if param.device.type == "cpu": + gpu_param = param.data.to(device) + dist.broadcast(gpu_param, src=0, group=bridge_group) + else: + # Already on GPU (Receiver) + dist.broadcast(param.data, src=0, group=bridge_group) + + if is_sender: + duration = time.time() - t0 + # Estimate size + total_bytes = sum(p.numel() * p.element_size() for p in model.parameters()) + bw = (total_bytes / 1e9) / duration if duration > 0 else 0 + logging.info(f"Broadcast complete. Time: {duration:.4f}s | BW: {bw:.2f} GB/s") + else: + logging.info("Weights received.") + sys.stdout.flush() + +# ----------------------------------------------------------- +# Reward Function +# ----------------------------------------------------------- +def compute_reward(token_ids, tokenizer): + target_id = tokenizer.encode("the", add_special_tokens=False)[0] + matches = (token_ids == target_id).float() + rewards = matches.sum(dim=1) + rewards = (rewards * 0.5) - 0.5 + return rewards + +# ----------------------------------------------------------- +# Trainer Loop (FSDP) +# ----------------------------------------------------------- +def run_trainer(rank, world_size, train_group, bridge_group): + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + torch.cuda.set_device(local_rank) + device = torch.device("cuda", local_rank) + + logging.info("Initializing FSDP Actor Model (Qwen2.5-0.5B)...") + tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-0.5B") + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + + model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen2.5-0.5B").to(device) + + # --- FSDP WRAPPER --- + if Qwen2DecoderLayer is not None: + auto_wrap_policy = functools.partial( + transformer_auto_wrap_policy, + transformer_layer_cls={Qwen2DecoderLayer}, + ) + else: + auto_wrap_policy = None + logging.warning("Qwen2DecoderLayer not found, using default FSDP wrapping.") + + model = FSDP( + model, + auto_wrap_policy=auto_wrap_policy, + process_group=train_group, + device_id=torch.cuda.current_device() + ) + + dataset = get_wikitext_dataset(tokenizer, seq_len=128) + sampler = DistributedSampler(dataset, num_replicas=dist.get_world_size(group=train_group), rank=dist.get_rank(group=train_group), shuffle=True) + + BATCH_SIZE = 4 + dataloader = DataLoader(dataset, batch_size=BATCH_SIZE, sampler=sampler, num_workers=2) + optimizer = torch.optim.AdamW(model.parameters(), lr=1e-5) + + num_epochs = 2 + + for epoch in range(1, num_epochs + 1): + model.train() + sampler.set_epoch(epoch) + epoch_start = time.time() + total_reward = 0.0 + + for i, batch_ids in enumerate(dataloader): + if i > 50: break + + batch_ids = batch_ids.to(device) + optimizer.zero_grad() + + outputs = model(batch_ids, labels=batch_ids) + logits = outputs.logits + + shift_logits = logits[..., :-1, :].contiguous() + shift_labels = batch_ids[..., 1:].contiguous() + loss_fct = nn.CrossEntropyLoss(reduction='none') + neg_log_probs = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1)) + neg_log_probs = neg_log_probs.view(shift_labels.size()) + + rewards = compute_reward(batch_ids, tokenizer) + rewards_expanded = rewards.unsqueeze(1).expand_as(neg_log_probs) + rl_loss = (neg_log_probs * rewards_expanded).mean() + + rl_loss.backward() + optimizer.step() + + total_reward += rewards.mean().item() + + if i % 20 == 0 and rank == 0: + logging.info(f"Batch {i}/{len(dataloader)} | RL Loss: {rl_loss.item():.4f} | Avg Reward: {rewards.mean().item():.2f}") + + epoch_duration = time.time() - epoch_start + if rank == 0: + avg_reward = total_reward / (i+1) + logging.info(f"--- RL EPOCH {epoch} STATS ---") + logging.info(f"Duration: {epoch_duration:.2f} seconds") + logging.info(f"Avg Reward: {avg_reward:.4f}") + logging.info(f"-------------------------") + sys.stdout.flush() + + # --- BRIDGE SYNC (FSDP Gathering) --- + # FSDP shards parameters. We must gather them to Rank 0 before broadcasting. + save_policy = FullStateDictConfig(offload_to_cpu=True, rank0_only=True) + with FSDP.state_dict_type(model, StateDictType.FULL_STATE_DICT, save_policy): + full_state = model.state_dict() + + if rank == 0: + logging.info("Gathering complete. Broadcasting...") + # Load into a temporary standard model to simplify broadcasting logic + cpu_model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen2.5-0.5B") + cpu_model.load_state_dict(full_state) + + broadcast_model_to_inference(cpu_model, rank, bridge_group, is_sender=True) + del cpu_model + +# ----------------------------------------------------------- +# Inference Loop +# ----------------------------------------------------------- +def run_inference(rank, bridge_group): + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + torch.cuda.set_device(local_rank) + device = torch.device("cuda", local_rank) + + tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-0.5B") + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + + logging.info("Initializing Inference Model...") + model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen2.5-0.5B").to(device) + model.eval() + + num_epochs = 2 + for epoch in range(1, num_epochs + 1): + logging.info(f"Waiting for RL model update (Epoch {epoch})...") + sys.stdout.flush() + + # Receive broadcast (Blocks until Training sends) + broadcast_model_to_inference(model, rank, bridge_group, is_sender=False) + + logging.info("Running inference test...") + test_input = "The AI scientist discovered" + inputs = tokenizer(test_input, return_tensors="pt").to(device) + + with torch.no_grad(): + outputs = model.generate( + **inputs, + max_new_tokens=30, + do_sample=True, + temperature=0.8, + repetition_penalty=1.2 + ) + + generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True) + clean_text = generated_text.replace("\n", "\\n") + + logging.info(f"--- [INFERENCE RESULT EPOCH {epoch}] ---") + logging.info(f"Output: {clean_text}") + logging.info("---------------------------------------") + sys.stdout.flush() + +# ----------------------------------------------------------- +# Main +# ----------------------------------------------------------- +def main(): + dist.init_process_group(backend="nccl", init_method="env://", timeout=timedelta(minutes=60)) + rank = dist.get_rank() + log_file = setup_logging(rank) + + if rank == 0: + logging.info(f"NCCL_SOCKET_IFNAME: {os.environ.get('NCCL_SOCKET_IFNAME', 'Not Set')}") + + train_ranks = list(range(0, TRAIN_WORLD_SIZE)) + train_group = dist.new_group(ranks=train_ranks, backend="nccl") + + bridge_ranks = [0, INFERENCE_MASTER_RANK] + bridge_group = dist.new_group(ranks=bridge_ranks) + + if rank in train_ranks: + my_bridge = bridge_group if rank == 0 else None + run_trainer(rank, TRAIN_WORLD_SIZE, train_group, my_bridge) + elif rank == INFERENCE_MASTER_RANK: + run_inference(rank, bridge_group) + else: + logging.info("Idling...") + pass + + logging.info("Waiting for all ranks to complete...") + dist.barrier() + dist.destroy_process_group() + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/fsdp-qwen-uccl.py b/fsdp-qwen-uccl.py new file mode 100644 index 0000000..fe44252 --- /dev/null +++ b/fsdp-qwen-uccl.py @@ -0,0 +1,394 @@ +from __future__ import annotations +import os +import sys +import time +import math +import logging +import socket +import functools +import itertools +from datetime import datetime, timedelta +import torch +import torch.nn as nn +import torch.distributed as dist +from torch.utils.data import DataLoader +from torch.utils.data.distributed import DistributedSampler +from transformers import AutoModelForCausalLM, AutoTokenizer +from datasets import load_dataset + +# --- FSDP Imports --- +from torch.distributed.fsdp import ( + FullyShardedDataParallel as FSDP, + StateDictType, + FullStateDictConfig, +) +from torch.distributed.fsdp.wrap import ( + transformer_auto_wrap_policy, +) + +# UCCL Import +try: + from uccl import collective +except ImportError: + print("Error: 'uccl' library not found. Please ensure it is installed.") + sys.exit(1) + +# Import Qwen Block for wrapping policy +try: + from transformers.models.qwen2.modeling_qwen2 import Qwen2DecoderLayer +except ImportError: + # Fallback if transformers version is old, though Qwen2 usually requires new version + Qwen2DecoderLayer = None + +# --- SUPPRESS WARNINGS --- +import warnings +warnings.filterwarnings("ignore") +os.environ["TORCH_CPP_LOG_LEVEL"] = "ERROR" +os.environ["TORCH_DISTRIBUTED_DEBUG"] = "OFF" + +# ----------------------------------------------------------- +# Configuration +# ----------------------------------------------------------- +TRAIN_NODES = 2 +GPUS_PER_NODE = 8 +TRAIN_WORLD_SIZE = TRAIN_NODES * GPUS_PER_NODE +INFERENCE_MASTER_RANK = TRAIN_WORLD_SIZE + +# ----------------------------------------------------------- +# Logging Setup +# ----------------------------------------------------------- +def setup_logging(rank: int) -> str: + log_dir = "logs" + os.makedirs(log_dir, exist_ok=True) + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + log_file = os.path.join(log_dir, f"rank_{rank}_{timestamp}.log") + + logger = logging.getLogger() + logger.setLevel(logging.INFO) + + for h in list(logger.handlers): + logger.removeHandler(h) + + fh = logging.FileHandler(log_file) + fh.setLevel(logging.INFO) + + ch = logging.StreamHandler(sys.stdout) + ch.setLevel(logging.INFO) + + formatter = logging.Formatter(fmt="%(asctime)s | Rank %(rank)d | %(levelname)s | %(message)s", datefmt="%H:%M:%S") + fh.setFormatter(formatter) + ch.setFormatter(formatter) + + class RankFilter(logging.Filter): + def filter(self, record): + record.rank = rank + return True + + fh.addFilter(RankFilter()) + ch.addFilter(RankFilter()) + + logger.addHandler(fh) + logger.addHandler(ch) + return log_file + +# ----------------------------------------------------------- +# Dataset +# ----------------------------------------------------------- +def get_wikitext_dataset(tokenizer: AutoTokenizer, seq_len: int = 128): + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + if local_rank != 0: + import datasets + datasets.logging.set_verbosity_error() + + logging.info("Loading WikiText-2 dataset...") + dataset = load_dataset("wikitext", "wikitext-2-raw-v1", split="train") + + def tokenize_function(examples): + return tokenizer(examples["text"]) + + tokenized_datasets = dataset.map(tokenize_function, batched=True, num_proc=4, remove_columns=["text"]) + all_input_ids = list(itertools.chain(*tokenized_datasets["input_ids"])) + data_tensor = torch.tensor(all_input_ids, dtype=torch.long) + + class WikiTextDataset(torch.utils.data.Dataset): + def __init__(self, data, seq_len): + self.data = data + self.seq_len = seq_len + self.num_samples = (len(self.data) - 1) // self.seq_len + def __len__(self): return self.num_samples + def __getitem__(self, idx): + start = idx * self.seq_len + end = start + self.seq_len + return self.data[start : end] + + logging.info(f"Dataset ready. Samples: {(len(data_tensor)-1)//seq_len}") + return WikiTextDataset(data_tensor, seq_len) + +# ----------------------------------------------------------- +# Broadcast Helper (UCCL RDMA) +# ----------------------------------------------------------- +def broadcast_model_to_inference(model, rank, bridge_group, is_sender=False): + """ + Sends model weights using UCCL Collective (RDMA). + Rank 0 sends to Rank 16. + """ + SRC_RANK = 0 + DST_RANK = INFERENCE_MASTER_RANK + + # Only participate if sender or receiver + if rank != SRC_RANK and rank != DST_RANK: + return + + try: + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + device = torch.device("cuda", local_rank) + except: + device = torch.device("cuda") + + if is_sender: + logging.info("UCCL: Starting Broadcast (Sender)...") + else: + logging.info("UCCL: Waiting for Broadcast (Receiver)...") + sys.stdout.flush() + + # If sender (Rank 0), 'model' is likely a CPU copy from FSDP gather + # If receiver (Rank 16), 'model' is a standard GPU model + state_dict = model.state_dict() + items = list(state_dict.items()) + + t0 = time.time() + + for name, tensor in items: + # 1. Move to GPU if needed (UCCL RDMA requirement) + if not tensor.is_cuda: + tensor_gpu = tensor.cuda(device, non_blocking=True) + else: + tensor_gpu = tensor + + # 2. Ensure contiguous memory layout + if not tensor_gpu.is_contiguous(): + tensor_gpu = tensor_gpu.contiguous() + + # 3. Register (Pin) memory for zero-copy RDMA + collective.register_tensor(tensor_gpu) + + if is_sender: + collective.send(tensor_gpu, dst=DST_RANK) + else: + collective.recv(tensor_gpu, src=SRC_RANK) + + # Copy back logic for receiver if needed + # (Usually not needed if model was already on GPU, but safe to check) + if not tensor.is_cuda: + tensor.copy_(tensor_gpu.cpu()) + elif tensor.data_ptr() != tensor_gpu.data_ptr(): + tensor.copy_(tensor_gpu) + + duration = time.time() - t0 + # Estimate size + total_bytes = sum(p.numel() * p.element_size() for p in model.parameters()) + bw = (total_bytes / 1e9) / duration if duration > 0 else 0 + + if is_sender: + logging.info(f"UCCL Broadcast complete. Time: {duration:.4f}s | BW: {bw:.2f} GB/s") + else: + logging.info(f"UCCL Receive complete. Time: {duration:.4f}s") + sys.stdout.flush() + +# ----------------------------------------------------------- +# Reward Function +# ----------------------------------------------------------- +def compute_reward(token_ids, tokenizer): + target_id = tokenizer.encode("the", add_special_tokens=False)[0] + matches = (token_ids == target_id).float() + rewards = matches.sum(dim=1) + rewards = (rewards * 0.5) - 0.5 + return rewards + +# ----------------------------------------------------------- +# Trainer Loop (FSDP) +# ----------------------------------------------------------- +def run_trainer(rank, world_size, train_group, bridge_group): + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + torch.cuda.set_device(local_rank) + device = torch.device("cuda", local_rank) + + logging.info("Initializing FSDP Actor Model (Qwen2.5-0.5B)...") + tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-0.5B") + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + + model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen2.5-0.5B").to(device) + + # --- FSDP WRAPPER --- + if Qwen2DecoderLayer is not None: + auto_wrap_policy = functools.partial( + transformer_auto_wrap_policy, + transformer_layer_cls={Qwen2DecoderLayer}, + ) + else: + auto_wrap_policy = None + logging.warning("Qwen2DecoderLayer not found, using default FSDP wrapping.") + + # Pass the NCCL train_group to FSDP for training comms + model = FSDP( + model, + auto_wrap_policy=auto_wrap_policy, + process_group=train_group, + device_id=torch.cuda.current_device() + ) + + dataset = get_wikitext_dataset(tokenizer, seq_len=128) + sampler = DistributedSampler(dataset, num_replicas=dist.get_world_size(group=train_group), rank=dist.get_rank(group=train_group), shuffle=True) + + BATCH_SIZE = 4 + dataloader = DataLoader(dataset, batch_size=BATCH_SIZE, sampler=sampler, num_workers=2) + optimizer = torch.optim.AdamW(model.parameters(), lr=1e-5) + + num_epochs = 2 + + for epoch in range(1, num_epochs + 1): + model.train() + sampler.set_epoch(epoch) + epoch_start = time.time() + total_reward = 0.0 + + for i, batch_ids in enumerate(dataloader): + if i > 50: break + + batch_ids = batch_ids.to(device) + optimizer.zero_grad() + + outputs = model(batch_ids, labels=batch_ids) + logits = outputs.logits + + shift_logits = logits[..., :-1, :].contiguous() + shift_labels = batch_ids[..., 1:].contiguous() + loss_fct = nn.CrossEntropyLoss(reduction='none') + neg_log_probs = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1)) + neg_log_probs = neg_log_probs.view(shift_labels.size()) + + rewards = compute_reward(batch_ids, tokenizer) + rewards_expanded = rewards.unsqueeze(1).expand_as(neg_log_probs) + rl_loss = (neg_log_probs * rewards_expanded).mean() + + rl_loss.backward() + optimizer.step() + + total_reward += rewards.mean().item() + + if i % 20 == 0 and rank == 0: + logging.info(f"Batch {i}/{len(dataloader)} | RL Loss: {rl_loss.item():.4f} | Avg Reward: {rewards.mean().item():.2f}") + + epoch_duration = time.time() - epoch_start + if rank == 0: + avg_reward = total_reward / (i+1) + logging.info(f"--- RL EPOCH {epoch} STATS ---") + logging.info(f"Duration: {epoch_duration:.2f} seconds") + logging.info(f"Avg Reward: {avg_reward:.4f}") + logging.info(f"-------------------------") + sys.stdout.flush() + + # --- BRIDGE SYNC (FSDP Gathering) --- + save_policy = FullStateDictConfig(offload_to_cpu=True, rank0_only=True) + with FSDP.state_dict_type(model, StateDictType.FULL_STATE_DICT, save_policy): + full_state = model.state_dict() + + if rank == 0: + logging.info("Gathering complete. Broadcasting via UCCL...") + # Load into a temporary standard model for broadcasting + cpu_model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen2.5-0.5B") + cpu_model.load_state_dict(full_state) + + broadcast_model_to_inference(cpu_model, rank, bridge_group, is_sender=True) + del cpu_model + +# ----------------------------------------------------------- +# Inference Loop +# ----------------------------------------------------------- +def run_inference(rank, bridge_group): + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + torch.cuda.set_device(local_rank) + device = torch.device("cuda", local_rank) + + tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-0.5B") + if tokenizer.pad_token is None: + tokenizer.pad_token = tokenizer.eos_token + + logging.info("Initializing Inference Model...") + model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen2.5-0.5B").to(device) + model.eval() + + num_epochs = 2 + for epoch in range(1, num_epochs + 1): + logging.info(f"Waiting for RL model update (Epoch {epoch})...") + sys.stdout.flush() + + # Receive broadcast (Blocks until Training sends) + broadcast_model_to_inference(model, rank, bridge_group, is_sender=False) + + logging.info("Running inference test...") + test_input = "The AI scientist discovered" + inputs = tokenizer(test_input, return_tensors="pt").to(device) + + with torch.no_grad(): + outputs = model.generate( + **inputs, + max_new_tokens=30, + do_sample=True, + temperature=0.8, + repetition_penalty=1.2 + ) + + generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True) + clean_text = generated_text.replace("\n", "\\n") + + logging.info(f"--- [INFERENCE RESULT EPOCH {epoch}] ---") + logging.info(f"Output: {clean_text}") + logging.info("---------------------------------------") + sys.stdout.flush() + +# ----------------------------------------------------------- +# Main +# ----------------------------------------------------------- +def main(): + # Sync Interfaces: Copy NCCL to GLOO if needed + if "NCCL_SOCKET_IFNAME" in os.environ and "GLOO_SOCKET_IFNAME" not in os.environ: + os.environ["GLOO_SOCKET_IFNAME"] = os.environ["NCCL_SOCKET_IFNAME"] + + # 1. Init GLOO (Required for UCCL Control Plane) + dist.init_process_group(backend="gloo", init_method="env://", timeout=timedelta(minutes=60)) + rank = dist.get_rank() + log_file = setup_logging(rank) + + if rank == 0: + logging.info(f"NCCL_SOCKET_IFNAME: {os.environ.get('NCCL_SOCKET_IFNAME', 'Not Set')}") + + # 2. Init UCCL (Data Plane) + logging.info("Initializing UCCL Collective...") + collective.init_collective(num_cpus=4) + + # 3. Setup NCCL Group (Training Plane) for FSDP performance + train_ranks = list(range(0, TRAIN_WORLD_SIZE)) + train_group = dist.new_group(ranks=train_ranks, backend="nccl") + + bridge_ranks = [0, INFERENCE_MASTER_RANK] + # Bridge group for logical grouping, but actual transfer uses UCCL + bridge_group = dist.new_group(ranks=bridge_ranks, backend="gloo") + + if rank in train_ranks: + run_trainer(rank, TRAIN_WORLD_SIZE, train_group, bridge_group) + elif rank == INFERENCE_MASTER_RANK: + run_inference(rank, bridge_group) + else: + logging.info("Idling...") + pass + + logging.info("Waiting for all ranks to complete...") + dist.barrier() + + collective.finalize_collective() + dist.destroy_process_group() + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/fsdp_rl_broadcast.py b/fsdp_rl_broadcast.py new file mode 100644 index 0000000..25c3df6 --- /dev/null +++ b/fsdp_rl_broadcast.py @@ -0,0 +1,331 @@ +from __future__ import annotations +import os +import sys +import time +import math +import logging +import socket +import functools +import itertools +from datetime import datetime, timedelta +import torch +import torch.nn as nn +import torch.distributed as dist +from torch.utils.data import DataLoader +from torch.utils.data.distributed import DistributedSampler +from transformers import GPT2LMHeadModel, GPT2Tokenizer +from transformers.models.gpt2.modeling_gpt2 import GPT2Block +from datasets import load_dataset + +# FSDP Imports +from torch.distributed.fsdp import ( + FullyShardedDataParallel as FSDP, + StateDictType, + FullStateDictConfig, +) +from torch.distributed.fsdp.wrap import ( + transformer_auto_wrap_policy, +) + +# ----------------------------------------------------------- +# Configuration +# ----------------------------------------------------------- +TRAIN_NODES = 2 +GPUS_PER_NODE = 8 +TRAIN_WORLD_SIZE = TRAIN_NODES * GPUS_PER_NODE +INFERENCE_MASTER_RANK = TRAIN_WORLD_SIZE + +# ----------------------------------------------------------- +# Logging Setup +# ----------------------------------------------------------- +def setup_logging(rank: int) -> str: + log_dir = "logs" + os.makedirs(log_dir, exist_ok=True) + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + log_file = os.path.join(log_dir, f"rank_{rank}_{timestamp}.log") + + logger = logging.getLogger() + logger.setLevel(logging.INFO) + for h in list(logger.handlers): + logger.removeHandler(h) + + fh = logging.FileHandler(log_file) + fh.setLevel(logging.INFO) + ch = logging.StreamHandler(sys.stdout) + ch.setLevel(logging.INFO) + formatter = logging.Formatter(fmt="%(asctime)s | Rank %(rank)d | %(levelname)s | %(message)s", datefmt="%H:%M:%S") + + class RankFilter(logging.Filter): + def filter(self, record): + record.rank = rank + return True + + fh.addFilter(RankFilter()) + ch.addFilter(RankFilter()) + fh.setFormatter(formatter) + ch.setFormatter(formatter) + logger.addHandler(fh) + logger.addHandler(ch) + return log_file + +# ----------------------------------------------------------- +# Dataset (WikiText-2 used as Prompts) +# ----------------------------------------------------------- +def get_wikitext_dataset(tokenizer: GPT2Tokenizer, seq_len: int = 128): + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + if local_rank != 0: + import datasets + datasets.logging.set_verbosity_error() + + logging.info("Loading WikiText-2 dataset...") + dataset = load_dataset("wikitext", "wikitext-2-raw-v1", split="train") + + def tokenize_function(examples): + return tokenizer(examples["text"]) + + tokenized_datasets = dataset.map(tokenize_function, batched=True, num_proc=4, remove_columns=["text"]) + all_input_ids = list(itertools.chain(*tokenized_datasets["input_ids"])) + data_tensor = torch.tensor(all_input_ids, dtype=torch.long) + + class WikiTextDataset(torch.utils.data.Dataset): + def __init__(self, data, seq_len): + self.data = data + self.seq_len = seq_len + self.num_samples = (len(self.data) - 1) // self.seq_len + def __len__(self): return self.num_samples + def __getitem__(self, idx): + start = idx * self.seq_len + end = start + self.seq_len + # In RL, we treat 'x' as the trajectory + return self.data[start : end] + + logging.info(f"Dataset ready. Samples: {(len(data_tensor)-1)//seq_len}") + return WikiTextDataset(data_tensor, seq_len) + +# ----------------------------------------------------------- +# Broadcast Helper +# ----------------------------------------------------------- +def broadcast_model_to_inference(model, rank, bridge_group, is_sender=False): + if bridge_group is None: return + + try: + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + device = torch.device("cuda", local_rank) + except: device = torch.device("cuda") + + if is_sender: logging.info("Broadcasting RL weights to Inference Node...") + + t0 = time.time() + with torch.no_grad(): + for param in model.parameters(): + if param.device.type == "cpu": + gpu_param = param.data.to(device) + dist.broadcast(gpu_param, src=0, group=bridge_group) + else: + dist.broadcast(param.data, src=0, group=bridge_group) + + if is_sender: + duration = time.time() - t0 + logging.info(f"Broadcast complete. Time taken: {duration:.2f}s") + else: + logging.info("RL Weights received from Training Cluster.") + +# ----------------------------------------------------------- +# Simple Reward Function +# ----------------------------------------------------------- +def compute_reward(token_ids, tokenizer): + """ + Simulated Reward Function. + Goal: Encourage the usage of the word "the". + Reward = +1.0 for every 'the' in the sequence. + Reward = -0.1 baseline penalty to encourage density. + """ + # Token ID for "the" (with space prefix usually) in GPT2 is 262 + target_id = 262 + + # Create reward tensor matching batch size + # token_ids shape: [Batch, Seq] + matches = (token_ids == target_id).float() + + # Sum matches per row to get reward per sample + rewards = matches.sum(dim=1) + + # Normalize rewards slightly to prevent explosion + rewards = (rewards * 0.5) - 0.5 + + return rewards + +# ----------------------------------------------------------- +# RL Trainer Loop (Policy Gradient) +# ----------------------------------------------------------- +def run_trainer(rank, world_size, train_group, bridge_group): + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + torch.cuda.set_device(local_rank) + device = torch.device("cuda", local_rank) + + tokenizer = GPT2Tokenizer.from_pretrained("gpt2") + tokenizer.pad_token = tokenizer.eos_token + + logging.info("Initializing FSDP Actor Model...") + model = GPT2LMHeadModel.from_pretrained("gpt2").to(device) + gpt2_auto_wrap_policy = functools.partial(transformer_auto_wrap_policy, transformer_layer_cls={GPT2Block}) + + model = FSDP(model, auto_wrap_policy=gpt2_auto_wrap_policy, process_group=train_group, device_id=torch.cuda.current_device()) + + dataset = get_wikitext_dataset(tokenizer, seq_len=128) + sampler = DistributedSampler(dataset, num_replicas=dist.get_world_size(group=train_group), rank=dist.get_rank(group=train_group), shuffle=True) + + BATCH_SIZE = 4 + dataloader = DataLoader(dataset, batch_size=BATCH_SIZE, sampler=sampler, num_workers=2) + optimizer = torch.optim.AdamW(model.parameters(), lr=1e-5) # Lower LR for RL stability + + num_epochs = 2 + total_training_start = time.time() + + for epoch in range(1, num_epochs + 1): + model.train() + sampler.set_epoch(epoch) + total_reward = 0.0 + + epoch_start = time.time() + + for i, batch_ids in enumerate(dataloader): + batch_ids = batch_ids.to(device) + optimizer.zero_grad() + + # 1. Forward Pass (Get Logits) + outputs = model(batch_ids, labels=batch_ids) + logits = outputs.logits + + # 2. Calculate Log Probs of the sequence + # Shift logits and labels for next-token prediction alignment + shift_logits = logits[..., :-1, :].contiguous() + shift_labels = batch_ids[..., 1:].contiguous() + + # CrossEntropy is -log_prob. + # We want log_prob, so we take negative CrossEntropy (without reduction first) + loss_fct = nn.CrossEntropyLoss(reduction='none') + # Shape: [Batch, Seq_len] + neg_log_probs = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1)) + neg_log_probs = neg_log_probs.view(shift_labels.size()) + + # log_probs = -neg_log_probs + + # 3. Calculate Reward (The RL Part) + # We evaluate the batch_ids themselves as the "trajectory" + rewards = compute_reward(batch_ids, tokenizer) # Shape: [Batch] + + # 4. Policy Gradient Loss + # Loss = - (Log_Prob * Reward) + # We need to broadcast reward to sequence length + rewards_expanded = rewards.unsqueeze(1).expand_as(neg_log_probs) + + # Since neg_log_probs is POSITIVE (it's loss), minimizing (neg_log_probs * reward) + # is equivalent to Maximizing (log_probs * reward) + # If reward is +5, we want to minimize loss (make log_probs higher). + # If reward is -5, we want to maximize loss (make log_probs lower). + rl_loss = (neg_log_probs * rewards_expanded).mean() + + rl_loss.backward() + optimizer.step() + + total_reward += rewards.mean().item() + + if i % 20 == 0 and rank == 0: + logging.info(f"Batch {i}/{len(dataloader)} | RL Loss: {rl_loss.item():.4f} | Avg Reward: {rewards.mean().item():.2f}") + + epoch_duration = time.time() - epoch_start + total_tokens_processed = len(dataloader) * BATCH_SIZE * 128 * dist.get_world_size(group=train_group) + throughput = total_tokens_processed / epoch_duration + + if rank == 0: + avg_reward = total_reward / len(dataloader) + logging.info(f"--- RL EPOCH {epoch} STATS ---") + logging.info(f"Duration: {epoch_duration:.2f} seconds") + logging.info(f"Throughput: {throughput:.2f} tokens/sec") + logging.info(f"Avg Reward: {avg_reward:.4f}") + logging.info(f"-------------------------") + + # --- BRIDGE SYNC --- + save_policy = FullStateDictConfig(offload_to_cpu=True, rank0_only=True) + with FSDP.state_dict_type(model, StateDictType.FULL_STATE_DICT, save_policy): + full_state = model.state_dict() + + if rank == 0: + cpu_model = GPT2LMHeadModel.from_pretrained("gpt2") + cpu_model.load_state_dict(full_state) + broadcast_model_to_inference(cpu_model, rank, bridge_group, is_sender=True) + del cpu_model + + total_training_time = time.time() - total_training_start + if rank == 0: + logging.info(f"Total Session Time: {total_training_time:.2f} seconds") + +# ----------------------------------------------------------- +# Inference Loop (With Better Sampling) +# ----------------------------------------------------------- +def run_inference(rank, bridge_group): + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + torch.cuda.set_device(local_rank) + device = torch.device("cuda", local_rank) + + tokenizer = GPT2Tokenizer.from_pretrained("gpt2") + tokenizer.pad_token = tokenizer.eos_token + + logging.info("Initializing Inference Model...") + model = GPT2LMHeadModel.from_pretrained("gpt2").to(device) + model.eval() + + num_epochs = 2 + for epoch in range(1, num_epochs + 1): + logging.info(f"Waiting for RL model update (Epoch {epoch})...") + broadcast_model_to_inference(model, rank, bridge_group, is_sender=False) + + logging.info("Running inference test...") + test_input = "The AI scientist discovered" + inputs = tokenizer(test_input, return_tensors="pt").to(device) + + # IMPROVED GENERATION PARAMS + with torch.no_grad(): + outputs = model.generate( + **inputs, + max_new_tokens=30, + do_sample=True, # Enable Sampling to fix loops + temperature=0.8, # Creativity + repetition_penalty=1.2 # Stop "of of of of" + ) + + generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True) + logging.info(f"--- [INFERENCE RESULT EPOCH {epoch}] ---") + logging.info(f"Output: {generated_text}") + +# ----------------------------------------------------------- +# Main +# ----------------------------------------------------------- +def main(): + dist.init_process_group(backend="nccl", init_method="env://", timeout=timedelta(minutes=60)) + rank = dist.get_rank() + log_file = setup_logging(rank) + + if rank == 0: + logging.info(f"NCCL_SOCKET_IFNAME: {os.environ.get('NCCL_SOCKET_IFNAME', 'Not Set')}") + + train_ranks = list(range(0, TRAIN_WORLD_SIZE)) + train_group = dist.new_group(ranks=train_ranks) + bridge_ranks = [0, INFERENCE_MASTER_RANK] + bridge_group = dist.new_group(ranks=bridge_ranks) + + if rank in train_ranks: + my_bridge = bridge_group if rank == 0 else None + run_trainer(rank, TRAIN_WORLD_SIZE, train_group, my_bridge) + elif rank == INFERENCE_MASTER_RANK: + run_inference(rank, bridge_group) + else: + pass + + logging.info("Waiting for all ranks to complete...") + dist.barrier() + dist.destroy_process_group() + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/fsdp_rl_uccl.py b/fsdp_rl_uccl.py new file mode 100644 index 0000000..1a457bd --- /dev/null +++ b/fsdp_rl_uccl.py @@ -0,0 +1,376 @@ +from __future__ import annotations +import os +import sys +import time +import math +import logging +import socket +import functools +import itertools +from datetime import datetime, timedelta +import torch +import torch.nn as nn +import torch.distributed as dist +from torch.utils.data import DataLoader +from torch.utils.data.distributed import DistributedSampler +from transformers import GPT2LMHeadModel, GPT2Tokenizer +from transformers.models.gpt2.modeling_gpt2 import GPT2Block +from datasets import load_dataset + +# UCCL Import +try: + from uccl import collective +except ImportError: + print("Error: 'uccl' library not found. Please ensure it is installed.") + sys.exit(1) + +# --- SUPPRESS WARNINGS --- +import warnings +warnings.filterwarnings("ignore") +os.environ["TORCH_CPP_LOG_LEVEL"] = "ERROR" +os.environ["TORCH_DISTRIBUTED_DEBUG"] = "OFF" + +# FSDP Imports +from torch.distributed.fsdp import ( + FullyShardedDataParallel as FSDP, + StateDictType, + FullStateDictConfig, +) +from torch.distributed.fsdp.wrap import ( + transformer_auto_wrap_policy, +) + +# ----------------------------------------------------------- +# Configuration +# ----------------------------------------------------------- +TRAIN_NODES = 2 +GPUS_PER_NODE = 8 +TRAIN_WORLD_SIZE = TRAIN_NODES * GPUS_PER_NODE +INFERENCE_MASTER_RANK = TRAIN_WORLD_SIZE + +# ----------------------------------------------------------- +# Logging Setup +# ----------------------------------------------------------- +def setup_logging(rank: int) -> str: + log_dir = "logs" + os.makedirs(log_dir, exist_ok=True) + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + log_file = os.path.join(log_dir, f"rank_{rank}_{timestamp}.log") + + logger = logging.getLogger() + logger.setLevel(logging.INFO) + for h in list(logger.handlers): + logger.removeHandler(h) + + fh = logging.FileHandler(log_file) + fh.setLevel(logging.INFO) + # Force flush to prevent missing logs + ch = logging.StreamHandler(sys.stdout) + ch.setLevel(logging.INFO) + formatter = logging.Formatter(fmt="%(asctime)s | Rank %(rank)d | %(levelname)s | %(message)s", datefmt="%H:%M:%S") + + class RankFilter(logging.Filter): + def filter(self, record): + record.rank = rank + return True + + fh.addFilter(RankFilter()) + ch.addFilter(RankFilter()) + fh.setFormatter(formatter) + ch.setFormatter(formatter) + logger.addHandler(fh) + logger.addHandler(ch) + return log_file + +# ----------------------------------------------------------- +# Dataset +# ----------------------------------------------------------- +def get_wikitext_dataset(tokenizer: GPT2Tokenizer, seq_len: int = 128): + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + if local_rank != 0: + import datasets + datasets.logging.set_verbosity_error() + + logging.info("Loading WikiText-2 dataset...") + dataset = load_dataset("wikitext", "wikitext-2-raw-v1", split="train") + + def tokenize_function(examples): + return tokenizer(examples["text"]) + + tokenized_datasets = dataset.map(tokenize_function, batched=True, num_proc=4, remove_columns=["text"]) + all_input_ids = list(itertools.chain(*tokenized_datasets["input_ids"])) + data_tensor = torch.tensor(all_input_ids, dtype=torch.long) + + class WikiTextDataset(torch.utils.data.Dataset): + def __init__(self, data, seq_len): + self.data = data + self.seq_len = seq_len + self.num_samples = (len(self.data) - 1) // self.seq_len + def __len__(self): return self.num_samples + def __getitem__(self, idx): + start = idx * self.seq_len + end = start + self.seq_len + return self.data[start : end] + + logging.info(f"Dataset ready. Samples: {(len(data_tensor)-1)//seq_len}") + return WikiTextDataset(data_tensor, seq_len) + +# ----------------------------------------------------------- +# UCCL Collective Broadcast Helper +# ----------------------------------------------------------- +def broadcast_model_uccl(model, rank, is_sender=False): + """ + Uses uccl.collective to send/recv model weights. + Rank 0 sends to Rank 16. + """ + # Define Source and Destination + SRC_RANK = 0 + DST_RANK = INFERENCE_MASTER_RANK + + # We only care if we are the sender or the receiver + if rank != SRC_RANK and rank != DST_RANK: + return + + # Determine Device for RDMA (must be GPU) + try: + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + device = torch.device("cuda", local_rank) + except: + device = torch.device("cuda") + + if is_sender: + logging.info("UCCL: Starting Broadcast (Sender)...") + else: + logging.info("UCCL: Waiting for Broadcast (Receiver)...") + + state_dict = model.state_dict() + items = list(state_dict.items()) + total_bytes = sum(t.numel() * t.element_size() for _, t in items) + + t0 = time.perf_counter() + + for name, tensor in items: + # RDMA requires contiguous GPU memory + # 1. Move to GPU if needed + if not tensor.is_cuda: + tensor_gpu = tensor.cuda(device, non_blocking=True) + else: + tensor_gpu = tensor + + # 2. Ensure contiguous + if not tensor_gpu.is_contiguous(): + tensor_gpu = tensor_gpu.contiguous() + + # 3. Register (Pin) memory for max performance + # Note: In a loop like this, registration adds overhead. + # Ideally, we register once, but FSDP creates new tensors per epoch. + collective.register_tensor(tensor_gpu) + + if is_sender: + # Send to Inference Node + collective.send(tensor_gpu, dst=DST_RANK) + else: + # Receive from Training Node + # We must recv into the GPU tensor, then copy back if the model expects CPU (rare for inference) + collective.recv(tensor_gpu, src=SRC_RANK) + + # If the original model was on CPU (unlikely for inference), copy back + if not tensor.is_cuda: + tensor.copy_(tensor_gpu.cpu()) + elif tensor.data_ptr() != tensor_gpu.data_ptr(): + # If we created a new contiguous buffer, copy back to model param + tensor.copy_(tensor_gpu) + + duration = time.perf_counter() - t0 + bw = (total_bytes / 1e9) / duration + + if is_sender: + logging.info(f"UCCL Broadcast Complete. Time: {duration:.3f}s | BW: {bw:.2f} GB/s") + else: + logging.info(f"UCCL Receive Complete. Updated Model.") + +# ----------------------------------------------------------- +# RL Trainer Loop +# ----------------------------------------------------------- +def compute_reward(token_ids, tokenizer): + target_id = 262 # "the" + matches = (token_ids == target_id).float() + rewards = matches.sum(dim=1) + rewards = (rewards * 0.5) - 0.5 + return rewards + +def run_trainer(rank, world_size, train_group): + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + torch.cuda.set_device(local_rank) + device = torch.device("cuda", local_rank) + + tokenizer = GPT2Tokenizer.from_pretrained("gpt2") + tokenizer.pad_token = tokenizer.eos_token + + logging.info("Initializing FSDP Actor Model...") + model = GPT2LMHeadModel.from_pretrained("gpt2").to(device) + gpt2_auto_wrap_policy = functools.partial(transformer_auto_wrap_policy, transformer_layer_cls={GPT2Block}) + + model = FSDP(model, auto_wrap_policy=gpt2_auto_wrap_policy, process_group=train_group, device_id=torch.cuda.current_device()) + + dataset = get_wikitext_dataset(tokenizer, seq_len=128) + sampler = DistributedSampler(dataset, num_replicas=dist.get_world_size(group=train_group), rank=dist.get_rank(group=train_group), shuffle=True) + + BATCH_SIZE = 4 + dataloader = DataLoader(dataset, batch_size=BATCH_SIZE, sampler=sampler, num_workers=2) + optimizer = torch.optim.AdamW(model.parameters(), lr=1e-5) + + num_epochs = 2 + total_training_start = time.time() + + for epoch in range(1, num_epochs + 1): + model.train() + sampler.set_epoch(epoch) + total_reward = 0.0 + epoch_start = time.time() + + for i, batch_ids in enumerate(dataloader): + if i > 40: break + batch_ids = batch_ids.to(device) + optimizer.zero_grad() + + outputs = model(batch_ids, labels=batch_ids) + logits = outputs.logits + + shift_logits = logits[..., :-1, :].contiguous() + shift_labels = batch_ids[..., 1:].contiguous() + loss_fct = nn.CrossEntropyLoss(reduction='none') + neg_log_probs = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1)) + neg_log_probs = neg_log_probs.view(shift_labels.size()) + + rewards = compute_reward(batch_ids, tokenizer) + rewards_expanded = rewards.unsqueeze(1).expand_as(neg_log_probs) + rl_loss = (neg_log_probs * rewards_expanded).mean() + + rl_loss.backward() + optimizer.step() + + total_reward += rewards.mean().item() + + if i % 20 == 0 and rank == 0: + logging.info(f"Batch {i}/{len(dataloader)} | RL Loss: {rl_loss.item():.4f} | Avg Reward: {rewards.mean().item():.2f}") + + epoch_duration = time.time() - epoch_start + total_tokens_processed = len(dataloader) * BATCH_SIZE * 128 * dist.get_world_size(group=train_group) + throughput = total_tokens_processed / epoch_duration + + if rank == 0: + avg_reward = total_reward / len(dataloader) + logging.info(f"--- RL EPOCH {epoch} STATS ---") + logging.info(f"Duration: {epoch_duration:.2f} seconds") + logging.info(f"Throughput: {throughput:.2f} tokens/sec") + logging.info(f"Avg Reward: {avg_reward:.4f}") + logging.info(f"-------------------------") + + # --- BRIDGE SYNC (UCCL Collective) --- + # 1. Gather full model to CPU on Rank 0 + save_policy = FullStateDictConfig(offload_to_cpu=True, rank0_only=True) + with FSDP.state_dict_type(model, StateDictType.FULL_STATE_DICT, save_policy): + full_state = model.state_dict() + + # 2. Broadcast via UCCL + if rank == 0: + logging.info("Preparing weights for UCCL broadcast...") + cpu_model = GPT2LMHeadModel.from_pretrained("gpt2") + cpu_model.load_state_dict(full_state) + + broadcast_model_uccl(cpu_model, rank, is_sender=True) + del cpu_model + + total_training_time = time.time() - total_training_start + if rank == 0: + logging.info(f"Total Session Time: {total_training_time:.2f} seconds") + +# ----------------------------------------------------------- +# Inference Loop +# ----------------------------------------------------------- +def run_inference(rank): + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + torch.cuda.set_device(local_rank) + device = torch.device("cuda", local_rank) + + tokenizer = GPT2Tokenizer.from_pretrained("gpt2") + tokenizer.pad_token = tokenizer.eos_token + + logging.info("Initializing Inference Model...") + model = GPT2LMHeadModel.from_pretrained("gpt2").to(device) + model.eval() + + num_epochs = 2 + for epoch in range(1, num_epochs + 1): + logging.info(f"Waiting for UCCL model update (Epoch {epoch})...") + + # Receive via UCCL Collective + broadcast_model_uccl(model, rank, is_sender=False) + + logging.info("Running inference test...") + test_input = "The AI scientist discovered" + inputs = tokenizer(test_input, return_tensors="pt").to(device) + + with torch.no_grad(): + outputs = model.generate( + **inputs, + max_new_tokens=30, + do_sample=True, + temperature=0.8, + repetition_penalty=1.2 + ) + + generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True) + logging.info(f"--- [INFERENCE RESULT EPOCH {epoch}] ---") + logging.info(f"Output: {generated_text}") + +# ----------------------------------------------------------- +# Main +# ----------------------------------------------------------- +def main(): + # Setup GLOO interface to match NCCL if set (crucial for multi-NIC nodes) + # UCCL needs GLOO to work, and GLOO needs to know which interface to use. + if "NCCL_SOCKET_IFNAME" in os.environ and "GLOO_SOCKET_IFNAME" not in os.environ: + os.environ["GLOO_SOCKET_IFNAME"] = os.environ["NCCL_SOCKET_IFNAME"] + + # 1. Init Standard Distributed (GLOO for UCCL Control Plane) + # Changed from 'nccl' to 'gloo' because UCCL CollectiveContext requires it. + dist.init_process_group(backend="gloo", init_method="env://", timeout=timedelta(minutes=60)) + rank = dist.get_rank() + world_size = dist.get_world_size() + log_file = setup_logging(rank) + + if rank == 0: + logging.info(f"NCCL_SOCKET_IFNAME: {os.environ.get('NCCL_SOCKET_IFNAME', 'Not Set')}") + logging.info(f"GLOO_SOCKET_IFNAME: {os.environ.get('GLOO_SOCKET_IFNAME', 'Not Set')}") + + # 2. Init UCCL Collective (This wires up RDMA connections automatically) + # This replaces the manual p2p handshake + logging.info("Initializing UCCL Collective...") + collective.init_collective(num_cpus=4) + + # 3. Setup Process Groups for Training (Explicit NCCL for GPU speed) + # FSDP requires NCCL for performance, so we create a new group explicitly using NCCL backend + # instead of inheriting the default GLOO backend. + train_ranks = list(range(0, TRAIN_WORLD_SIZE)) + train_group = dist.new_group(ranks=train_ranks, backend="nccl") + + # 4. Branch Logic + if rank in train_ranks: + run_trainer(rank, TRAIN_WORLD_SIZE, train_group) + elif rank == INFERENCE_MASTER_RANK: + run_inference(rank) + else: + logging.info("Idling...") + pass + + logging.info("Waiting for all ranks to complete...") + dist.barrier() + + # Cleanup + collective.finalize_collective() + dist.destroy_process_group() + +if __name__ == "__main__": + main() diff --git a/junk/fsdp.py b/junk/fsdp.py new file mode 100644 index 0000000..e045a2e --- /dev/null +++ b/junk/fsdp.py @@ -0,0 +1,484 @@ +from __future__ import annotations +import torch, time, os, sys +import torch.distributed as dist +import logging +from datetime import datetime +import math + +from transformers import GPT2LMHeadModel, GPT2Tokenizer +from datasets import load_dataset +from torch.utils.data import DataLoader +from uccl import p2p +import torch.nn as nn +from torch.distributed.fsdp import FullyShardedDataParallel as FSDP +from torch.distributed.fsdp.wrap import transformer_auto_wrap_policy +from transformers.models.gpt2.modeling_gpt2 import GPT2Block +from torch.utils.data.distributed import DistributedSampler + +# --------------------------- +# Logging +# --------------------------- +def setup_logging(rank): + """Setup logging with timestamps and rank info""" + log_dir = "logs" + os.makedirs(log_dir, exist_ok=True) + + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + log_file = f"{log_dir}/rank_{rank}_{timestamp}.log" + + logging.basicConfig( + level=logging.INFO, + format='%(asctime)s | Rank %(rank)d | %(levelname)s | %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + handlers=[ + logging.FileHandler(log_file), + logging.StreamHandler(sys.stdout) + ] + ) + + # Add rank to all log records + old_factory = logging.getLogRecordFactory() + def record_factory(*args, **kwargs): + record = old_factory(*args, **kwargs) + record.rank = rank + return record + logging.setLogRecordFactory(record_factory) + + logging.info(f"Logging initialized. Log file: {log_file}") + return log_file + + +# --------------------------- +# RDMA send/recv helpers +# --------------------------- +def broadcast_model(ep, conn_ids, model, rank): + """Send model to multiple receivers with detailed logging""" + state_dict = model.state_dict() + items = list(state_dict.items()) + total_tensors = len(items) + total_size_mb = sum(t.numel() * t.element_size() for _, t in items) / 1e6 + + logging.info("="*80) + logging.info(f"BROADCAST START - Sending to {len(conn_ids)} receivers") + logging.info(f"Total tensors: {total_tensors}") + logging.info(f"Total size: {total_size_mb:.2f} MB") + logging.info("="*80) + + broadcast_start = time.perf_counter() + + # Keep references to temporary GPU tensors to prevent GC before send completes + temp_tensors = [] + + for idx, (name, tensor) in enumerate(items, 1): + # ensure tensor on GPU for RDMA + if not tensor.is_cuda: + tensor = tensor.cuda() + temp_tensors.append(tensor) # Keep reference + + # Ensure contiguous memory for RDMA + if not tensor.is_contiguous(): + tensor = tensor.contiguous() + temp_tensors.append(tensor) + + size_bytes = tensor.numel() * tensor.element_size() + ptr = tensor.data_ptr() + + # Register memory + ok, mr_id = ep.reg(ptr, size_bytes) + assert ok, f"Failed to register tensor {name}" + + # Send to all receivers + for receiver_idx, conn_id in enumerate(conn_ids, 1): + ok = ep.send(conn_id, mr_id, ptr, size_bytes) + assert ok, f"Send failed for {name} to receiver {receiver_idx}" + + if idx % 20 == 0 or idx == total_tensors: + progress_pct = (idx / total_tensors) * 100 + logging.info(f"Progress: {progress_pct:.1f}% ({idx}/{total_tensors})") + + total_time = time.perf_counter() - broadcast_start + avg_bandwidth = (total_size_mb / 1000) / total_time if total_time > 0 else 0 + + logging.info("="*80) + logging.info(f"BROADCAST COMPLETE") + logging.info(f"Total time: {total_time:.2f}s") + logging.info(f"Average bandwidth: {avg_bandwidth:.2f} GB/s") + logging.info("="*80) + + +def recv_model(ep, conn_id, model, rank): + """Receive model from broadcaster with detailed logging""" + state_dict = model.state_dict() + items = list(state_dict.items()) + total_tensors = len(items) + total_size_mb = sum(t.numel() * t.element_size() for _, t in items) / 1e6 + + logging.info("="*80) + logging.info(f"RECEIVE START") + logging.info(f"Total tensors: {total_tensors}") + logging.info(f"Total size: {total_size_mb:.2f} MB") + logging.info("="*80) + + recv_start = time.perf_counter() + + for idx, (name, tensor) in enumerate(items, 1): + # allocate recv tensor on GPU + recv_tensor = torch.empty_like(tensor, device="cuda") + if not recv_tensor.is_contiguous(): + recv_tensor = recv_tensor.contiguous() + + size_bytes = recv_tensor.numel() * recv_tensor.element_size() + ptr = recv_tensor.data_ptr() + + # Register memory + ok, mr_id = ep.reg(ptr, size_bytes) + assert ok, f"Failed to register tensor {name}" + + # Receive tensor + ok = ep.recv(conn_id, mr_id, ptr, size_bytes) + assert ok, f"Receive failed for {name}" + + # Copy into model parameter / buffer + with torch.no_grad(): + if name in model.state_dict(): + model.state_dict()[name].copy_(recv_tensor) + else: + logging.warning(f"Key {name} not found in local model, skipping.") + + if idx % 20 == 0 or idx == total_tensors: + progress_pct = (idx / total_tensors) * 100 + logging.info(f"Progress: {progress_pct:.1f}% ({idx}/{total_tensors})") + + total_time = time.perf_counter() - recv_start + avg_bandwidth = (total_size_mb / 1000) / total_time if total_time > 0 else 0 + + logging.info("="*80) + logging.info(f"RECEIVE COMPLETE") + logging.info(f"Total time: {total_time:.2f}s") + logging.info(f"Average bandwidth: {avg_bandwidth:.2f} GB/s") + logging.info("="*80) + + +# --------------------------- +# Dataset preparation +# --------------------------- +def prepare_dataset(tokenizer, max_length=128, num_samples=200): + """Load and prepare WikiText-2 dataset""" + logging.info("Loading WikiText-2 dataset from Hugging Face...") + + # Disable HF caching logs for cleaner output + logging.getLogger("datasets").setLevel(logging.ERROR) + + dataset = load_dataset("wikitext", "wikitext-2-raw-v1", split="train") + dataset = dataset.filter(lambda x: len(x["text"].strip()) > 0) + + if len(dataset) > num_samples: + dataset = dataset.select(range(num_samples)) + + def tokenize_function(examples): + return tokenizer( + examples["text"], + truncation=True, + padding="max_length", + max_length=max_length + ) + + tokenized_dataset = dataset.map( + tokenize_function, + batched=True, + remove_columns=dataset.column_names + ) + + tokenized_dataset.set_format(type='torch', columns=['input_ids', 'attention_mask']) + return tokenized_dataset + + +# --------------------------- +# Training (rank 1..N) +# --------------------------- +def run_training(model, tokenizer, train_group, trainer_ranks, num_epochs=2, batch_size=4, lr=5e-5): + """Training loop for trainer ranks""" + logging.info("="*80) + logging.info("TRAINING NODE - Starting training on WikiText-2") + logging.info(f"Model class: {model.__class__.__name__}") + logging.info("="*80) + + train_dataset = prepare_dataset(tokenizer, max_length=128, num_samples=200) + + # FIX: Calculate local rank relative to the trainer group, NOT global world size + # If we use global world size, trainers will skip data indices meant for Rank 0 and Rank N + global_rank = dist.get_rank() + rank_in_group = trainer_ranks.index(global_rank) + num_trainers = len(trainer_ranks) + + sampler = DistributedSampler( + train_dataset, + num_replicas=num_trainers, + rank=rank_in_group, + shuffle=True + ) + + train_dataloader = DataLoader( + train_dataset, + batch_size=batch_size, + sampler=sampler + ) + + total_steps = len(train_dataloader) * num_epochs + + # Ensure model is in train mode + model.train() + optimizer = torch.optim.AdamW(model.parameters(), lr=lr) + scaler = torch.cuda.amp.GradScaler(enabled=True) + + total_train_start = time.perf_counter() + global_step = 0 + epoch_losses = [] + + for epoch in range(num_epochs): + epoch_start = time.perf_counter() + epoch_loss = 0.0 + + logging.info(f"EPOCH {epoch + 1}/{num_epochs}") + sampler.set_epoch(epoch) + + for batch_idx, batch in enumerate(train_dataloader): + step_start = time.perf_counter() + global_step += 1 + + input_ids = batch["input_ids"].cuda(non_blocking=True) + attention_mask = batch["attention_mask"].cuda(non_blocking=True) + + optimizer.zero_grad() + with torch.cuda.amp.autocast(): + outputs = model( + input_ids=input_ids, + attention_mask=attention_mask, + labels=input_ids + ) + loss = outputs.loss + + # Handle FSDP loss which might be sharded or 1D + loss_scalar = loss.mean() + + scaler.scale(loss_scalar).backward() + scaler.step(optimizer) + scaler.update() + + loss_value = loss_scalar.item() + epoch_loss += loss_value + + if (batch_idx + 1) % 10 == 0: + perplexity = math.exp(loss_value) if loss_value < 100 else float('inf') + logging.info( + f"Step {global_step}/{total_steps} | " + f"Loss: {loss_value:.4f} | PPL: {perplexity:.2f}" + ) + + avg_epoch_loss = epoch_loss / max(1, len(train_dataloader)) + epoch_losses.append(avg_epoch_loss) + logging.info(f"End of Epoch {epoch+1} - Avg Loss: {avg_epoch_loss:.4f}") + + # --------------------------- + # FSDP Saving Mechanism + # --------------------------- + # Only one rank per FSDP group (usually rank 0 of the group) should coordinate the save, + # but we need to set barrier so all finish. + dist.barrier(group=train_group) + + # For simplicity in this script, we are saving the local shard (state_dict) + # To save full model, we need FullStateDictConfig. + # Here we just save the state_dict as requested, but handle the FSDP access carefully. + try: + if isinstance(model, FSDP): + # To get the full state dict, we need a context manager + # Note: This gathers weights to CPU on rank 0 of the group + with FSDP.state_dict_type(model, torch.distributed.fsdp.StateDictType.FULL_STATE_DICT): + sd = model.state_dict() + else: + sd = model.state_dict() + + # Only the first trainer saves the checkpoint to avoid file corruption + if rank_in_group == 0: + checkpoint_dir = "checkpoints" + os.makedirs(checkpoint_dir, exist_ok=True) + checkpoint_path = f"{checkpoint_dir}/wikitext2_fsdp_model.pt" + + torch.save({ + 'model_state_dict': sd, + 'optimizer_state_dict': optimizer.state_dict(), + 'epoch_losses': epoch_losses, + }, checkpoint_path) + logging.info(f"Model checkpoint saved to: {checkpoint_path}") + + except Exception as e: + logging.error(f"Error saving checkpoint: {e}") + + logging.info("Training Complete.") + + +# --------------------------- +# Inference (rank last) +# --------------------------- +def run_inference(model, tokenizer, num_samples=5): + logging.info("="*80) + logging.info("INFERENCE NODE - Starting inference") + logging.info("="*80) + + model.eval() + prompts = ["The future of AI is"] + + total_tokens = 0 + start_time = time.perf_counter() + + with torch.no_grad(): + for i in range(num_samples): + prompt = prompts[i % len(prompts)] + inputs = tokenizer(prompt, return_tensors="pt") + input_ids = inputs["input_ids"].cuda() + + output_ids = model.generate( + input_ids, + max_length=50, + do_sample=True, + pad_token_id=tokenizer.eos_token_id + ) + + generated_text = tokenizer.decode(output_ids[0], skip_special_tokens=True) + logging.info(f"Sample {i+1}: {generated_text}") + + logging.info(f"Inference done in {time.perf_counter() - start_time:.2f}s") + + +# --------------------------- +# Main (all ranks) +# --------------------------- +def main(): + # init from environment (torchrun) + dist.init_process_group(backend="nccl") + rank = dist.get_rank() + world_size = dist.get_world_size() + + log_file = setup_logging(rank) + + # FIX: Ensure we have enough ranks + assert world_size >= 3, "Need at least 3 ranks: 1 Broadcaster, 1+ Trainers, 1 Inference" + + # FIX: Setup CUDA device correctly before any P2P / Torch ops + if torch.cuda.is_available(): + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + torch.cuda.set_device(local_rank) + local_gpu = torch.cuda.current_device() + logging.info(f"CUDA device set to GPU {local_gpu}") + else: + logging.error("CUDA required") + sys.exit(1) + + # --------------------------- + # Define Process Groups + # --------------------------- + # We have 3 roles: + # Rank 0: Broadcaster + # Rank 1 to N-2: Trainers + # Rank N-1: Inference + + trainer_ranks = list(range(1, world_size - 1)) + # Create a specific process group for trainers. + # FSDP MUST run on this group, otherwise it waits for Rank 0/N-1 (deadlock). + logging.info(f"Initializing Trainer Process Group for ranks: {trainer_ranks}") + trainer_group = dist.new_group(ranks=trainer_ranks) + + # P2P Setup + ep = p2p.Endpoint(local_gpu, 4) + local_md = ep.get_metadata() + + # Exchange metadata + all_metadata = [None] * world_size + dist.all_gather_object(all_metadata, local_md) + + # --------------------------- + # Logic Branching + # --------------------------- + if rank == 0: + # Broadcaster + logging.info("BROADCASTER MODE") + conn_ids = [] + + receiver_ranks = [r for r in range(1, world_size)] + for receiver_rank in receiver_ranks: + ip, port, r_gpu = p2p.Endpoint.parse_metadata(all_metadata[receiver_rank]) + ok, conn_id = ep.connect(ip, r_gpu, remote_port=port) + if ok: + conn_ids.append((receiver_rank, conn_id)) + + model = GPT2LMHeadModel.from_pretrained("gpt2").cuda() + # Pass list of conn_ids only + broadcast_model(ep, [c for _, c in conn_ids], model, rank) + + logging.info("Broadcast complete. Waiting for others to finish...") + # Broadcaster waits for everyone before destroying group + dist.barrier() + + elif rank in trainer_ranks: + # Trainer(s) + logging.info(f"TRAINING NODE (Rank {rank})") + + # Accept connection from broadcaster + ok, r_ip, r_gpu, conn_id = ep.accept() + assert ok, "Accept failed" + + base_model = GPT2LMHeadModel.from_pretrained("gpt2").cuda() + tokenizer = GPT2Tokenizer.from_pretrained("gpt2") + tokenizer.pad_token = tokenizer.eos_token + + # Receive weights via RDMA + recv_model(ep, conn_id, base_model, rank) + + # FIX: Initialize FSDP with the specific trainer_group + # If we don't pass process_group, it uses global group -> deadlock + auto_wrap_policy = transformer_auto_wrap_policy({GPT2Block}) + + model = FSDP( + base_model, + auto_wrap_policy=auto_wrap_policy, + process_group=trainer_group, # CRITICAL FIX + device_id=local_gpu + ) + + run_training( + model, tokenizer, + train_group=trainer_group, + trainer_ranks=trainer_ranks + ) + + logging.info("Training node finished.") + dist.barrier() + + elif rank == world_size - 1: + # Inference node + logging.info("INFERENCE NODE") + + ok, r_ip, r_gpu, conn_id = ep.accept() + assert ok, "Accept failed" + + model = GPT2LMHeadModel.from_pretrained("gpt2").cuda() + tokenizer = GPT2Tokenizer.from_pretrained("gpt2") + tokenizer.pad_token = tokenizer.eos_token + + recv_model(ep, conn_id, model, rank) + run_inference(model, tokenizer, num_samples=5) + + logging.info("Inference node finished.") + dist.barrier() + + dist.destroy_process_group() + logging.info("Process complete.") + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + sys.exit(0) + except Exception as e: + logging.error(f"Fatal error: {e}", exc_info=True) + sys.exit(1) \ No newline at end of file diff --git a/junk/fsdp_training.py b/junk/fsdp_training.py new file mode 100644 index 0000000..ffc5880 --- /dev/null +++ b/junk/fsdp_training.py @@ -0,0 +1,338 @@ +from __future__ import annotations +import os +import sys +import time +import math +import logging +import socket +import functools +import itertools +from datetime import datetime +import torch +import torch.nn as nn +import torch.distributed as dist +from torch.utils.data import DataLoader +from torch.utils.data.distributed import DistributedSampler +from transformers import GPT2LMHeadModel, GPT2Tokenizer +from transformers.models.gpt2.modeling_gpt2 import GPT2Block +from datasets import load_dataset + +# FSDP Imports +from torch.distributed.fsdp import ( + FullyShardedDataParallel as FSDP, + StateDictType, + FullStateDictConfig, +) +from torch.distributed.fsdp.wrap import ( + transformer_auto_wrap_policy, +) + +# ----------------------------------------------------------- +# Logging Setup +# ----------------------------------------------------------- +def setup_logging(rank: int) -> str: + """Setup logging to file (per rank) and console.""" + log_dir = "logs" + os.makedirs(log_dir, exist_ok=True) + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + log_file = os.path.join(log_dir, f"rank_{rank}_{timestamp}.log") + + logger = logging.getLogger() + logger.setLevel(logging.DEBUG) + + # Clear existing handlers + for h in list(logger.handlers): + logger.removeHandler(h) + + fh = logging.FileHandler(log_file) + fh.setLevel(logging.DEBUG) + + ch = logging.StreamHandler(sys.stdout) + ch.setLevel(logging.INFO) + + formatter = logging.Formatter( + fmt="%(asctime)s | Rank %(rank)d | %(levelname)s | %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + + class RankFilter(logging.Filter): + def filter(self, record): + record.rank = rank + return True + + fh.addFilter(RankFilter()) + ch.addFilter(RankFilter()) + fh.setFormatter(formatter) + ch.setFormatter(formatter) + + logger.addHandler(fh) + logger.addHandler(ch) + return log_file + +# ----------------------------------------------------------- +# GPU/Node Logging Helper +# ----------------------------------------------------------- +def log_gpu_info(): + """Log how many GPUs each node has and how ranks map to GPUs.""" + try: + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + except (TypeError, ValueError): + local_rank = 0 + node_rank = os.environ.get("NODE_RANK", "unknown") + rank = dist.get_rank() + world_size = dist.get_world_size() + num_gpus = torch.cuda.device_count() + hostname = socket.gethostname() + + # One summary line per node (LOCAL_RANK == 0) + if local_rank == 0: + logging.info( + f"[NODE SUMMARY] host={hostname} node_rank={node_rank} " + f"num_gpus_visible={num_gpus} world_size={world_size}" + ) + + # Detailed log for every rank + logging.info( + f"[RANK MAP] global_rank={rank} node_rank={node_rank} " + f"local_rank={local_rank} cuda_device={local_rank} host={hostname}" + ) + +# ----------------------------------------------------------- +# Dataset: WikiText-2 +# ----------------------------------------------------------- +def get_wikitext_dataset(tokenizer: GPT2Tokenizer, seq_len: int = 128): + """ + Loads WikiText-2, tokenizes it, and chunks it into sequences of seq_len. + """ + # Only main process on each node should ideally manage downloads, + # but 'datasets' uses file locking, so it's safe to call on all ranks. + # We suppress progress bars on non-zero local ranks to keep logs clean. + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + if local_rank != 0: + import datasets + datasets.logging.set_verbosity_error() + + logging.info("Loading WikiText-2 dataset...") + # Using 'wikitext-2-raw-v1' (no pre-processing) + dataset = load_dataset("wikitext", "wikitext-2-raw-v1", split="train") + + logging.info("Tokenizing WikiText-2...") + + def tokenize_function(examples): + return tokenizer(examples["text"]) + + # Tokenize the dataset + tokenized_datasets = dataset.map( + tokenize_function, + batched=True, + num_proc=4, + remove_columns=["text"] + ) + + logging.info("Flattening and chunking dataset...") + # Flatten all input_ids into one long list + all_input_ids = list(itertools.chain(*tokenized_datasets["input_ids"])) + + # Create tensor + # We need to ensure we can create samples of (seq_len) plus 1 token for label + total_tokens = len(all_input_ids) + + # Convert to tensor + data_tensor = torch.tensor(all_input_ids, dtype=torch.long) + + class WikiTextDataset(torch.utils.data.Dataset): + def __init__(self, data, seq_len): + self.data = data + self.seq_len = seq_len + # Calculate how many full blocks we can make + # We need block_size + 1 (for next token prediction) + self.num_samples = (len(self.data) - 1) // self.seq_len + + def __len__(self): + return self.num_samples + + def __getitem__(self, idx): + start = idx * self.seq_len + end = start + self.seq_len + + # x is input, y is target (shifted by 1) + x = self.data[start : end] + y = self.data[start+1 : end+1] + return x, y + + logging.info(f"Dataset created: {total_tokens} tokens, {(total_tokens-1)//seq_len} samples.") + return WikiTextDataset(data_tensor, seq_len) + +# ----------------------------------------------------------- +# Training Helpers +# ----------------------------------------------------------- +def train_one_epoch(model, dataloader, optimizer, device, epoch, rank): + model.train() + total_loss = 0.0 + num_batches = len(dataloader) + + logging.info(f"Starting epoch {epoch} with {num_batches} batches") + + for batch_idx, (input_ids, labels) in enumerate(dataloader): + input_ids = input_ids.to(device) + labels = labels.to(device) + + optimizer.zero_grad() + outputs = model(input_ids, labels=labels) + loss = outputs.loss + loss.backward() + optimizer.step() + + total_loss += loss.item() + + if batch_idx % 10 == 0: + logging.info( + f"Epoch {epoch} | Batch {batch_idx}/{num_batches} | " + f"Loss: {loss.item():.4f}" + ) + + avg_loss = total_loss / max(1, num_batches) + logging.info(f"Finished epoch {epoch} | Average loss: {avg_loss:.4f}") + return avg_loss + +def save_checkpoint_fsdp(model, optimizer, epoch, rank): + """ + Saves a consolidated checkpoint on rank 0. + FSDP requires a specific context manager to gather parameters + from all shards into a single state_dict. + """ + # Policy: gather full state dict to CPU on rank 0 + save_policy = FullStateDictConfig(offload_to_cpu=True, rank0_only=True) + + # Use the context manager to gather the full state dict + with FSDP.state_dict_type(model, StateDictType.FULL_STATE_DICT, save_policy): + cpu_state = model.state_dict() + + # Optimizer saving is complex in FSDP (sharded vs full). + # For simplicity in this 'exact same functionality' port, we save the full model weights. + # Saving full optimizer state requires FSDP.optim_state_dict + scattering logic on load, + # so we stick to model weights primarily for this example. + + if rank == 0: + ckpt_dir = "checkpoints" + os.makedirs(ckpt_dir, exist_ok=True) + ckpt_path = os.path.join(ckpt_dir, f"fsdp_gpt2_wikitext2.pt") + + state = { + "epoch": epoch, + "model_state_dict": cpu_state, + # Note: Standard optimizer.state_dict() may not work as expected in FSDP without specific calls. + # We save it here for structure consistency, but restoring it requires FSDP.load_optim_state_dict + "optimizer_state_dict": optimizer.state_dict(), + } + torch.save(state, ckpt_path) + logging.info(f"Checkpoint saved at: {ckpt_path}") + +# ----------------------------------------------------------- +# Main Training Loop +# ----------------------------------------------------------- +def main(): + # Initialize process group + dist.init_process_group(backend="nccl", init_method="env://") + rank = dist.get_rank() + world_size = dist.get_world_size() + + # Setup logging + log_file = setup_logging(rank) + logging.info(f"Logging initialized. Log file: {log_file}") + logging.info(f"Init: rank={rank}, world_size={world_size}") + + # log GPU/node info + log_gpu_info() + + if not torch.cuda.is_available(): + raise RuntimeError("CUDA/ROCm is required for this script.") + + # Assign GPU + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + torch.cuda.set_device(local_rank) + device = torch.device("cuda", local_rank) + logging.info(f"Using CUDA device index {local_rank}") + + # Load model + tokenizer + logging.info("Loading GPT-2 model and tokenizer...") + tokenizer = GPT2Tokenizer.from_pretrained("gpt2") + tokenizer.pad_token = tokenizer.eos_token + + # 1. Instantiate model on CPU (standard practice for FSDP to avoid GPU OOM on init, + # though for GPT2-small it fits fine). We move it to device automatically via FSDP + # or manually before depending on strategy. Here we move to device to match DDP flow + # closely, but FSDP handles sharding. + model = GPT2LMHeadModel.from_pretrained("gpt2").to(device) + + # 2. Define Auto Wrapping Policy + # This tells FSDP to wrap every GPT2Block individually, which enables + # sharding parameters and clearing gradients layer-by-layer. + gpt2_auto_wrap_policy = functools.partial( + transformer_auto_wrap_policy, + transformer_layer_cls={GPT2Block}, + ) + + # 3. Wrap with FSDP + model = FSDP( + model, + auto_wrap_policy=gpt2_auto_wrap_policy, + device_id=torch.cuda.current_device(), # Important: bind to specific GPU + ) + + logging.info(f"Model wrapped with FSDP.") + + # Dataset (UPDATED) + dataset = get_wikitext_dataset(tokenizer, seq_len=128) + sampler = DistributedSampler(dataset, num_replicas=world_size, rank=rank, shuffle=True) + dataloader = DataLoader(dataset, batch_size=4, sampler=sampler, num_workers=2) + + optimizer = torch.optim.AdamW(model.parameters(), lr=5e-5) + + num_epochs = 2 + + # Track total training time + total_start_time = time.time() + + for epoch in range(1, num_epochs + 1): + epoch_start_time = time.time() + + sampler.set_epoch(epoch) + avg_loss = train_one_epoch(model, dataloader, optimizer, device, epoch, rank) + + epoch_end_time = time.time() + epoch_duration = epoch_end_time - epoch_start_time + + if rank == 0: + logging.info( + f"Epoch {epoch} summary: avg_loss={avg_loss:.4f} | " + f"Time: {epoch_duration:.2f}s" + ) + + total_end_time = time.time() + total_duration = total_end_time - total_start_time + + if rank == 0: + logging.info(f"Total training time: {total_duration:.2f}s") + + save_checkpoint_fsdp(model, optimizer, num_epochs, rank) + + logging.info("Destroying process group...") + dist.destroy_process_group() + logging.info("Process finished.") + logging.info(f"Log file: {log_file}") + +# ----------------------------------------------------------- +# Entry Point +# ----------------------------------------------------------- +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + logging.warning("Interrupted by user") + except Exception as e: + logging.error(f"Fatal error: {e}", exc_info=True) + sys.exit(1) + +# torchrun --nnodes=2 --nproc_per_node=8 --node_rank=0 --master_addr=10.162.224.131 --master_port=29500 fsdp_training.py(amd 1) +# torchrun --nnodes=2 --nproc_per_node=8 --node_rank=1 --master_addr=10.162.224.131 --master_port=29500 fsdp_training.py(amd 2) \ No newline at end of file diff --git a/junk/fsdp_training_inference.py b/junk/fsdp_training_inference.py new file mode 100644 index 0000000..74a67b5 --- /dev/null +++ b/junk/fsdp_training_inference.py @@ -0,0 +1,274 @@ +from __future__ import annotations +import os +import sys +import time +import math +import logging +import socket +import functools +import itertools +from datetime import datetime, timedelta +import torch +import torch.nn as nn +import torch.distributed as dist +from torch.utils.data import DataLoader +from torch.utils.data.distributed import DistributedSampler +from transformers import GPT2LMHeadModel, GPT2Tokenizer +from transformers.models.gpt2.modeling_gpt2 import GPT2Block +from datasets import load_dataset +import warnings +warnings.filterwarnings("ignore", category=UserWarning) +# FSDP Imports +from torch.distributed.fsdp import ( + FullyShardedDataParallel as FSDP, + StateDictType, + FullStateDictConfig, +) +from torch.distributed.fsdp.wrap import ( + transformer_auto_wrap_policy, +) + +# ----------------------------------------------------------- +# Configuration +# ----------------------------------------------------------- +TRAIN_NODES = 2 +GPUS_PER_NODE = 8 +TRAIN_WORLD_SIZE = TRAIN_NODES * GPUS_PER_NODE +INFERENCE_MASTER_RANK = TRAIN_WORLD_SIZE + +# ----------------------------------------------------------- +# Logging Setup +# ----------------------------------------------------------- +def setup_logging(rank: int) -> str: + log_dir = "logs" + os.makedirs(log_dir, exist_ok=True) + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + log_file = os.path.join(log_dir, f"rank_{rank}_{timestamp}.log") + + logger = logging.getLogger() + logger.setLevel(logging.INFO) # Set to INFO to reduce clutter for timing + for h in list(logger.handlers): + logger.removeHandler(h) + + fh = logging.FileHandler(log_file) + fh.setLevel(logging.INFO) + ch = logging.StreamHandler(sys.stdout) + ch.setLevel(logging.INFO) + formatter = logging.Formatter(fmt="%(asctime)s | Rank %(rank)d | %(levelname)s | %(message)s", datefmt="%H:%M:%S") + + class RankFilter(logging.Filter): + def filter(self, record): + record.rank = rank + return True + + fh.addFilter(RankFilter()) + ch.addFilter(RankFilter()) + fh.setFormatter(formatter) + ch.setFormatter(formatter) + logger.addHandler(fh) + logger.addHandler(ch) + return log_file + +# ----------------------------------------------------------- +# Dataset +# ----------------------------------------------------------- +def get_wikitext_dataset(tokenizer: GPT2Tokenizer, seq_len: int = 128): + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + if local_rank != 0: + import datasets + datasets.logging.set_verbosity_error() + + logging.info("Loading WikiText-2 dataset...") + dataset = load_dataset("wikitext", "wikitext-2-raw-v1", split="train") + + def tokenize_function(examples): + return tokenizer(examples["text"]) + + tokenized_datasets = dataset.map(tokenize_function, batched=True, num_proc=4, remove_columns=["text"]) + all_input_ids = list(itertools.chain(*tokenized_datasets["input_ids"])) + data_tensor = torch.tensor(all_input_ids, dtype=torch.long) + + class WikiTextDataset(torch.utils.data.Dataset): + def __init__(self, data, seq_len): + self.data = data + self.seq_len = seq_len + self.num_samples = (len(self.data) - 1) // self.seq_len + def __len__(self): return self.num_samples + def __getitem__(self, idx): + start = idx * self.seq_len + end = start + self.seq_len + return self.data[start : end], self.data[start+1 : end+1] + + logging.info(f"Dataset ready. Samples: {(len(data_tensor)-1)//seq_len}") + return WikiTextDataset(data_tensor, seq_len) + +# ----------------------------------------------------------- +# Broadcast Helper +# ----------------------------------------------------------- +def broadcast_model_to_inference(model, rank, bridge_group, is_sender=False): + if bridge_group is None: return + + try: + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + device = torch.device("cuda", local_rank) + except: device = torch.device("cuda") + + if is_sender: logging.info("Broadcasting weights to Inference Node...") + + t0 = time.time() + with torch.no_grad(): + for param in model.parameters(): + if param.device.type == "cpu": + gpu_param = param.data.to(device) + dist.broadcast(gpu_param, src=0, group=bridge_group) + else: + dist.broadcast(param.data, src=0, group=bridge_group) + + if is_sender: + duration = time.time() - t0 + logging.info(f"Broadcast complete. Time taken: {duration:.2f}s") + else: + logging.info("Weights received from Training Cluster.") + +# ----------------------------------------------------------- +# Trainer Loop with Timing +# ----------------------------------------------------------- +def run_trainer(rank, world_size, train_group, bridge_group): + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + torch.cuda.set_device(local_rank) + device = torch.device("cuda", local_rank) + + tokenizer = GPT2Tokenizer.from_pretrained("gpt2") + tokenizer.pad_token = tokenizer.eos_token + + logging.info("Initializing FSDP Model...") + model = GPT2LMHeadModel.from_pretrained("gpt2").to(device) + gpt2_auto_wrap_policy = functools.partial(transformer_auto_wrap_policy, transformer_layer_cls={GPT2Block}) + + model = FSDP(model, auto_wrap_policy=gpt2_auto_wrap_policy, process_group=train_group, device_id=torch.cuda.current_device()) + + dataset = get_wikitext_dataset(tokenizer, seq_len=128) + sampler = DistributedSampler(dataset, num_replicas=dist.get_world_size(group=train_group), rank=dist.get_rank(group=train_group), shuffle=True) + + # Batch size per GPU + BATCH_SIZE = 4 + dataloader = DataLoader(dataset, batch_size=BATCH_SIZE, sampler=sampler, num_workers=2) + optimizer = torch.optim.AdamW(model.parameters(), lr=5e-5) + + num_epochs = 2 + + # Comparative Study Metrics + total_training_start = time.time() + + for epoch in range(1, num_epochs + 1): + model.train() + sampler.set_epoch(epoch) + total_loss = 0.0 + + # --- EPOCH TIMER START --- + epoch_start = time.time() + + for i, (ids, labels) in enumerate(dataloader): + ids, labels = ids.to(device), labels.to(device) + optimizer.zero_grad() + output = model(ids, labels=labels) + output.loss.backward() + optimizer.step() + total_loss += output.loss.item() + + if i % 20 == 0 and rank == 0: + logging.info(f"Batch {i}/{len(dataloader)} Loss: {output.loss.item():.4f}") + + # --- EPOCH TIMER END --- + epoch_duration = time.time() - epoch_start + + # Throughput Calculation + # Total samples processed = num_batches * batch_size * num_gpus (implied by distributed sampler dividing data) + # Actually DistributedSampler divides dataset, so len(dataloader) is local batches + # Total tokens = local_batches * batch_size * seq_len * world_size + total_tokens_processed = len(dataloader) * BATCH_SIZE * 128 * dist.get_world_size(group=train_group) + throughput = total_tokens_processed / epoch_duration + + if rank == 0: + avg_loss = total_loss / len(dataloader) + logging.info(f"--- EPOCH {epoch} STATS ---") + logging.info(f"Duration: {epoch_duration:.2f} seconds") + logging.info(f"Throughput: {throughput:.2f} tokens/sec") + logging.info(f"Avg Loss: {avg_loss:.4f}") + logging.info(f"-------------------------") + + # --- BRIDGE SYNC --- + save_policy = FullStateDictConfig(offload_to_cpu=True, rank0_only=True) + with FSDP.state_dict_type(model, StateDictType.FULL_STATE_DICT, save_policy): + full_state = model.state_dict() + + if rank == 0: + cpu_model = GPT2LMHeadModel.from_pretrained("gpt2") + cpu_model.load_state_dict(full_state) + broadcast_model_to_inference(cpu_model, rank, bridge_group, is_sender=True) + del cpu_model + + total_training_time = time.time() - total_training_start + if rank == 0: + logging.info(f"Total Session Time: {total_training_time:.2f} seconds") + +# ----------------------------------------------------------- +# Inference Loop +# ----------------------------------------------------------- +def run_inference(rank, bridge_group): + local_rank = int(os.environ.get("LOCAL_RANK", 0)) + torch.cuda.set_device(local_rank) + device = torch.device("cuda", local_rank) + + tokenizer = GPT2Tokenizer.from_pretrained("gpt2") + tokenizer.pad_token = tokenizer.eos_token + + logging.info("Initializing Inference Model...") + model = GPT2LMHeadModel.from_pretrained("gpt2").to(device) + model.eval() + + num_epochs = 2 + for epoch in range(1, num_epochs + 1): + logging.info(f"Waiting for model update from Training Cluster (Epoch {epoch})...") + broadcast_model_to_inference(model, rank, bridge_group, is_sender=False) + + logging.info("Running inference test...") + test_input = "The AI scientist discovered" + inputs = tokenizer(test_input, return_tensors="pt").to(device) + with torch.no_grad(): + outputs = model.generate(**inputs, max_new_tokens=25) + + generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True) + logging.info(f"--- [INFERENCE RESULT EPOCH {epoch}] ---") + logging.info(f"Output: {generated_text}") + +# ----------------------------------------------------------- +# Main +# ----------------------------------------------------------- +def main(): + dist.init_process_group(backend="nccl", init_method="env://", timeout=timedelta(minutes=60)) + rank = dist.get_rank() + log_file = setup_logging(rank) + + if rank == 0: + logging.info(f"NCCL_SOCKET_IFNAME: {os.environ.get('NCCL_SOCKET_IFNAME', 'Not Set')}") + + train_ranks = list(range(0, TRAIN_WORLD_SIZE)) + train_group = dist.new_group(ranks=train_ranks) + bridge_ranks = [0, INFERENCE_MASTER_RANK] + bridge_group = dist.new_group(ranks=bridge_ranks) + + if rank in train_ranks: + my_bridge = bridge_group if rank == 0 else None + run_trainer(rank, TRAIN_WORLD_SIZE, train_group, my_bridge) + elif rank == INFERENCE_MASTER_RANK: + run_inference(rank, bridge_group) + else: + pass + + logging.info("Waiting for all ranks to complete...") + dist.barrier() + dist.destroy_process_group() + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/junk/gpu_transfer.py b/junk/gpu_transfer.py new file mode 100644 index 0000000..114a312 --- /dev/null +++ b/junk/gpu_transfer.py @@ -0,0 +1,94 @@ +from __future__ import annotations +import torch, time, os, sys +import torch.distributed as dist + +from transformers import GPT2LMHeadModel +from transformers import AutoModelForCausalLM, AutoTokenizer + +MODEL_NAME = "Qwen/Qwen1.5-0.5B" +from uccl import p2p + +def send_model(ep, conn_id, model): + print(f"[Client] Sending {len(list(model.state_dict().items()))} tensors...") + for name, tensor in model.state_dict().items(): + if not tensor.is_cuda: + tensor = tensor.cuda() + size_bytes = tensor.numel() * tensor.element_size() + ptr = tensor.data_ptr() + ok, mr_id = ep.reg(ptr, size_bytes) + assert ok, f"[Client] Failed to register tensor {name}" + ok = ep.send(conn_id, mr_id, ptr, size_bytes) + assert ok, f"[Client] Send failed for {name}" + print(f"[Client] Sent {name} ({size_bytes/1e6:.2f} MB)") + print("[Client] Model transfer complete.") + + +def recv_model(ep, conn_id, model): + print(f"[Server] Receiving {len(list(model.state_dict().items()))} tensors...") + for name, tensor in model.state_dict().items(): + recv_tensor = torch.empty_like(tensor, device="cuda") + size_bytes = recv_tensor.numel() * recv_tensor.element_size() + ptr = recv_tensor.data_ptr() + ok, mr_id = ep.reg(ptr, size_bytes) + assert ok, f"[Server] Failed to register tensor {name}" + ok = ep.recv(conn_id, mr_id, ptr, size_bytes) + assert ok, f"[Server] Receive failed for {name}" + model.state_dict()[name].copy_(recv_tensor) + print(f"[Server] Received {name} ({size_bytes/1e6:.2f} MB)") + print("[Server] Model transfer complete.") + + +def main(): + dist.init_process_group(backend="gloo") + rank = dist.get_rank() + world_size = dist.get_world_size() + assert world_size == 2, "Run with two ranks (client/server)." + + local_gpu = rank + torch.cuda.set_device(local_gpu) + + ep = p2p.Endpoint(local_gpu, 4) + local_md = ep.get_metadata() + + # exchange metadata + if rank == 0: + dist.send(torch.ByteTensor(list(local_md)), dst=1) + remote_md = torch.zeros(len(local_md), dtype=torch.uint8) + dist.recv(remote_md, src=1) + else: + remote_md = torch.zeros(len(local_md), dtype=torch.uint8) + dist.recv(remote_md, src=0) + dist.send(torch.ByteTensor(list(local_md)), dst=0) + remote_metadata = bytes(remote_md.tolist()) + + if rank == 0: + ip, port, r_gpu = p2p.Endpoint.parse_metadata(remote_metadata) + ok, conn_id = ep.connect(ip, r_gpu, remote_port=port) + assert ok, "[Client] connect failed" + + #model = GPT2LMHeadModel.from_pretrained("gpt2").cuda() + tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME) + model = AutoModelForCausalLM.from_pretrained(MODEL_NAME, torch_dtype=torch.float16).cuda() + + start = time.perf_counter() + send_model(ep, conn_id, model) + print(f"[Client] Transfer finished in {time.perf_counter()-start:.2f}s") + + else: + ok, r_ip, r_gpu, conn_id = ep.accept() + assert ok, "[Server] accept failed" + + model = GPT2LMHeadModel.from_pretrained("gpt2").cuda() + start = time.perf_counter() + recv_model(ep, conn_id, model) + print(f"[Server] Transfer finished in {time.perf_counter()-start:.2f}s") + + dist.destroy_process_group() + + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + sys.exit(0) + diff --git a/junk/gpu_transfer_fsdp.py b/junk/gpu_transfer_fsdp.py new file mode 100644 index 0000000..97452ed --- /dev/null +++ b/junk/gpu_transfer_fsdp.py @@ -0,0 +1,188 @@ +# gpu_transfer_wikitext2_fsdp.py +# Option A: Broadcaster (rank 0) + Multi-GPU FSDP Trainers (ranks 1..N) + +from __future__ import annotations +import torch, time, os, sys +import torch.distributed as dist +import logging +from datetime import datetime +import math +import functools + +from transformers import GPT2LMHeadModel, GPT2Tokenizer +from datasets import load_dataset +from torch.utils.data import DataLoader + +# FSDP imports +from torch.distributed.fsdp import FullyShardedDataParallel as FSDP +from torch.distributed.fsdp.wrap import transformer_auto_wrap_policy +from torch.distributed.fsdp import StateDictType, FullStateDictConfig + +# UCCl P2P +from uccl import p2p + + +# ===================================================================== +# LOGGING SETUP +# ===================================================================== + +def setup_logging(rank): + """Setup logging with timestamps and rank info""" + log_dir = "logs" + os.makedirs(log_dir, exist_ok=True) + + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + log_file = f"{log_dir}/rank_{rank}_{timestamp}.log" + + logging.basicConfig( + level=logging.INFO, + format='%(asctime)s | Rank %(rank)d | %(levelname)s | %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + handlers=[ + logging.FileHandler(log_file), + logging.StreamHandler(sys.stdout) + ] + ) + + # Add rank info to log entries + old_factory = logging.getLogRecordFactory() + def record_factory(*args, **kwargs): + record = old_factory(*args, **kwargs) + record.rank = rank + return record + logging.setLogRecordFactory(record_factory) + + logging.info(f"Logging initialized. Log file: {log_file}") + return log_file + + +# ===================================================================== +# BROADCAST MODEL (Rank 0) +# ===================================================================== + +def broadcast_model(ep, conn_ids, model, rank): + """Send model to multiple receivers with detailed logging""" + state_dict = model.state_dict() + total_tensors = len(list(state_dict.items())) + total_size_mb = sum(t.numel() * t.element_size() for t in state_dict.values()) / 1e6 + + logging.info("="*80) + logging.info(f"BROADCAST START - Sending to {len(conn_ids)} receivers") + logging.info(f"Total tensors: {total_tensors}") + logging.info(f"Total size: {total_size_mb:.2f} MB") + logging.info("="*80) + + broadcast_start = time.perf_counter() + + for idx, (name, tensor) in enumerate(state_dict.items(), 1): + if not tensor.is_cuda: + tensor = tensor.cuda() + + size_bytes = tensor.numel() * tensor.element_size() + ptr = tensor.data_ptr() + + ok, mr_id = ep.reg(ptr, size_bytes) + assert ok, f"Failed to register tensor {name}" + + # Send tensor to all training ranks + for receiver_idx, conn_id in enumerate(conn_ids, 1): + ok = ep.send(conn_id, mr_id, ptr, size_bytes) + assert ok, f"Send failed for {name} to receiver {receiver_idx}" + + if idx % 20 == 0 or idx == total_tensors: + progress_pct = (idx / total_tensors) * 100 + logging.info(f"Progress: {progress_pct:.1f}% ({idx}/{total_tensors})") + + total_time = time.perf_counter() - broadcast_start + avg_bandwidth = (total_size_mb / 1000) / total_time # GB/s + + logging.info("="*80) + logging.info(f"BROADCAST COMPLETE") + logging.info(f"Total time: {total_time:.2f}s") + logging.info(f"Average bandwidth: {avg_bandwidth:.2f} GB/s") + logging.info("="*80) + + +# ===================================================================== +# RECEIVE MODEL (Ranks 1..N) +# ===================================================================== + +def recv_model(ep, conn_id, model, rank): + """Receive model from broadcaster with detailed logging""" + state_dict = model.state_dict() + total_tensors = len(list(state_dict.items())) + total_size_mb = sum(t.numel() * t.element_size() for t in state_dict.values()) / 1e6 + + logging.info("="*80) + logging.info("RECEIVE START") + logging.info(f"Total tensors: {total_tensors}") + logging.info(f"Total size: {total_size_mb:.2f} MB") + logging.info("="*80) + + recv_start = time.perf_counter() + + for idx, (name, tensor) in enumerate(state_dict.items(), 1): + recv_tensor = torch.empty_like(tensor, device="cuda") + size_bytes = recv_tensor.numel() * recv_tensor.element_size() + ptr = recv_tensor.data_ptr() + + ok, mr_id = ep.reg(ptr, size_bytes) + assert ok, f"Failed to register tensor {name}" + + ok = ep.recv(conn_id, mr_id, ptr, size_bytes) + assert ok, f"Receive failed for {name}" + + model.state_dict()[name].copy_(recv_tensor) + + if idx % 20 == 0 or idx == total_tensors: + logging.info(f"Progress: {idx}/{total_tensors} tensors received") + + total_time = time.perf_counter() - recv_start + avg_bandwidth = (total_size_mb / 1000) / total_time + + logging.info("="*80) + logging.info(f"RECEIVE COMPLETE") + logging.info(f"Total time: {total_time:.2f}s") + logging.info(f"Average bandwidth: {avg_bandwidth:.2f} GB/s") + logging.info("="*80) + + +# ===================================================================== +# DATASET PREPARATION +# ===================================================================== + +def prepare_dataset(tokenizer, max_length=128, num_samples=200): + """Load and prepare WikiText-2 dataset""" + logging.info("="*80) + logging.info("LOADING DATASET") + logging.info("="*80) + + dataset = load_dataset("wikitext", "wikitext-2-raw-v1", split="train") + logging.info(f"Dataset loaded: {len(dataset)} examples") + + dataset = dataset.filter(lambda x: len(x["text"].strip()) > 0) + logging.info(f"After filtering: {len(dataset)} examples") + + if len(dataset) > num_samples: + dataset = dataset.select(range(num_samples)) + logging.info(f"Using subset: {num_samples} examples") + + logging.info("Tokenizing...") + def tokenize(examples): + return tokenizer( + examples["text"], + truncation=True, + padding="max_length", + max_length=max_length + ) + + tokenized_dataset = dataset.map( + tokenize, + batched=True, + remove_columns=dataset.column_names + ) + + tokenized_dataset.set_format(type='torch', columns=['input_ids','attention_mask']) + + logging.info("Dataset ready.") + return tokenized_dataset diff --git a/junk/logAmd4.txt b/junk/logAmd4.txt new file mode 100644 index 0000000..c1e7e3c --- /dev/null +++ b/junk/logAmd4.txt @@ -0,0 +1,3831 @@ +[Gloo] Rank [Gloo] Rank 41 is connected to is connected to 2323 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 2323 + +[Gloo] Rank 7 is connected to 23 peer ranks. Expected number of connected peer ranks is : 23 +[Gloo] Rank [Gloo] Rank 0[Gloo] Rank is connected to 223[Gloo] Rank 5 is connected to [Gloo] Rank peer ranks. is connected to 623Expected number of connected peer ranks is : 323 is connected to peer ranks. 23 is connected to peer ranks. 23Expected number of connected peer ranks is : +23Expected number of connected peer ranks is : peer ranks. 23 peer ranks. 23Expected number of connected peer ranks is : +Expected number of connected peer ranks is : +2323 + +23:43:59 | Rank 7 | INFO | Initializing UCCL Collective... +Creating Engine with GPU index: 7, CPUs: 4 +23:43:59 | Rank 4 | INFO | Initializing UCCL Collective... +Creating Engine with GPU index: 4, CPUs: 4 +23:43:59 | Rank 1 | INFO | Initializing UCCL Collective... +Creating Engine with GPU index: 1, CPUs: 4 +23:43:59 | Rank 2 | INFO | Initializing UCCL Collective... +Creating Engine with GPU index: 2, CPUs: 4 +23:43:59 | Rank 5 | INFO | Initializing UCCL Collective... +Creating Engine with GPU index: 5, CPUs: 4 +23:43:59 | Rank 6 | INFO | Initializing UCCL Collective... +Creating Engine with GPU index: 6, CPUs: 4 +23:43:59 | Rank 0 | INFO | NCCL_SOCKET_IFNAME: enp49s0f1np1 +23:43:59 | Rank 0 | INFO | Initializing UCCL Collective... +Creating Engine with GPU index: 0, CPUs: 4 +23:43:59 | Rank 3 | INFO | Initializing UCCL Collective... +Creating Engine with GPU index: 3, CPUs: 4 +NCCL_IB_GID_INDEX set by environment to 3. +Found IB devices (ibv_get_device_list + NCCL_IB_HCA filter, ordered by libibverbs): + dev_idx 0: bnxt_re0 (1/1) + dev_idx 1: bnxt_re1 (1/1) + dev_idx 2: bnxt_re2 (1/1) + dev_idx 3: bnxt_re3 (1/1) + dev_idx 4: bnxt_re4 (1/1) + dev_idx 5: bnxt_re5 (1/1) + dev_idx 6: bnxt_re7 (1/1) + dev_idx 7: bnxt_re8 (1/1) +Found 8 GPUs (get_gpu_cards, ordered by GPU rank): + GPU 0: /sys/devices/pci0000:70/0000:70:01.1/0000:71:00.0/0000:72:00.0/0000:73:00.0/0000:74:00.0/0000:75:00.0 + GPU 1: /sys/devices/pci0000:00/0000:00:01.1/0000:01:00.0/0000:02:00.0/0000:03:00.0/0000:04:00.0/0000:05:00.0 + GPU 2: /sys/devices/pci0000:60/0000:60:01.1/0000:61:00.0/0000:62:00.0/0000:63:00.0/0000:64:00.0/0000:65:00.0 + GPU 3: /sys/devices/pci0000:10/0000:10:01.1/0000:11:00.0/0000:12:00.0/0000:13:00.0/0000:14:00.0/0000:15:00.0 + GPU 4: /sys/devices/pci0000:f0/0000:f0:01.1/0000:f1:00.0/0000:f2:00.0/0000:f3:00.0/0000:f4:00.0/0000:f5:00.0 + GPU 5: /sys/devices/pci0000:80/0000:80:01.1/0000:81:00.0/0000:82:00.0/0000:83:00.0/0000:84:00.0/0000:85:00.0 + GPU 6: /sys/devices/pci0000:e0/0000:e0:01.1/0000:e1:00.0/0000:e2:00.0/0000:e3:00.0/0000:e4:00.0/0000:e5:00.0 + GPU 7: /sys/devices/pci0000:90/0000:90:01.1/0000:91:00.0/0000:92:00.0/0000:93:00.0/0000:94:00.0/0000:95:00.0 +Found 8 RDMA NICs (get_rdma_nics + NCCL_IB_HCA filter, ordered by dev_idx in rdma_ctl->devices_[]): + RDMA NIC bnxt_re0: /sys/devices/pci0000:70/0000:70:01.1/0000:71:00.0/0000:72:01.0/0000:76:00.0, dev_idx: 0 + RDMA NIC bnxt_re1: /sys/devices/pci0000:00/0000:00:01.1/0000:01:00.0/0000:02:01.0/0000:06:00.0, dev_idx: 1 + RDMA NIC bnxt_re2: /sys/devices/pci0000:60/0000:60:01.1/0000:61:00.0/0000:62:01.0/0000:66:00.0, dev_idx: 2 + RDMA NIC bnxt_re3: /sys/devices/pci0000:10/0000:10:01.1/0000:11:00.0/0000:12:01.0/0000:16:00.0, dev_idx: 3 + RDMA NIC bnxt_re4: /sys/devices/pci0000:f0/0000:f0:01.1/0000:f1:00.0/0000:f2:01.0/0000:f6:00.0, dev_idx: 4 + RDMA NIC bnxt_re5: /sys/devices/pci0000:80/0000:80:01.1/0000:81:00.0/0000:82:01.0/0000:86:00.0, dev_idx: 5 + RDMA NIC bnxt_re7: /sys/devices/pci0000:e0/0000:e0:01.1/0000:e1:00.0/0000:e2:01.0/0000:e6:00.0, dev_idx: 6 + RDMA NIC bnxt_re8: /sys/devices/pci0000:90/0000:90:01.1/0000:91:00.0/0000:92:01.0/0000:96:00.0, dev_idx: 7 +Detected best GPU-NIC mapping: + GPU 0 -> NIC bnxt_re0, dev_idx: 0 + GPU 1 -> NIC bnxt_re1, dev_idx: 1 + GPU 2 -> NIC bnxt_re2, dev_idx: 2 + GPU 3 -> NIC bnxt_re3, dev_idx: 3 + GPU 4 -> NIC bnxt_re4, dev_idx: 4 + GPU 5 -> NIC bnxt_re5, dev_idx: 5 + GPU 6 -> NIC bnxt_re7, dev_idx: 6 + GPU 7 -> NIC bnxt_re8, dev_idx: 7 +Lazy creation of engine, GPU index: 3 +NCCL_IB_GID_INDEX set by environment to 3. +Found IB devices (ibv_get_device_list + NCCL_IB_HCA filter, ordered by libibverbs): + dev_idx 0: bnxt_re0 (1/1) + dev_idx 1: bnxt_re1 (1/1) + dev_idx 2: bnxt_re2 (1/1) + dev_idx 3: bnxt_re3 (1/1) + dev_idx 4: bnxt_re4 (1/1) + dev_idx 5: bnxt_re5 (1/1) + dev_idx 6: bnxt_re7 (1/1) + dev_idx 7: bnxt_re8 (1/1) +Found 8 GPUs (get_gpu_cards, ordered by GPU rank): + GPU 0: /sys/devices/pci0000:70/0000:70:01.1/0000:71:00.0/0000:72:00.0/0000:73:00.0/0000:74:00.0/0000:75:00.0 + GPU 1: /sys/devices/pci0000:00/0000:00:01.1/0000:01:00.0/0000:02:00.0/0000:03:00.0/0000:04:00.0/0000:05:00.0 + GPU 2: /sys/devices/pci0000:60/0000:60:01.1/0000:61:00.0/0000:62:00.0/0000:63:00.0/0000:64:00.0/0000:65:00.0 + GPU 3: /sys/devices/pci0000:10/0000:10:01.1/0000:11:00.0/0000:12:00.0/0000:13:00.0/0000:14:00.0/0000:15:00.0 + GPU 4: /sys/devices/pci0000:f0/0000:f0:01.1/0000:f1:00.0/0000:f2:00.0/0000:f3:00.0/0000:f4:00.0/0000:f5:00.0 + GPU 5: /sys/devices/pci0000:80/0000:80:01.1/0000:81:00.0/0000:82:00.0/0000:83:00.0/0000:84:00.0/0000:85:00.0 + GPU 6: /sys/devices/pci0000:e0/0000:e0:01.1/0000:e1:00.0/0000:e2:00.0/0000:e3:00.0/0000:e4:00.0/0000:e5:00.0 + GPU 7: /sys/devices/pci0000:90/0000:90:01.1/0000:91:00.0/0000:92:00.0/0000:93:00.0/0000:94:00.0/0000:95:00.0 +Found 8 RDMA NICs (get_rdma_nics + NCCL_IB_HCA filter, ordered by dev_idx in rdma_ctl->devices_[]): + RDMA NIC bnxt_re0: /sys/devices/pci0000:70/0000:70:01.1/0000:71:00.0/0000:72:01.0/0000:76:00.0, dev_idx: 0 + RDMA NIC bnxt_re1: /sys/devices/pci0000:00/0000:00:01.1/0000:01:00.0/0000:02:01.0/0000:06:00.0, dev_idx: 1 + RDMA NIC bnxt_re2: /sys/devices/pci0000:60/0000:60:01.1/0000:61:00.0/0000:62:01.0/0000:66:00.0, dev_idx: 2 + RDMA NIC bnxt_re3: /sys/devices/pci0000:10/0000:10:01.1/0000:11:00.0/0000:12:01.0/0000:16:00.0, dev_idx: 3 + RDMA NIC bnxt_re4: /sys/devices/pci0000:f0/0000:f0:01.1/0000:f1:00.0/0000:f2:01.0/0000:f6:00.0, dev_idx: 4 + RDMA NIC bnxt_re5: /sys/devices/pci0000:80/0000:80:01.1/0000:81:00.0/0000:82:01.0/0000:86:00.0, dev_idx: 5 + RDMA NIC bnxt_re7: /sys/devices/pci0000:e0/0000:e0:01.1/0000:e1:00.0/0000:e2:01.0/0000:e6:00.0, dev_idx: 6 + RDMA NIC bnxt_re8: /sys/devices/pci0000:90/0000:90:01.1/0000:91:00.0/0000:92:01.0/0000:96:00.0, dev_idx: 7 +Detected best GPU-NIC mapping: + GPU 0 -> NIC bnxt_re0, dev_idx: 0 + GPU 1 -> NIC bnxt_re1, dev_idx: 1 + GPU 2 -> NIC bnxt_re2, dev_idx: 2 + GPU 3 -> NIC bnxt_re3, dev_idx: 3 + GPU 4 -> NIC bnxt_re4, dev_idx: 4 + GPU 5 -> NIC bnxt_re5, dev_idx: 5 + GPU 6 -> NIC bnxt_re7, dev_idx: 6 + GPU 7 -> NIC bnxt_re8, dev_idx: 7 +Lazy creation of engine, GPU index: 1 +P2P listening on port 41181 +Engine initialized for GPU 3 +UDS socket initialized at /tmp/uccl_gpu_3.sock +Endpoint initialized successfully +P2P listening on port 32893 +Engine initialized for GPU 1 +UDS socket initialized at /tmp/uccl_gpu_1.sock +Endpoint initialized successfully +NCCL_IB_GID_INDEX set by environment to 3. +Found IB devices (ibv_get_device_list + NCCL_IB_HCA filter, ordered by libibverbs): + dev_idx 0: bnxt_re0 (1/1) + dev_idx 1: bnxt_re1 (1/1) + dev_idx 2: bnxt_re2 (1/1) + dev_idx 3: bnxt_re3 (1/1) + dev_idx 4: bnxt_re4 (1/1) + dev_idx 5: bnxt_re5 (1/1) + dev_idx 6: bnxt_re7 (1/1) + dev_idx 7: bnxt_re8 (1/1) +Found 8 GPUs (get_gpu_cards, ordered by GPU rank): + GPU 0: /sys/devices/pci0000:70/0000:70:01.1/0000:71:00.0/0000:72:00.0/0000:73:00.0/0000:74:00.0/0000:75:00.0 + GPU 1: /sys/devices/pci0000:00/0000:00:01.1/0000:01:00.0/0000:02:00.0/0000:03:00.0/0000:04:00.0/0000:05:00.0 + GPU 2: /sys/devices/pci0000:60/0000:60:01.1/0000:61:00.0/0000:62:00.0/0000:63:00.0/0000:64:00.0/0000:65:00.0 + GPU 3: /sys/devices/pci0000:10/0000:10:01.1/0000:11:00.0/0000:12:00.0/0000:13:00.0/0000:14:00.0/0000:15:00.0 + GPU 4: /sys/devices/pci0000:f0/0000:f0:01.1/0000:f1:00.0/0000:f2:00.0/0000:f3:00.0/0000:f4:00.0/0000:f5:00.0 + GPU 5: /sys/devices/pci0000:80/0000:80:01.1/0000:81:00.0/0000:82:00.0/0000:83:00.0/0000:84:00.0/0000:85:00.0 + GPU 6: /sys/devices/pci0000:e0/0000:e0:01.1/0000:e1:00.0/0000:e2:00.0/0000:e3:00.0/0000:e4:00.0/0000:e5:00.0 + GPU 7: /sys/devices/pci0000:90/0000:90:01.1/0000:91:00.0/0000:92:00.0/0000:93:00.0/0000:94:00.0/0000:95:00.0 +Found 8 RDMA NICs (get_rdma_nics + NCCL_IB_HCA filter, ordered by dev_idx in rdma_ctl->devices_[]): + RDMA NIC bnxt_re0: /sys/devices/pci0000:70/0000:70:01.1/0000:71:00.0/0000:72:01.0/0000:76:00.0, dev_idx: 0 + RDMA NIC bnxt_re1: /sys/devices/pci0000:00/0000:00:01.1/0000:01:00.0/0000:02:01.0/0000:06:00.0, dev_idx: 1 + RDMA NIC bnxt_re2: /sys/devices/pci0000:60/0000:60:01.1/0000:61:00.0/0000:62:01.0/0000:66:00.0, dev_idx: 2 + RDMA NIC bnxt_re3: /sys/devices/pci0000:10/0000:10:01.1/0000:11:00.0/0000:12:01.0/0000:16:00.0, dev_idx: 3 + RDMA NIC bnxt_re4: /sys/devices/pci0000:f0/0000:f0:01.1/0000:f1:00.0/0000:f2:01.0/0000:f6:00.0, dev_idx: 4 + RDMA NIC bnxt_re5: /sys/devices/pci0000:80/0000:80:01.1/0000:81:00.0/0000:82:01.0/0000:86:00.0, dev_idx: 5 + RDMA NIC bnxt_re7: /sys/devices/pci0000:e0/0000:e0:01.1/0000:e1:00.0/0000:e2:01.0/0000:e6:00.0, dev_idx: 6 + RDMA NIC bnxt_re8: /sys/devices/pci0000:90/0000:90:01.1/0000:91:00.0/0000:92:01.0/0000:96:00.0, dev_idx: 7 +Detected best GPU-NIC mapping: + GPU 0 -> NIC bnxt_re0, dev_idx: 0 + GPU 1 -> NIC bnxt_re1, dev_idx: 1 + GPU 2 -> NIC bnxt_re2, dev_idx: 2 + GPU 3 -> NIC bnxt_re3, dev_idx: 3 + GPU 4 -> NIC bnxt_re4, dev_idx: 4 + GPU 5 -> NIC bnxt_re5, dev_idx: 5 + GPU 6 -> NIC bnxt_re7, dev_idx: 6 + GPU 7 -> NIC bnxt_re8, dev_idx: 7 +Lazy creation of engine, GPU index: 6 +NCCL_IB_GID_INDEX set by environment to 3. +NCCL_IB_GID_INDEX set by environment to 3. +Found IB devices (ibv_get_device_list + NCCL_IB_HCA filter, ordered by libibverbs): + dev_idx 0: bnxt_re0 (1/1) + dev_idx 1: bnxt_re1 (1/1) + dev_idx 2: bnxt_re2 (1/1) + dev_idx 3: bnxt_re3 (1/1) + dev_idx 4: bnxt_re4 (1/1) + dev_idx 5: bnxt_re5 (1/1) + dev_idx 6: bnxt_re7 (1/1) + dev_idx 7: bnxt_re8 (1/1) +Found 8 GPUs (get_gpu_cards, ordered by GPU rank): + GPU 0: /sys/devices/pci0000:70/0000:70:01.1/0000:71:00.0/0000:72:00.0/0000:73:00.0/0000:74:00.0/0000:75:00.0 + GPU 1: /sys/devices/pci0000:00/0000:00:01.1/0000:01:00.0/0000:02:00.0/0000:03:00.0/0000:04:00.0/0000:05:00.0 + GPU 2: /sys/devices/pci0000:60/0000:60:01.1/0000:61:00.0/0000:62:00.0/0000:63:00.0/0000:64:00.0/0000:65:00.0 + GPU 3: /sys/devices/pci0000:10/0000:10:01.1/0000:11:00.0/0000:12:00.0/0000:13:00.0/0000:14:00.0/0000:15:00.0 + GPU 4: /sys/devices/pci0000:f0/0000:f0:01.1/0000:f1:00.0/0000:f2:00.0/0000:f3:00.0/0000:f4:00.0/0000:f5:00.0 + GPU 5: /sys/devices/pci0000:80/0000:80:01.1/0000:81:00.0/0000:82:00.0/0000:83:00.0/0000:84:00.0/0000:85:00.0 + GPU 6: /sys/devices/pci0000:e0/0000:e0:01.1/0000:e1:00.0/0000:e2:00.0/0000:e3:00.0/0000:e4:00.0/0000:e5:00.0 + GPU 7: /sys/devices/pci0000:90/0000:90:01.1/0000:91:00.0/0000:92:00.0/0000:93:00.0/0000:94:00.0/0000:95:00.0 +Found 8 RDMA NICs (get_rdma_nics + NCCL_IB_HCA filter, ordered by dev_idx in rdma_ctl->devices_[]): + RDMA NIC bnxt_re0: /sys/devices/pci0000:70/0000:70:01.1/0000:71:00.0/0000:72:01.0/0000:76:00.0, dev_idx: 0 + RDMA NIC bnxt_re1: /sys/devices/pci0000:00/0000:00:01.1/0000:01:00.0/0000:02:01.0/0000:06:00.0, dev_idx: 1 + RDMA NIC bnxt_re2: /sys/devices/pci0000:60/0000:60:01.1/0000:61:00.0/0000:62:01.0/0000:66:00.0, dev_idx: 2 + RDMA NIC bnxt_re3: /sys/devices/pci0000:10/0000:10:01.1/0000:11:00.0/0000:12:01.0/0000:16:00.0, dev_idx: 3 + RDMA NIC bnxt_re4: /sys/devices/pci0000:f0/0000:f0:01.1/0000:f1:00.0/0000:f2:01.0/0000:f6:00.0, dev_idx: 4 + RDMA NIC bnxt_re5: /sys/devices/pci0000:80/0000:80:01.1/0000:81:00.0/0000:82:01.0/0000:86:00.0, dev_idx: 5 + RDMA NIC bnxt_re7: /sys/devices/pci0000:e0/0000:e0:01.1/0000:e1:00.0/0000:e2:01.0/0000:e6:00.0, dev_idx: 6 + RDMA NIC bnxt_re8: /sys/devices/pci0000:90/0000:90:01.1/0000:91:00.0/0000:92:01.0/0000:96:00.0, dev_idx: 7 +Detected best GPU-NIC mapping: + GPU 0 -> NIC bnxt_re0, dev_idx: 0 + GPU 1 -> NIC bnxt_re1, dev_idx: 1 + GPU 2 -> NIC bnxt_re2, dev_idx: 2 + GPU 3 -> NIC bnxt_re3, dev_idx: 3 + GPU 4 -> NIC bnxt_re4, dev_idx: 4 + GPU 5 -> NIC bnxt_re5, dev_idx: 5 + GPU 6 -> NIC bnxt_re7, dev_idx: 6 + GPU 7 -> NIC bnxt_re8, dev_idx: 7 +Lazy creation of engine, GPU index: 2 +Found IB devices (ibv_get_device_list + NCCL_IB_HCA filter, ordered by libibverbs): + dev_idx 0: bnxt_re0 (1/1) + dev_idx 1: bnxt_re1 (1/1) + dev_idx 2: bnxt_re2 (1/1) + dev_idx 3: bnxt_re3 (1/1) + dev_idx 4: bnxt_re4 (1/1) + dev_idx 5: bnxt_re5 (1/1) + dev_idx 6: bnxt_re7 (1/1) + dev_idx 7: bnxt_re8 (1/1) +Found 8 GPUs (get_gpu_cards, ordered by GPU rank): + GPU 0: /sys/devices/pci0000:70/0000:70:01.1/0000:71:00.0/0000:72:00.0/0000:73:00.0/0000:74:00.0/0000:75:00.0 + GPU 1: /sys/devices/pci0000:00/0000:00:01.1/0000:01:00.0/0000:02:00.0/0000:03:00.0/0000:04:00.0/0000:05:00.0 + GPU 2: /sys/devices/pci0000:60/0000:60:01.1/0000:61:00.0/0000:62:00.0/0000:63:00.0/0000:64:00.0/0000:65:00.0 + GPU 3: /sys/devices/pci0000:10/0000:10:01.1/0000:11:00.0/0000:12:00.0/0000:13:00.0/0000:14:00.0/0000:15:00.0 + GPU 4: /sys/devices/pci0000:f0/0000:f0:01.1/0000:f1:00.0/0000:f2:00.0/0000:f3:00.0/0000:f4:00.0/0000:f5:00.0 + GPU 5: /sys/devices/pci0000:80/0000:80:01.1/0000:81:00.0/0000:82:00.0/0000:83:00.0/0000:84:00.0/0000:85:00.0 + GPU 6: /sys/devices/pci0000:e0/0000:e0:01.1/0000:e1:00.0/0000:e2:00.0/0000:e3:00.0/0000:e4:00.0/0000:e5:00.0 + GPU 7: /sys/devices/pci0000:90/0000:90:01.1/0000:91:00.0/0000:92:00.0/0000:93:00.0/0000:94:00.0/0000:95:00.0 +Found 8 RDMA NICs (get_rdma_nics + NCCL_IB_HCA filter, ordered by dev_idx in rdma_ctl->devices_[]): + RDMA NIC bnxt_re0: /sys/devices/pci0000:70/0000:70:01.1/0000:71:00.0/0000:72:01.0/0000:76:00.0, dev_idx: 0 + RDMA NIC bnxt_re1: /sys/devices/pci0000:00/0000:00:01.1/0000:01:00.0/0000:02:01.0/0000:06:00.0, dev_idx: 1 + RDMA NIC bnxt_re2: /sys/devices/pci0000:60/0000:60:01.1/0000:61:00.0/0000:62:01.0/0000:66:00.0, dev_idx: 2 + RDMA NIC bnxt_re3: /sys/devices/pci0000:10/0000:10:01.1/0000:11:00.0/0000:12:01.0/0000:16:00.0, dev_idx: 3 + RDMA NIC bnxt_re4: /sys/devices/pci0000:f0/0000:f0:01.1/0000:f1:00.0/0000:f2:01.0/0000:f6:00.0, dev_idx: 4 + RDMA NIC bnxt_re5: /sys/devices/pci0000:80/0000:80:01.1/0000:81:00.0/0000:82:01.0/0000:86:00.0, dev_idx: 5 + RDMA NIC bnxt_re7: /sys/devices/pci0000:e0/0000:e0:01.1/0000:e1:00.0/0000:e2:01.0/0000:e6:00.0, dev_idx: 6 + RDMA NIC bnxt_re8: /sys/devices/pci0000:90/0000:90:01.1/0000:91:00.0/0000:92:01.0/0000:96:00.0, dev_idx: 7 +Detected best GPU-NIC mapping: + GPU 0 -> NIC bnxt_re0, dev_idx: 0 + GPU 1 -> NIC bnxt_re1, dev_idx: 1 + GPU 2 -> NIC bnxt_re2, dev_idx: 2 + GPU 3 -> NIC bnxt_re3, dev_idx: 3 + GPU 4 -> NIC bnxt_re4, dev_idx: 4 + GPU 5 -> NIC bnxt_re5, dev_idx: 5 + GPU 6 -> NIC bnxt_re7, dev_idx: 6 + GPU 7 -> NIC bnxt_re8, dev_idx: 7 +Lazy creation of engine, GPU index: 7 +NCCL_IB_GID_INDEX set by environment to 3. +Found IB devices (ibv_get_device_list + NCCL_IB_HCA filter, ordered by libibverbs): + dev_idx 0: bnxt_re0 (1/1) + dev_idx 1: bnxt_re1 (1/1) + dev_idx 2: bnxt_re2 (1/1) + dev_idx 3: bnxt_re3 (1/1) + dev_idx 4: bnxt_re4 (1/1) + dev_idx 5: bnxt_re5 (1/1) + dev_idx 6: bnxt_re7 (1/1) + dev_idx 7: bnxt_re8 (1/1) +Found 8 GPUs (get_gpu_cards, ordered by GPU rank): + GPU 0: /sys/devices/pci0000:70/0000:70:01.1/0000:71:00.0/0000:72:00.0/0000:73:00.0/0000:74:00.0/0000:75:00.0 + GPU 1: /sys/devices/pci0000:00/0000:00:01.1/0000:01:00.0/0000:02:00.0/0000:03:00.0/0000:04:00.0/0000:05:00.0 + GPU 2: /sys/devices/pci0000:60/0000:60:01.1/0000:61:00.0/0000:62:00.0/0000:63:00.0/0000:64:00.0/0000:65:00.0 + GPU 3: /sys/devices/pci0000:10/0000:10:01.1/0000:11:00.0/0000:12:00.0/0000:13:00.0/0000:14:00.0/0000:15:00.0 + GPU 4: /sys/devices/pci0000:f0/0000:f0:01.1/0000:f1:00.0/0000:f2:00.0/0000:f3:00.0/0000:f4:00.0/0000:f5:00.0 + GPU 5: /sys/devices/pci0000:80/0000:80:01.1/0000:81:00.0/0000:82:00.0/0000:83:00.0/0000:84:00.0/0000:85:00.0 + GPU 6: /sys/devices/pci0000:e0/0000:e0:01.1/0000:e1:00.0/0000:e2:00.0/0000:e3:00.0/0000:e4:00.0/0000:e5:00.0 + GPU 7: /sys/devices/pci0000:90/0000:90:01.1/0000:91:00.0/0000:92:00.0/0000:93:00.0/0000:94:00.0/0000:95:00.0 +Found 8 RDMA NICs (get_rdma_nics + NCCL_IB_HCA filter, ordered by dev_idx in rdma_ctl->devices_[]): + RDMA NIC bnxt_re0: /sys/devices/pci0000:70/0000:70:01.1/0000:71:00.0/0000:72:01.0/0000:76:00.0, dev_idx: 0 + RDMA NIC bnxt_re1: /sys/devices/pci0000:00/0000:00:01.1/0000:01:00.0/0000:02:01.0/0000:06:00.0, dev_idx: 1 + RDMA NIC bnxt_re2: /sys/devices/pci0000:60/0000:60:01.1/0000:61:00.0/0000:62:01.0/0000:66:00.0, dev_idx: 2 + RDMA NIC bnxt_re3: /sys/devices/pci0000:10/0000:10:01.1/0000:11:00.0/0000:12:01.0/0000:16:00.0, dev_idx: 3 + RDMA NIC bnxt_re4: /sys/devices/pci0000:f0/0000:f0:01.1/0000:f1:00.0/0000:f2:01.0/0000:f6:00.0, dev_idx: 4 + RDMA NIC bnxt_re5: /sys/devices/pci0000:80/0000:80:01.1/0000:81:00.0/0000:82:01.0/0000:86:00.0, dev_idx: 5 + RDMA NIC bnxt_re7: /sys/devices/pci0000:e0/0000:e0:01.1/0000:e1:00.0/0000:e2:01.0/0000:e6:00.0, dev_idx: 6 + RDMA NIC bnxt_re8: /sys/devices/pci0000:90/0000:90:01.1/0000:91:00.0/0000:92:01.0/0000:96:00.0, dev_idx: 7 +Detected best GPU-NIC mapping: + GPU 0 -> NIC bnxt_re0, dev_idx: 0 + GPU 1 -> NIC bnxt_re1, dev_idx: 1 + GPU 2 -> NIC bnxt_re2, dev_idx: 2 + GPU 3 -> NIC bnxt_re3, dev_idx: 3 + GPU 4 -> NIC bnxt_re4, dev_idx: 4 + GPU 5 -> NIC bnxt_re5, dev_idx: 5 + GPU 6 -> NIC bnxt_re7, dev_idx: 6 + GPU 7 -> NIC bnxt_re8, dev_idx: 7 +Lazy creation of engine, GPU index: 4 +P2P listening on port 37641 +Engine initialized for GPU 6 +UDS socket initialized at /tmp/uccl_gpu_6.sock +Endpoint initialized successfully +P2P listening on port 43815 +Engine initialized for GPU 2 +UDS socket initialized at /tmp/uccl_gpu_2.sock +Endpoint initialized successfully +P2P listening on port 35403 +Engine initialized for GPU 7 +UDS socket initialized at /tmp/uccl_gpu_7.sock +Endpoint initialized successfully +P2P listening on port 43637 +Engine initialized for GPU 4 +UDS socket initialized at /tmp/uccl_gpu_4.sock +Endpoint initialized successfully +NCCL_IB_GID_INDEX set by environment to 3. +Found IB devices (ibv_get_device_list + NCCL_IB_HCA filter, ordered by libibverbs): + dev_idx 0: bnxt_re0 (1/1) + dev_idx 1: bnxt_re1 (1/1) + dev_idx 2: bnxt_re2 (1/1) + dev_idx 3: bnxt_re3 (1/1) + dev_idx 4: bnxt_re4 (1/1) + dev_idx 5: bnxt_re5 (1/1) + dev_idx 6: bnxt_re7 (1/1) + dev_idx 7: bnxt_re8 (1/1) +Found 8 GPUs (get_gpu_cards, ordered by GPU rank): + GPU 0: /sys/devices/pci0000:70/0000:70:01.1/0000:71:00.0/0000:72:00.0/0000:73:00.0/0000:74:00.0/0000:75:00.0 + GPU 1: /sys/devices/pci0000:00/0000:00:01.1/0000:01:00.0/0000:02:00.0/0000:03:00.0/0000:04:00.0/0000:05:00.0 + GPU 2: /sys/devices/pci0000:60/0000:60:01.1/0000:61:00.0/0000:62:00.0/0000:63:00.0/0000:64:00.0/0000:65:00.0 + GPU 3: /sys/devices/pci0000:10/0000:10:01.1/0000:11:00.0/0000:12:00.0/0000:13:00.0/0000:14:00.0/0000:15:00.0 + GPU 4: /sys/devices/pci0000:f0/0000:f0:01.1/0000:f1:00.0/0000:f2:00.0/0000:f3:00.0/0000:f4:00.0/0000:f5:00.0 + GPU 5: /sys/devices/pci0000:80/0000:80:01.1/0000:81:00.0/0000:82:00.0/0000:83:00.0/0000:84:00.0/0000:85:00.0 + GPU 6: /sys/devices/pci0000:e0/0000:e0:01.1/0000:e1:00.0/0000:e2:00.0/0000:e3:00.0/0000:e4:00.0/0000:e5:00.0 + GPU 7: /sys/devices/pci0000:90/0000:90:01.1/0000:91:00.0/0000:92:00.0/0000:93:00.0/0000:94:00.0/0000:95:00.0 +Found 8 RDMA NICs (get_rdma_nics + NCCL_IB_HCA filter, ordered by dev_idx in rdma_ctl->devices_[]): + RDMA NIC bnxt_re0: /sys/devices/pci0000:70/0000:70:01.1/0000:71:00.0/0000:72:01.0/0000:76:00.0, dev_idx: 0 + RDMA NIC bnxt_re1: /sys/devices/pci0000:00/0000:00:01.1/0000:01:00.0/0000:02:01.0/0000:06:00.0, dev_idx: 1 + RDMA NIC bnxt_re2: /sys/devices/pci0000:60/0000:60:01.1/0000:61:00.0/0000:62:01.0/0000:66:00.0, dev_idx: 2 + RDMA NIC bnxt_re3: /sys/devices/pci0000:10/0000:10:01.1/0000:11:00.0/0000:12:01.0/0000:16:00.0, dev_idx: 3 + RDMA NIC bnxt_re4: /sys/devices/pci0000:f0/0000:f0:01.1/0000:f1:00.0/0000:f2:01.0/0000:f6:00.0, dev_idx: 4 + RDMA NIC bnxt_re5: /sys/devices/pci0000:80/0000:80:01.1/0000:81:00.0/0000:82:01.0/0000:86:00.0, dev_idx: 5 + RDMA NIC bnxt_re7: /sys/devices/pci0000:e0/0000:e0:01.1/0000:e1:00.0/0000:e2:01.0/0000:e6:00.0, dev_idx: 6 + RDMA NIC bnxt_re8: /sys/devices/pci0000:90/0000:90:01.1/0000:91:00.0/0000:92:01.0/0000:96:00.0, dev_idx: 7 +Detected best GPU-NIC mapping: + GPU 0 -> NIC bnxt_re0, dev_idx: 0 + GPU 1 -> NIC bnxt_re1, dev_idx: 1 + GPU 2 -> NIC bnxt_re2, dev_idx: 2 + GPU 3 -> NIC bnxt_re3, dev_idx: 3 + GPU 4 -> NIC bnxt_re4, dev_idx: 4 + GPU 5 -> NIC bnxt_re5, dev_idx: 5 + GPU 6 -> NIC bnxt_re7, dev_idx: 6 + GPU 7 -> NIC bnxt_re8, dev_idx: 7 +Lazy creation of engine, GPU index: 0 +NCCL_IB_GID_INDEX set by environment to 3. +Found IB devices (ibv_get_device_list + NCCL_IB_HCA filter, ordered by libibverbs): + dev_idx 0: bnxt_re0 (1/1) + dev_idx 1: bnxt_re1 (1/1) + dev_idx 2: bnxt_re2 (1/1) + dev_idx 3: bnxt_re3 (1/1) + dev_idx 4: bnxt_re4 (1/1) + dev_idx 5: bnxt_re5 (1/1) + dev_idx 6: bnxt_re7 (1/1) + dev_idx 7: bnxt_re8 (1/1) +Found 8 GPUs (get_gpu_cards, ordered by GPU rank): + GPU 0: /sys/devices/pci0000:70/0000:70:01.1/0000:71:00.0/0000:72:00.0/0000:73:00.0/0000:74:00.0/0000:75:00.0 + GPU 1: /sys/devices/pci0000:00/0000:00:01.1/0000:01:00.0/0000:02:00.0/0000:03:00.0/0000:04:00.0/0000:05:00.0 + GPU 2: /sys/devices/pci0000:60/0000:60:01.1/0000:61:00.0/0000:62:00.0/0000:63:00.0/0000:64:00.0/0000:65:00.0 + GPU 3: /sys/devices/pci0000:10/0000:10:01.1/0000:11:00.0/0000:12:00.0/0000:13:00.0/0000:14:00.0/0000:15:00.0 + GPU 4: /sys/devices/pci0000:f0/0000:f0:01.1/0000:f1:00.0/0000:f2:00.0/0000:f3:00.0/0000:f4:00.0/0000:f5:00.0 + GPU 5: /sys/devices/pci0000:80/0000:80:01.1/0000:81:00.0/0000:82:00.0/0000:83:00.0/0000:84:00.0/0000:85:00.0 + GPU 6: /sys/devices/pci0000:e0/0000:e0:01.1/0000:e1:00.0/0000:e2:00.0/0000:e3:00.0/0000:e4:00.0/0000:e5:00.0 + GPU 7: /sys/devices/pci0000:90/0000:90:01.1/0000:91:00.0/0000:92:00.0/0000:93:00.0/0000:94:00.0/0000:95:00.0 +Found 8 RDMA NICs (get_rdma_nics + NCCL_IB_HCA filter, ordered by dev_idx in rdma_ctl->devices_[]): + RDMA NIC bnxt_re0: /sys/devices/pci0000:70/0000:70:01.1/0000:71:00.0/0000:72:01.0/0000:76:00.0, dev_idx: 0 + RDMA NIC bnxt_re1: /sys/devices/pci0000:00/0000:00:01.1/0000:01:00.0/0000:02:01.0/0000:06:00.0, dev_idx: 1 + RDMA NIC bnxt_re2: /sys/devices/pci0000:60/0000:60:01.1/0000:61:00.0/0000:62:01.0/0000:66:00.0, dev_idx: 2 + RDMA NIC bnxt_re3: /sys/devices/pci0000:10/0000:10:01.1/0000:11:00.0/0000:12:01.0/0000:16:00.0, dev_idx: 3 + RDMA NIC bnxt_re4: /sys/devices/pci0000:f0/0000:f0:01.1/0000:f1:00.0/0000:f2:01.0/0000:f6:00.0, dev_idx: 4 + RDMA NIC bnxt_re5: /sys/devices/pci0000:80/0000:80:01.1/0000:81:00.0/0000:82:01.0/0000:86:00.0, dev_idx: 5 + RDMA NIC bnxt_re7: /sys/devices/pci0000:e0/0000:e0:01.1/0000:e1:00.0/0000:e2:01.0/0000:e6:00.0, dev_idx: 6 + RDMA NIC bnxt_re8: /sys/devices/pci0000:90/0000:90:01.1/0000:91:00.0/0000:92:01.0/0000:96:00.0, dev_idx: 7 +Detected best GPU-NIC mapping: + GPU 0 -> NIC bnxt_re0, dev_idx: 0 + GPU 1 -> NIC bnxt_re1, dev_idx: 1 + GPU 2 -> NIC bnxt_re2, dev_idx: 2 + GPU 3 -> NIC bnxt_re3, dev_idx: 3 + GPU 4 -> NIC bnxt_re4, dev_idx: 4 + GPU 5 -> NIC bnxt_re5, dev_idx: 5 + GPU 6 -> NIC bnxt_re7, dev_idx: 6 + GPU 7 -> NIC bnxt_re8, dev_idx: 7 +Lazy creation of engine, GPU index: 5 +P2P listening on port 37555 +Engine initialized for GPU 0 +UDS socket initialized at /tmp/uccl_gpu_0.sock +Endpoint initialized successfully +P2P listening on port 44883 +Engine initialized for GPU 5 +UDS socket initialized at /tmp/uccl_gpu_5.sock +Endpoint initialized successfully +[Rank 6] Rank 0 is local (IP: 10.162.224.133) +[Rank 6] Rank 1 is local (IP: 10.162.224.133) +[Rank 6] Rank 2 is local (IP: 10.162.224.133) +[Rank 6] Rank 3 is local (IP: 10.162.224.133) +[Rank 6] Rank 4 is local (IP: 10.162.224.133) +[Rank 6] Rank 5 is local (IP: 10.162.224.133) +[Rank 6] Rank 7 is local (IP: 10.162.224.133) +[Rank 6] Rank 8 is remote (IP: 10.162.224.132) +[Rank 6] Rank 9 is remote (IP: 10.162.224.132) +[Rank 6] Rank 10 is remote (IP: 10.162.224.132) +[Rank 6] Rank 11 is remote (IP: 10.162.224.132) +[Rank 6] Rank 12 is remote (IP: 10.162.224.132) +[Rank 6] Rank 13 is remote (IP: 10.162.224.132) +[Rank 6] Rank 14 is remote (IP: 10.162.224.132) +[Rank 6] Rank 15 is remote (IP: 10.162.224.132) +[Rank 6] Rank 16 is remote (IP: 10.162.224.129) +[Rank 6] Rank 17 is remote (IP: 10.162.224.129) +[Rank 6] Rank 18 is remote (IP: 10.162.224.129) +[Rank 6] Rank 19 is remote (IP: 10.162.224.129) +[Rank 6] Rank 20 is remote (IP: 10.162.224.129) +[Rank 6] Rank 21 is remote (IP: 10.162.224.129) +[Rank 6] Rank 22 is remote (IP: 10.162.224.129) +[Rank 6] Rank 23 is remote (IP: 10.162.224.129) +Connecting to remote GPU 0 +[Rank 6] Connected locally to rank 0 for sending (conn_id=0)Connecting to remote GPU 1 + +Connecting to remote GPU 2 +Connecting to remote GPU 3 +[Rank 6] Connected locally to rank 1 for sending (conn_id=1) +Connecting to remote GPU 4 +Connecting to remote GPU Connecting to remote GPU 7 +5 +Attempting to connect to 10.162.224.132:0 via port 43233 +Attempting to connect to 10.162.224.132:1 via port 44619 +Attempting to connect to 10.162.224.132:2 via port 38595 +Attempting to connect to 10.162.224.132:3 via port 37259 +Attempting to connect to 10.162.224.132:4 via port 45953 +Attempting to connect to 10.162.224.132:5 via port 40167 +Attempting to connect to 10.162.224.132:6 via port 38657 +Attempting to connect to 10.162.224.132:7 via port 42115 +Attempting to connect to 10.162.224.129:0 via port 39565 +Attempting to connect to 10.162.224.129:1 via port 37401 +Attempting to connect to 10.162.224.129:2 via port 35855 +Attempting to connect to 10.162.224.129:3 via port 46353 +[Rank 4] Rank 0 is local (IP: 10.162.224.133)Attempting to connect to +10.162.224.129:[Rank 4] Rank 1 is local (IP: 10.162.224.133)4 + via port [Rank 4] Rank 2 is local (IP: 10.162.224.133)46687 + +[Rank 4] Rank 3 is local (IP: 10.162.224.133) +[Rank 4] Rank 5 is local (IP: 10.162.224.133) +[Rank 4] Rank 6 is local (IP: 10.162.224.133) +[Rank 4] Rank 7 is local (IP: 10.162.224.133) +Attempting to connect to [Rank 4] Rank 8 is remote (IP: 10.162.224.132)10.162.224.129 +:[Rank 4] Rank 9 is remote (IP: 10.162.224.132)5 + via port [Rank 4] Rank 10 is remote (IP: 10.162.224.132)34053 + +[Rank 4] Rank 11 is remote (IP: 10.162.224.132) +Attempting to connect to [Rank 4] Rank 12 is remote (IP: 10.162.224.132)10.162.224.129 +[Rank 4] Rank 13 is remote (IP: 10.162.224.132): +6[Rank 4] Rank 14 is remote (IP: 10.162.224.132) via port +35349[Rank 4] Rank 15 is remote (IP: 10.162.224.132) + +[Rank 4] Rank 16 is remote (IP: 10.162.224.129)Attempting to connect to +10.162.224.129[Rank 4] Rank 17 is remote (IP: 10.162.224.129): +Waiting to accept UDS connection[Rank 4] Rank 18 is remote (IP: 10.162.224.129) + +[Rank 4] Rank 19 is remote (IP: 10.162.224.129) +[Rank 4] Rank 20 is remote (IP: 10.162.224.129) +7[Rank 4] Rank 21 is remote (IP: 10.162.224.129) via port +Waiting to accept UDS connection[Rank 4] Rank 22 is remote (IP: 10.162.224.129) + +Waiting to accept UDS connection[Rank 4] Rank 23 is remote (IP: 10.162.224.129) + +39309 +Waiting to accept UDS connectionWaiting to accept UDS connection + +Waiting to accept UDS connectionConnecting to remote GPU +Connecting to remote GPU Waiting to accept UDS connection0 + +1 +Waiting to accept incoming connection...[Rank 4] Connected locally to rank 0 for sending (conn_id=0) +[Rank 4] Connected locally to rank 1 for sending (conn_id=1)Waiting to accept incoming connection... +Waiting to accept incoming connection... + +Connecting to remote GPU Waiting to accept incoming connection... +Connecting to remote GPU +32 + +Waiting to accept incoming connection... +Waiting to accept incoming connection... +Connecting to remote GPU 5 +Connecting to remote GPU 6 +Waiting to accept incoming connection...Connecting to remote GPU +7 +Attempting to connect to 10.162.224.132:0 via port 43233 +Waiting to accept incoming connection... +Waiting to accept incoming connection... +Attempting to connect to 10.162.224.132:1 via port 44619 +Waiting to accept incoming connection... +Waiting to accept incoming connection... +Waiting to accept incoming connection...Attempting to connect to +10.162.224.132:2 via port 38595 +Attempting to connect to 10.162.224.132Waiting to accept incoming connection...Attempting to connect to +10.162.224.132::4 via port 3 via port 37259 +45953 +Waiting to accept incoming connection...Attempting to connect to +10.162.224.132Waiting to accept incoming connection...: +5 via port Attempting to connect to 10.162.224.132:Waiting to accept incoming connection... +6 via port 38657 +40167 +Attempting to connect to 10.162.224.132:7 via port 42115 +Attempting to connect to 10.162.224.129:0 via port 39565 +Attempting to connect to 10.162.224.129:1 via port 37401 +Attempting to connect to 10.162.224.129:2 via port 35855 +Attempting to connect to 10.162.224.129:3 via port 46353 +Attempting to connect to 10.162.224.129:4 via port 46687 +Attempting to connect to 10.162.224.129:5 via port 34053 +Attempting to connect to 10.162.224.129:6 via port 35349 +Attempting to connect to 10.162.224.129:7 via port 39309 +Waiting to accept UDS connection +Waiting to accept UDS connection +Waiting to accept UDS connection +Waiting to accept UDS connection +Waiting to accept UDS connection +Waiting to accept UDS connection +Waiting to accept UDS connection +Waiting to accept incoming connection... +Waiting to accept incoming connection... +Waiting to accept incoming connection... +Waiting to accept incoming connection... +Waiting to accept incoming connection... +Waiting to accept incoming connection... +Waiting to accept incoming connection... +Waiting to accept incoming connection... +Waiting to accept incoming connection... +Waiting to accept incoming connection... +Waiting to accept incoming connection... +Waiting to accept incoming connection... +Waiting to accept incoming connection... +Waiting to accept incoming connection... +Waiting to accept incoming connection... +Waiting to accept incoming connection... +[Rank 1] Rank 0 is local (IP: 10.162.224.133) +[Rank 1] Rank 2 is local (IP: 10.162.224.133) +[Rank 1] Rank 3 is local (IP: 10.162.224.133) +[Rank 1] Rank 4 is local (IP: 10.162.224.133) +[Rank 1] Rank 5 is local (IP: 10.162.224.133) +[Rank 1] Rank 6 is local (IP: 10.162.224.133) +[Rank 1] Rank 7 is local (IP: 10.162.224.133) +[Rank 1] Rank 8 is remote (IP: 10.162.224.132) +[Rank 1] Rank 9 is remote (IP: 10.162.224.132) +[Rank 1] Rank 10 is remote (IP: 10.162.224.132) +[Rank 1] Rank 11 is remote (IP: 10.162.224.132) +[Rank 1] Rank 12 is remote (IP: 10.162.224.132) +[Rank 1] Rank 13 is remote (IP: 10.162.224.132) +[Rank 1] Rank 14 is remote (IP: 10.162.224.132) +[Rank 1] Rank 15 is remote (IP: 10.162.224.132) +[Rank 1] Rank 16 is remote (IP: 10.162.224.129) +[Rank 1] Rank 17 is remote (IP: 10.162.224.129) +[Rank 1] Rank 18 is remote (IP: 10.162.224.129) +[Rank 1] Rank 19 is remote (IP: 10.162.224.129) +[Rank 1] Rank 20 is remote (IP: 10.162.224.129) +[Rank 1] Rank 21 is remote (IP: 10.162.224.129) +[Rank 1] Rank 22 is remote (IP: 10.162.224.129) +[Rank 1] Rank 23 is remote (IP: 10.162.224.129) +Connecting to remote GPU 0 +[Rank 1] Connected locally to rank 0 for sending (conn_id=0)Connecting to remote GPU +2 +Connecting to remote GPU 3[Rank 1] Connected locally to rank 2 for sending (conn_id=1) +[Rank 0] Rank 1 is local (IP: 10.162.224.133) +[Rank 0] Rank 2 is local (IP: 10.162.224.133) +[Rank 0] Rank 3 is local (IP: 10.162.224.133) +[Rank 0] Rank 4 is local (IP: 10.162.224.133) +[Rank 0] Rank 5 is local (IP: 10.162.224.133) +[Rank 0] Rank 6 is local (IP: 10.162.224.133) +[Rank 0] Rank 7 is local (IP: 10.162.224.133) +[Rank 0] Rank 8 is remote (IP: 10.162.224.132) +[Rank 0] Rank 9 is remote (IP: 10.162.224.132) +[Rank 0] Rank 10 is remote (IP: 10.162.224.132) +[Rank 0] Rank 11 is remote (IP: 10.162.224.132) + +[Rank 0] Rank 12 is remote (IP: 10.162.224.132) +[Rank 0] Rank 13 is remote (IP: 10.162.224.132) +[Rank 0] Rank 14 is remote (IP: 10.162.224.132) +[Rank 0] Rank 15 is remote (IP: 10.162.224.132) +[Rank 0] Rank 16 is remote (IP: 10.162.224.129) +[Rank 0] Rank 17 is remote (IP: 10.162.224.129) +[Rank 0] Rank 18 is remote (IP: 10.162.224.129) +[Rank 0] Rank 19 is remote (IP: 10.162.224.129) +[Rank 0] Rank 20 is remote (IP: 10.162.224.129) +[Rank 0] Rank 21 is remote (IP: 10.162.224.129) +Connecting to remote GPU [Rank 0] Rank 22 is remote (IP: 10.162.224.129) +4[Rank 0] Rank 23 is remote (IP: 10.162.224.129) + +Connecting to remote GPU 5 +Connecting to remote GPU Connecting to remote GPU Connecting to remote GPU 62 + +1 +[Rank 0] Connected locally to rank 2 for sending (conn_id=0)[Rank 0] Connected locally to rank 1 for sending (conn_id=1)Connecting to remote GPU 3 +Connecting to remote GPU + +Connecting to remote GPU 4 +7 +Attempting to connect to 10.162.224.132[Rank 0] Connected locally to rank 3 for sending (conn_id=2):Connecting to remote GPU Attempting to connect to 510.162.224.132 +[Rank 0] Connected locally to rank 4 for sending (conn_id=3):Connecting to remote GPU 1Connecting to remote GPU via port +744619[Rank 0] Connected locally to rank 5 for sending (conn_id=4) + + +0Attempting to connect to 10.162.224.132:1Attempting to connect to 10.162.224.132:5 via port via port +40167 +[Rank 0] Connected locally to rank 7 for sending (conn_id=5) via port Attempting to connect to 44619[Rank 3] Rank 0 is local (IP: 10.162.224.133)10.162.224.132 + + +Attempting to connect to :[Rank 3] Rank 1 is local (IP: 10.162.224.133)10.162.224.1322 +: via port [Rank 3] Rank 2 is local (IP: 10.162.224.133)438595 +[Rank 3] Rank 4 is local (IP: 10.162.224.133) via port + +45953[Rank 3] Rank 5 is local (IP: 10.162.224.133) + +Attempting to connect to [Rank 3] Rank 6 is local (IP: 10.162.224.133) +[Rank 3] Rank 7 is local (IP: 10.162.224.133)Attempting to connect to +10.162.224.13210.162.224.132[Rank 3] Rank 8 is remote (IP: 10.162.224.132):: +35[Rank 3] Rank 9 is remote (IP: 10.162.224.132) via port + via port 3725940167 + +[Rank 3] Rank 10 is remote (IP: 10.162.224.132) +Attempting to connect to [Rank 3] Rank 11 is remote (IP: 10.162.224.132)Attempting to connect to 10.162.224.12910.162.224.132 +::3[Rank 3] Rank 12 is remote (IP: 10.162.224.132) via port 2 +37259 via port [Rank 3] Rank 13 is remote (IP: 10.162.224.132) +35855 +Attempting to connect to +10.162.224.129Attempting to connect to [Rank 3] Rank 14 is remote (IP: 10.162.224.132):10.162.224.132[Rank 5] Rank 0 is local (IP: 10.162.224.133) +1[Rank 3] Rank 15 is remote (IP: 10.162.224.132): via port +7 +37401[Rank 3] Rank 16 is remote (IP: 10.162.224.129) via port +[Rank 5] Rank 1 is local (IP: 10.162.224.133) +[Rank 5] Rank 2 is local (IP: 10.162.224.133) + +[Rank 5] Rank 3 is local (IP: 10.162.224.133)42115[Rank 3] Rank 17 is remote (IP: 10.162.224.129) +Attempting to connect to +10.162.224.132 +Attempting to connect to [Rank 5] Rank 4 is local (IP: 10.162.224.133):2[Rank 3] Rank 18 is remote (IP: 10.162.224.129) via port 10.162.224.129 +38595 +:[Rank 3] Rank 19 is remote (IP: 10.162.224.129) +[Rank 5] Rank 6 is local (IP: 10.162.224.133)2 + via port +[Rank 3] Rank 20 is remote (IP: 10.162.224.129)Attempting to connect to 35855[Rank 5] Rank 7 is local (IP: 10.162.224.133) +10.162.224.129 + +[Rank 3] Rank 21 is remote (IP: 10.162.224.129) +:[Rank 3] Rank 22 is remote (IP: 10.162.224.129) +3[Rank 3] Rank 23 is remote (IP: 10.162.224.129) via port +46353Attempting to connect to +[Rank 5] Rank 8 is remote (IP: 10.162.224.132) +[Rank 5] Rank 9 is remote (IP: 10.162.224.132)[Rank 2] Rank 0 is local (IP: 10.162.224.133) + +[Rank 5] Rank 10 is remote (IP: 10.162.224.132) +[Rank 2] Rank 1 is local (IP: 10.162.224.133)[Rank 5] Rank 11 is remote (IP: 10.162.224.132) + +10.162.224.129[Rank 2] Rank 3 is local (IP: 10.162.224.133)[Rank 5] Rank 12 is remote (IP: 10.162.224.132)6 +: +0 +[Rank 5] Rank 13 is remote (IP: 10.162.224.132)[Rank 2] Rank 4 is local (IP: 10.162.224.133) via port +Attempting to connect to +39565[Rank 5] Rank 14 is remote (IP: 10.162.224.132)10.162.224.132:6Waiting to accept UDS connection +[Rank 2] Rank 5 is local (IP: 10.162.224.133)Waiting to accept UDS connection + + + +[Rank 5] Rank 15 is remote (IP: 10.162.224.132)[Rank 2] Rank 6 is local (IP: 10.162.224.133)Connecting to remote GPU +Waiting to accept UDS connection +[Rank 0] Connected locally to rank 6 for sending (conn_id=14)Attempting to connect to [Rank 5] Rank 16 is remote (IP: 10.162.224.129) +[Rank 5] Rank 17 is remote (IP: 10.162.224.129) +0[Rank 5] Rank 18 is remote (IP: 10.162.224.129) + + +[Rank 0] Accepted local connection from rank 6 (GPU 6) for receiving (conn_id=15)[Rank 5] Rank 19 is remote (IP: 10.162.224.129)[Rank 2] Rank 7 is local (IP: 10.162.224.133)Attempting to connect to Connecting to remote GPU + +Waiting to accept UDS connection + +10.162.224.129Attempting to connect to [Rank 5] Rank 20 is remote (IP: 10.162.224.129)10.162.224.1292 +:[Rank 0] Accepted local connection from rank 4 (GPU 4) for receiving (conn_id=16) + +[Rank 5] Rank 21 is remote (IP: 10.162.224.129)[Rank 2] Rank 8 is remote (IP: 10.162.224.132)6:4[Rank 3] Connected locally to rank 0 for sending (conn_id=0) via port Connecting to remote GPU 46687 + + +[Rank 2] Rank 9 is remote (IP: 10.162.224.132) via port +35349 +[Rank 2] Rank 10 is remote (IP: 10.162.224.132) +1Waiting to accept incoming connection... + +[Rank 5] Rank 22 is remote (IP: 10.162.224.129) + +Waiting to accept incoming connection...10.162.224.129[Rank 2] Rank 11 is remote (IP: 10.162.224.132) + +Connecting to remote GPU : +[Rank 2] Rank 12 is remote (IP: 10.162.224.132)[Rank 5] Rank 23 is remote (IP: 10.162.224.129) +Connecting to remote GPU +[Rank 2] Rank 13 is remote (IP: 10.162.224.132)46 + via port +[Rank 2] Rank 14 is remote (IP: 10.162.224.132)Waiting to accept incoming connection...Waiting to accept incoming connection... +[Rank 3] Connected locally to rank 1 for sending (conn_id=1) +Attempting to connect to 10.162.224.129Connecting to remote GPU : + +[Rank 2] Rank 15 is remote (IP: 10.162.224.132)Waiting to accept UDS connectionAttempting to connect to + +7Connecting to remote GPU +Attempting to connect to Connecting to remote GPU [Rank 2] Rank 16 is remote (IP: 10.162.224.129)10.162.224.13210.162.224.132Attempting to connect to [Rank 0] Accepted local connection from rank 1 (GPU 1) for receiving (conn_id=20)Connecting to remote GPU + +2:[Rank 2] Rank 17 is remote (IP: 10.162.224.129) +Waiting to accept incoming connection...7 +0 +10.162.224.132Waiting to accept incoming connection... via port :[Rank 2] Rank 18 is remote (IP: 10.162.224.129) + +42115[Rank 5] Connected locally to rank 2 for sending (conn_id=0)1 +Waiting to accept incoming connection...1 + +Attempting to connect to 10.162.224.129: via port 0[Rank 2] Rank 19 is remote (IP: 10.162.224.129)44619[Rank 5] Connected locally to rank 0 for sending (conn_id=1) + via port +395654 + +Waiting to accept UDS connection[Rank 2] Rank 20 is remote (IP: 10.162.224.129)Connecting to remote GPU + + + +4 +Waiting to accept incoming connection...[Rank 2] Rank 21 is remote (IP: 10.162.224.129) +Attempting to connect to [Rank 2] Rank 22 is remote (IP: 10.162.224.129) +10.162.224.132 +Connecting to remote GPU :Attempting to connect to [Rank 2] Rank 23 is remote (IP: 10.162.224.129)7 +6Waiting to accept incoming connection... + via port +Attempting to connect to 3865710.162.224.129Waiting to accept UDS connection + +10.162.224.129Attempting to connect to :: +10.162.224.12915Waiting to accept incoming connection...: via port 37401 via port + +340532Attempting to connect to [Rank 0] Accepted local connection from rank 3 (GPU 3) for receiving (conn_id=25) +Connecting to remote GPU via port 10.162.224.132Waiting to accept incoming connection... + +Waiting to accept incoming connection...0 +Waiting to accept incoming connection...35855: + +5Attempting to connect to 43233Connecting to remote GPU via port 10.162.224.129 + +Waiting to accept incoming connection... + via port Waiting to accept incoming connection...338657 +40167 +:Attempting to connect to Waiting to accept UDS connection[Rank 2] Connected locally to rank 0 for sending (conn_id=0)10.162.224.129 +:6 via port Attempting to connect to 3534910.162.224.132Connecting to remote GPU +::4 + via port Attempting to connect to Waiting to accept incoming connection...4595310.162.224.1294 +5 + +: +Waiting to accept incoming connection... via port Waiting to accept incoming connection...401672 + + + + via port Connecting to remote GPU Attempting to connect to [Rank 7] Rank 0 is local (IP: 10.162.224.133)35855[Rank 2] Connected locally to rank 4 for sending (conn_id=1)Connecting to remote GPU +65Waiting to accept UDS connection +10.162.224.129Attempting to connect to +:[Rank 2] Connected locally to rank 3 for sending (conn_id=2)10.162.224.132 +[Rank 7] Rank 1 is local (IP: 10.162.224.133) +7: + +Attempting to connect to Attempting to connect to [Rank 7] Rank 2 is local (IP: 10.162.224.133)10.162.224.129Waiting to accept incoming connection... +10.162.224.132 via port :[Rank 2] Connected locally to rank 6 for sending (conn_id=3)Attempting to connect to +10.162.224.132:[Rank 0] Accepted local connection from rank 5 (GPU 5) for receiving (conn_id=33) +04 + + via port [Rank 7] Rank 3 is local (IP: 10.162.224.133) +[Rank 7] Rank 4 is local (IP: 10.162.224.133) +Attempting to connect to [Rank 7] Rank 5 is local (IP: 10.162.224.133)10.162.224.13239309 + via port :Waiting to accept incoming connection... +43233Attempting to connect to + +10.162.224.129[Rank 7] Rank 6 is local (IP: 10.162.224.133)Attempting to connect to 45953:1 + via port 10.162.224.13246687 + +5Attempting to connect to [Rank 7] Rank 8 is remote (IP: 10.162.224.132)Waiting to accept UDS connection44619 via port 10.162.224.132: +4 +[Rank 7] Rank 9 is remote (IP: 10.162.224.132) via port + +340531:45953[Rank 7] Rank 10 is remote (IP: 10.162.224.132)Connecting to remote GPU +6 via port +2Waiting to accept incoming connection... +37401 +Waiting to accept UDS connection via port +Attempting to connect to 10.162.224.129 +:[Rank 7] Rank 11 is remote (IP: 10.162.224.132)Attempting to connect to Waiting to accept incoming connection...0 + +10.162.224.132 +38595 via port [Rank 7] Rank 12 is remote (IP: 10.162.224.132) +Attempting to connect to [Rank 7] Rank 13 is remote (IP: 10.162.224.132)10.162.224.132: +:Attempting to connect to 6[Rank 7] Rank 14 is remote (IP: 10.162.224.132) +43956510.162.224.132: via port Waiting to accept incoming connection... +38657Waiting to accept UDS connectionAttempting to connect to via port [Rank 7] Rank 15 is remote (IP: 10.162.224.132)45953 + + +Waiting to accept UDS connection10.162.224.129Attempting to connect to Attempting to connect to + + +:Waiting to accept UDS connection710.162.224.132 + via port 10.162.224.132:Waiting to accept UDS connection39309::[Rank 7] Rank 16 is remote (IP: 10.162.224.129)57 + via port 0 via port 40167 +[Rank 7] Rank 17 is remote (IP: 10.162.224.129) via port +42115 + +Waiting to accept incoming connection...Waiting to accept incoming connection... +Waiting to accept incoming connection...43233 +Waiting to accept incoming connection... +[Rank 7] Rank 18 is remote (IP: 10.162.224.129) + +Attempting to connect to +Waiting to accept UDS connection10.162.224.1290Attempting to connect to via port 10.162.224.132 +:43233[Rank 0] Accepted local connection from rank 2 (GPU 2) for receiving (conn_id=41)[Rank 7] Rank 19 is remote (IP: 10.162.224.129)Waiting to accept incoming connection... + +Waiting to accept incoming connection...:Attempting to connect to 2 + + + + +Waiting to accept incoming connection...3[Rank 7] Rank 20 is remote (IP: 10.162.224.129)10.162.224.129 + via port Waiting to accept incoming connection... + via port 38595: +463534Waiting to accept UDS connectionWaiting to accept UDS connection + + via port [Rank 7] Rank 21 is remote (IP: 10.162.224.129) + +[Rank 7] Rank 22 is remote (IP: 10.162.224.129) + +Attempting to connect to Attempting to connect to [Rank 7] Rank 23 is remote (IP: 10.162.224.129)10.162.224.129Waiting to accept UDS connection10.162.224.13246687: + +:1 via port [Rank 2] Accepted local connection from rank 1 (GPU 1) for receiving (conn_id=9)6Waiting to accept UDS connection +44619 via port 35349 + + +Attempting to connect to Waiting to accept incoming connection...Waiting to accept incoming connection...Attempting to connect to +10.162.224.132 + +Waiting to accept UDS connection +Connecting to remote GPU Attempting to connect to 10.162.224.1323:10.162.224.129Connecting to remote GPU 0 +Connecting to remote GPU :2 + +73: via port 37259Waiting to accept incoming connection...3Connecting to remote GPU + + via port via port 1Waiting to accept incoming connection...[Rank 0] Accepted local connection from rank 7 (GPU 7) for receiving (conn_id=45)Waiting to accept UDS connection[Rank 7] Connected locally to rank 2 for sending (conn_id=0)[Rank 7] Connected locally to rank 0 for sending (conn_id=1)4211546353 + + + + +Attempting to connect to +Connecting to remote GPU +Waiting to accept incoming connection...10.162.224.129 +3Waiting to accept UDS connection +:Waiting to accept incoming connection... + +1 via port 37401 +Connecting to remote GPU Waiting to accept UDS connection4 + + +Attempting to connect to Waiting to accept incoming connection...Attempting to connect to Attempting to connect to 10.162.224.129 +Waiting to accept incoming connection...10.162.224.13210.162.224.132:Waiting to accept incoming connection... +::0 +370 via port via port via port via port 39565 +37259Attempting to connect to 3930943233 + + +10.162.224.132Attempting to connect to Waiting to accept incoming connection...Connecting to remote GPU :10.162.224.129 +62:Attempting to connect to + via port 310.162.224.12938595Attempting to connect to via port : +10.162.224.132463537:Waiting to accept incoming connection... + via port 2 via port +38595Waiting to accept incoming connection...Waiting to accept incoming connection... + + +Attempting to connect to 39309Waiting to accept UDS connection10.162.224.132Waiting to accept incoming connection... + +: +Waiting to accept UDS connection3Attempting to connect to +Waiting to accept incoming connection... via port 10.162.224.132Waiting to accept UDS connection +37259:Waiting to accept incoming connection... + +Connecting to remote GPU 01 +Attempting to connect to [Rank 2] Accepted local connection from rank 0 (GPU 0) for receiving (conn_id=16)[Rank 2] Accepted local connection from rank 5 (GPU 5) for receiving (conn_id=17) via port +Waiting to accept UDS connection10.162.224.132 + +43233Waiting to accept incoming connection... +: + +Waiting to accept incoming connection...7Attempting to connect to Waiting to accept incoming connection... +[Rank 2] Connected locally to rank 1 for sending (conn_id=18) via port 10.162.224.129 +Waiting to accept incoming connection... +42115 +:Attempting to connect to Waiting to accept incoming connection...410.162.224.129 via port +: +46687Waiting to accept incoming connection...1 + via port +Attempting to connect to Attempting to connect to 10.162.224.1293740110.162.224.132:Waiting to accept incoming connection... +:5 +7 via port Waiting to accept incoming connection... via port Attempting to connect to 34053 +4211510.162.224.132 +:Attempting to connect to +Waiting to accept incoming connection...410.162.224.132Waiting to accept UDS connection + via port Waiting to accept incoming connection...45953: + + +6Connecting to remote GPU Attempting to connect to Waiting to accept incoming connection... via port 510.162.224.129 +38657 +:Waiting to accept UDS connection +Waiting to accept incoming connection...5 + +Waiting to accept incoming connection... via port Waiting to accept UDS connection[Rank 2] Connected locally to rank 5 for sending (conn_id=26)Waiting to accept incoming connection... +34053 + + +Waiting to accept incoming connection... +Waiting to accept UDS connectionConnecting to remote GPU + +5Waiting to accept incoming connection...Waiting to accept incoming connection... +Waiting to accept incoming connection...[Rank 2] Accepted local connection from rank 7 (GPU 7) for receiving (conn_id=27) + + +Attempting to connect to Attempting to connect to Attempting to connect to +10.162.224.13210.162.224.132Waiting to accept incoming connection...10.162.224.129:: +:46 via port 3 via port Attempting to connect to 10.162.224.129 via port :4668737259386576 + + +Waiting to accept incoming connection...Waiting to accept incoming connection... via port +Attempting to connect to +35349Waiting to accept incoming connection...10.162.224.129 +Attempting to connect to +:Attempting to connect to 10.162.224.129310.162.224.129: via port Waiting to accept incoming connection...:746353 + +Attempting to connect to 0Attempting to connect to via port 10.162.224.129 via port 10.162.224.12939309:39565: +Waiting to accept incoming connection... +3Attempting to connect to + via port Waiting to accept incoming connection...10.162.224.129Waiting to accept UDS connection46353 +: + +55Waiting to accept incoming connection...Connecting to remote GPU via port via port +340537 +Attempting to connect to 34053 +Waiting to accept UDS connection10.162.224.129 +Attempting to connect to +:Attempting to connect to 10.162.224.132[Rank 2] Connected locally to rank 7 for sending (conn_id=35)Waiting to accept incoming connection...610.162.224.129: + + via port Waiting to accept incoming connection...:635349 +1 via port 37401 via port +38657 + +Waiting to accept incoming connection...Attempting to connect to Attempting to connect to +10.162.224.129Waiting to accept incoming connection...10.162.224.129: +:Waiting to accept incoming connection...26 + via port via port 3585535349 + +Waiting to accept UDS connectionAttempting to connect to +10.162.224.129:2 via port 35855 +Waiting to accept incoming connection...Waiting to accept incoming connection... + +Waiting to accept incoming connection...Waiting to accept incoming connection... + +Waiting to accept incoming connection...Waiting to accept UDS connection + +Waiting to accept incoming connection...Waiting to accept incoming connection... + +Waiting to accept UDS connectionWaiting to accept UDS connection + +Waiting to accept UDS connectionWaiting to accept incoming connection... + +Waiting to accept incoming connection... +Waiting to accept incoming connection... +Waiting to accept incoming connection... +Waiting to accept UDS connection +Attempting to connect to 10.162.224.129:4 via port 46687 +Attempting to connect to 10.162.224.132:Waiting to accept incoming connection... +Attempting to connect to 10.162.224.129:7 via port Attempting to connect to 10.162.224.132:5Waiting to accept incoming connection... +0Waiting to accept incoming connection... +Waiting to accept incoming connection... +Waiting to accept incoming connection... +1 via port 44619 +Waiting to accept UDS connection + via port 40167 +Waiting to accept UDS connection +Waiting to accept incoming connection... +39309 + via port 39565 +Waiting to accept incoming connection... +[Rank 4] Connected locally to rank 2 for sending (conn_id=34)[Rank 2] Accepted local connection from rank 4 (GPU 4) for receiving (conn_id=43) +[Rank 4] Accepted local connection from rank 2 (GPU 2) for receiving (conn_id=37)[Rank 4] Connected locally to rank 6 for sending (conn_id=36) + +[Rank 4] Connected locally to rank 7 for sending (conn_id=39) +[Rank 4] Connected locally to rank 5 for sending (conn_id=40)[Rank 4] Accepted local connection from rank 0 (GPU 0) for receiving (conn_id=35) +[Rank 4] Connected locally to rank 3 for sending (conn_id=38) + + +[Rank 1] Connected locally to rank 3 for sending (conn_id=34)[Rank 4] Accepted local connection from rank 1 (GPU 1) for receiving (conn_id=41) +[Rank 1] Accepted local connection from rank 6 (GPU 6) for receiving (conn_id=37)[Rank 1] Accepted local connection from rank 4 (GPU 4) for receiving (conn_id=35) + +[Rank 1] Accepted local connection from rank 0 (GPU 0) for receiving (conn_id=38) +[Rank 1] Accepted local connection from rank 3 (GPU 3) for receiving (conn_id=36) +[Rank 1] Connected locally to rank 7 for sending (conn_id=41)[Rank 1] Connected locally to rank 5 for sending (conn_id=43) +[Rank 1] Connected locally to rank 6 for sending (conn_id=39) +[Rank 1] Accepted local connection from rank 2 (GPU 2) for receiving (conn_id=40)[Rank 1] Connected locally to rank 4 for sending (conn_id=42) + + + +[Rank 6] Connected locally to rank 3 for sending (conn_id=34)[Rank 2] Accepted local connection from rank 6 (GPU 6) for receiving (conn_id=44)[Rank 6] Accepted local connection from rank 2 (GPU 2) for receiving (conn_id=36) +[Rank 4] Accepted local connection from rank 6 (GPU 6) for receiving (conn_id=42) + +[Rank 6] Accepted local connection from rank 4 (GPU 4) for receiving (conn_id=40)[Rank 6] Accepted local connection from rank 1 (GPU 1) for receiving (conn_id=41)[Rank 6] Accepted local connection from rank 0 (GPU 0) for receiving (conn_id=35) +[Rank 6] Connected locally to rank 5 for sending (conn_id=42) + + +[Rank 6] Connected locally to rank 2 for sending (conn_id=39)[Rank 6] Connected locally to rank 7 for sending (conn_id=38)[Rank 6] Connected locally to rank 4 for sending (conn_id=37) + + + +[Rank 7] Connected locally to rank 1 for sending (conn_id=34)[Rank 5] Connected locally to rank 1 for sending (conn_id=34)[Rank 5] Accepted local connection from rank 1 (GPU 1) for receiving (conn_id=38)[Rank 7] Accepted local connection from rank 0 (GPU 0) for receiving (conn_id=35)[Rank 3] Connected locally to rank 6 for sending (conn_id=34)[Rank 6] Accepted local connection from rank 3 (GPU 3) for receiving (conn_id=43)[Rank 5] Accepted local connection from rank 2 (GPU 2) for receiving (conn_id=36)[Rank 7] Accepted local connection from rank 3 (GPU 3) for receiving (conn_id=44)[Rank 1] Accepted local connection from rank 5 (GPU 5) for receiving (conn_id=44) +[Rank 4] Accepted local connection from rank 7 (GPU 7) for receiving (conn_id=43)[Rank 3] Accepted local connection from rank 7 (GPU 7) for receiving (conn_id=43)[Rank 3] Accepted local connection from rank 4 (GPU 4) for receiving (conn_id=37)[Rank 3] Connected locally to rank 4 for sending (conn_id=40)[Rank 7] Connected locally to rank 3 for sending (conn_id=40)[Rank 3] Connected locally to rank 7 for sending (conn_id=42)[Rank 5] Accepted local connection from rank 7 (GPU 7) for receiving (conn_id=43) +[Rank 6] Accepted local connection from rank 5 (GPU 5) for receiving (conn_id=44) +[Rank 1] Accepted local connection from rank 7 (GPU 7) for receiving (conn_id=45) +[Rank 7] Accepted local connection from rank 5 (GPU 5) for receiving (conn_id=45)[Rank 4] Accepted local connection from rank 3 (GPU 3) for receiving (conn_id=45) +[Rank 3] Connected locally to rank 5 for sending (conn_id=45)[Rank 5] Accepted local connection from rank 0 (GPU 0) for receiving (conn_id=35) +[Rank 6] Accepted local connection from rank 7 (GPU 7) for receiving (conn_id=45) + + +[Rank 7] Accepted local connection from rank 4 (GPU 4) for receiving (conn_id=37)[Rank 5] Accepted local connection from rank 6 (GPU 6) for receiving (conn_id=39) +[Rank 7] Accepted local connection from rank 1 (GPU 1) for receiving (conn_id=38) +[Rank 2] Accepted local connection from rank 3 (GPU 3) for receiving (conn_id=45)[Rank 5] Connected locally to rank 3 for sending (conn_id=40) +[Rank 3] Accepted local connection from rank 6 (GPU 6) for receiving (conn_id=39) + + +[Rank 4] Accepted local connection from rank 5 (GPU 5) for receiving (conn_id=44) + +[Rank 5] Connected locally to rank 6 for sending (conn_id=41) +[Rank 7] Accepted local connection from rank 6 (GPU 6) for receiving (conn_id=39) + + +[Rank 7] Connected locally to rank 6 for sending (conn_id=41)[Rank 5] Connected locally to rank 4 for sending (conn_id=42) + + +[Rank 3] Accepted local connection from rank 0 (GPU 0) for receiving (conn_id=35)[Rank 7] Connected locally to rank 5 for sending (conn_id=42) +[Rank 5] Accepted local connection from rank 4 (GPU 4) for receiving (conn_id=37) +[Rank 7] Connected locally to rank 4 for sending (conn_id=43) +[Rank 5] Accepted local connection from rank 3 (GPU 3) for receiving (conn_id=44)[Rank 3] Accepted local connection from rank 5 (GPU 5) for receiving (conn_id=41) +[Rank 5] Connected locally to rank 7 for sending (conn_id=45) +[Rank 7] Accepted local connection from rank 2 (GPU 2) for receiving (conn_id=36) + + + +[Rank 3] Connected locally to rank 2 for sending (conn_id=44) + +[Rank 3] Accepted local connection from rank 2 (GPU 2) for receiving (conn_id=38) + +[Rank 3] Accepted local connection from rank 1 (GPU 1) for receiving (conn_id=36) + + + + + + + +[Rank 6] Connected remotely to rank 9 for sending (conn_id=3) +[Rank 6] Connected remotely to rank 12 for sending (conn_id=6) +[Rank 6] Connected remotely to rank 17 for sending (conn_id=11) +[Rank 6] Connected remotely to rank 18 for sending (conn_id=12) +[Rank 6] Connected remotely to rank 19 for sending (conn_id=13) +[Rank 6] Connected remotely to rank 20 for sending (conn_id=14) +[Rank 6] Connected remotely to rank 22 for sending (conn_id=16) +[Rank 6] Connected remotely to rank 23 for sending (conn_id=17) +[Rank 4] Connected remotely to rank 9 for sending (conn_id=3) +[Rank 4] Connected remotely to rank 10 for sending (conn_id=4) +[Rank 4] Connected remotely to rank 12 for sending (conn_id=6) +[Rank 4] Connected remotely to rank 17 for sending (conn_id=10) +[Rank 4] Connected remotely to rank 16 for sending (conn_id=11) +[Rank 4] Connected remotely to rank 15 for sending (conn_id=9) +[Rank 4] Connected remotely to rank 18 for sending (conn_id=12)[Rank 4] Connected remotely to rank 19 for sending (conn_id=13) + +[Rank 4] Connected remotely to rank 20 for sending (conn_id=14)[Rank 4] Connected remotely to rank 21 for sending (conn_id=15) + +[Rank 4] Connected remotely to rank 23 for sending (conn_id=17) +[Rank 1] Connected remotely to rank 9 for sending (conn_id=2) +[Rank 0] Connected remotely to rank 9 for sending (conn_id=7) +[Rank 0] Connected remotely to rank 12 for sending (conn_id=8) +[Rank 0] Connected remotely to rank 10 for sending (conn_id=12) +[Rank 1] Connected remotely to rank 18 for sending (conn_id=7) +[Rank 1] Connected remotely to rank 16 for sending (conn_id=8) +[Rank 1] Connected remotely to rank 22 for sending (conn_id=9) +[Rank 0] Connected remotely to rank 15 for sending (conn_id=21) +[Rank 0] Connected remotely to rank 16 for sending (conn_id=22) +[Rank 0] Accepted remote connection from rank 8 (IP 10.162.224.132, GPU 0) for receiving (conn_id=23) +[Rank 0] Accepted remote connection from rank 21 (IP 10.162.224.129, GPU 5) for receiving (conn_id=24) +[Rank 1] Connected remotely to rank 21 for sending (conn_id=14) +[Rank 0] Accepted remote connection from rank 19 (IP 10.162.224.129, GPU 3) for receiving (conn_id=26) +[Rank 0] Accepted remote connection from rank 11 (IP 10.162.224.132, GPU 3) for receiving (conn_id=27) +[Rank 5] Connected remotely to rank 17 for sending (conn_id=2) +[Rank 3] Connected remotely to rank 18 for sending (conn_id=4) +[Rank 0] Accepted remote connection from rank 16 (IP 10.162.224.129, GPU 0) for receiving (conn_id=28) +[Rank 0] Connected remotely to rank 14 for sending (conn_id=29)[Rank 0] Connected remotely to rank 22 for sending (conn_id=30) + +[Rank 0] Accepted remote connection from rank 17 (IP 10.162.224.129, GPU 1) for receiving (conn_id=31) +[Rank 1] Connected remotely to rank 12 for sending (conn_id=19) +[Rank 0] Accepted remote connection from rank 9 (IP 10.162.224.132, GPU 1) for receiving (conn_id=32) +[Rank 0] Accepted remote connection from rank 20 (IP 10.162.224.129, GPU 4) for receiving (conn_id=34) +[Rank 0] Accepted remote connection from rank 14 (IP 10.162.224.132, GPU 6) for receiving (conn_id=35) +[Rank 1] Connected remotely to rank 23 for sending (conn_id=21) +[Rank 1] Connected remotely to rank 20 for sending (conn_id=22) +[Rank 5] Connected remotely to rank 9 for sending (conn_id=5) +[Rank 0] Connected remotely to rank 21 for sending (conn_id=36)[Rank 2] Connected remotely to rank 8 for sending (conn_id=4) + +[Rank 0] Accepted remote connection from rank 22 (IP 10.162.224.129, GPU 6) for receiving (conn_id=37) +[Rank 2] Connected remotely to rank 12 for sending (conn_id=5) +[Rank 1] Connected remotely to rank 17 for sending (conn_id=23) +[Rank 0] Accepted remote connection from rank 12 (IP 10.162.224.132, GPU 4) for receiving (conn_id=38) +[Rank 5] Connected remotely to rank 12 for sending (conn_id=6) +[Rank 0] Connected remotely to rank 23 for sending (conn_id=39) +[Rank 0] Accepted remote connection from rank 10 (IP 10.162.224.132, GPU 2) for receiving (conn_id=40) +[Rank 0] Accepted remote connection from rank 13 (IP 10.162.224.132, GPU 5) for receiving (conn_id=43)[Rank 0] Connected remotely to rank 8 for sending (conn_id=42) + +[Rank 5] Connected remotely to rank 19 for sending (conn_id=9) +[Rank 2] Connected remotely to rank 10 for sending (conn_id=8) +[Rank 5] Connected remotely to rank 22 for sending (conn_id=10) +[Rank 1] Connected remotely to rank 19 for sending (conn_id=30) +[Rank 2] Connected remotely to rank 15 for sending (conn_id=11) +[Rank 3] Connected remotely to rank 19 for sending (conn_id=15) +[Rank 2] Connected remotely to rank 23 for sending (conn_id=15) +[Rank 5] Connected remotely to rank 20 for sending (conn_id=18) +[Rank 7] Connected remotely to rank 10 for sending (conn_id=3)[Rank 7] Connected remotely to rank 15 for sending (conn_id=5) +[Rank 7] Connected remotely to rank 17 for sending (conn_id=6) + +[Rank 2] Connected remotely to rank 21 for sending (conn_id=23)[Rank 7] Connected remotely to rank 12 for sending (conn_id=7) + +[Rank 7] Connected remotely to rank 21 for sending (conn_id=8) +[Rank 2] Connected remotely to rank 20 for sending (conn_id=29) +[Rank 3] Connected remotely to rank 22 for sending (conn_id=29) +[Rank 7] Connected remotely to rank 19 for sending (conn_id=10)[Rank 3] Connected remotely to rank 16 for sending (conn_id=30) +[Rank 5] Connected remotely to rank 23 for sending (conn_id=27) + +[Rank 2] Connected remotely to rank 19 for sending (conn_id=34) +[Rank 5] Connected remotely to rank 21 for sending (conn_id=28) +[Rank 7] Connected remotely to rank 18 for sending (conn_id=14)[Rank 2] Connected remotely to rank 22 for sending (conn_id=37) + +[Rank 2] Connected remotely to rank 18 for sending (conn_id=38) +[Rank 2] Accepted remote connection from rank 11 (IP 10.162.224.132, GPU 3) for receiving (conn_id=41) +[Rank 7] Connected remotely to rank 20 for sending (conn_id=22) +[Rank 7] Connected remotely to rank 9 for sending (conn_id=28) +[Rank 7] Connected remotely to rank 16 for sending (conn_id=33) +[Rank 6] Connected remotely to rank 8 for sending (conn_id=2) +[Rank 6] Connected remotely to rank 10 for sending (conn_id=4) +[Rank 6] Connected remotely to rank 11 for sending (conn_id=5) +[Rank 6] Connected remotely to rank 13 for sending (conn_id=7) +[Rank 6] Connected remotely to rank 15 for sending (conn_id=9)[Rank 6] Connected remotely to rank 14 for sending (conn_id=8) + +[Rank 6] Connected remotely to rank 16 for sending (conn_id=10) +[Rank 6] Connected remotely to rank 21 for sending (conn_id=15) +[Rank 6] Accepted remote connection from rank 10 (IP 10.162.224.132, GPU 2) for receiving (conn_id=18) +[Rank 6] Accepted remote connection from rank 16 (IP 10.162.224.129, GPU 0) for receiving (conn_id=19)[Rank 6] Accepted remote connection from rank 12 (IP 10.162.224.132, GPU 4) for receiving (conn_id=20) + +[Rank 6] Accepted remote connection from rank 8 (IP 10.162.224.132, GPU 0) for receiving (conn_id=21) +[Rank 6] Accepted remote connection from rank 19 (IP 10.162.224.129, GPU 3) for receiving (conn_id=22)[Rank 6] Accepted remote connection from rank 9 (IP 10.162.224.132, GPU 1) for receiving (conn_id=23) + +[Rank 4] Connected remotely to rank 8 for sending (conn_id=2) +[Rank 6] Accepted remote connection from rank 11 (IP 10.162.224.132, GPU 3) for receiving (conn_id=24) +[Rank 6] Accepted remote connection from rank 14 (IP 10.162.224.132, GPU 6) for receiving (conn_id=25) +[Rank 6] Accepted remote connection from rank 13 (IP 10.162.224.132, GPU 5) for receiving (conn_id=26) +[Rank 6] Accepted remote connection from rank 22 (IP 10.162.224.129, GPU 6) for receiving (conn_id=27) +[Rank 6] Accepted remote connection from rank 17 (IP 10.162.224.129, GPU 1) for receiving (conn_id=28)[Rank 6] Accepted remote connection from rank 20 (IP 10.162.224.129, GPU 4) for receiving (conn_id=29)[Rank 4] Connected remotely to rank 11 for sending (conn_id=5)[Rank 6] Accepted remote connection from rank 15 (IP 10.162.224.132, GPU 7) for receiving (conn_id=30) + + + +[Rank 6] Accepted remote connection from rank 23 (IP 10.162.224.129, GPU 7) for receiving (conn_id=32)[Rank 6] Accepted remote connection from rank 21 (IP 10.162.224.129, GPU 5) for receiving (conn_id=31) + +[Rank 6] Accepted remote connection from rank 18 (IP 10.162.224.129, GPU 2) for receiving (conn_id=33) +[Rank 4] Connected remotely to rank 13 for sending (conn_id=8)[Rank 4] Connected remotely to rank 14 for sending (conn_id=7) + +[Rank 6] Full mesh established: 7 local connections, 16 remote connections +[Rank 4] Connected remotely to rank 22 for sending (conn_id=16) +[Rank 4] Accepted remote connection from rank 20 (IP 10.162.224.129, GPU 4) for receiving (conn_id=18) +[Rank 4] Accepted remote connection from rank 13 (IP 10.162.224.132, GPU 5) for receiving (conn_id=20) +[Rank 4] Accepted remote connection from rank 9 (IP 10.162.224.132, GPU 1) for receiving (conn_id=19) +[Rank 4] Accepted remote connection from rank 15 (IP 10.162.224.132, GPU 7) for receiving (conn_id=23)[Rank 4] Accepted remote connection from rank 12 (IP 10.162.224.132, GPU 4) for receiving (conn_id=21) + +[Rank 4] Accepted remote connection from rank 19 (IP 10.162.224.129, GPU 3) for receiving (conn_id=22)[Rank 4] Accepted remote connection from rank 10 (IP 10.162.224.132, GPU 2) for receiving (conn_id=25)[Rank 4] Accepted remote connection from rank 8 (IP 10.162.224.132, GPU 0) for receiving (conn_id=24) + + +[Rank 4] Accepted remote connection from rank 11 (IP 10.162.224.132, GPU 3) for receiving (conn_id=27)[Rank 4] Accepted remote connection from rank 16 (IP 10.162.224.129, GPU 0) for receiving (conn_id=26) + +[Rank 4] Accepted remote connection from rank 17 (IP 10.162.224.129, GPU 1) for receiving (conn_id=28) +[Rank 4] Accepted remote connection from rank 18 (IP 10.162.224.129, GPU 2) for receiving (conn_id=29)[Rank 4] Accepted remote connection from rank 14 (IP 10.162.224.132, GPU 6) for receiving (conn_id=30)[Rank 4] Accepted remote connection from rank 21 (IP 10.162.224.129, GPU 5) for receiving (conn_id=33) + +[Rank 4] Accepted remote connection from rank 22 (IP 10.162.224.129, GPU 6) for receiving (conn_id=31) + +[Rank 4] Accepted remote connection from rank 23 (IP 10.162.224.129, GPU 7) for receiving (conn_id=32) +[Rank 4] Full mesh established: 7 local connections, 16 remote connections +[Rank 0] Connected remotely to rank 13 for sending (conn_id=6) +[Rank 1] Connected remotely to rank 10 for sending (conn_id=3) +[Rank 0] Connected remotely to rank 11 for sending (conn_id=9)[Rank 0] Connected remotely to rank 18 for sending (conn_id=10) + +[Rank 0] Connected remotely to rank 17 for sending (conn_id=11) +[Rank 1] Connected remotely to rank 13 for sending (conn_id=4)[Rank 1] Connected remotely to rank 15 for sending (conn_id=6)[Rank 1] Connected remotely to rank 11 for sending (conn_id=5)[Rank 0] Connected remotely to rank 19 for sending (conn_id=13) + + + +[Rank 0] Connected remotely to rank 20 for sending (conn_id=17) +[Rank 1] Accepted remote connection from rank 22 (IP 10.162.224.129, GPU 6) for receiving (conn_id=10) +[Rank 0] Accepted remote connection from rank 23 (IP 10.162.224.129, GPU 7) for receiving (conn_id=18) +[Rank 1] Accepted remote connection from rank 12 (IP 10.162.224.132, GPU 4) for receiving (conn_id=11) +[Rank 0] Accepted remote connection from rank 18 (IP 10.162.224.129, GPU 2) for receiving (conn_id=19) +[Rank 1] Accepted remote connection from rank 13 (IP 10.162.224.132, GPU 5) for receiving (conn_id=12) +[Rank 1] Accepted remote connection from rank 14 (IP 10.162.224.132, GPU 6) for receiving (conn_id=13) +[Rank 3] Connected remotely to rank 14 for sending (conn_id=3)[Rank 3] Connected remotely to rank 9 for sending (conn_id=2) + +[Rank 1] Accepted remote connection from rank 9 (IP 10.162.224.132, GPU 1) for receiving (conn_id=15) +[Rank 1] Connected remotely to rank 8 for sending (conn_id=16)[Rank 1] Accepted remote connection from rank 16 (IP 10.162.224.129, GPU 0) for receiving (conn_id=17) +[Rank 1] Accepted remote connection from rank 17 (IP 10.162.224.129, GPU 1) for receiving (conn_id=18) + +[Rank 5] Connected remotely to rank 13 for sending (conn_id=3) +[Rank 3] Connected remotely to rank 13 for sending (conn_id=5)[Rank 1] Accepted remote connection from rank 20 (IP 10.162.224.129, GPU 4) for receiving (conn_id=20) + +[Rank 5] Connected remotely to rank 18 for sending (conn_id=4) +[Rank 3] Connected remotely to rank 12 for sending (conn_id=6) +[Rank 3] Connected remotely to rank 10 for sending (conn_id=7) +[Rank 1] Connected remotely to rank 14 for sending (conn_id=24) +[Rank 2] Connected remotely to rank 16 for sending (conn_id=6) +[Rank 5] Connected remotely to rank 15 for sending (conn_id=7) +[Rank 2] Accepted remote connection from rank 23 (IP 10.162.224.129, GPU 7) for receiving (conn_id=7) +[Rank 5] Accepted remote connection from rank 10 (IP 10.162.224.132, GPU 2) for receiving (conn_id=8) +[Rank 1] Accepted remote connection from rank 8 (IP 10.162.224.132, GPU 0) for receiving (conn_id=25) +[Rank 1] Accepted remote connection from rank 15 (IP 10.162.224.132, GPU 7) for receiving (conn_id=26) +[Rank 1] Accepted remote connection from rank 11 (IP 10.162.224.132, GPU 3) for receiving (conn_id=27) +[Rank 0] Accepted remote connection from rank 15 (IP 10.162.224.132, GPU 7) for receiving (conn_id=44) +[Rank 1] Accepted remote connection from rank 21 (IP 10.162.224.129, GPU 5) for receiving (conn_id=28) +[Rank 3] Connected remotely to rank 20 for sending (conn_id=8) +[Rank 2] Connected remotely to rank 9 for sending (conn_id=10) +[Rank 1] Accepted remote connection from rank 19 (IP 10.162.224.129, GPU 3) for receiving (conn_id=29) +[Rank 5] Accepted remote connection from rank 8 (IP 10.162.224.132, GPU 0) for receiving (conn_id=11) +[Rank 0] Full mesh established: 7 local connections, 16 remote connections +[Rank 5] Accepted remote connection from rank 13 (IP 10.162.224.132, GPU 5) for receiving (conn_id=12) +[Rank 3] Connected remotely to rank 11 for sending (conn_id=9) +[Rank 3] Accepted remote connection from rank 14 (IP 10.162.224.132, GPU 6) for receiving (conn_id=10)[Rank 1] Accepted remote connection from rank 10 (IP 10.162.224.132, GPU 2) for receiving (conn_id=31) +[Rank 3] Accepted remote connection from rank 11 (IP 10.162.224.132, GPU 3) for receiving (conn_id=11) + +[Rank 3] Accepted remote connection from rank 10 (IP 10.162.224.132, GPU 2) for receiving (conn_id=12) +[Rank 2] Connected remotely to rank 17 for sending (conn_id=12) +[Rank 1] Accepted remote connection from rank 18 (IP 10.162.224.129, GPU 2) for receiving (conn_id=32) +[Rank 3] Accepted remote connection from rank 8 (IP 10.162.224.132, GPU 0) for receiving (conn_id=13) +[Rank 1] Accepted remote connection from rank 23 (IP 10.162.224.129, GPU 7) for receiving (conn_id=33) +[Rank 5] Connected remotely to rank 16 for sending (conn_id=13) +[Rank 3] Connected remotely to rank 23 for sending (conn_id=14) +[Rank 7] Connected remotely to rank 8 for sending (conn_id=2) +[Rank 5] Connected remotely to rank 10 for sending (conn_id=14) +[Rank 2] Connected remotely to rank 11 for sending (conn_id=13)[Rank 2] Accepted remote connection from rank 18 (IP 10.162.224.129, GPU 2) for receiving (conn_id=14) + +[Rank 3] Accepted remote connection from rank 9 (IP 10.162.224.132, GPU 1) for receiving (conn_id=18)[Rank 5] Accepted remote connection from rank 12 (IP 10.162.224.132, GPU 4) for receiving (conn_id=15)[Rank 5] Accepted remote connection from rank 9 (IP 10.162.224.132, GPU 1) for receiving (conn_id=16) +[Rank 3] Accepted remote connection from rank 13 (IP 10.162.224.132, GPU 5) for receiving (conn_id=16)[Rank 3] Accepted remote connection from rank 12 (IP 10.162.224.132, GPU 4) for receiving (conn_id=17)[Rank 3] Accepted remote connection from rank 17 (IP 10.162.224.129, GPU 1) for receiving (conn_id=20) + + + +[Rank 2] Accepted remote connection from rank 21 (IP 10.162.224.129, GPU 5) for receiving (conn_id=20) +[Rank 5] Connected remotely to rank 8 for sending (conn_id=17) +[Rank 3] Accepted remote connection from rank 22 (IP 10.162.224.129, GPU 6) for receiving (conn_id=19) +[Rank 2] Accepted remote connection from rank 19 (IP 10.162.224.129, GPU 3) for receiving (conn_id=19)[Rank 2] Connected remotely to rank 13 for sending (conn_id=22) + +[Rank 2] Accepted remote connection from rank 16 (IP 10.162.224.129, GPU 0) for receiving (conn_id=21) +[Rank 3] Accepted remote connection from rank 15 (IP 10.162.224.132, GPU 7) for receiving (conn_id=21) +[Rank 7] Connected remotely to rank 11 for sending (conn_id=4) +[Rank 3] Accepted remote connection from rank 19 (IP 10.162.224.129, GPU 3) for receiving (conn_id=22)[Rank 5] Accepted remote connection from rank 15 (IP 10.162.224.132, GPU 7) for receiving (conn_id=19) + +[Rank 3] Connected remotely to rank 15 for sending (conn_id=23) +[Rank 5] Accepted remote connection from rank 14 (IP 10.162.224.132, GPU 6) for receiving (conn_id=20)[Rank 2] Accepted remote connection from rank 17 (IP 10.162.224.129, GPU 1) for receiving (conn_id=24) +[Rank 1] Full mesh established: 7 local connections, 16 remote connections + + +[Rank 2] Accepted remote connection from rank 20 (IP 10.162.224.129, GPU 4) for receiving (conn_id=25) +[Rank 3] Accepted remote connection from rank 23 (IP 10.162.224.129, GPU 7) for receiving (conn_id=24) +[Rank 5] Connected remotely to rank 14 for sending (conn_id=21) +[Rank 3] Accepted remote connection from rank 21 (IP 10.162.224.129, GPU 5) for receiving (conn_id=25) +[Rank 5] Accepted remote connection from rank 17 (IP 10.162.224.129, GPU 1) for receiving (conn_id=22)[Rank 2] Accepted remote connection from rank 9 (IP 10.162.224.132, GPU 1) for receiving (conn_id=28) +[Rank 3] Connected remotely to rank 8 for sending (conn_id=26) +[Rank 5] Accepted remote connection from rank 19 (IP 10.162.224.129, GPU 3) for receiving (conn_id=23) +[Rank 3] Accepted remote connection from rank 20 (IP 10.162.224.129, GPU 4) for receiving (conn_id=27)[Rank 5] Accepted remote connection from rank 11 (IP 10.162.224.132, GPU 3) for receiving (conn_id=24) + + +[Rank 3] Accepted remote connection from rank 16 (IP 10.162.224.129, GPU 0) for receiving (conn_id=28) +[Rank 5] Connected remotely to rank 11 for sending (conn_id=25) +[Rank 2] Accepted remote connection from rank 12 (IP 10.162.224.132, GPU 4) for receiving (conn_id=31)[Rank 5] Accepted remote connection from rank 20 (IP 10.162.224.129, GPU 4) for receiving (conn_id=26) +[Rank 2] Accepted remote connection from rank 22 (IP 10.162.224.129, GPU 6) for receiving (conn_id=30) +[Rank 7] Connected remotely to rank 14 for sending (conn_id=9) +[Rank 2] Accepted remote connection from rank 10 (IP 10.162.224.132, GPU 2) for receiving (conn_id=32) + +[Rank 7] Accepted remote connection from rank 11 (IP 10.162.224.132, GPU 3) for receiving (conn_id=11)[Rank 3] Accepted remote connection from rank 18 (IP 10.162.224.129, GPU 2) for receiving (conn_id=31)[Rank 2] Accepted remote connection from rank 15 (IP 10.162.224.132, GPU 7) for receiving (conn_id=33) + + +[Rank 3] Connected remotely to rank 21 for sending (conn_id=32) +[Rank 7] Accepted remote connection from rank 15 (IP 10.162.224.132, GPU 7) for receiving (conn_id=12)[Rank 5] Accepted remote connection from rank 22 (IP 10.162.224.129, GPU 6) for receiving (conn_id=29) + +[Rank 5] Accepted remote connection from rank 21 (IP 10.162.224.129, GPU 5) for receiving (conn_id=30) +[Rank 3] Connected remotely to rank 17 for sending (conn_id=33) +[Rank 2] Connected remotely to rank 14 for sending (conn_id=36) +[Rank 5] Accepted remote connection from rank 18 (IP 10.162.224.129, GPU 2) for receiving (conn_id=32)[Rank 5] Accepted remote connection from rank 16 (IP 10.162.224.129, GPU 0) for receiving (conn_id=31) +[Rank 7] Connected remotely to rank 22 for sending (conn_id=13)[Rank 7] Accepted remote connection from rank 8 (IP 10.162.224.132, GPU 0) for receiving (conn_id=16)[Rank 5] Accepted remote connection from rank 23 (IP 10.162.224.129, GPU 7) for receiving (conn_id=33) + +[Rank 2] Accepted remote connection from rank 8 (IP 10.162.224.132, GPU 0) for receiving (conn_id=39)[Rank 7] Accepted remote connection from rank 13 (IP 10.162.224.132, GPU 5) for receiving (conn_id=15) +[Rank 7] Accepted remote connection from rank 20 (IP 10.162.224.129, GPU 4) for receiving (conn_id=17) +[Rank 2] Accepted remote connection from rank 13 (IP 10.162.224.132, GPU 5) for receiving (conn_id=40) + + + +[Rank 7] Accepted remote connection from rank 17 (IP 10.162.224.129, GPU 1) for receiving (conn_id=18)[Rank 2] Accepted remote connection from rank 14 (IP 10.162.224.132, GPU 6) for receiving (conn_id=42)[Rank 7] Accepted remote connection from rank 16 (IP 10.162.224.129, GPU 0) for receiving (conn_id=19) +[Rank 7] Accepted remote connection from rank 9 (IP 10.162.224.132, GPU 1) for receiving (conn_id=21) +[Rank 7] Accepted remote connection from rank 12 (IP 10.162.224.132, GPU 4) for receiving (conn_id=20)[Rank 7] Accepted remote connection from rank 19 (IP 10.162.224.129, GPU 3) for receiving (conn_id=23)[Rank 7] Accepted remote connection from rank 21 (IP 10.162.224.129, GPU 5) for receiving (conn_id=24)[Rank 7] Accepted remote connection from rank 22 (IP 10.162.224.129, GPU 6) for receiving (conn_id=26)[Rank 7] Accepted remote connection from rank 18 (IP 10.162.224.129, GPU 2) for receiving (conn_id=27) +[Rank 7] Connected remotely to rank 13 for sending (conn_id=29)[Rank 7] Accepted remote connection from rank 10 (IP 10.162.224.132, GPU 2) for receiving (conn_id=30) + + +[Rank 7] Connected remotely to rank 23 for sending (conn_id=31) +[Rank 7] Accepted remote connection from rank 23 (IP 10.162.224.129, GPU 7) for receiving (conn_id=32) +[Rank 7] Accepted remote connection from rank 14 (IP 10.162.224.132, GPU 6) for receiving (conn_id=25) +[Rank 3] Full mesh established: 7 local connections, 16 remote connections + + + + + +[Rank 5] Full mesh established: 7 local connections, 16 remote connections +[Rank 2] Full mesh established: 7 local connections, 16 remote connections +[Rank 7] Full mesh established: 7 local connections, 16 remote connections +23:44:18 | Rank 4 | INFO | Initializing DDP Actor Model... +23:44:18 | Rank 6 | INFO | Initializing DDP Actor Model... +23:44:18 | Rank 1 | INFO | Initializing DDP Actor Model... +23:44:18 | Rank 3 | INFO | Initializing DDP Actor Model... +23:44:18 | Rank 5 | INFO | Initializing DDP Actor Model... +23:44:18 | Rank 2 | INFO | Initializing DDP Actor Model... +23:44:18 | Rank 7 | INFO | Initializing DDP Actor Model... +23:44:18 | Rank 0 | INFO | Initializing DDP Actor Model... +chi-mi325x-pod2-101:1948888:1948888 [0] NCCL INFO Kernel version: 5.15.0-160-generic +chi-mi325x-pod2-101:1948888:1948888 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to enp49s0f1np1 +chi-mi325x-pod2-101:1948888:1948888 [0] NCCL INFO Bootstrap : Using enp49s0f1np1:10.162.224.133<0> +chi-mi325x-pod2-101:1948888:1948888 [0] NCCL INFO ROCr version 1.15 +chi-mi325x-pod2-101:1948888:1948888 [0] NCCL INFO Dmabuf feature disabled without NCCL_DMABUF_ENABLE=1 +chi-mi325x-pod2-101:1948888:1948888 [0] NCCL INFO Kernel version: 5.15.0-160-generic +chi-mi325x-pod2-101:1948888:1948888 [0] NCCL INFO RCCL version : 2.22.3-HEAD:7d8d67c +HIP version : 6.4.43484-123eb5128 +ROCm version : 6.4.2.0-120-e7d83f5 +Hostname : chi-mi325x-pod2-101.ord.vultr.cpe.ice.amd.com +Librccl path : /home/yangzhou/.local/lib/python3.10/site-packages/torch/lib/librccl.so +chi-mi325x-pod2-101:1948888:1948888 [0] NCCL INFO Comm config Blocking set to 1 +chi-mi325x-pod2-101:1948892:1948892 [4] NCCL INFO ROCr version 1.15 +chi-mi325x-pod2-101:1948894:1948894 [6] NCCL INFO ROCr version 1.15 +chi-mi325x-pod2-101:1948895:1948895 [7] NCCL INFO ROCr version 1.15 +chi-mi325x-pod2-101:1948892:1948892 [4] NCCL INFO Dmabuf feature disabled without NCCL_DMABUF_ENABLE=1 +chi-mi325x-pod2-101:1948893:1948893 [5] NCCL INFO ROCr version 1.15 +chi-mi325x-pod2-101:1948894:1948894 [6] NCCL INFO Dmabuf feature disabled without NCCL_DMABUF_ENABLE=1 +chi-mi325x-pod2-101:1948895:1948895 [7] NCCL INFO Dmabuf feature disabled without NCCL_DMABUF_ENABLE=1 +chi-mi325x-pod2-101:1948893:1948893 [5] NCCL INFO Dmabuf feature disabled without NCCL_DMABUF_ENABLE=1 +chi-mi325x-pod2-101:1948892:1948892 [4] NCCL INFO Kernel version: 5.15.0-160-generic +chi-mi325x-pod2-101:1948889:1948889 [1] NCCL INFO ROCr version 1.15 +chi-mi325x-pod2-101:1948889:1948889 [1] NCCL INFO Dmabuf feature disabled without NCCL_DMABUF_ENABLE=1 +chi-mi325x-pod2-101:1948894:1948894 [6] NCCL INFO Kernel version: 5.15.0-160-generic +chi-mi325x-pod2-101:1948895:1948895 [7] NCCL INFO Kernel version: 5.15.0-160-generic +chi-mi325x-pod2-101:1948893:1948893 [5] NCCL INFO Kernel version: 5.15.0-160-generic +chi-mi325x-pod2-101:1948889:1948889 [1] NCCL INFO Kernel version: 5.15.0-160-generic +chi-mi325x-pod2-101:1948892:1948892 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to enp49s0f1np1 +chi-mi325x-pod2-101:1948893:1948893 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to enp49s0f1np1 +chi-mi325x-pod2-101:1948894:1948894 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to enp49s0f1np1 +chi-mi325x-pod2-101:1948895:1948895 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to enp49s0f1np1 +chi-mi325x-pod2-101:1948889:1948889 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to enp49s0f1np1 +chi-mi325x-pod2-101:1948895:1948895 [7] NCCL INFO Bootstrap : Using enp49s0f1np1:10.162.224.133<0> +chi-mi325x-pod2-101:1948893:1948893 [5] NCCL INFO Bootstrap : Using enp49s0f1np1:10.162.224.133<0> +chi-mi325x-pod2-101:1948894:1948894 [6] NCCL INFO Bootstrap : Using enp49s0f1np1:10.162.224.133<0> +chi-mi325x-pod2-101:1948892:1948892 [4] NCCL INFO Bootstrap : Using enp49s0f1np1:10.162.224.133<0> +chi-mi325x-pod2-101:1948895:1948895 [7] NCCL INFO RCCL version : 2.22.3-HEAD:7d8d67c +HIP version : 6.4.43484-123eb5128 +ROCm version : 6.4.2.0-120-e7d83f5 +Hostname : chi-mi325x-pod2-101.ord.vultr.cpe.ice.amd.com +Librccl path : /home/yangzhou/.local/lib/python3.10/site-packages/torch/lib/librccl.so +chi-mi325x-pod2-101:1948893:1948893 [5] NCCL INFO RCCL version : 2.22.3-HEAD:7d8d67c +HIP version : 6.4.43484-123eb5128 +ROCm version : 6.4.2.0-120-e7d83f5 +Hostname : chi-mi325x-pod2-101.ord.vultr.cpe.ice.amd.com +Librccl path : /home/yangzhou/.local/lib/python3.10/site-packages/torch/lib/librccl.so +chi-mi325x-pod2-101:1948894:1948894 [6] NCCL INFO RCCL version : 2.22.3-HEAD:7d8d67c +HIP version : 6.4.43484-123eb5128 +ROCm version : 6.4.2.0-120-e7d83f5 +Hostname : chi-mi325x-pod2-101.ord.vultr.cpe.ice.amd.com +Librccl path : /home/yangzhou/.local/lib/python3.10/site-packages/torch/lib/librccl.so +chi-mi325x-pod2-101:1948892:1948892 [4] NCCL INFO RCCL version : 2.22.3-HEAD:7d8d67c +HIP version : 6.4.43484-123eb5128 +ROCm version : 6.4.2.0-120-e7d83f5 +Hostname : chi-mi325x-pod2-101.ord.vultr.cpe.ice.amd.com +Librccl path : /home/yangzhou/.local/lib/python3.10/site-packages/torch/lib/librccl.so +chi-mi325x-pod2-101:1948889:1948889 [1] NCCL INFO Bootstrap : Using enp49s0f1np1:10.162.224.133<0> +chi-mi325x-pod2-101:1948895:1948895 [7] NCCL INFO Comm config Blocking set to 1 +chi-mi325x-pod2-101:1948892:1948892 [4] NCCL INFO Comm config Blocking set to 1 +chi-mi325x-pod2-101:1948893:1948893 [5] NCCL INFO Comm config Blocking set to 1 +chi-mi325x-pod2-101:1948894:1948894 [6] NCCL INFO Comm config Blocking set to 1 +chi-mi325x-pod2-101:1948889:1948889 [1] NCCL INFO RCCL version : 2.22.3-HEAD:7d8d67c +HIP version : 6.4.43484-123eb5128 +ROCm version : 6.4.2.0-120-e7d83f5 +Hostname : chi-mi325x-pod2-101.ord.vultr.cpe.ice.amd.com +Librccl path : /home/yangzhou/.local/lib/python3.10/site-packages/torch/lib/librccl.so +chi-mi325x-pod2-101:1948889:1948889 [1] NCCL INFO Comm config Blocking set to 1 +chi-mi325x-pod2-101:1948891:1948891 [3] NCCL INFO ROCr version 1.15 +chi-mi325x-pod2-101:1948891:1948891 [3] NCCL INFO Dmabuf feature disabled without NCCL_DMABUF_ENABLE=1 +chi-mi325x-pod2-101:1948891:1948891 [3] NCCL INFO Kernel version: 5.15.0-160-generic +chi-mi325x-pod2-101:1948891:1948891 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to enp49s0f1np1 +chi-mi325x-pod2-101:1948891:1948891 [3] NCCL INFO Bootstrap : Using enp49s0f1np1:10.162.224.133<0> +chi-mi325x-pod2-101:1948891:1948891 [3] NCCL INFO RCCL version : 2.22.3-HEAD:7d8d67c +HIP version : 6.4.43484-123eb5128 +ROCm version : 6.4.2.0-120-e7d83f5 +Hostname : chi-mi325x-pod2-101.ord.vultr.cpe.ice.amd.com +Librccl path : /home/yangzhou/.local/lib/python3.10/site-packages/torch/lib/librccl.so +chi-mi325x-pod2-101:1948891:1948891 [3] NCCL INFO Comm config Blocking set to 1 +chi-mi325x-pod2-101:1948890:1948890 [2] NCCL INFO ROCr version 1.15 +chi-mi325x-pod2-101:1948890:1948890 [2] NCCL INFO Dmabuf feature disabled without NCCL_DMABUF_ENABLE=1 +chi-mi325x-pod2-101:1948890:1948890 [2] NCCL INFO Kernel version: 5.15.0-160-generic +chi-mi325x-pod2-101:1948890:1948890 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to enp49s0f1np1 +chi-mi325x-pod2-101:1948890:1948890 [2] NCCL INFO Bootstrap : Using enp49s0f1np1:10.162.224.133<0> +chi-mi325x-pod2-101:1948890:1948890 [2] NCCL INFO RCCL version : 2.22.3-HEAD:7d8d67c +HIP version : 6.4.43484-123eb5128 +ROCm version : 6.4.2.0-120-e7d83f5 +Hostname : chi-mi325x-pod2-101.ord.vultr.cpe.ice.amd.com +Librccl path : /home/yangzhou/.local/lib/python3.10/site-packages/torch/lib/librccl.so +chi-mi325x-pod2-101:1948890:1948890 [2] NCCL INFO Comm config Blocking set to 1 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO NET/Plugin: Could not find: librccl-net.so. Using internal network plugin. +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to enp49s0f1np1 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO NCCL_IB_HCA set to ^enp,eth,docker,lo,mlx5_0,mlx5_1 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO NET/Plugin: Could not find: librccl-net.so. Using internal network plugin. +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to enp49s0f1np1 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO NCCL_IB_HCA set to ^enp,eth,docker,lo,mlx5_0,mlx5_1 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO NET/Plugin: Could not find: librccl-net.so. Using internal network plugin. +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO NET/Plugin: Could not find: librccl-net.so. Using internal network plugin. +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to enp49s0f1np1 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to enp49s0f1np1 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO NCCL_IB_HCA set to ^enp,eth,docker,lo,mlx5_0,mlx5_1 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO NCCL_IB_HCA set to ^enp,eth,docker,lo,mlx5_0,mlx5_1 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO NET/Plugin: Could not find: librccl-net.so. Using internal network plugin. +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to enp49s0f1np1 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO NCCL_IB_HCA set to ^enp,eth,docker,lo,mlx5_0,mlx5_1 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO NET/Plugin: Could not find: librccl-net.so. Using internal network plugin. +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to enp49s0f1np1 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO NCCL_IB_HCA set to ^enp,eth,docker,lo,mlx5_0,mlx5_1 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO NET/IB : Using [0]bnxt_re0:1/RoCE [1]bnxt_re1:1/RoCE [2]bnxt_re2:1/RoCE [3]bnxt_re3:1/RoCE [4]bnxt_re4:1/RoCE [5]bnxt_re5:1/RoCE [6]bnxt_re7:1/RoCE [7]bnxt_re8:1/RoCE [RO]; OOB enp49s0f1np1:10.162.224.133<0> +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Found /sys/kernel/mm/memory_peers/amdkfd/version +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO /longer_pathname_so_that_rpms_can_support_packaging_the_debug_info_for_all_os_profiles/src/out/rhel-8.8/8.8/build/rccl/hipify/src/transport/net_ib.cc:708 -> 2 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Using network IB +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO [node_id = 3; gpu_id = 51110; unique_id = 6877879602702818491; location_id = 1280; bdf = 1280; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO [node_id = 2; gpu_id = 9091; unique_id = 8105839402969962122; location_id = 29952; bdf = 29952; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO [node_id = 6; gpu_id = 33762; unique_id = 8999893907506446937; location_id = 62720; bdf = 62720; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO [node_id = 7; gpu_id = 26567; unique_id = 10867919794612444986; location_id = 34048; bdf = 34048; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO [node_id = 9; gpu_id = 43978; unique_id = 12870402980068231451; location_id = 38144; bdf = 38144; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO [node_id = 4; gpu_id = 61326; unique_id = 15451093352642011365; location_id = 25856; bdf = 25856; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO [node_id = 8; gpu_id = 20463; unique_id = 16006804536255338307; location_id = 58624; bdf = 58624; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO [node_id = 5; gpu_id = 2987; unique_id = 16430629804050718901; location_id = 5376; bdf = 5376; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO NET/Plugin: Could not find: librccl-net.so. Using internal network plugin. +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to enp49s0f1np1 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO initialized internal alternative rsmi functionality +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO NCCL_IB_HCA set to ^enp,eth,docker,lo,mlx5_0,mlx5_1 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO NET/IB : Using [0]bnxt_re0:1/RoCE [1]bnxt_re1:1/RoCE [2]bnxt_re2:1/RoCE [3]bnxt_re3:1/RoCE [4]bnxt_re4:1/RoCE [5]bnxt_re5:1/RoCE [6]bnxt_re7:1/RoCE [7]bnxt_re8:1/RoCE [RO]; OOB enp49s0f1np1:10.162.224.133<0> +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Found /sys/kernel/mm/memory_peers/amdkfd/version +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO NET/IB : Using [0]bnxt_re0:1/RoCE [1]bnxt_re1:1/RoCE [2]bnxt_re2:1/RoCE [3]bnxt_re3:1/RoCE [4]bnxt_re4:1/RoCE [5]bnxt_re5:1/RoCE [6]bnxt_re7:1/RoCE [7]bnxt_re8:1/RoCE [RO]; OOB enp49s0f1np1:10.162.224.133<0> +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO /longer_pathname_so_that_rpms_can_support_packaging_the_debug_info_for_all_os_profiles/src/out/rhel-8.8/8.8/build/rccl/hipify/src/transport/net_ib.cc:708 -> 2 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Using network IB +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Found /sys/kernel/mm/memory_peers/amdkfd/version +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO /longer_pathname_so_that_rpms_can_support_packaging_the_debug_info_for_all_os_profiles/src/out/rhel-8.8/8.8/build/rccl/hipify/src/transport/net_ib.cc:708 -> 2 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Using network IB +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO [node_id = 3; gpu_id = 51110; unique_id = 6877879602702818491; location_id = 1280; bdf = 1280; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO [node_id = 2; gpu_id = 9091; unique_id = 8105839402969962122; location_id = 29952; bdf = 29952; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO [node_id = 6; gpu_id = 33762; unique_id = 8999893907506446937; location_id = 62720; bdf = 62720; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO [node_id = 7; gpu_id = 26567; unique_id = 10867919794612444986; location_id = 34048; bdf = 34048; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO [node_id = 9; gpu_id = 43978; unique_id = 12870402980068231451; location_id = 38144; bdf = 38144; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO [node_id = 4; gpu_id = 61326; unique_id = 15451093352642011365; location_id = 25856; bdf = 25856; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO [node_id = 8; gpu_id = 20463; unique_id = 16006804536255338307; location_id = 58624; bdf = 58624; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO [node_id = 3; gpu_id = 51110; unique_id = 6877879602702818491; location_id = 1280; bdf = 1280; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO [node_id = 2; gpu_id = 9091; unique_id = 8105839402969962122; location_id = 29952; bdf = 29952; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO [node_id = 5; gpu_id = 2987; unique_id = 16430629804050718901; location_id = 5376; bdf = 5376; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO [node_id = 6; gpu_id = 33762; unique_id = 8999893907506446937; location_id = 62720; bdf = 62720; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO [node_id = 7; gpu_id = 26567; unique_id = 10867919794612444986; location_id = 34048; bdf = 34048; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO [node_id = 9; gpu_id = 43978; unique_id = 12870402980068231451; location_id = 38144; bdf = 38144; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO [node_id = 4; gpu_id = 61326; unique_id = 15451093352642011365; location_id = 25856; bdf = 25856; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO [node_id = 8; gpu_id = 20463; unique_id = 16006804536255338307; location_id = 58624; bdf = 58624; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO [node_id = 5; gpu_id = 2987; unique_id = 16430629804050718901; location_id = 5376; bdf = 5376; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO initialized internal alternative rsmi functionality +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO initialized internal alternative rsmi functionality +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO NET/IB : Using [0]bnxt_re0:1/RoCE [1]bnxt_re1:1/RoCE [2]bnxt_re2:1/RoCE [3]bnxt_re3:1/RoCE [4]bnxt_re4:1/RoCE [5]bnxt_re5:1/RoCE [6]bnxt_re7:1/RoCE [7]bnxt_re8:1/RoCE [RO]; OOB enp49s0f1np1:10.162.224.133<0> +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Found /sys/kernel/mm/memory_peers/amdkfd/version +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO /longer_pathname_so_that_rpms_can_support_packaging_the_debug_info_for_all_os_profiles/src/out/rhel-8.8/8.8/build/rccl/hipify/src/transport/net_ib.cc:708 -> 2 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Using network IB +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO [node_id = 3; gpu_id = 51110; unique_id = 6877879602702818491; location_id = 1280; bdf = 1280; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO [node_id = 2; gpu_id = 9091; unique_id = 8105839402969962122; location_id = 29952; bdf = 29952; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO [node_id = 6; gpu_id = 33762; unique_id = 8999893907506446937; location_id = 62720; bdf = 62720; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO [node_id = 7; gpu_id = 26567; unique_id = 10867919794612444986; location_id = 34048; bdf = 34048; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO [node_id = 9; gpu_id = 43978; unique_id = 12870402980068231451; location_id = 38144; bdf = 38144; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO [node_id = 4; gpu_id = 61326; unique_id = 15451093352642011365; location_id = 25856; bdf = 25856; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO [node_id = 8; gpu_id = 20463; unique_id = 16006804536255338307; location_id = 58624; bdf = 58624; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO [node_id = 5; gpu_id = 2987; unique_id = 16430629804050718901; location_id = 5376; bdf = 5376; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO initialized internal alternative rsmi functionality +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO NET/IB : Using [0]bnxt_re0:1/RoCE [1]bnxt_re1:1/RoCE [2]bnxt_re2:1/RoCE [3]bnxt_re3:1/RoCE [4]bnxt_re4:1/RoCE [5]bnxt_re5:1/RoCE [6]bnxt_re7:1/RoCE [7]bnxt_re8:1/RoCE [RO]; OOB enp49s0f1np1:10.162.224.133<0> +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO NET/IB : Using [0]bnxt_re0:1/RoCE [1]bnxt_re1:1/RoCE [2]bnxt_re2:1/RoCE [3]bnxt_re3:1/RoCE [4]bnxt_re4:1/RoCE [5]bnxt_re5:1/RoCE [6]bnxt_re7:1/RoCE [7]bnxt_re8:1/RoCE [RO]; OOB enp49s0f1np1:10.162.224.133<0> +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Found /sys/kernel/mm/memory_peers/amdkfd/version +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Found /sys/kernel/mm/memory_peers/amdkfd/version +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO /longer_pathname_so_that_rpms_can_support_packaging_the_debug_info_for_all_os_profiles/src/out/rhel-8.8/8.8/build/rccl/hipify/src/transport/net_ib.cc:708 -> 2 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Using network IB +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO /longer_pathname_so_that_rpms_can_support_packaging_the_debug_info_for_all_os_profiles/src/out/rhel-8.8/8.8/build/rccl/hipify/src/transport/net_ib.cc:708 -> 2 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Using network IB +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO [node_id = 3; gpu_id = 51110; unique_id = 6877879602702818491; location_id = 1280; bdf = 1280; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO [node_id = 3; gpu_id = 51110; unique_id = 6877879602702818491; location_id = 1280; bdf = 1280; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO [node_id = 2; gpu_id = 9091; unique_id = 8105839402969962122; location_id = 29952; bdf = 29952; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO [node_id = 2; gpu_id = 9091; unique_id = 8105839402969962122; location_id = 29952; bdf = 29952; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO [node_id = 6; gpu_id = 33762; unique_id = 8999893907506446937; location_id = 62720; bdf = 62720; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO [node_id = 6; gpu_id = 33762; unique_id = 8999893907506446937; location_id = 62720; bdf = 62720; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO [node_id = 7; gpu_id = 26567; unique_id = 10867919794612444986; location_id = 34048; bdf = 34048; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO [node_id = 7; gpu_id = 26567; unique_id = 10867919794612444986; location_id = 34048; bdf = 34048; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO [node_id = 9; gpu_id = 43978; unique_id = 12870402980068231451; location_id = 38144; bdf = 38144; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO [node_id = 9; gpu_id = 43978; unique_id = 12870402980068231451; location_id = 38144; bdf = 38144; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO [node_id = 4; gpu_id = 61326; unique_id = 15451093352642011365; location_id = 25856; bdf = 25856; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO [node_id = 4; gpu_id = 61326; unique_id = 15451093352642011365; location_id = 25856; bdf = 25856; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO [node_id = 8; gpu_id = 20463; unique_id = 16006804536255338307; location_id = 58624; bdf = 58624; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO [node_id = 8; gpu_id = 20463; unique_id = 16006804536255338307; location_id = 58624; bdf = 58624; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO [node_id = 5; gpu_id = 2987; unique_id = 16430629804050718901; location_id = 5376; bdf = 5376; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO [node_id = 5; gpu_id = 2987; unique_id = 16430629804050718901; location_id = 5376; bdf = 5376; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO initialized internal alternative rsmi functionality +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO initialized internal alternative rsmi functionality +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO NET/Plugin: Could not find: librccl-net.so. Using internal network plugin. +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to enp49s0f1np1 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO NCCL_IB_HCA set to ^enp,eth,docker,lo,mlx5_0,mlx5_1 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO NET/IB : Using [0]bnxt_re0:1/RoCE [1]bnxt_re1:1/RoCE [2]bnxt_re2:1/RoCE [3]bnxt_re3:1/RoCE [4]bnxt_re4:1/RoCE [5]bnxt_re5:1/RoCE [6]bnxt_re7:1/RoCE [7]bnxt_re8:1/RoCE [RO]; OOB enp49s0f1np1:10.162.224.133<0> +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Found /sys/kernel/mm/memory_peers/amdkfd/version +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO NET/IB : Using [0]bnxt_re0:1/RoCE [1]bnxt_re1:1/RoCE [2]bnxt_re2:1/RoCE [3]bnxt_re3:1/RoCE [4]bnxt_re4:1/RoCE [5]bnxt_re5:1/RoCE [6]bnxt_re7:1/RoCE [7]bnxt_re8:1/RoCE [RO]; OOB enp49s0f1np1:10.162.224.133<0> +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO /longer_pathname_so_that_rpms_can_support_packaging_the_debug_info_for_all_os_profiles/src/out/rhel-8.8/8.8/build/rccl/hipify/src/transport/net_ib.cc:708 -> 2 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Using network IB +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Found /sys/kernel/mm/memory_peers/amdkfd/version +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO /longer_pathname_so_that_rpms_can_support_packaging_the_debug_info_for_all_os_profiles/src/out/rhel-8.8/8.8/build/rccl/hipify/src/transport/net_ib.cc:708 -> 2 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Using network IB +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO [node_id = 3; gpu_id = 51110; unique_id = 6877879602702818491; location_id = 1280; bdf = 1280; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO [node_id = 2; gpu_id = 9091; unique_id = 8105839402969962122; location_id = 29952; bdf = 29952; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO [node_id = 6; gpu_id = 33762; unique_id = 8999893907506446937; location_id = 62720; bdf = 62720; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO [node_id = 7; gpu_id = 26567; unique_id = 10867919794612444986; location_id = 34048; bdf = 34048; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO [node_id = 9; gpu_id = 43978; unique_id = 12870402980068231451; location_id = 38144; bdf = 38144; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO [node_id = 4; gpu_id = 61326; unique_id = 15451093352642011365; location_id = 25856; bdf = 25856; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO [node_id = 3; gpu_id = 51110; unique_id = 6877879602702818491; location_id = 1280; bdf = 1280; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO [node_id = 8; gpu_id = 20463; unique_id = 16006804536255338307; location_id = 58624; bdf = 58624; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO [node_id = 2; gpu_id = 9091; unique_id = 8105839402969962122; location_id = 29952; bdf = 29952; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO [node_id = 5; gpu_id = 2987; unique_id = 16430629804050718901; location_id = 5376; bdf = 5376; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO [node_id = 6; gpu_id = 33762; unique_id = 8999893907506446937; location_id = 62720; bdf = 62720; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO [node_id = 7; gpu_id = 26567; unique_id = 10867919794612444986; location_id = 34048; bdf = 34048; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO [node_id = 9; gpu_id = 43978; unique_id = 12870402980068231451; location_id = 38144; bdf = 38144; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO [node_id = 4; gpu_id = 61326; unique_id = 15451093352642011365; location_id = 25856; bdf = 25856; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO [node_id = 8; gpu_id = 20463; unique_id = 16006804536255338307; location_id = 58624; bdf = 58624; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO [node_id = 5; gpu_id = 2987; unique_id = 16430629804050718901; location_id = 5376; bdf = 5376; domain = 0; partition = 0], +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO initialized internal alternative rsmi functionality +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO initialized internal alternative rsmi functionality +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO ncclCommInitRank comm 0x648bcbd0 rank 5 nranks 16 cudaDev 5 nvmlDev 4 busId 85000 commId 0x41dbbbc5b0645f0a - Init START +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO ncclCommInitRank comm 0x507e6620 rank 4 nranks 16 cudaDev 4 nvmlDev 7 busId f5000 commId 0x41dbbbc5b0645f0a - Init START +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO ncclCommInitRank comm 0x4647b6e0 rank 7 nranks 16 cudaDev 7 nvmlDev 5 busId 95000 commId 0x41dbbbc5b0645f0a - Init START +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO ncclCommInitRank comm 0x7ec7cef0 rank 6 nranks 16 cudaDev 6 nvmlDev 6 busId e5000 commId 0x41dbbbc5b0645f0a - Init START +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO ncclCommInitRank comm 0x7e4c8c40 rank 2 nranks 16 cudaDev 2 nvmlDev 2 busId 65000 commId 0x41dbbbc5b0645f0a - Init START +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO ncclCommInitRank comm 0x48e1d6c0 rank 3 nranks 16 cudaDev 3 nvmlDev 1 busId 15000 commId 0x41dbbbc5b0645f0a - Init START +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO ncclCommInitRank comm 0x80bc0b40 rank 0 nranks 16 cudaDev 0 nvmlDev 3 busId 75000 commId 0x41dbbbc5b0645f0a - Init START +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO ncclCommInitRank comm 0x60eef880 rank 1 nranks 16 cudaDev 1 nvmlDev 0 busId 5000 commId 0x41dbbbc5b0645f0a - Init START +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO initialized internal alternative rsmi functionality +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO initialized internal alternative rsmi functionality +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO initialized internal alternative rsmi functionality +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO initialized internal alternative rsmi functionality +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO initialized internal alternative rsmi functionality +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO initialized internal alternative rsmi functionality +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO initialized internal alternative rsmi functionality +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO initialized internal alternative rsmi functionality +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO NCCL_NET_GDR_LEVEL set by environment to PXB +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Setting affinity for GPU 3 to ffffffff,ffffffff,00000000,00000000,ffffffff,ffffffff +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO NCCL_NET_GDR_LEVEL set by environment to PXB +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Setting affinity for GPU 2 to ffffffff,ffffffff,00000000,00000000,ffffffff,ffffffff +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO NCCL_NET_GDR_LEVEL set by environment to PXB +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Setting affinity for GPU 4 to ffffffff,ffffffff,00000000,00000000,ffffffff,ffffffff,00000000,00000000 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO NCCL_NET_GDR_LEVEL set by environment to PXB +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO NCCL_NET_GDR_LEVEL set by environment to PXB +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Setting affinity for GPU 5 to ffffffff,ffffffff,00000000,00000000,ffffffff,ffffffff,00000000,00000000 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Setting affinity for GPU 6 to ffffffff,ffffffff,00000000,00000000,ffffffff,ffffffff,00000000,00000000 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO NCCL_NET_GDR_LEVEL set by environment to PXB +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO NCCL_NET_GDR_LEVEL set by environment to PXB +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Setting affinity for GPU 0 to ffffffff,ffffffff,00000000,00000000,ffffffff,ffffffff +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Setting affinity for GPU 7 to ffffffff,ffffffff,00000000,00000000,ffffffff,ffffffff,00000000,00000000 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO NCCL_NET_GDR_LEVEL set by environment to PXB +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Setting affinity for GPU 1 to ffffffff,ffffffff,00000000,00000000,ffffffff,ffffffff +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO comm 0x60eef880 rank 1 nRanks 16 nNodes 2 localRanks 8 localRank 1 MNNVL 0 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO comm 0x4647b6e0 rank 7 nRanks 16 nNodes 2 localRanks 8 localRank 7 MNNVL 0 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO comm 0x48e1d6c0 rank 3 nRanks 16 nNodes 2 localRanks 8 localRank 3 MNNVL 0 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO comm 0x7e4c8c40 rank 2 nRanks 16 nNodes 2 localRanks 8 localRank 2 MNNVL 0 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO comm 0x80bc0b40 rank 0 nRanks 16 nNodes 2 localRanks 8 localRank 0 MNNVL 0 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO comm 0x648bcbd0 rank 5 nRanks 16 nNodes 2 localRanks 8 localRank 5 MNNVL 0 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO comm 0x7ec7cef0 rank 6 nRanks 16 nNodes 2 localRanks 8 localRank 6 MNNVL 0 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 00/64 : 0 5 7 6 4 12 14 15 13 8 10 11 9 1 3 2 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 01/64 : 0 7 4 6 14 12 15 8 13 10 9 11 3 1 2 5 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 02/64 : 0 6 3 4 5 13 12 11 14 8 9 15 10 2 7 1 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO comm 0x507e6620 rank 4 nRanks 16 nNodes 2 localRanks 8 localRank 4 MNNVL 0 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 03/64 : 0 4 1 5 2 3 6 7 15 14 11 10 13 9 12 8 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Trees [0] 0/-1/-1->5->2 [1] 4/-1/-1->5->1 [2] -1/-1/-1->5->4 [3] 3/-1/-1->5->6 [4] 7/13/-1->5->-1 [5] 6/-1/-1->5->7 [6] 1/-1/-1->5->0 [7] 2/-1/-1->5->3 [8] -1/-1/-1->5->4 [9] 3/-1/-1->5->6 [10] 7/13/-1->5->-1 [11] 6/-1/-1->5->7 [12] 1/-1/-1->5->0 [13] 2/-1/-1->5->3 [14] 0/-1/-1->5->2 [15] 4/-1/-1->5->1 [16] 7/13/-1->5->-1 [17] 6/-1/-1->5->7 [18] 1/-1/-1->5->0 [19] 2/-1/-1->5->3 [20] 0/-1/-1->5->2 [21] 4/-1/-1->5->1 [22] -1/-1/-1->5->4 [23] 3/-1/-1->5->6 [24] 1/-1/-1->5->0 [25] 2/-1/-1->5->3 [26] 0/-1/-1->5->2 [27] 4/-1/-1->5->1 [28] -1/-1/-1->5->4 [29] 3/-1/-1->5->6 [30] 7/13/-1->5->-1 [31] 6/-1/-1->5->7 [32] 4/-1/-1->5->1 [33] 0/-1/-1->5->2 [34] 3/-1/-1->5->6 [35] -1/-1/-1->5->4 [36] 6/-1/-1->5->7 [37] 7/-1/-1->5->13 [38] 2/-1/-1->5->3 [39] 1/-1/-1->5->0 [40] 3/-1/-1->5->6 [41] -1/-1/-1->5->4 [42] 6/-1/-1->5->7 [43] 7/-1/-1->5->13 [44] 2/-1/-1->5->3 [45] 1/-1/-1->5->0 [46] 4/-1/-1->5->1 [47] 0/-1/-1->5->2 [48] 6/-1/-1->5->7 [49] 7/-1/-1->5->13 [50] 2/-1/-1->5->3 [51] 1/-1/-1->5->0 [52] 4/-1/-1->5->1 [53] 0/-1/-1->5->2 [54] 3/-1/-1->5->6 [55] -1/-1/-1->5->4 [56] 2/-1/-1->5->3 [57] 1/-1/-1->5->0 [58] 4/-1/-1->5->1 [59] 0/-1/-1->5->2 [60] 3/-1/-1->5->6 [61] -1/-1/-1->5->4 [62] 6/-1/-1->5->7 [63] 7/-1/-1->5->13 comm 0x648bcbd0 nRanks 16 busId 85000 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Trees [0] 4/-1/-1->7->6 [1] 2/-1/-1->7->0 [2] 6/-1/-1->7->1 [3] -1/-1/-1->7->3 [4] 3/-1/-1->7->5 [5] 5/15/-1->7->-1 [6] 0/-1/-1->7->4 [7] 1/-1/-1->7->2 [8] 6/-1/-1->7->1 [9] -1/-1/-1->7->3 [10] 3/-1/-1->7->5 [11] 5/15/-1->7->-1 [12] 0/-1/-1->7->4 [13] 1/-1/-1->7->2 [14] 4/-1/-1->7->6 [15] 2/-1/-1->7->0 [16] 3/-1/-1->7->5 [17] 5/15/-1->7->-1 [18] 0/-1/-1->7->4 [19] 1/-1/-1->7->2 [20] 4/-1/-1->7->6 [21] 2/-1/-1->7->0 [22] 6/-1/-1->7->1 [23] -1/-1/-1->7->3 [24] 0/-1/-1->7->4 [25] 1/-1/-1->7->2 [26] 4/-1/-1->7->6 [27] 2/-1/-1->7->0 [28] 6/-1/-1->7->1 [29] -1/-1/-1->7->3 [30] 3/-1/-1->7->5 [31] 5/15/-1->7->-1 [32] 2/-1/-1->7->0 [33] 4/-1/-1->7->6 [34] -1/-1/-1->7->3 [35] 6/-1/-1->7->1 [36] 5/-1/-1->7->15 [37] 3/-1/-1->7->5 [38] 1/-1/-1->7->2 [39] 0/-1/-1->7->4 [40] -1/-1/-1->7->3 [41] 6/-1/-1->7->1 [42] 5/-1/-1->7->15 [43] 3/-1/-1->7->5 [44] 1/-1/-1->7->2 [45] 0/-1/-1->7->4 [46] 2/-1/-1->7->0 [47] 4/-1/-1->7->6 [48] 5/-1/-1->7->15 [49] 3/-1/-1->7->5 [50] 1/-1/-1->7->2 [51] 0/-1/-1->7->4 [52] 2/-1/-1->7->0 [53] 4/-1/-1->7->6 [54] -1/-1/-1->7->3 [55] 6/-1/-1->7->1 [56] 1/-1/-1->7->2 [57] 0/-1/-1->7->4 [58] 2/-1/-1->7->0 [59] 4/-1/-1->7->6 [60] -1/-1/-1->7->3 [61] 6/-1/-1->7->1 [62] 5/-1/-1->7->15 [63] 3/-1/-1->7->5 comm 0x4647b6e0 nRanks 16 busId 95000 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 04/64 : 0 1 7 3 11 15 9 8 12 10 14 13 5 6 2 4 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Trees [0] 7/-1/-1->6->0 [1] -1/-1/-1->6->2 [2] 3/-1/-1->6->7 [3] 5/-1/-1->6->1 [4] 1/-1/-1->6->3 [5] 2/-1/-1->6->5 [6] 4/14/-1->6->-1 [7] 0/-1/-1->6->4 [8] 3/-1/-1->6->7 [9] 5/-1/-1->6->1 [10] 1/-1/-1->6->3 [11] 2/-1/-1->6->5 [12] 4/14/-1->6->-1 [13] 0/-1/-1->6->4 [14] 7/-1/-1->6->0 [15] -1/-1/-1->6->2 [16] 1/-1/-1->6->3 [17] 2/-1/-1->6->5 [18] 4/14/-1->6->-1 [19] 0/-1/-1->6->4 [20] 7/-1/-1->6->0 [21] -1/-1/-1->6->2 [22] 3/-1/-1->6->7 [23] 5/-1/-1->6->1 [24] 4/14/-1->6->-1 [25] 0/-1/-1->6->4 [26] 7/-1/-1->6->0 [27] -1/-1/-1->6->2 [28] 3/-1/-1->6->7 [29] 5/-1/-1->6->1 [30] 1/-1/-1->6->3 [31] 2/-1/-1->6->5 [32] -1/-1/-1->6->2 [33] 7/-1/-1->6->0 [34] 5/-1/-1->6->1 [35] 3/-1/-1->6->7 [36] 2/-1/-1->6->5 [37] 1/-1/-1->6->3 [38] 0/-1/-1->6->4 [39] 4/-1/-1->6->14 [40] 5/-1/-1->6->1 [41] 3/-1/-1->6->7 [42] 2/-1/-1->6->5 [43] 1/-1/-1->6->3 [44] 0/-1/-1->6->4 [45] 4/-1/-1->6->14 [46] -1/-1/-1->6->2 [47] 7/-1/-1->6->0 [48] 2/-1/-1->6->5 [49] 1/-1/-1->6->3 [50] 0/-1/-1->6->4 [51] 4/-1/-1->6->14 [52] -1/-1/-1->6->2 [53] 7/-1/-1->6->0 [54] 5/-1/-1->6->1 [55] 3/-1/-1->6->7 [56] 0/-1/-1->6->4 [57] 4/-1/-1->6->14 [58] -1/-1/-1->6->2 [59] 7/-1/-1->6->0 [60] 5/-1/-1->6->1 [61] 3/-1/-1->6->7 [62] 2/-1/-1->6->5 [63] 1/-1/-1->6->3 comm 0x7ec7cef0 nRanks 16 busId e5000 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Trees [0] 3/9/-1->1->-1 [1] 5/-1/-1->1->3 [2] 7/-1/-1->1->0 [3] 6/-1/-1->1->4 [4] 0/-1/-1->1->6 [5] 4/-1/-1->1->2 [6] 2/-1/-1->1->5 [7] -1/-1/-1->1->7 [8] 7/-1/-1->1->0 [9] 6/-1/-1->1->4 [10] 0/-1/-1->1->6 [11] 4/-1/-1->1->2 [12] 2/-1/-1->1->5 [13] -1/-1/-1->1->7 [14] 3/9/-1->1->-1 [15] 5/-1/-1->1->3 [16] 0/-1/-1->1->6 [17] 4/-1/-1->1->2 [18] 2/-1/-1->1->5 [19] -1/-1/-1->1->7 [20] 3/9/-1->1->-1 [21] 5/-1/-1->1->3 [22] 7/-1/-1->1->0 [23] 6/-1/-1->1->4 [24] 2/-1/-1->1->5 [25] -1/-1/-1->1->7 [26] 3/9/-1->1->-1 [27] 5/-1/-1->1->3 [28] 7/-1/-1->1->0 [29] 6/-1/-1->1->4 [30] 0/-1/-1->1->6 [31] 4/-1/-1->1->2 [32] 5/-1/-1->1->3 [33] 3/-1/-1->1->9 [34] 6/-1/-1->1->4 [35] 7/-1/-1->1->0 [36] 4/-1/-1->1->2 [37] 0/-1/-1->1->6 [38] -1/-1/-1->1->7 [39] 2/-1/-1->1->5 [40] 6/-1/-1->1->4 [41] 7/-1/-1->1->0 [42] 4/-1/-1->1->2 [43] 0/-1/-1->1->6 [44] -1/-1/-1->1->7 [45] 2/-1/-1->1->5 [46] 5/-1/-1->1->3 [47] 3/-1/-1->1->9 [48] 4/-1/-1->1->2 [49] 0/-1/-1->1->6 [50] -1/-1/-1->1->7 [51] 2/-1/-1->1->5 [52] 5/-1/-1->1->3 [53] 3/-1/-1->1->9 [54] 6/-1/-1->1->4 [55] 7/-1/-1->1->0 [56] -1/-1/-1->1->7 [57] 2/-1/-1->1->5 [58] 5/-1/-1->1->3 [59] 3/-1/-1->1->9 [60] 6/-1/-1->1->4 [61] 7/-1/-1->1->0 [62] 4/-1/-1->1->2 [63] 0/-1/-1->1->6 comm 0x60eef880 nRanks 16 busId 5000 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO P2P Chunksize set to 131072 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO P2P Chunksize set to 131072 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 05/64 : 0 2 6 1 9 14 10 8 11 12 13 15 7 5 4 3 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Trees [0] 2/-1/-1->3->1 [1] 1/11/-1->3->-1 [2] 4/-1/-1->3->6 [3] 7/-1/-1->3->5 [4] 6/-1/-1->3->7 [5] 0/-1/-1->3->4 [6] -1/-1/-1->3->2 [7] 5/-1/-1->3->0 [8] 4/-1/-1->3->6 [9] 7/-1/-1->3->5 [10] 6/-1/-1->3->7 [11] 0/-1/-1->3->4 [12] -1/-1/-1->3->2 [13] 5/-1/-1->3->0 [14] 2/-1/-1->3->1 [15] 1/11/-1->3->-1 [16] 6/-1/-1->3->7 [17] 0/-1/-1->3->4 [18] -1/-1/-1->3->2 [19] 5/-1/-1->3->0 [20] 2/-1/-1->3->1 [21] 1/11/-1->3->-1 [22] 4/-1/-1->3->6 [23] 7/-1/-1->3->5 [24] -1/-1/-1->3->2 [25] 5/-1/-1->3->0 [26] 2/-1/-1->3->1 [27] 1/11/-1->3->-1 [28] 4/-1/-1->3->6 [29] 7/-1/-1->3->5 [30] 6/-1/-1->3->7 [31] 0/-1/-1->3->4 [32] 1/-1/-1->3->11 [33] 2/-1/-1->3->1 [34] 7/-1/-1->3->5 [35] 4/-1/-1->3->6 [36] 0/-1/-1->3->4 [37] 6/-1/-1->3->7 [38] 5/-1/-1->3->0 [39] -1/-1/-1->3->2 [40] 7/-1/-1->3->5 [41] 4/-1/-1->3->6 [42] 0/-1/-1->3->4 [43] 6/-1/-1->3->7 [44] 5/-1/-1->3->0 [45] -1/-1/-1->3->2 [46] 1/-1/-1->3->11 [47] 2/-1/-1->3->1 [48] 0/-1/-1->3->4 [49] 6/-1/-1->3->7 [50] 5/-1/-1->3->0 [51] -1/-1/-1->3->2 [52] 1/-1/-1->3->11 [53] 2/-1/-1->3->1 [54] 7/-1/-1->3->5 [55] 4/-1/-1->3->6 [56] 5/-1/-1->3->0 [57] -1/-1/-1->3->2 [58] 1/-1/-1->3->11 [59] 2/-1/-1->3->1 [60] 7/-1/-1->3->5 [61] 4/-1/-1->3->6 [62] 0/-1/-1->3->4 [63] 6/-1/-1->3->7 comm 0x48e1d6c0 nRanks 16 busId 15000 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Trees [0] -1/-1/-1->4->7 [1] 0/-1/-1->4->5 [2] 5/-1/-1->4->3 [3] 1/-1/-1->4->2 [4] 2/-1/-1->4->0 [5] 3/-1/-1->4->1 [6] 7/-1/-1->4->6 [7] 6/12/-1->4->-1 [8] 5/-1/-1->4->3 [9] 1/-1/-1->4->2 [10] 2/-1/-1->4->0 [11] 3/-1/-1->4->1 [12] 7/-1/-1->4->6 [13] 6/12/-1->4->-1 [14] -1/-1/-1->4->7 [15] 0/-1/-1->4->5 [16] 2/-1/-1->4->0 [17] 3/-1/-1->4->1 [18] 7/-1/-1->4->6 [19] 6/12/-1->4->-1 [20] -1/-1/-1->4->7 [21] 0/-1/-1->4->5 [22] 5/-1/-1->4->3 [23] 1/-1/-1->4->2 [24] 7/-1/-1->4->6 [25] 6/12/-1->4->-1 [26] -1/-1/-1->4->7 [27] 0/-1/-1->4->5 [28] 5/-1/-1->4->3 [29] 1/-1/-1->4->2 [30] 2/-1/-1->4->0 [31] 3/-1/-1->4->1 [32] 0/-1/-1->4->5 [33] -1/-1/-1->4->7 [34] 1/-1/-1->4->2 [35] 5/-1/-1->4->3 [36] 3/-1/-1->4->1 [37] 2/-1/-1->4->0 [38] 6/-1/-1->4->12 [39] 7/-1/-1->4->6 [40] 1/-1/-1->4->2 [41] 5/-1/-1->4->3 [42] 3/-1/-1->4->1 [43] 2/-1/-1->4->0 [44] 6/-1/-1->4->12 [45] 7/-1/-1->4->6 [46] 0/-1/-1->4->5 [47] -1/-1/-1->4->7 [48] 3/-1/-1->4->1 [49] 2/-1/-1->4->0 [50] 6/-1/-1->4->12 [51] 7/-1/-1->4->6 [52] 0/-1/-1->4->5 [53] -1/-1/-1->4->7 [54] 1/-1/-1->4->2 [55] 5/-1/-1->4->3 [56] 6/-1/-1->4->12 [57] 7/-1/-1->4->6 [58] 0/-1/-1->4->5 [59] -1/-1/-1->4->7 [60] 1/-1/-1->4->2 [61] 5/-1/-1->4->3 [62] 3/-1/-1->4->1 [63] 2/-1/-1->4->0 comm 0x507e6620 nRanks 16 busId f5000 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO P2P Chunksize set to 131072 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO P2P Chunksize set to 131072 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Trees [0] 5/-1/-1->2->3 [1] 6/-1/-1->2->7 [2] 0/10/-1->2->-1 [3] 4/-1/-1->2->0 [4] -1/-1/-1->2->4 [5] 1/-1/-1->2->6 [6] 3/-1/-1->2->1 [7] 7/-1/-1->2->5 [8] 0/10/-1->2->-1 [9] 4/-1/-1->2->0 [10] -1/-1/-1->2->4 [11] 1/-1/-1->2->6 [12] 3/-1/-1->2->1 [13] 7/-1/-1->2->5 [14] 5/-1/-1->2->3 [15] 6/-1/-1->2->7 [16] -1/-1/-1->2->4 [17] 1/-1/-1->2->6 [18] 3/-1/-1->2->1 [19] 7/-1/-1->2->5 [20] 5/-1/-1->2->3 [21] 6/-1/-1->2->7 [22] 0/10/-1->2->-1 [23] 4/-1/-1->2->0 [24] 3/-1/-1->2->1 [25] 7/-1/-1->2->5 [26] 5/-1/-1->2->3 [27] 6/-1/-1->2->7 [28] 0/10/-1->2->-1 [29] 4/-1/-1->2->0 [30] -1/-1/-1->2->4 [31] 1/-1/-1->2->6 [32] 6/-1/-1->2->7 [33] 5/-1/-1->2->3 [34] 4/-1/-1->2->0 [35] 0/-1/-1->2->10 [36] 1/-1/-1->2->6 [37] -1/-1/-1->2->4 [38] 7/-1/-1->2->5 [39] 3/-1/-1->2->1 [40] 4/-1/-1->2->0 [41] 0/-1/-1->2->10 [42] 1/-1/-1->2->6 [43] -1/-1/-1->2->4 [44] 7/-1/-1->2->5 [45] 3/-1/-1->2->1 [46] 6/-1/-1->2->7 [47] 5/-1/-1->2->3 [48] 1/-1/-1->2->6 [49] -1/-1/-1->2->4 [50] 7/-1/-1->2->5 [51] 3/-1/-1->2->1 [52] 6/-1/-1->2->7 [53] 5/-1/-1->2->3 [54] 4/-1/-1->2->0 [55] 0/-1/-1->2->10 [56] 7/-1/-1->2->5 [57] 3/-1/-1->2->1 [58] 6/-1/-1->2->7 [59] 5/-1/-1->2->3 [60] 4/-1/-1->2->0 [61] 0/-1/-1->2->10 [62] 1/-1/-1->2->6 [63] -1/-1/-1->2->4 comm 0x7e4c8c40 nRanks 16 busId 65000 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 06/64 : 0 3 5 1 4 7 2 10 15 12 9 13 11 8 14 6 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO P2P Chunksize set to 131072 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO P2P Chunksize set to 131072 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO P2P Chunksize set to 131072 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 07/64 : 0 8 15 11 13 14 9 10 12 4 2 1 6 5 3 7 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 08/64 : 0 7 4 6 14 12 15 8 13 10 9 11 3 1 2 5 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 09/64 : 0 6 3 4 5 13 12 11 14 8 9 15 10 2 7 1 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 10/64 : 0 4 1 5 2 3 6 7 15 14 11 10 13 9 12 8 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 11/64 : 0 1 7 3 11 15 9 8 12 10 14 13 5 6 2 4 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 12/64 : 0 2 6 1 9 14 10 8 11 12 13 15 7 5 4 3 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 13/64 : 0 3 5 1 4 7 2 10 15 12 9 13 11 8 14 6 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 14/64 : 0 8 15 11 13 14 9 10 12 4 2 1 6 5 3 7 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 15/64 : 0 5 7 6 4 12 14 15 13 8 10 11 9 1 3 2 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 16/64 : 0 6 3 4 5 13 12 11 14 8 9 15 10 2 7 1 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 17/64 : 0 4 1 5 2 3 6 7 15 14 11 10 13 9 12 8 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 18/64 : 0 1 7 3 11 15 9 8 12 10 14 13 5 6 2 4 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 19/64 : 0 2 6 1 9 14 10 8 11 12 13 15 7 5 4 3 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 20/64 : 0 3 5 1 4 7 2 10 15 12 9 13 11 8 14 6 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 21/64 : 0 8 15 11 13 14 9 10 12 4 2 1 6 5 3 7 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 22/64 : 0 5 7 6 4 12 14 15 13 8 10 11 9 1 3 2 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 23/64 : 0 7 4 6 14 12 15 8 13 10 9 11 3 1 2 5 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 24/64 : 0 4 1 5 2 3 6 7 15 14 11 10 13 9 12 8 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 25/64 : 0 1 7 3 11 15 9 8 12 10 14 13 5 6 2 4 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 26/64 : 0 2 6 1 9 14 10 8 11 12 13 15 7 5 4 3 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 27/64 : 0 3 5 1 4 7 2 10 15 12 9 13 11 8 14 6 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 28/64 : 0 8 15 11 13 14 9 10 12 4 2 1 6 5 3 7 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 29/64 : 0 5 7 6 4 12 14 15 13 8 10 11 9 1 3 2 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 30/64 : 0 7 4 6 14 12 15 8 13 10 9 11 3 1 2 5 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 31/64 : 0 6 3 4 5 13 12 11 14 8 9 15 10 2 7 1 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 32/64 : 0 1 7 3 11 15 9 8 12 10 14 13 5 6 2 4 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 33/64 : 0 2 6 1 9 14 10 8 11 12 13 15 7 5 4 3 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 34/64 : 0 3 5 1 4 7 2 10 15 12 9 13 11 8 14 6 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 35/64 : 0 8 15 11 13 14 9 10 12 4 2 1 6 5 3 7 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 36/64 : 0 5 7 6 4 12 14 15 13 8 10 11 9 1 3 2 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 37/64 : 0 7 4 6 14 12 15 8 13 10 9 11 3 1 2 5 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 38/64 : 0 6 3 4 5 13 12 11 14 8 9 15 10 2 7 1 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 39/64 : 0 4 1 5 2 3 6 7 15 14 11 10 13 9 12 8 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 40/64 : 0 2 6 1 9 14 10 8 11 12 13 15 7 5 4 3 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 41/64 : 0 3 5 1 4 7 2 10 15 12 9 13 11 8 14 6 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 42/64 : 0 8 15 11 13 14 9 10 12 4 2 1 6 5 3 7 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 43/64 : 0 5 7 6 4 12 14 15 13 8 10 11 9 1 3 2 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 44/64 : 0 7 4 6 14 12 15 8 13 10 9 11 3 1 2 5 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 45/64 : 0 6 3 4 5 13 12 11 14 8 9 15 10 2 7 1 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 46/64 : 0 4 1 5 2 3 6 7 15 14 11 10 13 9 12 8 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 47/64 : 0 1 7 3 11 15 9 8 12 10 14 13 5 6 2 4 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 48/64 : 0 3 5 1 4 7 2 10 15 12 9 13 11 8 14 6 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 49/64 : 0 8 15 11 13 14 9 10 12 4 2 1 6 5 3 7 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 50/64 : 0 5 7 6 4 12 14 15 13 8 10 11 9 1 3 2 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 51/64 : 0 7 4 6 14 12 15 8 13 10 9 11 3 1 2 5 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 52/64 : 0 6 3 4 5 13 12 11 14 8 9 15 10 2 7 1 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 53/64 : 0 4 1 5 2 3 6 7 15 14 11 10 13 9 12 8 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 54/64 : 0 1 7 3 11 15 9 8 12 10 14 13 5 6 2 4 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 55/64 : 0 2 6 1 9 14 10 8 11 12 13 15 7 5 4 3 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 56/64 : 0 8 15 11 13 14 9 10 12 4 2 1 6 5 3 7 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 57/64 : 0 5 7 6 4 12 14 15 13 8 10 11 9 1 3 2 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 58/64 : 0 7 4 6 14 12 15 8 13 10 9 11 3 1 2 5 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 59/64 : 0 6 3 4 5 13 12 11 14 8 9 15 10 2 7 1 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 60/64 : 0 4 1 5 2 3 6 7 15 14 11 10 13 9 12 8 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 61/64 : 0 1 7 3 11 15 9 8 12 10 14 13 5 6 2 4 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 62/64 : 0 2 6 1 9 14 10 8 11 12 13 15 7 5 4 3 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 63/64 : 0 3 5 1 4 7 2 10 15 12 9 13 11 8 14 6 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Trees [0] 6/-1/-1->0->5 [1] 7/-1/-1->0->4 [2] 1/-1/-1->0->2 [3] 2/8/-1->0->-1 [4] 4/-1/-1->0->1 [5] -1/-1/-1->0->3 [6] 5/-1/-1->0->7 [7] 3/-1/-1->0->6 [8] 1/-1/-1->0->2 [9] 2/8/-1->0->-1 [10] 4/-1/-1->0->1 [11] -1/-1/-1->0->3 [12] 5/-1/-1->0->7 [13] 3/-1/-1->0->6 [14] 6/-1/-1->0->5 [15] 7/-1/-1->0->4 [16] 4/-1/-1->0->1 [17] -1/-1/-1->0->3 [18] 5/-1/-1->0->7 [19] 3/-1/-1->0->6 [20] 6/-1/-1->0->5 [21] 7/-1/-1->0->4 [22] 1/-1/-1->0->2 [23] 2/8/-1->0->-1 [24] 5/-1/-1->0->7 [25] 3/-1/-1->0->6 [26] 6/-1/-1->0->5 [27] 7/-1/-1->0->4 [28] 1/-1/-1->0->2 [29] 2/8/-1->0->-1 [30] 4/-1/-1->0->1 [31] -1/-1/-1->0->3 [32] 7/-1/-1->0->4 [33] 6/-1/-1->0->5 [34] 2/-1/-1->0->8 [35] 1/-1/-1->0->2 [36] -1/-1/-1->0->3 [37] 4/-1/-1->0->1 [38] 3/-1/-1->0->6 [39] 5/-1/-1->0->7 [40] 2/-1/-1->0->8 [41] 1/-1/-1->0->2 [42] -1/-1/-1->0->3 [43] 4/-1/-1->0->1 [44] 3/-1/-1->0->6 [45] 5/-1/-1->0->7 [46] 7/-1/-1->0->4 [47] 6/-1/-1->0->5 [48] -1/-1/-1->0->3 [49] 4/-1/-1->0->1 [50] 3/-1/-1->0->6 [51] 5/-1/-1->0->7 [52] 7/-1/-1->0->4 [53] 6/-1/-1->0->5 [54] 2/-1/-1->0->8 [55] 1/-1/-1->0->2 [56] 3/-1/-1->0->6 [57] 5/-1/-1->0->7 [58] 7/-1/-1->0->4 [59] 6/-1/-1->0->5 [60] 2/-1/-1->0->8 [61] 1/-1/-1->0->2 [62] -1/-1/-1->0->3 [63] 4/-1/-1->0->1 comm 0x80bc0b40 nRanks 16 busId 75000 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO P2P Chunksize set to 131072 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 04/0 : 0[75000] -> 1[5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 11/0 : 0[75000] -> 1[5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 03/0 : 6[e5000] -> 7[95000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 18/0 : 0[75000] -> 1[5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 10/0 : 6[e5000] -> 7[95000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 02/0 : 3[15000] -> 4[f5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 25/0 : 0[75000] -> 1[5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 09/0 : 3[15000] -> 4[f5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 32/0 : 0[75000] -> 1[5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 16/0 : 3[15000] -> 4[f5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 47/0 : 0[75000] -> 1[5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 31/0 : 3[15000] -> 4[f5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 54/0 : 0[75000] -> 1[5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 38/0 : 3[15000] -> 4[f5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 61/0 : 0[75000] -> 1[5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 04/0 : 5[85000] -> 6[e5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 45/0 : 3[15000] -> 4[f5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 11/0 : 5[85000] -> 6[e5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 52/0 : 3[15000] -> 4[f5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 01/0 : 1[5000] -> 2[65000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 18/0 : 5[85000] -> 6[e5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 17/0 : 6[e5000] -> 7[95000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 24/0 : 6[e5000] -> 7[95000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 39/0 : 6[e5000] -> 7[95000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 46/0 : 6[e5000] -> 7[95000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 02/0 : 4[f5000] -> 5[85000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 53/0 : 6[e5000] -> 7[95000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 60/0 : 6[e5000] -> 7[95000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 03/0 : 2[65000] -> 3[15000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 59/0 : 3[15000] -> 4[f5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 08/0 : 1[5000] -> 2[65000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 25/0 : 5[85000] -> 6[e5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 23/0 : 1[5000] -> 2[65000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 32/0 : 5[85000] -> 6[e5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 30/0 : 1[5000] -> 2[65000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 47/0 : 5[85000] -> 6[e5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 37/0 : 1[5000] -> 2[65000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 54/0 : 5[85000] -> 6[e5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 44/0 : 1[5000] -> 2[65000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 61/0 : 5[85000] -> 6[e5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 09/0 : 4[f5000] -> 5[85000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 51/0 : 1[5000] -> 2[65000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 16/0 : 4[f5000] -> 5[85000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 58/0 : 1[5000] -> 2[65000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 10/0 : 2[65000] -> 3[15000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 31/0 : 4[f5000] -> 5[85000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 17/0 : 2[65000] -> 3[15000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 38/0 : 4[f5000] -> 5[85000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 24/0 : 2[65000] -> 3[15000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 45/0 : 4[f5000] -> 5[85000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 39/0 : 2[65000] -> 3[15000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 52/0 : 4[f5000] -> 5[85000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 46/0 : 2[65000] -> 3[15000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 59/0 : 4[f5000] -> 5[85000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 53/0 : 2[65000] -> 3[15000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 60/0 : 2[65000] -> 3[15000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 05/0 : 0[75000] -> 2[65000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 12/0 : 0[75000] -> 2[65000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 19/0 : 0[75000] -> 2[65000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 26/0 : 0[75000] -> 2[65000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 33/0 : 0[75000] -> 2[65000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 00/0 : 1[5000] -> 3[15000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 40/0 : 0[75000] -> 2[65000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 55/0 : 0[75000] -> 2[65000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 62/0 : 0[75000] -> 2[65000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 15/0 : 1[5000] -> 3[15000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 22/0 : 1[5000] -> 3[15000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 29/0 : 1[5000] -> 3[15000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 36/0 : 1[5000] -> 3[15000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 43/0 : 1[5000] -> 3[15000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 50/0 : 1[5000] -> 3[15000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 00/0 : 5[85000] -> 7[95000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 57/0 : 1[5000] -> 3[15000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 01/0 : 4[f5000] -> 6[e5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 06/0 : 3[15000] -> 5[85000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 08/0 : 4[f5000] -> 6[e5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 13/0 : 3[15000] -> 5[85000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 04/0 : 2[65000] -> 4[f5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 23/0 : 4[f5000] -> 6[e5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 11/0 : 2[65000] -> 4[f5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 30/0 : 4[f5000] -> 6[e5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 18/0 : 2[65000] -> 4[f5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 37/0 : 4[f5000] -> 6[e5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 25/0 : 2[65000] -> 4[f5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 20/0 : 3[15000] -> 5[85000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 32/0 : 2[65000] -> 4[f5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 15/0 : 5[85000] -> 7[95000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 27/0 : 3[15000] -> 5[85000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 22/0 : 5[85000] -> 7[95000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 34/0 : 3[15000] -> 5[85000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 29/0 : 5[85000] -> 7[95000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 41/0 : 3[15000] -> 5[85000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 36/0 : 5[85000] -> 7[95000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 48/0 : 3[15000] -> 5[85000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 43/0 : 5[85000] -> 7[95000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 63/0 : 3[15000] -> 5[85000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 44/0 : 4[f5000] -> 6[e5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 50/0 : 5[85000] -> 7[95000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 51/0 : 4[f5000] -> 6[e5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 57/0 : 5[85000] -> 7[95000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 47/0 : 2[65000] -> 4[f5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 06/0 : 1[5000] -> 4[f5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 58/0 : 4[f5000] -> 6[e5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 54/0 : 2[65000] -> 4[f5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 61/0 : 2[65000] -> 4[f5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 06/0 : 0[75000] -> 3[15000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 01/0 : 2[65000] -> 5[85000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 08/0 : 2[65000] -> 5[85000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 13/0 : 0[75000] -> 3[15000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 23/0 : 2[65000] -> 5[85000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 30/0 : 2[65000] -> 5[85000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 13/0 : 1[5000] -> 4[f5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 20/0 : 1[5000] -> 4[f5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 27/0 : 1[5000] -> 4[f5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 34/0 : 1[5000] -> 4[f5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 41/0 : 1[5000] -> 4[f5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 20/0 : 0[75000] -> 3[15000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 48/0 : 1[5000] -> 4[f5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 37/0 : 2[65000] -> 5[85000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 27/0 : 0[75000] -> 3[15000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 44/0 : 2[65000] -> 5[85000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 34/0 : 0[75000] -> 3[15000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 51/0 : 2[65000] -> 5[85000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 03/0 : 3[15000] -> 6[e5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 41/0 : 0[75000] -> 3[15000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 58/0 : 2[65000] -> 5[85000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 10/0 : 3[15000] -> 6[e5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 48/0 : 0[75000] -> 3[15000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 17/0 : 3[15000] -> 6[e5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 63/0 : 0[75000] -> 3[15000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 24/0 : 3[15000] -> 6[e5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 39/0 : 3[15000] -> 6[e5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 63/0 : 1[5000] -> 4[f5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 46/0 : 3[15000] -> 6[e5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 53/0 : 3[15000] -> 6[e5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 05/0 : 2[65000] -> 6[e5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 60/0 : 3[15000] -> 6[e5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 03/0 : 0[75000] -> 4[f5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 07/0 : 3[15000] -> 7[95000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 10/0 : 0[75000] -> 4[f5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 14/0 : 3[15000] -> 7[95000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 06/0 : 4[f5000] -> 7[95000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 17/0 : 0[75000] -> 4[f5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 21/0 : 3[15000] -> 7[95000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 13/0 : 4[f5000] -> 7[95000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 12/0 : 2[65000] -> 6[e5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 24/0 : 0[75000] -> 4[f5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 20/0 : 4[f5000] -> 7[95000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 19/0 : 2[65000] -> 6[e5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 39/0 : 0[75000] -> 4[f5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 27/0 : 4[f5000] -> 7[95000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 26/0 : 2[65000] -> 6[e5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 46/0 : 0[75000] -> 4[f5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 34/0 : 4[f5000] -> 7[95000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 33/0 : 2[65000] -> 6[e5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 53/0 : 0[75000] -> 4[f5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 41/0 : 4[f5000] -> 7[95000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 60/0 : 0[75000] -> 4[f5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 40/0 : 2[65000] -> 6[e5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 55/0 : 2[65000] -> 6[e5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 28/0 : 3[15000] -> 7[95000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 35/0 : 3[15000] -> 7[95000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 42/0 : 3[15000] -> 7[95000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 49/0 : 3[15000] -> 7[95000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 56/0 : 3[15000] -> 7[95000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 48/0 : 4[f5000] -> 7[95000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 63/0 : 4[f5000] -> 7[95000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 62/0 : 2[65000] -> 6[e5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 03/0 : 1[5000] -> 5[85000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 02/0 : 2[65000] -> 7[95000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 09/0 : 2[65000] -> 7[95000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 16/0 : 2[65000] -> 7[95000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 31/0 : 2[65000] -> 7[95000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 38/0 : 2[65000] -> 7[95000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 10/0 : 1[5000] -> 5[85000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 17/0 : 1[5000] -> 5[85000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 24/0 : 1[5000] -> 5[85000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 39/0 : 1[5000] -> 5[85000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 46/0 : 1[5000] -> 5[85000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 53/0 : 1[5000] -> 5[85000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 45/0 : 2[65000] -> 7[95000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 60/0 : 1[5000] -> 5[85000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 52/0 : 2[65000] -> 7[95000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 00/0 : 0[75000] -> 5[85000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 59/0 : 2[65000] -> 7[95000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 07/0 : 12[f5000] -> 4[f5000] [receive] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 14/0 : 12[f5000] -> 4[f5000] [receive] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 21/0 : 12[f5000] -> 4[f5000] [receive] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 28/0 : 12[f5000] -> 4[f5000] [receive] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 35/0 : 12[f5000] -> 4[f5000] [receive] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 42/0 : 12[f5000] -> 4[f5000] [receive] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 15/0 : 0[75000] -> 5[85000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 49/0 : 12[f5000] -> 4[f5000] [receive] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 56/0 : 12[f5000] -> 4[f5000] [receive] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 00/0 : 4[f5000] -> 12[f5000] [send] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 15/0 : 4[f5000] -> 12[f5000] [send] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 22/0 : 4[f5000] -> 12[f5000] [send] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 29/0 : 4[f5000] -> 12[f5000] [send] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 36/0 : 4[f5000] -> 12[f5000] [send] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 43/0 : 4[f5000] -> 12[f5000] [send] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 50/0 : 4[f5000] -> 12[f5000] [send] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 57/0 : 4[f5000] -> 12[f5000] [send] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 22/0 : 0[75000] -> 5[85000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 01/0 : 11[15000] -> 3[15000] [receive] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 08/0 : 11[15000] -> 3[15000] [receive] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 23/0 : 11[15000] -> 3[15000] [receive] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 30/0 : 11[15000] -> 3[15000] [receive] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 37/0 : 11[15000] -> 3[15000] [receive] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 44/0 : 11[15000] -> 3[15000] [receive] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 51/0 : 11[15000] -> 3[15000] [receive] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 58/0 : 11[15000] -> 3[15000] [receive] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 04/0 : 3[15000] -> 11[15000] [send] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 11/0 : 3[15000] -> 11[15000] [send] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 18/0 : 3[15000] -> 11[15000] [send] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 25/0 : 3[15000] -> 11[15000] [send] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 32/0 : 3[15000] -> 11[15000] [send] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 47/0 : 3[15000] -> 11[15000] [send] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 54/0 : 3[15000] -> 11[15000] [send] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 61/0 : 3[15000] -> 11[15000] [send] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 07/0 : 1[5000] -> 6[e5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 14/0 : 1[5000] -> 6[e5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 21/0 : 1[5000] -> 6[e5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 28/0 : 1[5000] -> 6[e5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 35/0 : 1[5000] -> 6[e5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 29/0 : 0[75000] -> 5[85000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 36/0 : 0[75000] -> 5[85000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 43/0 : 0[75000] -> 5[85000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 42/0 : 1[5000] -> 6[e5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 50/0 : 0[75000] -> 5[85000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 57/0 : 0[75000] -> 5[85000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 02/0 : 10[65000] -> 2[65000] [receive] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 09/0 : 10[65000] -> 2[65000] [receive] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 16/0 : 10[65000] -> 2[65000] [receive] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 31/0 : 10[65000] -> 2[65000] [receive] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 38/0 : 10[65000] -> 2[65000] [receive] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 45/0 : 10[65000] -> 2[65000] [receive] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 52/0 : 10[65000] -> 2[65000] [receive] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 59/0 : 10[65000] -> 2[65000] [receive] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 06/0 : 2[65000] -> 10[65000] [send] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 13/0 : 2[65000] -> 10[65000] [send] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 20/0 : 2[65000] -> 10[65000] [send] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 27/0 : 2[65000] -> 10[65000] [send] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 34/0 : 2[65000] -> 10[65000] [send] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 41/0 : 2[65000] -> 10[65000] [send] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 48/0 : 2[65000] -> 10[65000] [send] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 63/0 : 2[65000] -> 10[65000] [send] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 49/0 : 1[5000] -> 6[e5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 56/0 : 1[5000] -> 6[e5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 04/0 : 1[5000] -> 7[95000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 04/0 : 13[85000] -> 5[85000] [receive] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 11/0 : 13[85000] -> 5[85000] [receive] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 18/0 : 13[85000] -> 5[85000] [receive] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 25/0 : 13[85000] -> 5[85000] [receive] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 32/0 : 13[85000] -> 5[85000] [receive] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 47/0 : 13[85000] -> 5[85000] [receive] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 54/0 : 13[85000] -> 5[85000] [receive] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 61/0 : 13[85000] -> 5[85000] [receive] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 02/0 : 5[85000] -> 13[85000] [send] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 09/0 : 5[85000] -> 13[85000] [send] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 16/0 : 5[85000] -> 13[85000] [send] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 31/0 : 5[85000] -> 13[85000] [send] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 38/0 : 5[85000] -> 13[85000] [send] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 45/0 : 5[85000] -> 13[85000] [send] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 52/0 : 5[85000] -> 13[85000] [send] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 59/0 : 5[85000] -> 13[85000] [send] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 11/0 : 1[5000] -> 7[95000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 02/0 : 0[75000] -> 6[e5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 18/0 : 1[5000] -> 7[95000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 09/0 : 0[75000] -> 6[e5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 25/0 : 1[5000] -> 7[95000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 16/0 : 0[75000] -> 6[e5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 32/0 : 1[5000] -> 7[95000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 31/0 : 0[75000] -> 6[e5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 38/0 : 0[75000] -> 6[e5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 45/0 : 0[75000] -> 6[e5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 52/0 : 0[75000] -> 6[e5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 59/0 : 0[75000] -> 6[e5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 47/0 : 1[5000] -> 7[95000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 54/0 : 1[5000] -> 7[95000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 61/0 : 1[5000] -> 7[95000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 00/0 : 9[5000] -> 1[5000] [receive] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 15/0 : 9[5000] -> 1[5000] [receive] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 22/0 : 9[5000] -> 1[5000] [receive] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 29/0 : 9[5000] -> 1[5000] [receive] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 36/0 : 9[5000] -> 1[5000] [receive] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 43/0 : 9[5000] -> 1[5000] [receive] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 50/0 : 9[5000] -> 1[5000] [receive] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 57/0 : 9[5000] -> 1[5000] [receive] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 05/0 : 1[5000] -> 9[5000] [send] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 12/0 : 1[5000] -> 9[5000] [send] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 19/0 : 1[5000] -> 9[5000] [send] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 26/0 : 1[5000] -> 9[5000] [send] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 33/0 : 1[5000] -> 9[5000] [send] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 40/0 : 1[5000] -> 9[5000] [send] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 55/0 : 1[5000] -> 9[5000] [send] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 62/0 : 1[5000] -> 9[5000] [send] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 01/0 : 0[75000] -> 7[95000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 06/0 : 14[e5000] -> 6[e5000] [receive] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 13/0 : 14[e5000] -> 6[e5000] [receive] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 20/0 : 14[e5000] -> 6[e5000] [receive] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 27/0 : 14[e5000] -> 6[e5000] [receive] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 34/0 : 14[e5000] -> 6[e5000] [receive] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 41/0 : 14[e5000] -> 6[e5000] [receive] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 48/0 : 14[e5000] -> 6[e5000] [receive] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 63/0 : 14[e5000] -> 6[e5000] [receive] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 01/0 : 6[e5000] -> 14[e5000] [send] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 08/0 : 6[e5000] -> 14[e5000] [send] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 23/0 : 6[e5000] -> 14[e5000] [send] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 08/0 : 0[75000] -> 7[95000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 30/0 : 6[e5000] -> 14[e5000] [send] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 37/0 : 6[e5000] -> 14[e5000] [send] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 44/0 : 6[e5000] -> 14[e5000] [send] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 51/0 : 6[e5000] -> 14[e5000] [send] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 58/0 : 6[e5000] -> 14[e5000] [send] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 23/0 : 0[75000] -> 7[95000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 30/0 : 0[75000] -> 7[95000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 37/0 : 0[75000] -> 7[95000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 44/0 : 0[75000] -> 7[95000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 51/0 : 0[75000] -> 7[95000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 58/0 : 0[75000] -> 7[95000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 03/0 : 8[75000] -> 0[75000] [receive] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 10/0 : 8[75000] -> 0[75000] [receive] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 05/0 : 15[95000] -> 7[95000] [receive] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 17/0 : 8[75000] -> 0[75000] [receive] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 12/0 : 15[95000] -> 7[95000] [receive] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 24/0 : 8[75000] -> 0[75000] [receive] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 19/0 : 15[95000] -> 7[95000] [receive] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 39/0 : 8[75000] -> 0[75000] [receive] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 26/0 : 15[95000] -> 7[95000] [receive] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 46/0 : 8[75000] -> 0[75000] [receive] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 33/0 : 15[95000] -> 7[95000] [receive] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 53/0 : 8[75000] -> 0[75000] [receive] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 40/0 : 15[95000] -> 7[95000] [receive] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 60/0 : 8[75000] -> 0[75000] [receive] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 55/0 : 15[95000] -> 7[95000] [receive] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 07/0 : 0[75000] -> 8[75000] [send] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 62/0 : 15[95000] -> 7[95000] [receive] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 14/0 : 0[75000] -> 8[75000] [send] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 03/0 : 7[95000] -> 15[95000] [send] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 21/0 : 0[75000] -> 8[75000] [send] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 10/0 : 7[95000] -> 15[95000] [send] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 28/0 : 0[75000] -> 8[75000] [send] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 17/0 : 7[95000] -> 15[95000] [send] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 35/0 : 0[75000] -> 8[75000] [send] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 24/0 : 7[95000] -> 15[95000] [send] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 39/0 : 7[95000] -> 15[95000] [send] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 42/0 : 0[75000] -> 8[75000] [send] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 46/0 : 7[95000] -> 15[95000] [send] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 49/0 : 0[75000] -> 8[75000] [send] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 53/0 : 7[95000] -> 15[95000] [send] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 56/0 : 0[75000] -> 8[75000] [send] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 60/0 : 7[95000] -> 15[95000] [send] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 04/0 : 4[f5000] -> 0[75000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 11/0 : 4[f5000] -> 0[75000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 18/0 : 4[f5000] -> 0[75000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 25/0 : 4[f5000] -> 0[75000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 32/0 : 4[f5000] -> 0[75000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 47/0 : 4[f5000] -> 0[75000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 54/0 : 4[f5000] -> 0[75000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 61/0 : 4[f5000] -> 0[75000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 01/0 : 5[85000] -> 0[75000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 08/0 : 5[85000] -> 0[75000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 23/0 : 5[85000] -> 0[75000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 30/0 : 5[85000] -> 0[75000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 37/0 : 5[85000] -> 0[75000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 44/0 : 5[85000] -> 0[75000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 51/0 : 5[85000] -> 0[75000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 58/0 : 5[85000] -> 0[75000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 06/0 : 6[e5000] -> 0[75000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 13/0 : 6[e5000] -> 0[75000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 20/0 : 6[e5000] -> 0[75000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 27/0 : 6[e5000] -> 0[75000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 34/0 : 6[e5000] -> 0[75000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 41/0 : 6[e5000] -> 0[75000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 48/0 : 6[e5000] -> 0[75000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 63/0 : 6[e5000] -> 0[75000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 07/0 : 7[95000] -> 0[75000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 14/0 : 7[95000] -> 0[75000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 21/0 : 7[95000] -> 0[75000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 28/0 : 7[95000] -> 0[75000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 35/0 : 7[95000] -> 0[75000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 42/0 : 7[95000] -> 0[75000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 49/0 : 7[95000] -> 0[75000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 56/0 : 7[95000] -> 0[75000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 02/0 : 7[95000] -> 1[5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 09/0 : 7[95000] -> 1[5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 16/0 : 7[95000] -> 1[5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 31/0 : 7[95000] -> 1[5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 38/0 : 7[95000] -> 1[5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 05/0 : 6[e5000] -> 1[5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 12/0 : 6[e5000] -> 1[5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 45/0 : 7[95000] -> 1[5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 19/0 : 6[e5000] -> 1[5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 52/0 : 7[95000] -> 1[5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 26/0 : 6[e5000] -> 1[5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 59/0 : 7[95000] -> 1[5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 33/0 : 6[e5000] -> 1[5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 06/0 : 7[95000] -> 2[65000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 40/0 : 6[e5000] -> 1[5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 13/0 : 7[95000] -> 2[65000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 55/0 : 6[e5000] -> 1[5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 20/0 : 7[95000] -> 2[65000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 62/0 : 6[e5000] -> 1[5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 06/0 : 5[85000] -> 1[5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 27/0 : 7[95000] -> 2[65000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 34/0 : 7[95000] -> 2[65000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 41/0 : 7[95000] -> 2[65000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 13/0 : 5[85000] -> 1[5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 48/0 : 7[95000] -> 2[65000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 20/0 : 5[85000] -> 1[5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 63/0 : 7[95000] -> 2[65000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 27/0 : 5[85000] -> 1[5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 04/0 : 7[95000] -> 3[15000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 04/0 : 6[e5000] -> 2[65000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 34/0 : 5[85000] -> 1[5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 41/0 : 5[85000] -> 1[5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 48/0 : 5[85000] -> 1[5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 11/0 : 6[e5000] -> 2[65000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 18/0 : 6[e5000] -> 2[65000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 25/0 : 6[e5000] -> 2[65000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 32/0 : 6[e5000] -> 2[65000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 11/0 : 7[95000] -> 3[15000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 47/0 : 6[e5000] -> 2[65000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 18/0 : 7[95000] -> 3[15000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 63/0 : 5[85000] -> 1[5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 54/0 : 6[e5000] -> 2[65000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 25/0 : 7[95000] -> 3[15000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 32/0 : 7[95000] -> 3[15000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 61/0 : 6[e5000] -> 2[65000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 47/0 : 7[95000] -> 3[15000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 54/0 : 7[95000] -> 3[15000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 03/0 : 5[85000] -> 2[65000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 02/0 : 6[e5000] -> 3[15000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 09/0 : 6[e5000] -> 3[15000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 16/0 : 6[e5000] -> 3[15000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 03/0 : 4[f5000] -> 1[5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 61/0 : 7[95000] -> 3[15000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 31/0 : 6[e5000] -> 3[15000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 01/0 : 7[95000] -> 4[f5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 38/0 : 6[e5000] -> 3[15000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 08/0 : 7[95000] -> 4[f5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 10/0 : 5[85000] -> 2[65000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 23/0 : 7[95000] -> 4[f5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 17/0 : 5[85000] -> 2[65000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 30/0 : 7[95000] -> 4[f5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 24/0 : 5[85000] -> 2[65000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 37/0 : 7[95000] -> 4[f5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 39/0 : 5[85000] -> 2[65000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 46/0 : 5[85000] -> 2[65000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 10/0 : 4[f5000] -> 1[5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 53/0 : 5[85000] -> 2[65000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 17/0 : 4[f5000] -> 1[5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 45/0 : 6[e5000] -> 3[15000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 24/0 : 4[f5000] -> 1[5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 52/0 : 6[e5000] -> 3[15000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 39/0 : 4[f5000] -> 1[5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 59/0 : 6[e5000] -> 3[15000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 46/0 : 4[f5000] -> 1[5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 44/0 : 7[95000] -> 4[f5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 53/0 : 4[f5000] -> 1[5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 51/0 : 7[95000] -> 4[f5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 60/0 : 5[85000] -> 2[65000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 58/0 : 7[95000] -> 4[f5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 60/0 : 4[f5000] -> 1[5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 05/0 : 7[95000] -> 5[85000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 05/0 : 3[15000] -> 0[75000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 12/0 : 3[15000] -> 0[75000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 19/0 : 3[15000] -> 0[75000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 12/0 : 7[95000] -> 5[85000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 26/0 : 3[15000] -> 0[75000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 19/0 : 7[95000] -> 5[85000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 33/0 : 3[15000] -> 0[75000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 26/0 : 7[95000] -> 5[85000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 40/0 : 3[15000] -> 0[75000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 33/0 : 7[95000] -> 5[85000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 55/0 : 3[15000] -> 0[75000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 40/0 : 7[95000] -> 5[85000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 62/0 : 3[15000] -> 0[75000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 55/0 : 7[95000] -> 5[85000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 07/0 : 5[85000] -> 3[15000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 00/0 : 6[e5000] -> 4[f5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 14/0 : 5[85000] -> 3[15000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 21/0 : 5[85000] -> 3[15000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 15/0 : 6[e5000] -> 4[f5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 22/0 : 6[e5000] -> 4[f5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 00/0 : 2[65000] -> 0[75000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 07/0 : 4[f5000] -> 2[65000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 29/0 : 6[e5000] -> 4[f5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 15/0 : 2[65000] -> 0[75000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 14/0 : 4[f5000] -> 2[65000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 36/0 : 6[e5000] -> 4[f5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 22/0 : 2[65000] -> 0[75000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 62/0 : 7[95000] -> 5[85000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 21/0 : 4[f5000] -> 2[65000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 43/0 : 6[e5000] -> 4[f5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 29/0 : 2[65000] -> 0[75000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 50/0 : 6[e5000] -> 4[f5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 28/0 : 5[85000] -> 3[15000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 36/0 : 2[65000] -> 0[75000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 57/0 : 6[e5000] -> 4[f5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 35/0 : 5[85000] -> 3[15000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 42/0 : 5[85000] -> 3[15000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 49/0 : 5[85000] -> 3[15000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 56/0 : 5[85000] -> 3[15000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 28/0 : 4[f5000] -> 2[65000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 00/0 : 7[95000] -> 6[e5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 35/0 : 4[f5000] -> 2[65000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 01/0 : 3[15000] -> 1[5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 43/0 : 2[65000] -> 0[75000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 42/0 : 4[f5000] -> 2[65000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 08/0 : 3[15000] -> 1[5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 50/0 : 2[65000] -> 0[75000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 49/0 : 4[f5000] -> 2[65000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 23/0 : 3[15000] -> 1[5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 57/0 : 2[65000] -> 0[75000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 56/0 : 4[f5000] -> 2[65000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 30/0 : 3[15000] -> 1[5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 15/0 : 7[95000] -> 6[e5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 37/0 : 3[15000] -> 1[5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 22/0 : 7[95000] -> 6[e5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 44/0 : 3[15000] -> 1[5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 29/0 : 7[95000] -> 6[e5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 51/0 : 3[15000] -> 1[5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 36/0 : 7[95000] -> 6[e5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 58/0 : 3[15000] -> 1[5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 43/0 : 7[95000] -> 6[e5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 50/0 : 7[95000] -> 6[e5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 57/0 : 7[95000] -> 6[e5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 07/0 : 6[e5000] -> 5[85000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 14/0 : 6[e5000] -> 5[85000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 21/0 : 6[e5000] -> 5[85000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 28/0 : 6[e5000] -> 5[85000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 35/0 : 6[e5000] -> 5[85000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 00/0 : 3[15000] -> 2[65000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 05/0 : 5[85000] -> 4[f5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 42/0 : 6[e5000] -> 5[85000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 15/0 : 3[15000] -> 2[65000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 12/0 : 5[85000] -> 4[f5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 02/0 : 1[5000] -> 0[75000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 05/0 : 4[f5000] -> 3[15000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 49/0 : 6[e5000] -> 5[85000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 22/0 : 3[15000] -> 2[65000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 07/0 : 2[65000] -> 1[5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 12/0 : 4[f5000] -> 3[15000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 29/0 : 3[15000] -> 2[65000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 56/0 : 6[e5000] -> 5[85000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 14/0 : 2[65000] -> 1[5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 19/0 : 4[f5000] -> 3[15000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 09/0 : 1[5000] -> 0[75000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 36/0 : 3[15000] -> 2[65000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 21/0 : 2[65000] -> 1[5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 26/0 : 4[f5000] -> 3[15000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 43/0 : 3[15000] -> 2[65000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 28/0 : 2[65000] -> 1[5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 33/0 : 4[f5000] -> 3[15000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 50/0 : 3[15000] -> 2[65000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 35/0 : 2[65000] -> 1[5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 57/0 : 3[15000] -> 2[65000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 42/0 : 2[65000] -> 1[5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 19/0 : 5[85000] -> 4[f5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 26/0 : 5[85000] -> 4[f5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 16/0 : 1[5000] -> 0[75000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 33/0 : 5[85000] -> 4[f5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 31/0 : 1[5000] -> 0[75000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948895:1950343 [7] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3. +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 40/0 : 5[85000] -> 4[f5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 38/0 : 1[5000] -> 0[75000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 40/0 : 4[f5000] -> 3[15000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 55/0 : 5[85000] -> 4[f5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 45/0 : 1[5000] -> 0[75000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 55/0 : 4[f5000] -> 3[15000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 52/0 : 1[5000] -> 0[75000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 49/0 : 2[65000] -> 1[5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 62/0 : 4[f5000] -> 3[15000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 56/0 : 2[65000] -> 1[5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 62/0 : 5[85000] -> 4[f5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950354 [3] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3. +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 59/0 : 1[5000] -> 0[75000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948892:1950352 [4] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3. +chi-mi325x-pod2-101:1948894:1950344 [6] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3. +chi-mi325x-pod2-101:1948890:1950350 [2] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3. +chi-mi325x-pod2-101:1948889:1950345 [1] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3. +chi-mi325x-pod2-101:1948893:1950342 [5] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3. +chi-mi325x-pod2-101:1948888:1950356 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3. +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Connected all rings +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Connected all rings +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Connected all rings +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Connected all rings +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 04/2 : 0[75000] -> 1[5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Connected all rings +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Connected all rings +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Connected all rings +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Connected all rings +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 11/2 : 0[75000] -> 1[5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 18/2 : 0[75000] -> 1[5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 03/2 : 6[e5000] -> 7[95000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 25/2 : 0[75000] -> 1[5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 10/2 : 6[e5000] -> 7[95000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 32/2 : 0[75000] -> 1[5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 02/2 : 3[15000] -> 4[f5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 17/2 : 6[e5000] -> 7[95000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 47/2 : 0[75000] -> 1[5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 09/2 : 3[15000] -> 4[f5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 24/2 : 6[e5000] -> 7[95000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 39/2 : 6[e5000] -> 7[95000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 54/2 : 0[75000] -> 1[5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 61/2 : 0[75000] -> 1[5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 16/2 : 3[15000] -> 4[f5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 04/2 : 5[85000] -> 6[e5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 31/2 : 3[15000] -> 4[f5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 11/2 : 5[85000] -> 6[e5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 38/2 : 3[15000] -> 4[f5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 01/2 : 1[5000] -> 2[65000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 18/2 : 5[85000] -> 6[e5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 45/2 : 3[15000] -> 4[f5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 08/2 : 1[5000] -> 2[65000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 25/2 : 5[85000] -> 6[e5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 52/2 : 3[15000] -> 4[f5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 23/2 : 1[5000] -> 2[65000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 02/2 : 4[f5000] -> 5[85000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 46/2 : 6[e5000] -> 7[95000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 30/2 : 1[5000] -> 2[65000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 09/2 : 4[f5000] -> 5[85000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 03/2 : 2[65000] -> 3[15000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 53/2 : 6[e5000] -> 7[95000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 16/2 : 4[f5000] -> 5[85000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 10/2 : 2[65000] -> 3[15000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 60/2 : 6[e5000] -> 7[95000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 31/2 : 4[f5000] -> 5[85000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 17/2 : 2[65000] -> 3[15000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 38/2 : 4[f5000] -> 5[85000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 24/2 : 2[65000] -> 3[15000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 39/2 : 2[65000] -> 3[15000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 59/2 : 3[15000] -> 4[f5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 46/2 : 2[65000] -> 3[15000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 32/2 : 5[85000] -> 6[e5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 47/2 : 5[85000] -> 6[e5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 37/2 : 1[5000] -> 2[65000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 54/2 : 5[85000] -> 6[e5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 44/2 : 1[5000] -> 2[65000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 61/2 : 5[85000] -> 6[e5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 51/2 : 1[5000] -> 2[65000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 58/2 : 1[5000] -> 2[65000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 45/2 : 4[f5000] -> 5[85000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 05/2 : 0[75000] -> 2[65000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 52/2 : 4[f5000] -> 5[85000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 12/2 : 0[75000] -> 2[65000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 59/2 : 4[f5000] -> 5[85000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 53/2 : 2[65000] -> 3[15000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 19/2 : 0[75000] -> 2[65000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 60/2 : 2[65000] -> 3[15000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 26/2 : 0[75000] -> 2[65000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 00/2 : 1[5000] -> 3[15000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 33/2 : 0[75000] -> 2[65000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 40/2 : 0[75000] -> 2[65000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 55/2 : 0[75000] -> 2[65000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 62/2 : 0[75000] -> 2[65000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 15/2 : 1[5000] -> 3[15000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 22/2 : 1[5000] -> 3[15000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 29/2 : 1[5000] -> 3[15000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 06/2 : 3[15000] -> 5[85000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 36/2 : 1[5000] -> 3[15000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 13/2 : 3[15000] -> 5[85000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 43/2 : 1[5000] -> 3[15000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 50/2 : 1[5000] -> 3[15000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 57/2 : 1[5000] -> 3[15000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 01/2 : 4[f5000] -> 6[e5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 08/2 : 4[f5000] -> 6[e5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 23/2 : 4[f5000] -> 6[e5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 04/2 : 2[65000] -> 4[f5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 11/2 : 2[65000] -> 4[f5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 20/2 : 3[15000] -> 5[85000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 00/2 : 5[85000] -> 7[95000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 18/2 : 2[65000] -> 4[f5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 27/2 : 3[15000] -> 5[85000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 15/2 : 5[85000] -> 7[95000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 34/2 : 3[15000] -> 5[85000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 22/2 : 5[85000] -> 7[95000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 41/2 : 3[15000] -> 5[85000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 29/2 : 5[85000] -> 7[95000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 48/2 : 3[15000] -> 5[85000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 36/2 : 5[85000] -> 7[95000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 43/2 : 5[85000] -> 7[95000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 63/2 : 3[15000] -> 5[85000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 30/2 : 4[f5000] -> 6[e5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 50/2 : 5[85000] -> 7[95000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 37/2 : 4[f5000] -> 6[e5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 06/2 : 1[5000] -> 4[f5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 25/2 : 2[65000] -> 4[f5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 44/2 : 4[f5000] -> 6[e5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 13/2 : 1[5000] -> 4[f5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 32/2 : 2[65000] -> 4[f5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 51/2 : 4[f5000] -> 6[e5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 47/2 : 2[65000] -> 4[f5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 58/2 : 4[f5000] -> 6[e5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 54/2 : 2[65000] -> 4[f5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 61/2 : 2[65000] -> 4[f5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 06/2 : 0[75000] -> 3[15000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 01/2 : 2[65000] -> 5[85000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 13/2 : 0[75000] -> 3[15000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 08/2 : 2[65000] -> 5[85000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 57/2 : 5[85000] -> 7[95000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 20/2 : 0[75000] -> 3[15000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 20/2 : 1[5000] -> 4[f5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 27/2 : 0[75000] -> 3[15000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 27/2 : 1[5000] -> 4[f5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 34/2 : 1[5000] -> 4[f5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 34/2 : 0[75000] -> 3[15000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 41/2 : 1[5000] -> 4[f5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 41/2 : 0[75000] -> 3[15000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 48/2 : 1[5000] -> 4[f5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 48/2 : 0[75000] -> 3[15000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 63/2 : 1[5000] -> 4[f5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 63/2 : 0[75000] -> 3[15000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 23/2 : 2[65000] -> 5[85000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 30/2 : 2[65000] -> 5[85000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 37/2 : 2[65000] -> 5[85000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 44/2 : 2[65000] -> 5[85000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 51/2 : 2[65000] -> 5[85000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 58/2 : 2[65000] -> 5[85000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 03/2 : 3[15000] -> 6[e5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 10/2 : 3[15000] -> 6[e5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 17/2 : 3[15000] -> 6[e5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 06/2 : 4[f5000] -> 7[95000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 24/2 : 3[15000] -> 6[e5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 13/2 : 4[f5000] -> 7[95000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 39/2 : 3[15000] -> 6[e5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 20/2 : 4[f5000] -> 7[95000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 46/2 : 3[15000] -> 6[e5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 05/2 : 2[65000] -> 6[e5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 27/2 : 4[f5000] -> 7[95000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 53/2 : 3[15000] -> 6[e5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 12/2 : 2[65000] -> 6[e5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 34/2 : 4[f5000] -> 7[95000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 60/2 : 3[15000] -> 6[e5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 19/2 : 2[65000] -> 6[e5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 41/2 : 4[f5000] -> 7[95000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 07/2 : 3[15000] -> 7[95000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 26/2 : 2[65000] -> 6[e5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 03/2 : 0[75000] -> 4[f5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 48/2 : 4[f5000] -> 7[95000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 14/2 : 3[15000] -> 7[95000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 33/2 : 2[65000] -> 6[e5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 10/2 : 0[75000] -> 4[f5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 40/2 : 2[65000] -> 6[e5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 21/2 : 3[15000] -> 7[95000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 17/2 : 0[75000] -> 4[f5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 28/2 : 3[15000] -> 7[95000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 24/2 : 0[75000] -> 4[f5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 35/2 : 3[15000] -> 7[95000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 39/2 : 0[75000] -> 4[f5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 42/2 : 3[15000] -> 7[95000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 46/2 : 0[75000] -> 4[f5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 49/2 : 3[15000] -> 7[95000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 53/2 : 0[75000] -> 4[f5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 56/2 : 3[15000] -> 7[95000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 60/2 : 0[75000] -> 4[f5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 63/2 : 4[f5000] -> 7[95000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 03/2 : 1[5000] -> 5[85000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 55/2 : 2[65000] -> 6[e5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 10/2 : 1[5000] -> 5[85000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 62/2 : 2[65000] -> 6[e5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 02/2 : 2[65000] -> 7[95000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 09/2 : 2[65000] -> 7[95000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 16/2 : 2[65000] -> 7[95000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 17/2 : 1[5000] -> 5[85000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 24/2 : 1[5000] -> 5[85000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 39/2 : 1[5000] -> 5[85000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 00/2 : 0[75000] -> 5[85000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 07/2 : 12[f5000] -> 4[f5000] [receive] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 14/2 : 12[f5000] -> 4[f5000] [receive] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 21/2 : 12[f5000] -> 4[f5000] [receive] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 28/2 : 12[f5000] -> 4[f5000] [receive] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 46/2 : 1[5000] -> 5[85000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 35/2 : 12[f5000] -> 4[f5000] [receive] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 42/2 : 12[f5000] -> 4[f5000] [receive] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 31/2 : 2[65000] -> 7[95000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 49/2 : 12[f5000] -> 4[f5000] [receive] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 56/2 : 12[f5000] -> 4[f5000] [receive] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 00/2 : 4[f5000] -> 12[f5000] [send] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 15/2 : 4[f5000] -> 12[f5000] [send] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 22/2 : 4[f5000] -> 12[f5000] [send] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 29/2 : 4[f5000] -> 12[f5000] [send] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 36/2 : 4[f5000] -> 12[f5000] [send] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 43/2 : 4[f5000] -> 12[f5000] [send] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 50/2 : 4[f5000] -> 12[f5000] [send] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 57/2 : 4[f5000] -> 12[f5000] [send] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 15/2 : 0[75000] -> 5[85000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 38/2 : 2[65000] -> 7[95000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 22/2 : 0[75000] -> 5[85000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 45/2 : 2[65000] -> 7[95000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 29/2 : 0[75000] -> 5[85000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 52/2 : 2[65000] -> 7[95000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 36/2 : 0[75000] -> 5[85000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 59/2 : 2[65000] -> 7[95000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 01/2 : 11[15000] -> 3[15000] [receive] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 08/2 : 11[15000] -> 3[15000] [receive] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 23/2 : 11[15000] -> 3[15000] [receive] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 30/2 : 11[15000] -> 3[15000] [receive] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 37/2 : 11[15000] -> 3[15000] [receive] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 44/2 : 11[15000] -> 3[15000] [receive] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 51/2 : 11[15000] -> 3[15000] [receive] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 58/2 : 11[15000] -> 3[15000] [receive] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 04/2 : 3[15000] -> 11[15000] [send] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 11/2 : 3[15000] -> 11[15000] [send] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 43/2 : 0[75000] -> 5[85000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 18/2 : 3[15000] -> 11[15000] [send] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 25/2 : 3[15000] -> 11[15000] [send] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 32/2 : 3[15000] -> 11[15000] [send] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 47/2 : 3[15000] -> 11[15000] [send] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 54/2 : 3[15000] -> 11[15000] [send] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 61/2 : 3[15000] -> 11[15000] [send] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 50/2 : 0[75000] -> 5[85000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 57/2 : 0[75000] -> 5[85000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 53/2 : 1[5000] -> 5[85000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 60/2 : 1[5000] -> 5[85000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 07/2 : 1[5000] -> 6[e5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 14/2 : 1[5000] -> 6[e5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 21/2 : 1[5000] -> 6[e5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 28/2 : 1[5000] -> 6[e5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 35/2 : 1[5000] -> 6[e5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 02/2 : 10[65000] -> 2[65000] [receive] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 09/2 : 10[65000] -> 2[65000] [receive] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 16/2 : 10[65000] -> 2[65000] [receive] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 31/2 : 10[65000] -> 2[65000] [receive] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 38/2 : 10[65000] -> 2[65000] [receive] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 45/2 : 10[65000] -> 2[65000] [receive] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 52/2 : 10[65000] -> 2[65000] [receive] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 59/2 : 10[65000] -> 2[65000] [receive] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 06/2 : 2[65000] -> 10[65000] [send] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 13/2 : 2[65000] -> 10[65000] [send] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 20/2 : 2[65000] -> 10[65000] [send] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 27/2 : 2[65000] -> 10[65000] [send] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 34/2 : 2[65000] -> 10[65000] [send] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 41/2 : 2[65000] -> 10[65000] [send] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 48/2 : 2[65000] -> 10[65000] [send] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 63/2 : 2[65000] -> 10[65000] [send] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 42/2 : 1[5000] -> 6[e5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 49/2 : 1[5000] -> 6[e5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 56/2 : 1[5000] -> 6[e5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 02/2 : 0[75000] -> 6[e5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 04/2 : 13[85000] -> 5[85000] [receive] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 11/2 : 13[85000] -> 5[85000] [receive] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 18/2 : 13[85000] -> 5[85000] [receive] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 25/2 : 13[85000] -> 5[85000] [receive] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 32/2 : 13[85000] -> 5[85000] [receive] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 47/2 : 13[85000] -> 5[85000] [receive] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 04/2 : 1[5000] -> 7[95000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 54/2 : 13[85000] -> 5[85000] [receive] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 61/2 : 13[85000] -> 5[85000] [receive] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 02/2 : 5[85000] -> 13[85000] [send] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 09/2 : 5[85000] -> 13[85000] [send] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 16/2 : 5[85000] -> 13[85000] [send] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 31/2 : 5[85000] -> 13[85000] [send] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 38/2 : 5[85000] -> 13[85000] [send] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 45/2 : 5[85000] -> 13[85000] [send] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 52/2 : 5[85000] -> 13[85000] [send] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 59/2 : 5[85000] -> 13[85000] [send] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 09/2 : 0[75000] -> 6[e5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 11/2 : 1[5000] -> 7[95000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 16/2 : 0[75000] -> 6[e5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 18/2 : 1[5000] -> 7[95000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 31/2 : 0[75000] -> 6[e5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 25/2 : 1[5000] -> 7[95000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 38/2 : 0[75000] -> 6[e5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 45/2 : 0[75000] -> 6[e5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 52/2 : 0[75000] -> 6[e5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 59/2 : 0[75000] -> 6[e5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 32/2 : 1[5000] -> 7[95000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 47/2 : 1[5000] -> 7[95000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 54/2 : 1[5000] -> 7[95000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 61/2 : 1[5000] -> 7[95000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 00/2 : 9[5000] -> 1[5000] [receive] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 15/2 : 9[5000] -> 1[5000] [receive] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 22/2 : 9[5000] -> 1[5000] [receive] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 29/2 : 9[5000] -> 1[5000] [receive] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 36/2 : 9[5000] -> 1[5000] [receive] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 43/2 : 9[5000] -> 1[5000] [receive] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 50/2 : 9[5000] -> 1[5000] [receive] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 57/2 : 9[5000] -> 1[5000] [receive] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 05/2 : 1[5000] -> 9[5000] [send] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 12/2 : 1[5000] -> 9[5000] [send] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 19/2 : 1[5000] -> 9[5000] [send] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 26/2 : 1[5000] -> 9[5000] [send] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 33/2 : 1[5000] -> 9[5000] [send] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 40/2 : 1[5000] -> 9[5000] [send] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 55/2 : 1[5000] -> 9[5000] [send] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 62/2 : 1[5000] -> 9[5000] [send] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 01/2 : 0[75000] -> 7[95000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 06/2 : 14[e5000] -> 6[e5000] [receive] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 13/2 : 14[e5000] -> 6[e5000] [receive] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 20/2 : 14[e5000] -> 6[e5000] [receive] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 27/2 : 14[e5000] -> 6[e5000] [receive] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 34/2 : 14[e5000] -> 6[e5000] [receive] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 41/2 : 14[e5000] -> 6[e5000] [receive] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 48/2 : 14[e5000] -> 6[e5000] [receive] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 63/2 : 14[e5000] -> 6[e5000] [receive] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 01/2 : 6[e5000] -> 14[e5000] [send] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 08/2 : 6[e5000] -> 14[e5000] [send] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 23/2 : 6[e5000] -> 14[e5000] [send] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 30/2 : 6[e5000] -> 14[e5000] [send] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 37/2 : 6[e5000] -> 14[e5000] [send] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 44/2 : 6[e5000] -> 14[e5000] [send] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 51/2 : 6[e5000] -> 14[e5000] [send] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 58/2 : 6[e5000] -> 14[e5000] [send] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 08/2 : 0[75000] -> 7[95000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 23/2 : 0[75000] -> 7[95000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 30/2 : 0[75000] -> 7[95000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 37/2 : 0[75000] -> 7[95000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 44/2 : 0[75000] -> 7[95000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 51/2 : 0[75000] -> 7[95000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 58/2 : 0[75000] -> 7[95000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 03/2 : 8[75000] -> 0[75000] [receive] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 05/2 : 15[95000] -> 7[95000] [receive] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 10/2 : 8[75000] -> 0[75000] [receive] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 12/2 : 15[95000] -> 7[95000] [receive] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 17/2 : 8[75000] -> 0[75000] [receive] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 19/2 : 15[95000] -> 7[95000] [receive] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 24/2 : 8[75000] -> 0[75000] [receive] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 26/2 : 15[95000] -> 7[95000] [receive] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 39/2 : 8[75000] -> 0[75000] [receive] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 33/2 : 15[95000] -> 7[95000] [receive] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 46/2 : 8[75000] -> 0[75000] [receive] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 40/2 : 15[95000] -> 7[95000] [receive] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 53/2 : 8[75000] -> 0[75000] [receive] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 55/2 : 15[95000] -> 7[95000] [receive] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 60/2 : 8[75000] -> 0[75000] [receive] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 62/2 : 15[95000] -> 7[95000] [receive] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 07/2 : 0[75000] -> 8[75000] [send] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 03/2 : 7[95000] -> 15[95000] [send] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 14/2 : 0[75000] -> 8[75000] [send] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 10/2 : 7[95000] -> 15[95000] [send] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 21/2 : 0[75000] -> 8[75000] [send] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 17/2 : 7[95000] -> 15[95000] [send] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 28/2 : 0[75000] -> 8[75000] [send] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 24/2 : 7[95000] -> 15[95000] [send] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 35/2 : 0[75000] -> 8[75000] [send] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 39/2 : 7[95000] -> 15[95000] [send] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 42/2 : 0[75000] -> 8[75000] [send] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 46/2 : 7[95000] -> 15[95000] [send] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 49/2 : 0[75000] -> 8[75000] [send] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 53/2 : 7[95000] -> 15[95000] [send] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 56/2 : 0[75000] -> 8[75000] [send] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 60/2 : 7[95000] -> 15[95000] [send] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 04/2 : 4[f5000] -> 0[75000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 11/2 : 4[f5000] -> 0[75000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 18/2 : 4[f5000] -> 0[75000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 25/2 : 4[f5000] -> 0[75000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 32/2 : 4[f5000] -> 0[75000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 47/2 : 4[f5000] -> 0[75000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 54/2 : 4[f5000] -> 0[75000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 61/2 : 4[f5000] -> 0[75000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 01/2 : 5[85000] -> 0[75000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 08/2 : 5[85000] -> 0[75000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 23/2 : 5[85000] -> 0[75000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 30/2 : 5[85000] -> 0[75000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 37/2 : 5[85000] -> 0[75000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 44/2 : 5[85000] -> 0[75000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 51/2 : 5[85000] -> 0[75000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 58/2 : 5[85000] -> 0[75000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 06/2 : 6[e5000] -> 0[75000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 13/2 : 6[e5000] -> 0[75000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 20/2 : 6[e5000] -> 0[75000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 27/2 : 6[e5000] -> 0[75000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 34/2 : 6[e5000] -> 0[75000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 41/2 : 6[e5000] -> 0[75000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 48/2 : 6[e5000] -> 0[75000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 63/2 : 6[e5000] -> 0[75000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 07/2 : 7[95000] -> 0[75000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 14/2 : 7[95000] -> 0[75000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 21/2 : 7[95000] -> 0[75000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 28/2 : 7[95000] -> 0[75000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 35/2 : 7[95000] -> 0[75000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 42/2 : 7[95000] -> 0[75000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 49/2 : 7[95000] -> 0[75000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 56/2 : 7[95000] -> 0[75000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 02/2 : 7[95000] -> 1[5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 09/2 : 7[95000] -> 1[5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 16/2 : 7[95000] -> 1[5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 31/2 : 7[95000] -> 1[5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 38/2 : 7[95000] -> 1[5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 45/2 : 7[95000] -> 1[5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 05/2 : 6[e5000] -> 1[5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 52/2 : 7[95000] -> 1[5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 59/2 : 7[95000] -> 1[5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 06/2 : 7[95000] -> 2[65000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 13/2 : 7[95000] -> 2[65000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 20/2 : 7[95000] -> 2[65000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 06/2 : 5[85000] -> 1[5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 12/2 : 6[e5000] -> 1[5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 19/2 : 6[e5000] -> 1[5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 26/2 : 6[e5000] -> 1[5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 33/2 : 6[e5000] -> 1[5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 27/2 : 7[95000] -> 2[65000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 40/2 : 6[e5000] -> 1[5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 34/2 : 7[95000] -> 2[65000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 55/2 : 6[e5000] -> 1[5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 13/2 : 5[85000] -> 1[5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 41/2 : 7[95000] -> 2[65000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 62/2 : 6[e5000] -> 1[5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 48/2 : 7[95000] -> 2[65000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 63/2 : 7[95000] -> 2[65000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 20/2 : 5[85000] -> 1[5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 04/2 : 7[95000] -> 3[15000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 27/2 : 5[85000] -> 1[5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 34/2 : 5[85000] -> 1[5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 04/2 : 6[e5000] -> 2[65000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 41/2 : 5[85000] -> 1[5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 11/2 : 6[e5000] -> 2[65000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 18/2 : 6[e5000] -> 2[65000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 25/2 : 6[e5000] -> 2[65000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 32/2 : 6[e5000] -> 2[65000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 47/2 : 6[e5000] -> 2[65000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 11/2 : 7[95000] -> 3[15000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 54/2 : 6[e5000] -> 2[65000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 18/2 : 7[95000] -> 3[15000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 61/2 : 6[e5000] -> 2[65000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 25/2 : 7[95000] -> 3[15000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 32/2 : 7[95000] -> 3[15000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 47/2 : 7[95000] -> 3[15000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 48/2 : 5[85000] -> 1[5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 54/2 : 7[95000] -> 3[15000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 63/2 : 5[85000] -> 1[5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 61/2 : 7[95000] -> 3[15000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 03/2 : 5[85000] -> 2[65000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 02/2 : 6[e5000] -> 3[15000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 09/2 : 6[e5000] -> 3[15000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 01/2 : 7[95000] -> 4[f5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 16/2 : 6[e5000] -> 3[15000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 08/2 : 7[95000] -> 4[f5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 31/2 : 6[e5000] -> 3[15000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 03/2 : 4[f5000] -> 1[5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 10/2 : 5[85000] -> 2[65000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 23/2 : 7[95000] -> 4[f5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 38/2 : 6[e5000] -> 3[15000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 17/2 : 5[85000] -> 2[65000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 30/2 : 7[95000] -> 4[f5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 45/2 : 6[e5000] -> 3[15000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 24/2 : 5[85000] -> 2[65000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 37/2 : 7[95000] -> 4[f5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 52/2 : 6[e5000] -> 3[15000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 39/2 : 5[85000] -> 2[65000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 44/2 : 7[95000] -> 4[f5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 59/2 : 6[e5000] -> 3[15000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 46/2 : 5[85000] -> 2[65000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 10/2 : 4[f5000] -> 1[5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 53/2 : 5[85000] -> 2[65000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 17/2 : 4[f5000] -> 1[5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 24/2 : 4[f5000] -> 1[5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 05/2 : 3[15000] -> 0[75000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 39/2 : 4[f5000] -> 1[5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 12/2 : 3[15000] -> 0[75000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 46/2 : 4[f5000] -> 1[5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 53/2 : 4[f5000] -> 1[5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 51/2 : 7[95000] -> 4[f5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 60/2 : 4[f5000] -> 1[5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 58/2 : 7[95000] -> 4[f5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 60/2 : 5[85000] -> 2[65000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 19/2 : 3[15000] -> 0[75000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 26/2 : 3[15000] -> 0[75000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 33/2 : 3[15000] -> 0[75000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 40/2 : 3[15000] -> 0[75000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 55/2 : 3[15000] -> 0[75000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 62/2 : 3[15000] -> 0[75000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 05/2 : 7[95000] -> 5[85000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 12/2 : 7[95000] -> 5[85000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 19/2 : 7[95000] -> 5[85000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 26/2 : 7[95000] -> 5[85000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 00/2 : 6[e5000] -> 4[f5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 33/2 : 7[95000] -> 5[85000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 15/2 : 6[e5000] -> 4[f5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 07/2 : 5[85000] -> 3[15000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 40/2 : 7[95000] -> 5[85000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 22/2 : 6[e5000] -> 4[f5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 14/2 : 5[85000] -> 3[15000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 55/2 : 7[95000] -> 5[85000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 29/2 : 6[e5000] -> 4[f5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 21/2 : 5[85000] -> 3[15000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 36/2 : 6[e5000] -> 4[f5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 28/2 : 5[85000] -> 3[15000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 43/2 : 6[e5000] -> 4[f5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 50/2 : 6[e5000] -> 4[f5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 07/2 : 4[f5000] -> 2[65000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 57/2 : 6[e5000] -> 4[f5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 14/2 : 4[f5000] -> 2[65000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 01/2 : 3[15000] -> 1[5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 00/2 : 2[65000] -> 0[75000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 21/2 : 4[f5000] -> 2[65000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 15/2 : 2[65000] -> 0[75000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 08/2 : 3[15000] -> 1[5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 28/2 : 4[f5000] -> 2[65000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 62/2 : 7[95000] -> 5[85000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 22/2 : 2[65000] -> 0[75000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 23/2 : 3[15000] -> 1[5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 29/2 : 2[65000] -> 0[75000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 30/2 : 3[15000] -> 1[5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 35/2 : 5[85000] -> 3[15000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 36/2 : 2[65000] -> 0[75000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 37/2 : 3[15000] -> 1[5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 42/2 : 5[85000] -> 3[15000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 43/2 : 2[65000] -> 0[75000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 44/2 : 3[15000] -> 1[5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 49/2 : 5[85000] -> 3[15000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 51/2 : 3[15000] -> 1[5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 56/2 : 5[85000] -> 3[15000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 58/2 : 3[15000] -> 1[5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 00/2 : 7[95000] -> 6[e5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 35/2 : 4[f5000] -> 2[65000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 42/2 : 4[f5000] -> 2[65000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 49/2 : 4[f5000] -> 2[65000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 56/2 : 4[f5000] -> 2[65000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 50/2 : 2[65000] -> 0[75000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 57/2 : 2[65000] -> 0[75000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 15/2 : 7[95000] -> 6[e5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 22/2 : 7[95000] -> 6[e5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 29/2 : 7[95000] -> 6[e5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 36/2 : 7[95000] -> 6[e5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 43/2 : 7[95000] -> 6[e5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 50/2 : 7[95000] -> 6[e5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 00/2 : 3[15000] -> 2[65000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 05/2 : 5[85000] -> 4[f5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 15/2 : 3[15000] -> 2[65000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 02/2 : 1[5000] -> 0[75000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 22/2 : 3[15000] -> 2[65000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 09/2 : 1[5000] -> 0[75000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 29/2 : 3[15000] -> 2[65000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 07/2 : 6[e5000] -> 5[85000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 36/2 : 3[15000] -> 2[65000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 14/2 : 6[e5000] -> 5[85000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 57/2 : 7[95000] -> 6[e5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 43/2 : 3[15000] -> 2[65000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 21/2 : 6[e5000] -> 5[85000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 50/2 : 3[15000] -> 2[65000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 28/2 : 6[e5000] -> 5[85000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 07/2 : 2[65000] -> 1[5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 12/2 : 5[85000] -> 4[f5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 57/2 : 3[15000] -> 2[65000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 35/2 : 6[e5000] -> 5[85000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 14/2 : 2[65000] -> 1[5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 19/2 : 5[85000] -> 4[f5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 26/2 : 5[85000] -> 4[f5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 33/2 : 5[85000] -> 4[f5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 16/2 : 1[5000] -> 0[75000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 40/2 : 5[85000] -> 4[f5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 31/2 : 1[5000] -> 0[75000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 55/2 : 5[85000] -> 4[f5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 62/2 : 5[85000] -> 4[f5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 05/2 : 4[f5000] -> 3[15000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 38/2 : 1[5000] -> 0[75000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 12/2 : 4[f5000] -> 3[15000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 45/2 : 1[5000] -> 0[75000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 42/2 : 6[e5000] -> 5[85000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 19/2 : 4[f5000] -> 3[15000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 21/2 : 2[65000] -> 1[5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 52/2 : 1[5000] -> 0[75000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 49/2 : 6[e5000] -> 5[85000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 26/2 : 4[f5000] -> 3[15000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 28/2 : 2[65000] -> 1[5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 56/2 : 6[e5000] -> 5[85000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 33/2 : 4[f5000] -> 3[15000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 35/2 : 2[65000] -> 1[5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 40/2 : 4[f5000] -> 3[15000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 42/2 : 2[65000] -> 1[5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 49/2 : 2[65000] -> 1[5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 56/2 : 2[65000] -> 1[5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 59/2 : 1[5000] -> 0[75000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 55/2 : 4[f5000] -> 3[15000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 62/2 : 4[f5000] -> 3[15000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 05/0 : 1[5000] -> 2[65000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 06/0 : 1[5000] -> 2[65000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 11/0 : 1[5000] -> 2[65000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 12/0 : 1[5000] -> 2[65000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 17/0 : 1[5000] -> 2[65000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 18/0 : 1[5000] -> 2[65000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 24/0 : 1[5000] -> 2[65000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 31/0 : 1[5000] -> 2[65000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 36/0 : 1[5000] -> 2[65000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 39/0 : 1[5000] -> 2[65000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 42/0 : 1[5000] -> 2[65000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 45/0 : 1[5000] -> 2[65000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 48/0 : 1[5000] -> 2[65000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 57/0 : 1[5000] -> 2[65000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 62/0 : 1[5000] -> 2[65000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 02/0 : 0[75000] -> 1[5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 08/0 : 0[75000] -> 1[5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 10/0 : 0[75000] -> 1[5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 16/0 : 0[75000] -> 1[5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 22/0 : 0[75000] -> 1[5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 28/0 : 0[75000] -> 1[5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 30/0 : 0[75000] -> 1[5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 35/0 : 0[75000] -> 1[5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 37/0 : 0[75000] -> 1[5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 41/0 : 0[75000] -> 1[5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 43/0 : 0[75000] -> 1[5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 49/0 : 0[75000] -> 1[5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 55/0 : 0[75000] -> 1[5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 63/0 : 0[75000] -> 1[5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 02/0 : 0[75000] -> 2[65000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 03/0 : 0[75000] -> 2[65000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 08/0 : 0[75000] -> 2[65000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 09/0 : 0[75000] -> 2[65000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 22/0 : 0[75000] -> 2[65000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 23/0 : 0[75000] -> 2[65000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 28/0 : 0[75000] -> 2[65000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 29/0 : 0[75000] -> 2[65000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 34/0 : 0[75000] -> 2[65000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 35/0 : 0[75000] -> 2[65000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 41/0 : 0[75000] -> 2[65000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 54/0 : 0[75000] -> 2[65000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 60/0 : 0[75000] -> 2[65000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 61/0 : 0[75000] -> 2[65000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 00/0 : 6[e5000] -> 7[95000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 05/0 : 3[15000] -> 4[f5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 02/0 : 6[e5000] -> 7[95000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 08/0 : 3[15000] -> 4[f5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 08/0 : 6[e5000] -> 7[95000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 11/0 : 3[15000] -> 4[f5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 14/0 : 6[e5000] -> 7[95000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 03/0 : 5[85000] -> 6[e5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 17/0 : 3[15000] -> 4[f5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 20/0 : 6[e5000] -> 7[95000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 05/0 : 5[85000] -> 6[e5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 22/0 : 3[15000] -> 4[f5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 22/0 : 6[e5000] -> 7[95000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 09/0 : 5[85000] -> 6[e5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 28/0 : 3[15000] -> 4[f5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 26/0 : 6[e5000] -> 7[95000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 01/0 : 4[f5000] -> 5[85000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 17/0 : 5[85000] -> 6[e5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 35/0 : 3[15000] -> 4[f5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 28/0 : 6[e5000] -> 7[95000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 08/0 : 4[f5000] -> 5[85000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 23/0 : 5[85000] -> 6[e5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 36/0 : 3[15000] -> 4[f5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 33/0 : 6[e5000] -> 7[95000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 15/0 : 4[f5000] -> 5[85000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 41/0 : 3[15000] -> 4[f5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 35/0 : 6[e5000] -> 7[95000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 21/0 : 4[f5000] -> 5[85000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 42/0 : 3[15000] -> 4[f5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 41/0 : 6[e5000] -> 7[95000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 22/0 : 4[f5000] -> 5[85000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 48/0 : 3[15000] -> 4[f5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 47/0 : 6[e5000] -> 7[95000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 00/0 : 2[65000] -> 3[15000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 27/0 : 4[f5000] -> 5[85000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 55/0 : 3[15000] -> 4[f5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 55/0 : 6[e5000] -> 7[95000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 06/0 : 2[65000] -> 3[15000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 28/0 : 4[f5000] -> 5[85000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 61/0 : 3[15000] -> 4[f5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 59/0 : 6[e5000] -> 7[95000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 12/0 : 2[65000] -> 3[15000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 62/0 : 3[15000] -> 4[f5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 61/0 : 6[e5000] -> 7[95000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 14/0 : 2[65000] -> 3[15000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 29/0 : 5[85000] -> 6[e5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 18/0 : 2[65000] -> 3[15000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 31/0 : 5[85000] -> 6[e5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 20/0 : 2[65000] -> 3[15000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 34/0 : 5[85000] -> 6[e5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 36/0 : 5[85000] -> 6[e5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 40/0 : 5[85000] -> 6[e5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 32/0 : 4[f5000] -> 5[85000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 42/0 : 5[85000] -> 6[e5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 35/0 : 4[f5000] -> 5[85000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 48/0 : 5[85000] -> 6[e5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 41/0 : 4[f5000] -> 5[85000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 46/0 : 4[f5000] -> 5[85000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 26/0 : 2[65000] -> 3[15000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 55/0 : 4[f5000] -> 5[85000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 33/0 : 2[65000] -> 3[15000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 58/0 : 4[f5000] -> 5[85000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 45/0 : 2[65000] -> 3[15000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 61/0 : 4[f5000] -> 5[85000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 47/0 : 2[65000] -> 3[15000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 51/0 : 2[65000] -> 3[15000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 60/0 : 5[85000] -> 6[e5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 57/0 : 2[65000] -> 3[15000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 62/0 : 5[85000] -> 6[e5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 59/0 : 2[65000] -> 3[15000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 01/0 : 1[5000] -> 3[15000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 14/0 : 1[5000] -> 3[15000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 20/0 : 1[5000] -> 3[15000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 21/0 : 1[5000] -> 3[15000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 26/0 : 1[5000] -> 3[15000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 27/0 : 1[5000] -> 3[15000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 32/0 : 1[5000] -> 3[15000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 33/0 : 1[5000] -> 3[15000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 04/0 : 5[85000] -> 7[95000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 06/0 : 4[f5000] -> 6[e5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 46/0 : 1[5000] -> 3[15000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 03/0 : 3[15000] -> 5[85000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 07/0 : 4[f5000] -> 6[e5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 47/0 : 1[5000] -> 3[15000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 07/0 : 3[15000] -> 5[85000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 12/0 : 4[f5000] -> 6[e5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 52/0 : 1[5000] -> 3[15000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 09/0 : 3[15000] -> 5[85000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 13/0 : 4[f5000] -> 6[e5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 03/0 : 2[65000] -> 4[f5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 53/0 : 1[5000] -> 3[15000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 19/0 : 3[15000] -> 5[85000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 18/0 : 4[f5000] -> 6[e5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 09/0 : 2[65000] -> 4[f5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 23/0 : 3[15000] -> 5[85000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 10/0 : 2[65000] -> 4[f5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 25/0 : 3[15000] -> 5[85000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 16/0 : 2[65000] -> 4[f5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 05/0 : 5[85000] -> 7[95000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 29/0 : 3[15000] -> 5[85000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 23/0 : 2[65000] -> 4[f5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 10/0 : 5[85000] -> 7[95000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 38/0 : 3[15000] -> 5[85000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 29/0 : 2[65000] -> 4[f5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 11/0 : 5[85000] -> 7[95000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 40/0 : 3[15000] -> 5[85000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 16/0 : 5[85000] -> 7[95000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 44/0 : 3[15000] -> 5[85000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 58/0 : 1[5000] -> 3[15000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 17/0 : 5[85000] -> 7[95000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 59/0 : 1[5000] -> 3[15000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 19/0 : 4[f5000] -> 6[e5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 30/0 : 5[85000] -> 7[95000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 50/0 : 3[15000] -> 5[85000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 24/0 : 4[f5000] -> 6[e5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 25/0 : 4[f5000] -> 6[e5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 38/0 : 4[f5000] -> 6[e5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 30/0 : 2[65000] -> 4[f5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 39/0 : 4[f5000] -> 6[e5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 34/0 : 2[65000] -> 4[f5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 45/0 : 4[f5000] -> 6[e5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 37/0 : 2[65000] -> 4[f5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 50/0 : 4[f5000] -> 6[e5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 40/0 : 2[65000] -> 4[f5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 54/0 : 3[15000] -> 5[85000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 43/0 : 2[65000] -> 4[f5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 31/0 : 5[85000] -> 7[95000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 56/0 : 3[15000] -> 5[85000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 49/0 : 2[65000] -> 4[f5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 37/0 : 5[85000] -> 7[95000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 60/0 : 3[15000] -> 5[85000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 60/0 : 2[65000] -> 4[f5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 42/0 : 5[85000] -> 7[95000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 03/0 : 1[5000] -> 4[f5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 48/0 : 5[85000] -> 7[95000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 05/0 : 1[5000] -> 4[f5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 49/0 : 5[85000] -> 7[95000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 09/0 : 1[5000] -> 4[f5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 56/0 : 4[f5000] -> 6[e5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 62/0 : 5[85000] -> 7[95000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 11/0 : 1[5000] -> 4[f5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 57/0 : 4[f5000] -> 6[e5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 63/0 : 5[85000] -> 7[95000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 17/0 : 1[5000] -> 4[f5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 23/0 : 1[5000] -> 4[f5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 29/0 : 1[5000] -> 4[f5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 63/0 : 2[65000] -> 4[f5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 31/0 : 1[5000] -> 4[f5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 00/0 : 2[65000] -> 5[85000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 05/0 : 0[75000] -> 3[15000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 07/0 : 2[65000] -> 5[85000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 07/0 : 0[75000] -> 3[15000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 13/0 : 2[65000] -> 5[85000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 11/0 : 0[75000] -> 3[15000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 14/0 : 2[65000] -> 5[85000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 17/0 : 0[75000] -> 3[15000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 19/0 : 2[65000] -> 5[85000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 19/0 : 0[75000] -> 3[15000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 25/0 : 0[75000] -> 3[15000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 31/0 : 0[75000] -> 3[15000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 36/0 : 1[5000] -> 4[f5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 36/0 : 0[75000] -> 3[15000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 40/0 : 1[5000] -> 4[f5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 38/0 : 0[75000] -> 3[15000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 42/0 : 1[5000] -> 4[f5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 42/0 : 0[75000] -> 3[15000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 54/0 : 1[5000] -> 4[f5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 44/0 : 0[75000] -> 3[15000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 60/0 : 1[5000] -> 4[f5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 50/0 : 0[75000] -> 3[15000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 20/0 : 2[65000] -> 5[85000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 62/0 : 1[5000] -> 4[f5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 56/0 : 0[75000] -> 3[15000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 25/0 : 2[65000] -> 5[85000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 62/0 : 0[75000] -> 3[15000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 26/0 : 2[65000] -> 5[85000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 33/0 : 2[65000] -> 5[85000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 38/0 : 2[65000] -> 5[85000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 47/0 : 2[65000] -> 5[85000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 02/0 : 3[15000] -> 6[e5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 04/0 : 3[15000] -> 6[e5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 50/0 : 2[65000] -> 5[85000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 08/0 : 3[15000] -> 6[e5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 53/0 : 2[65000] -> 5[85000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 16/0 : 3[15000] -> 6[e5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 56/0 : 2[65000] -> 5[85000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 59/0 : 2[65000] -> 5[85000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 22/0 : 3[15000] -> 6[e5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 28/0 : 3[15000] -> 6[e5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 30/0 : 3[15000] -> 6[e5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 01/0 : 2[65000] -> 6[e5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 35/0 : 3[15000] -> 6[e5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 37/0 : 3[15000] -> 6[e5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 41/0 : 3[15000] -> 6[e5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 00/0 : 4[f5000] -> 7[95000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 12/0 : 4[f5000] -> 7[95000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 14/0 : 4[f5000] -> 7[95000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 18/0 : 4[f5000] -> 7[95000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 24/0 : 4[f5000] -> 7[95000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 11/0 : 2[65000] -> 6[e5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 26/0 : 4[f5000] -> 7[95000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 15/0 : 2[65000] -> 6[e5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 33/0 : 4[f5000] -> 7[95000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 17/0 : 2[65000] -> 6[e5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 43/0 : 3[15000] -> 6[e5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 21/0 : 2[65000] -> 6[e5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 49/0 : 3[15000] -> 6[e5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 27/0 : 2[65000] -> 6[e5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 55/0 : 3[15000] -> 6[e5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 31/0 : 2[65000] -> 6[e5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 61/0 : 3[15000] -> 6[e5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 32/0 : 2[65000] -> 6[e5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 63/0 : 3[15000] -> 6[e5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 01/0 : 0[75000] -> 4[f5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 03/0 : 3[15000] -> 7[95000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 39/0 : 4[f5000] -> 7[95000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 04/0 : 3[15000] -> 7[95000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 45/0 : 4[f5000] -> 7[95000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 09/0 : 3[15000] -> 7[95000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 47/0 : 4[f5000] -> 7[95000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 04/0 : 0[75000] -> 4[f5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 10/0 : 3[15000] -> 7[95000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 51/0 : 4[f5000] -> 7[95000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 15/0 : 0[75000] -> 4[f5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 16/0 : 3[15000] -> 7[95000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 53/0 : 4[f5000] -> 7[95000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 36/0 : 2[65000] -> 6[e5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 16/0 : 0[75000] -> 4[f5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 23/0 : 3[15000] -> 7[95000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 57/0 : 4[f5000] -> 7[95000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 42/0 : 2[65000] -> 6[e5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 21/0 : 0[75000] -> 4[f5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 29/0 : 3[15000] -> 7[95000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 46/0 : 2[65000] -> 6[e5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 27/0 : 0[75000] -> 4[f5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 30/0 : 3[15000] -> 7[95000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 48/0 : 2[65000] -> 6[e5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 30/0 : 0[75000] -> 4[f5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 34/0 : 3[15000] -> 7[95000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 52/0 : 2[65000] -> 6[e5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 32/0 : 0[75000] -> 4[f5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 37/0 : 3[15000] -> 7[95000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 58/0 : 2[65000] -> 6[e5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 37/0 : 0[75000] -> 4[f5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 40/0 : 3[15000] -> 7[95000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 43/0 : 0[75000] -> 4[f5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 43/0 : 3[15000] -> 7[95000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 54/0 : 3[15000] -> 7[95000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 60/0 : 3[15000] -> 7[95000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 59/0 : 4[f5000] -> 7[95000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 01/0 : 1[5000] -> 5[85000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 49/0 : 0[75000] -> 4[f5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 06/0 : 1[5000] -> 5[85000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 52/0 : 0[75000] -> 4[f5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 12/0 : 1[5000] -> 5[85000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 58/0 : 0[75000] -> 4[f5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 15/0 : 1[5000] -> 5[85000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 63/0 : 0[75000] -> 4[f5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 63/0 : 3[15000] -> 7[95000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 01/0 : 2[65000] -> 7[95000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 18/0 : 1[5000] -> 5[85000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 21/0 : 1[5000] -> 5[85000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 27/0 : 1[5000] -> 5[85000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 32/0 : 1[5000] -> 5[85000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 07/0 : 2[65000] -> 7[95000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 13/0 : 2[65000] -> 7[95000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 15/0 : 2[65000] -> 7[95000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 19/0 : 2[65000] -> 7[95000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 21/0 : 2[65000] -> 7[95000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 25/0 : 2[65000] -> 7[95000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 27/0 : 2[65000] -> 7[95000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 45/0 : 1[5000] -> 5[85000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 51/0 : 1[5000] -> 5[85000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 06/0 : 0[75000] -> 5[85000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 13/0 : 12[f5000] -> 4[f5000] [receive] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 19/0 : 12[f5000] -> 4[f5000] [receive] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 52/0 : 1[5000] -> 5[85000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 25/0 : 12[f5000] -> 4[f5000] [receive] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 38/0 : 12[f5000] -> 4[f5000] [receive] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 44/0 : 12[f5000] -> 4[f5000] [receive] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 50/0 : 12[f5000] -> 4[f5000] [receive] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 07/0 : 4[f5000] -> 12[f5000] [send] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 13/0 : 4[f5000] -> 12[f5000] [send] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 19/0 : 4[f5000] -> 12[f5000] [send] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 25/0 : 4[f5000] -> 12[f5000] [send] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 38/0 : 4[f5000] -> 12[f5000] [send] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 44/0 : 4[f5000] -> 12[f5000] [send] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 56/0 : 4[f5000] -> 12[f5000] [send] via NET/IB/4/GDRDMA comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 57/0 : 1[5000] -> 5[85000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 12/0 : 0[75000] -> 5[85000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 58/0 : 1[5000] -> 5[85000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 14/0 : 0[75000] -> 5[85000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 03/0 : 1[5000] -> 6[e5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 32/0 : 2[65000] -> 7[95000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 18/0 : 0[75000] -> 5[85000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 04/0 : 1[5000] -> 6[e5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 44/0 : 2[65000] -> 7[95000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 20/0 : 0[75000] -> 5[85000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 46/0 : 2[65000] -> 7[95000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 24/0 : 0[75000] -> 5[85000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 15/0 : 11[15000] -> 3[15000] [receive] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 21/0 : 11[15000] -> 3[15000] [receive] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 50/0 : 2[65000] -> 7[95000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 27/0 : 11[15000] -> 3[15000] [receive] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 32/0 : 11[15000] -> 3[15000] [receive] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 46/0 : 11[15000] -> 3[15000] [receive] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 52/0 : 11[15000] -> 3[15000] [receive] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 01/0 : 3[15000] -> 11[15000] [send] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 15/0 : 3[15000] -> 11[15000] [send] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 21/0 : 3[15000] -> 11[15000] [send] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 27/0 : 3[15000] -> 11[15000] [send] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 26/0 : 0[75000] -> 5[85000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 46/0 : 3[15000] -> 11[15000] [send] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 52/0 : 3[15000] -> 11[15000] [send] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 58/0 : 3[15000] -> 11[15000] [send] via NET/IB/3/GDRDMA comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 56/0 : 2[65000] -> 7[95000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 58/0 : 2[65000] -> 7[95000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 09/0 : 1[5000] -> 6[e5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 10/0 : 1[5000] -> 6[e5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 16/0 : 1[5000] -> 6[e5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 23/0 : 1[5000] -> 6[e5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 33/0 : 0[75000] -> 5[85000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 29/0 : 1[5000] -> 6[e5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 39/0 : 0[75000] -> 5[85000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 30/0 : 1[5000] -> 6[e5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 45/0 : 0[75000] -> 5[85000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 34/0 : 1[5000] -> 6[e5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 47/0 : 0[75000] -> 5[85000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 51/0 : 0[75000] -> 5[85000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 53/0 : 0[75000] -> 5[85000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 59/0 : 0[75000] -> 5[85000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 37/0 : 1[5000] -> 6[e5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 40/0 : 1[5000] -> 6[e5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 43/0 : 1[5000] -> 6[e5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 54/0 : 1[5000] -> 6[e5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 60/0 : 1[5000] -> 6[e5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 63/0 : 1[5000] -> 6[e5000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 02/0 : 1[5000] -> 7[95000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 08/0 : 10[65000] -> 2[65000] [receive] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 22/0 : 10[65000] -> 2[65000] [receive] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 28/0 : 10[65000] -> 2[65000] [receive] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 35/0 : 10[65000] -> 2[65000] [receive] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 41/0 : 10[65000] -> 2[65000] [receive] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 55/0 : 10[65000] -> 2[65000] [receive] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 61/0 : 10[65000] -> 2[65000] [receive] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 02/0 : 2[65000] -> 10[65000] [send] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 08/0 : 2[65000] -> 10[65000] [send] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 22/0 : 2[65000] -> 10[65000] [send] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 28/0 : 2[65000] -> 10[65000] [send] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 35/0 : 2[65000] -> 10[65000] [send] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 55/0 : 2[65000] -> 10[65000] [send] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 61/0 : 2[65000] -> 10[65000] [send] via NET/IB/2/GDRDMA comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 07/0 : 1[5000] -> 7[95000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 08/0 : 1[5000] -> 7[95000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 13/0 : 1[5000] -> 7[95000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 19/0 : 1[5000] -> 7[95000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 22/0 : 1[5000] -> 7[95000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 28/0 : 1[5000] -> 7[95000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 35/0 : 1[5000] -> 7[95000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 00/0 : 0[75000] -> 6[e5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 10/0 : 13[85000] -> 5[85000] [receive] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 16/0 : 13[85000] -> 5[85000] [receive] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 30/0 : 13[85000] -> 5[85000] [receive] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 37/0 : 13[85000] -> 5[85000] [receive] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 43/0 : 13[85000] -> 5[85000] [receive] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 49/0 : 13[85000] -> 5[85000] [receive] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 63/0 : 13[85000] -> 5[85000] [receive] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 04/0 : 5[85000] -> 13[85000] [send] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 10/0 : 5[85000] -> 13[85000] [send] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 30/0 : 5[85000] -> 13[85000] [send] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 37/0 : 5[85000] -> 13[85000] [send] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 43/0 : 5[85000] -> 13[85000] [send] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 49/0 : 5[85000] -> 13[85000] [send] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 63/0 : 5[85000] -> 13[85000] [send] via NET/IB/5/GDRDMA comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 07/0 : 0[75000] -> 6[e5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 13/0 : 0[75000] -> 6[e5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 14/0 : 0[75000] -> 6[e5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 19/0 : 0[75000] -> 6[e5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 38/0 : 1[5000] -> 7[95000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 20/0 : 0[75000] -> 6[e5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 41/0 : 1[5000] -> 7[95000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 25/0 : 0[75000] -> 6[e5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 44/0 : 1[5000] -> 7[95000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 26/0 : 0[75000] -> 6[e5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 50/0 : 1[5000] -> 7[95000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 33/0 : 0[75000] -> 6[e5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 55/0 : 1[5000] -> 7[95000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 44/0 : 0[75000] -> 6[e5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 56/0 : 1[5000] -> 7[95000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 47/0 : 0[75000] -> 6[e5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 50/0 : 0[75000] -> 6[e5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 53/0 : 0[75000] -> 6[e5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 56/0 : 0[75000] -> 6[e5000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 06/0 : 0[75000] -> 7[95000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 12/0 : 14[e5000] -> 6[e5000] [receive] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 18/0 : 14[e5000] -> 6[e5000] [receive] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 24/0 : 14[e5000] -> 6[e5000] [receive] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 39/0 : 14[e5000] -> 6[e5000] [receive] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 45/0 : 14[e5000] -> 6[e5000] [receive] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 51/0 : 14[e5000] -> 6[e5000] [receive] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 57/0 : 14[e5000] -> 6[e5000] [receive] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 06/0 : 6[e5000] -> 14[e5000] [send] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 12/0 : 6[e5000] -> 14[e5000] [send] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 14/0 : 9[5000] -> 1[5000] [receive] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 18/0 : 6[e5000] -> 14[e5000] [send] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 20/0 : 9[5000] -> 1[5000] [receive] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 24/0 : 6[e5000] -> 14[e5000] [send] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 39/0 : 6[e5000] -> 14[e5000] [send] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 26/0 : 9[5000] -> 1[5000] [receive] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 45/0 : 6[e5000] -> 14[e5000] [send] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 33/0 : 9[5000] -> 1[5000] [receive] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 57/0 : 6[e5000] -> 14[e5000] [send] via NET/IB/6/GDRDMA comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 47/0 : 9[5000] -> 1[5000] [receive] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 53/0 : 9[5000] -> 1[5000] [receive] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 59/0 : 9[5000] -> 1[5000] [receive] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 00/0 : 1[5000] -> 9[5000] [send] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 14/0 : 1[5000] -> 9[5000] [send] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 20/0 : 1[5000] -> 9[5000] [send] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 47/0 : 1[5000] -> 9[5000] [send] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 53/0 : 1[5000] -> 9[5000] [send] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 59/0 : 1[5000] -> 9[5000] [send] via NET/IB/1/GDRDMA comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 12/0 : 0[75000] -> 7[95000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 15/0 : 0[75000] -> 7[95000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 18/0 : 0[75000] -> 7[95000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 21/0 : 0[75000] -> 7[95000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 24/0 : 0[75000] -> 7[95000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 27/0 : 0[75000] -> 7[95000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 32/0 : 0[75000] -> 7[95000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 39/0 : 0[75000] -> 7[95000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 45/0 : 0[75000] -> 7[95000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 46/0 : 0[75000] -> 7[95000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 52/0 : 0[75000] -> 7[95000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 57/0 : 0[75000] -> 7[95000] via P2P/IPC comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 09/0 : 8[75000] -> 0[75000] [receive] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 11/0 : 15[95000] -> 7[95000] [receive] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 23/0 : 8[75000] -> 0[75000] [receive] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 17/0 : 15[95000] -> 7[95000] [receive] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 29/0 : 8[75000] -> 0[75000] [receive] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 31/0 : 15[95000] -> 7[95000] [receive] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 34/0 : 8[75000] -> 0[75000] [receive] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 36/0 : 15[95000] -> 7[95000] [receive] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 40/0 : 8[75000] -> 0[75000] [receive] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 42/0 : 15[95000] -> 7[95000] [receive] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 54/0 : 8[75000] -> 0[75000] [receive] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 48/0 : 15[95000] -> 7[95000] [receive] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 03/0 : 0[75000] -> 8[75000] [send] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 05/0 : 7[95000] -> 15[95000] [send] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 09/0 : 0[75000] -> 8[75000] [send] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 11/0 : 7[95000] -> 15[95000] [send] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 23/0 : 0[75000] -> 8[75000] [send] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 31/0 : 7[95000] -> 15[95000] [send] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 29/0 : 0[75000] -> 8[75000] [send] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 36/0 : 7[95000] -> 15[95000] [send] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 34/0 : 0[75000] -> 8[75000] [send] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 42/0 : 7[95000] -> 15[95000] [send] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 40/0 : 0[75000] -> 8[75000] [send] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 48/0 : 7[95000] -> 15[95000] [send] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 54/0 : 0[75000] -> 8[75000] [send] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 62/0 : 7[95000] -> 15[95000] [send] via NET/IB/7/GDRDMA comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Channel 60/0 : 0[75000] -> 8[75000] [send] via NET/IB/0/GDRDMA comm 0x80bc0b40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 01/0 : 4[f5000] -> 0[75000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 10/0 : 4[f5000] -> 0[75000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 15/0 : 4[f5000] -> 0[75000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 16/0 : 4[f5000] -> 0[75000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 21/0 : 4[f5000] -> 0[75000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 27/0 : 4[f5000] -> 0[75000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 30/0 : 4[f5000] -> 0[75000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 37/0 : 4[f5000] -> 0[75000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 43/0 : 4[f5000] -> 0[75000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 46/0 : 4[f5000] -> 0[75000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 49/0 : 4[f5000] -> 0[75000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 52/0 : 4[f5000] -> 0[75000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 58/0 : 4[f5000] -> 0[75000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 63/0 : 4[f5000] -> 0[75000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 00/0 : 5[85000] -> 0[75000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 06/0 : 5[85000] -> 0[75000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 12/0 : 5[85000] -> 0[75000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 14/0 : 5[85000] -> 0[75000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 18/0 : 5[85000] -> 0[75000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 20/0 : 5[85000] -> 0[75000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 24/0 : 5[85000] -> 0[75000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 26/0 : 5[85000] -> 0[75000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 33/0 : 5[85000] -> 0[75000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 39/0 : 5[85000] -> 0[75000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 45/0 : 5[85000] -> 0[75000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 47/0 : 5[85000] -> 0[75000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 53/0 : 5[85000] -> 0[75000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 57/0 : 5[85000] -> 0[75000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 59/0 : 5[85000] -> 0[75000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 00/0 : 6[e5000] -> 0[75000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 07/0 : 6[e5000] -> 0[75000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 14/0 : 6[e5000] -> 0[75000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 19/0 : 6[e5000] -> 0[75000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 25/0 : 6[e5000] -> 0[75000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 26/0 : 6[e5000] -> 0[75000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 33/0 : 6[e5000] -> 0[75000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 38/0 : 6[e5000] -> 0[75000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 44/0 : 6[e5000] -> 0[75000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 47/0 : 6[e5000] -> 0[75000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 50/0 : 6[e5000] -> 0[75000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 53/0 : 6[e5000] -> 0[75000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 56/0 : 6[e5000] -> 0[75000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 59/0 : 6[e5000] -> 0[75000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 01/0 : 7[95000] -> 0[75000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 06/0 : 7[95000] -> 0[75000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 12/0 : 7[95000] -> 0[75000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 15/0 : 7[95000] -> 0[75000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 18/0 : 7[95000] -> 0[75000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 24/0 : 7[95000] -> 0[75000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 27/0 : 7[95000] -> 0[75000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 32/0 : 7[95000] -> 0[75000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 39/0 : 7[95000] -> 0[75000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 45/0 : 7[95000] -> 0[75000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 46/0 : 7[95000] -> 0[75000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 51/0 : 7[95000] -> 0[75000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 52/0 : 7[95000] -> 0[75000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 57/0 : 7[95000] -> 0[75000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 58/0 : 7[95000] -> 0[75000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 07/0 : 7[95000] -> 1[5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 08/0 : 7[95000] -> 1[5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 13/0 : 7[95000] -> 1[5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 19/0 : 7[95000] -> 1[5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 22/0 : 7[95000] -> 1[5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 25/0 : 7[95000] -> 1[5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 28/0 : 7[95000] -> 1[5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 35/0 : 7[95000] -> 1[5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 41/0 : 7[95000] -> 1[5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 44/0 : 7[95000] -> 1[5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 50/0 : 7[95000] -> 1[5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 55/0 : 7[95000] -> 1[5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 56/0 : 7[95000] -> 1[5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 03/0 : 6[e5000] -> 1[5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 04/0 : 6[e5000] -> 1[5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 09/0 : 6[e5000] -> 1[5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 10/0 : 6[e5000] -> 1[5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 16/0 : 6[e5000] -> 1[5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 61/0 : 7[95000] -> 1[5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 23/0 : 6[e5000] -> 1[5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 01/0 : 7[95000] -> 2[65000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 29/0 : 6[e5000] -> 1[5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 07/0 : 7[95000] -> 2[65000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 30/0 : 6[e5000] -> 1[5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 15/0 : 7[95000] -> 2[65000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 34/0 : 6[e5000] -> 1[5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 19/0 : 7[95000] -> 2[65000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 21/0 : 7[95000] -> 2[65000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 37/0 : 6[e5000] -> 1[5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 43/0 : 6[e5000] -> 1[5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 49/0 : 6[e5000] -> 1[5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 54/0 : 6[e5000] -> 1[5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 60/0 : 6[e5000] -> 1[5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 63/0 : 6[e5000] -> 1[5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 25/0 : 7[95000] -> 2[65000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 32/0 : 7[95000] -> 2[65000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 38/0 : 7[95000] -> 2[65000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 44/0 : 7[95000] -> 2[65000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 01/0 : 5[85000] -> 1[5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 46/0 : 7[95000] -> 2[65000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 50/0 : 7[95000] -> 2[65000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 12/0 : 5[85000] -> 1[5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 52/0 : 7[95000] -> 2[65000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 15/0 : 5[85000] -> 1[5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 18/0 : 5[85000] -> 1[5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 56/0 : 7[95000] -> 2[65000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 58/0 : 7[95000] -> 2[65000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 21/0 : 5[85000] -> 1[5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 03/0 : 7[95000] -> 3[15000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 24/0 : 5[85000] -> 1[5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 09/0 : 7[95000] -> 3[15000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 32/0 : 5[85000] -> 1[5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 10/0 : 7[95000] -> 3[15000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 39/0 : 5[85000] -> 1[5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 16/0 : 7[95000] -> 3[15000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 45/0 : 5[85000] -> 1[5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 23/0 : 7[95000] -> 3[15000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 46/0 : 5[85000] -> 1[5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 01/0 : 6[e5000] -> 2[65000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 51/0 : 5[85000] -> 1[5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 05/0 : 6[e5000] -> 2[65000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 15/0 : 6[e5000] -> 2[65000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 17/0 : 6[e5000] -> 2[65000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 21/0 : 6[e5000] -> 2[65000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 29/0 : 7[95000] -> 3[15000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 27/0 : 6[e5000] -> 2[65000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 30/0 : 7[95000] -> 3[15000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 31/0 : 6[e5000] -> 2[65000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 34/0 : 7[95000] -> 3[15000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 52/0 : 5[85000] -> 1[5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 37/0 : 7[95000] -> 3[15000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 57/0 : 5[85000] -> 1[5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 40/0 : 7[95000] -> 3[15000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 58/0 : 5[85000] -> 1[5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 43/0 : 7[95000] -> 3[15000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 36/0 : 6[e5000] -> 2[65000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 42/0 : 6[e5000] -> 2[65000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 46/0 : 6[e5000] -> 2[65000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 48/0 : 6[e5000] -> 2[65000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 49/0 : 7[95000] -> 3[15000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 52/0 : 6[e5000] -> 2[65000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 60/0 : 7[95000] -> 3[15000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 58/0 : 6[e5000] -> 2[65000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 63/0 : 7[95000] -> 3[15000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 62/0 : 6[e5000] -> 2[65000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 00/0 : 7[95000] -> 4[f5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 06/0 : 7[95000] -> 4[f5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 12/0 : 7[95000] -> 4[f5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 04/0 : 6[e5000] -> 3[15000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 08/0 : 6[e5000] -> 3[15000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 10/0 : 6[e5000] -> 3[15000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 00/0 : 5[85000] -> 2[65000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 22/0 : 6[e5000] -> 3[15000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 28/0 : 6[e5000] -> 3[15000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 30/0 : 6[e5000] -> 3[15000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 14/0 : 7[95000] -> 4[f5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 35/0 : 6[e5000] -> 3[15000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 18/0 : 7[95000] -> 4[f5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 37/0 : 6[e5000] -> 3[15000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 07/0 : 5[85000] -> 2[65000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 20/0 : 7[95000] -> 4[f5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 13/0 : 5[85000] -> 2[65000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 24/0 : 7[95000] -> 4[f5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 14/0 : 5[85000] -> 2[65000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 26/0 : 7[95000] -> 4[f5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 19/0 : 5[85000] -> 2[65000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 33/0 : 7[95000] -> 4[f5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 20/0 : 5[85000] -> 2[65000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 41/0 : 6[e5000] -> 3[15000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 05/0 : 4[f5000] -> 1[5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 43/0 : 6[e5000] -> 3[15000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 09/0 : 4[f5000] -> 1[5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 49/0 : 6[e5000] -> 3[15000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 11/0 : 4[f5000] -> 1[5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 55/0 : 6[e5000] -> 3[15000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 39/0 : 7[95000] -> 4[f5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 61/0 : 6[e5000] -> 3[15000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 45/0 : 7[95000] -> 4[f5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 63/0 : 6[e5000] -> 3[15000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 47/0 : 7[95000] -> 4[f5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 25/0 : 5[85000] -> 2[65000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 53/0 : 7[95000] -> 4[f5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 07/0 : 3[15000] -> 0[75000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 26/0 : 5[85000] -> 2[65000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 57/0 : 7[95000] -> 4[f5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 11/0 : 3[15000] -> 0[75000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 33/0 : 5[85000] -> 2[65000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 13/0 : 3[15000] -> 0[75000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 23/0 : 4[f5000] -> 1[5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 38/0 : 5[85000] -> 2[65000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 17/0 : 3[15000] -> 0[75000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 29/0 : 4[f5000] -> 1[5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 25/0 : 3[15000] -> 0[75000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 31/0 : 4[f5000] -> 1[5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 31/0 : 3[15000] -> 0[75000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 34/0 : 4[f5000] -> 1[5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 36/0 : 3[15000] -> 0[75000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 36/0 : 4[f5000] -> 1[5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 38/0 : 3[15000] -> 0[75000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 40/0 : 4[f5000] -> 1[5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 59/0 : 7[95000] -> 4[f5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 42/0 : 3[15000] -> 0[75000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 44/0 : 3[15000] -> 0[75000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 44/0 : 5[85000] -> 2[65000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 48/0 : 3[15000] -> 0[75000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 47/0 : 5[85000] -> 2[65000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 50/0 : 3[15000] -> 0[75000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 50/0 : 5[85000] -> 2[65000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 56/0 : 3[15000] -> 0[75000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 56/0 : 5[85000] -> 2[65000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 06/0 : 6[e5000] -> 4[f5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 59/0 : 5[85000] -> 2[65000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 42/0 : 4[f5000] -> 1[5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 48/0 : 4[f5000] -> 1[5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 07/0 : 6[e5000] -> 4[f5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 54/0 : 4[f5000] -> 1[5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 12/0 : 6[e5000] -> 4[f5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 62/0 : 4[f5000] -> 1[5000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 13/0 : 6[e5000] -> 4[f5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 04/0 : 7[95000] -> 5[85000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 18/0 : 6[e5000] -> 4[f5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 10/0 : 7[95000] -> 5[85000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 19/0 : 6[e5000] -> 4[f5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 11/0 : 7[95000] -> 5[85000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 24/0 : 6[e5000] -> 4[f5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 16/0 : 7[95000] -> 5[85000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 17/0 : 7[95000] -> 5[85000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 25/0 : 6[e5000] -> 4[f5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 30/0 : 7[95000] -> 5[85000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 38/0 : 6[e5000] -> 4[f5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 39/0 : 6[e5000] -> 4[f5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 44/0 : 6[e5000] -> 4[f5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 45/0 : 6[e5000] -> 4[f5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 51/0 : 6[e5000] -> 4[f5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 56/0 : 6[e5000] -> 4[f5000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 31/0 : 7[95000] -> 5[85000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 00/0 : 3[15000] -> 1[5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 36/0 : 7[95000] -> 5[85000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 14/0 : 3[15000] -> 1[5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 37/0 : 7[95000] -> 5[85000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 15/0 : 3[15000] -> 1[5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 42/0 : 7[95000] -> 5[85000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 20/0 : 3[15000] -> 1[5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 43/0 : 7[95000] -> 5[85000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 21/0 : 3[15000] -> 1[5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 48/0 : 7[95000] -> 5[85000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 26/0 : 3[15000] -> 1[5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 27/0 : 3[15000] -> 1[5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 49/0 : 7[95000] -> 5[85000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 63/0 : 7[95000] -> 5[85000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 32/0 : 3[15000] -> 1[5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 02/0 : 2[65000] -> 0[75000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 33/0 : 3[15000] -> 1[5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 03/0 : 5[85000] -> 3[15000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 03/0 : 2[65000] -> 0[75000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 46/0 : 3[15000] -> 1[5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 09/0 : 5[85000] -> 3[15000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 08/0 : 2[65000] -> 0[75000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 47/0 : 3[15000] -> 1[5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 13/0 : 5[85000] -> 3[15000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 52/0 : 3[15000] -> 1[5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 19/0 : 5[85000] -> 3[15000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 03/0 : 4[f5000] -> 2[65000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 53/0 : 3[15000] -> 1[5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 23/0 : 5[85000] -> 3[15000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 04/0 : 4[f5000] -> 2[65000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 59/0 : 3[15000] -> 1[5000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 09/0 : 4[f5000] -> 2[65000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 10/0 : 4[f5000] -> 2[65000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 16/0 : 4[f5000] -> 2[65000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 09/0 : 2[65000] -> 0[75000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 23/0 : 4[f5000] -> 2[65000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 23/0 : 2[65000] -> 0[75000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 28/0 : 2[65000] -> 0[75000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 25/0 : 5[85000] -> 3[15000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 34/0 : 2[65000] -> 0[75000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 29/0 : 5[85000] -> 3[15000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 35/0 : 2[65000] -> 0[75000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 34/0 : 5[85000] -> 3[15000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 40/0 : 2[65000] -> 0[75000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 38/0 : 5[85000] -> 3[15000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 41/0 : 2[65000] -> 0[75000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 40/0 : 5[85000] -> 3[15000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 44/0 : 5[85000] -> 3[15000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 29/0 : 4[f5000] -> 2[65000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 30/0 : 4[f5000] -> 2[65000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 34/0 : 4[f5000] -> 2[65000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 37/0 : 4[f5000] -> 2[65000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 40/0 : 4[f5000] -> 2[65000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 54/0 : 2[65000] -> 0[75000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 43/0 : 4[f5000] -> 2[65000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 55/0 : 2[65000] -> 0[75000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 60/0 : 2[65000] -> 0[75000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 50/0 : 5[85000] -> 3[15000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 61/0 : 2[65000] -> 0[75000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 54/0 : 5[85000] -> 3[15000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 60/0 : 5[85000] -> 3[15000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 02/0 : 7[95000] -> 6[e5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 08/0 : 7[95000] -> 6[e5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 54/0 : 4[f5000] -> 2[65000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 60/0 : 4[f5000] -> 2[65000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 63/0 : 4[f5000] -> 2[65000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 04/0 : 1[5000] -> 0[75000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 08/0 : 1[5000] -> 0[75000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 14/0 : 7[95000] -> 6[e5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 10/0 : 1[5000] -> 0[75000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 20/0 : 7[95000] -> 6[e5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 26/0 : 7[95000] -> 6[e5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 28/0 : 7[95000] -> 6[e5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 33/0 : 7[95000] -> 6[e5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 22/0 : 1[5000] -> 0[75000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 28/0 : 1[5000] -> 0[75000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 30/0 : 1[5000] -> 0[75000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 35/0 : 1[5000] -> 0[75000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 37/0 : 1[5000] -> 0[75000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 35/0 : 7[95000] -> 6[e5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 41/0 : 1[5000] -> 0[75000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 41/0 : 7[95000] -> 6[e5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 06/0 : 3[15000] -> 2[65000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 47/0 : 7[95000] -> 6[e5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 53/0 : 7[95000] -> 6[e5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 55/0 : 7[95000] -> 6[e5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 12/0 : 3[15000] -> 2[65000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 59/0 : 7[95000] -> 6[e5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 14/0 : 3[15000] -> 2[65000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 03/0 : 6[e5000] -> 5[85000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 18/0 : 3[15000] -> 2[65000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 05/0 : 6[e5000] -> 5[85000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 43/0 : 1[5000] -> 0[75000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 20/0 : 3[15000] -> 2[65000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 09/0 : 6[e5000] -> 5[85000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 49/0 : 1[5000] -> 0[75000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 24/0 : 3[15000] -> 2[65000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 11/0 : 6[e5000] -> 5[85000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 55/0 : 1[5000] -> 0[75000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 26/0 : 3[15000] -> 2[65000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 17/0 : 6[e5000] -> 5[85000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 61/0 : 1[5000] -> 0[75000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 33/0 : 3[15000] -> 2[65000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Channel 61/0 : 7[95000] -> 6[e5000] via P2P/IPC comm 0x4647b6e0 nRanks 16 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Channel 63/0 : 1[5000] -> 0[75000] via P2P/IPC comm 0x60eef880 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 39/0 : 3[15000] -> 2[65000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 23/0 : 6[e5000] -> 5[85000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 01/0 : 5[85000] -> 4[f5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 45/0 : 3[15000] -> 2[65000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 05/0 : 2[65000] -> 1[5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 29/0 : 6[e5000] -> 5[85000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 02/0 : 5[85000] -> 4[f5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 47/0 : 3[15000] -> 2[65000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 06/0 : 2[65000] -> 1[5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 08/0 : 5[85000] -> 4[f5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 51/0 : 3[15000] -> 2[65000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 11/0 : 2[65000] -> 1[5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 15/0 : 5[85000] -> 4[f5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 53/0 : 3[15000] -> 2[65000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 21/0 : 5[85000] -> 4[f5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Channel 59/0 : 3[15000] -> 2[65000] via P2P/IPC comm 0x48e1d6c0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 02/0 : 4[f5000] -> 3[15000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 22/0 : 5[85000] -> 4[f5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 08/0 : 4[f5000] -> 3[15000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 11/0 : 4[f5000] -> 3[15000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 31/0 : 6[e5000] -> 5[85000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 17/0 : 4[f5000] -> 3[15000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 34/0 : 6[e5000] -> 5[85000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 22/0 : 4[f5000] -> 3[15000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 36/0 : 6[e5000] -> 5[85000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 12/0 : 2[65000] -> 1[5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 28/0 : 4[f5000] -> 3[15000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 40/0 : 6[e5000] -> 5[85000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 17/0 : 2[65000] -> 1[5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 48/0 : 6[e5000] -> 5[85000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 18/0 : 2[65000] -> 1[5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 27/0 : 5[85000] -> 4[f5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 54/0 : 6[e5000] -> 5[85000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 24/0 : 2[65000] -> 1[5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 28/0 : 5[85000] -> 4[f5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 60/0 : 6[e5000] -> 5[85000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 31/0 : 2[65000] -> 1[5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 32/0 : 5[85000] -> 4[f5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Channel 62/0 : 6[e5000] -> 5[85000] via P2P/IPC comm 0x7ec7cef0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 36/0 : 2[65000] -> 1[5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 35/0 : 5[85000] -> 4[f5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 41/0 : 5[85000] -> 4[f5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 31/0 : 4[f5000] -> 3[15000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 46/0 : 5[85000] -> 4[f5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 35/0 : 4[f5000] -> 3[15000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 36/0 : 4[f5000] -> 3[15000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 41/0 : 4[f5000] -> 3[15000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 42/0 : 4[f5000] -> 3[15000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 39/0 : 2[65000] -> 1[5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 48/0 : 4[f5000] -> 3[15000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 45/0 : 2[65000] -> 1[5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 48/0 : 2[65000] -> 1[5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 52/0 : 5[85000] -> 4[f5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 51/0 : 2[65000] -> 1[5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 58/0 : 5[85000] -> 4[f5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 57/0 : 2[65000] -> 1[5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Channel 61/0 : 5[85000] -> 4[f5000] via P2P/IPC comm 0x648bcbd0 nRanks 16 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Channel 62/0 : 2[65000] -> 1[5000] via P2P/IPC comm 0x7e4c8c40 nRanks 16 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Channel 61/0 : 4[f5000] -> 3[15000] via P2P/IPC comm 0x507e6620 nRanks 16 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Connected all trees +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Connected all trees +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Connected all trees +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Connected all trees +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Connected all trees +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Connected all trees +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Connected all trees +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Connected all trees +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 256 | 256 +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO 64 coll channels, 64 collnet channels, 0 nvls channels, 64 p2p channels, 2 p2p channels per peer +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 256 | 256 +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO 64 coll channels, 64 collnet channels, 0 nvls channels, 64 p2p channels, 2 p2p channels per peer +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO CC Off, Multi-GPU CC Off, workFifoBytes 1048576 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 256 | 256 +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO 64 coll channels, 64 collnet channels, 0 nvls channels, 64 p2p channels, 2 p2p channels per peer +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 256 | 256 +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO 64 coll channels, 64 collnet channels, 0 nvls channels, 64 p2p channels, 2 p2p channels per peer +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 256 | 256 +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO 64 coll channels, 64 collnet channels, 0 nvls channels, 64 p2p channels, 2 p2p channels per peer +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 256 | 256 +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO 64 coll channels, 64 collnet channels, 0 nvls channels, 64 p2p channels, 2 p2p channels per peer +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 256 | 256 +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO 64 coll channels, 64 collnet channels, 0 nvls channels, 64 p2p channels, 2 p2p channels per peer +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 256 | 256 +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO 64 coll channels, 64 collnet channels, 0 nvls channels, 64 p2p channels, 2 p2p channels per peer +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO TUNER/Plugin: Could not find: librccl-tuner.so librccl-net.so. Using internal tuner plugin. +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO ncclCommInitRank comm 0x507e6620 rank 4 nranks 16 cudaDev 4 nvmlDev 7 busId f5000 commId 0x41dbbbc5b0645f0a localSize 384 used 2156642912 bytes on core 65 - Init COMPLETE +chi-mi325x-pod2-101:1948892:1950267 [4] NCCL INFO Init timings: rank 4 nranks 16 total 9.95 (kernels 0.32, bootstrap 0.89, allgathers 0.25, topo 0.39, graphs 0.00, connections 8.10, rest 0.01) +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO TUNER/Plugin: Could not find: librccl-tuner.so librccl-net.so. Using internal tuner plugin. +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO TUNER/Plugin: Could not find: librccl-tuner.so librccl-net.so. Using internal tuner plugin. +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO ncclCommInitRank comm 0x7ec7cef0 rank 6 nranks 16 cudaDev 6 nvmlDev 6 busId e5000 commId 0x41dbbbc5b0645f0a localSize 384 used 2169946720 bytes on core 97 - Init COMPLETE +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO TUNER/Plugin: Could not find: librccl-tuner.so librccl-net.so. Using internal tuner plugin. +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO ncclCommInitRank comm 0x60eef880 rank 1 nranks 16 cudaDev 1 nvmlDev 0 busId 5000 commId 0x41dbbbc5b0645f0a localSize 384 used 2154807904 bytes on core 1 - Init COMPLETE +chi-mi325x-pod2-101:1948894:1950268 [6] NCCL INFO Init timings: rank 6 nranks 16 total 9.95 (kernels 0.31, bootstrap 0.90, allgathers 0.25, topo 0.38, graphs 0.00, connections 8.09, rest 0.02) +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO ncclCommInitRank comm 0x80bc0b40 rank 0 nranks 16 cudaDev 0 nvmlDev 3 busId 75000 commId 0x41dbbbc5b0645f0a localSize 384 used 2155004512 bytes on core 132 - Init COMPLETE +chi-mi325x-pod2-101:1948889:1950269 [1] NCCL INFO Init timings: rank 1 nranks 16 total 9.95 (kernels 0.32, bootstrap 0.89, allgathers 0.25, topo 0.39, graphs 0.00, connections 8.10, rest 0.01) +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO TUNER/Plugin: Could not find: librccl-tuner.so librccl-net.so. Using internal tuner plugin. +chi-mi325x-pod2-101:1948888:1950264 [0] NCCL INFO Init timings: rank 0 nranks 16 total 9.95 (kernels 0.29, bootstrap 0.92, allgathers 0.26, topo 0.37, graphs 0.00, connections 8.09, rest 0.02) +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO ncclCommInitRank comm 0x7e4c8c40 rank 2 nranks 16 cudaDev 2 nvmlDev 2 busId 65000 commId 0x41dbbbc5b0645f0a localSize 384 used 2172043872 bytes on core 163 - Init COMPLETE +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO TUNER/Plugin: Could not find: librccl-tuner.so librccl-net.so. Using internal tuner plugin. +chi-mi325x-pod2-101:1948890:1950272 [2] NCCL INFO Init timings: rank 2 nranks 16 total 9.90 (kernels 0.32, bootstrap 0.84, allgathers 0.25, topo 0.38, graphs 0.00, connections 8.10, rest 0.01) +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO ncclCommInitRank comm 0x48e1d6c0 rank 3 nranks 16 cudaDev 3 nvmlDev 1 busId 15000 commId 0x41dbbbc5b0645f0a localSize 384 used 2135671392 bytes on core 61 - Init COMPLETE +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO TUNER/Plugin: Could not find: librccl-tuner.so librccl-net.so. Using internal tuner plugin. +chi-mi325x-pod2-101:1948891:1950271 [3] NCCL INFO Init timings: rank 3 nranks 16 total 9.93 (kernels 0.31, bootstrap 0.88, allgathers 0.25, topo 0.39, graphs 0.00, connections 8.09, rest 0.02) +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO TUNER/Plugin: Could not find: librccl-tuner.so librccl-net.so. Using internal tuner plugin. +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO ncclCommInitRank comm 0x4647b6e0 rank 7 nranks 16 cudaDev 7 nvmlDev 5 busId 95000 commId 0x41dbbbc5b0645f0a localSize 384 used 2156642912 bytes on core 120 - Init COMPLETE +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO ncclCommInitRank comm 0x648bcbd0 rank 5 nranks 16 cudaDev 5 nvmlDev 4 busId 85000 commId 0x41dbbbc5b0645f0a localSize 384 used 2163655264 bytes on core 248 - Init COMPLETE +chi-mi325x-pod2-101:1948895:1950265 [7] NCCL INFO Init timings: rank 7 nranks 16 total 9.95 (kernels 0.33, bootstrap 0.88, allgathers 0.25, topo 0.38, graphs 0.00, connections 8.10, rest 0.00) +chi-mi325x-pod2-101:1948893:1950266 [5] NCCL INFO Init timings: rank 5 nranks 16 total 9.95 (kernels 0.32, bootstrap 0.89, allgathers 0.25, topo 0.38, graphs 0.00, connections 8.09, rest 0.02) +23:44:28 | Rank 7 | INFO | Loading WikiText-2 dataset... +23:44:28 | Rank 0 | INFO | Loading WikiText-2 dataset... +23:44:28 | Rank 6 | INFO | Loading WikiText-2 dataset... +23:44:28 | Rank 5 | INFO | Loading WikiText-2 dataset... +23:44:28 | Rank 4 | INFO | Loading WikiText-2 dataset... +23:44:29 | Rank 3 | INFO | Loading WikiText-2 dataset... +23:44:29 | Rank 1 | INFO | Loading WikiText-2 dataset... +23:44:29 | Rank 2 | INFO | Loading WikiText-2 dataset... +chi-mi325x-pod2-101:1948895:1950365 [7] NCCL INFO recvProxyProgress: issued GDR flush +chi-mi325x-pod2-101:1948889:1950362 [1] NCCL INFO recvProxyProgress: issued GDR flush +chi-mi325x-pod2-101:1948891:1950359 [3] NCCL INFO recvProxyProgress: issued GDR flush +chi-mi325x-pod2-101:1948890:1950360 [2] NCCL INFO recvProxyProgress: issued GDR flush +chi-mi325x-pod2-101:1948892:1950358 [4] NCCL INFO recvProxyProgress: issued GDR flush +chi-mi325x-pod2-101:1948893:1950361 [5] NCCL INFO recvProxyProgress: issued GDR flush +chi-mi325x-pod2-101:1948894:1950363 [6] NCCL INFO recvProxyProgress: issued GDR flush +23:44:30 | Rank 7 | INFO | Dataset ready. Samples: 18686 +23:44:30 | Rank 5 | INFO | Dataset ready. Samples: 18686 +23:44:30 | Rank 4 | INFO | Dataset ready. Samples: 18686 +23:44:30 | Rank 1 | INFO | Dataset ready. Samples: 18686 +23:44:31 | Rank 0 | INFO | Dataset ready. Samples: 18686 +23:44:31 | Rank 2 | INFO | Dataset ready. Samples: 18686 +23:44:31 | Rank 6 | INFO | Dataset ready. Samples: 18686 +23:44:31 | Rank 3 | INFO | Dataset ready. Samples: 18686 +chi-mi325x-pod2-101:1948888:1950364 [0] NCCL INFO recvProxyProgress: issued GDR flush +23:44:36 | Rank 0 | INFO | Batch 0/292 | RL Loss: 6.7720 | Avg Reward: 1.38 +23:45:43 | Rank 0 | INFO | Batch 20/292 | RL Loss: 7.6501 | Avg Reward: 1.88 +23:46:18 | Rank 0 | INFO | Batch 40/292 | RL Loss: 8.4536 | Avg Reward: 2.12 +23:46:19 | Rank 0 | INFO | --- RL EPOCH 1 STATS --- +23:46:19 | Rank 0 | INFO | Duration: 108.17 seconds +23:46:19 | Rank 0 | INFO | Throughput: 22113.00 tokens/sec +23:46:19 | Rank 0 | INFO | Avg Reward: 0.3489 +23:46:19 | Rank 0 | INFO | ------------------------- +23:46:19 | Rank 0 | INFO | Preparing weights for UCCL broadcast from DDP Master... +23:46:19 | Rank 0 | INFO | UCCL: Starting Broadcast (Sender)... +23:46:19 | Rank 0 | INFO | UCCL Broadcast Complete. Time: 0.034s | BW: 19.02 GB/s +23:46:22 | Rank 0 | INFO | Batch 0/292 | RL Loss: 10.0533 | Avg Reward: 2.75 +23:46:57 | Rank 0 | INFO | Batch 20/292 | RL Loss: 11.0915 | Avg Reward: 2.75 +23:47:33 | Rank 0 | INFO | Batch 40/292 | RL Loss: 12.3705 | Avg Reward: 3.00 +23:47:33 | Rank 1 | INFO | Waiting for all ranks to complete... +23:47:33 | Rank 3 | INFO | Waiting for all ranks to complete... +23:47:33 | Rank 0 | INFO | --- RL EPOCH 2 STATS --- +23:47:33 | Rank 0 | INFO | Duration: 74.47 seconds +23:47:33 | Rank 0 | INFO | Throughput: 32121.21 tokens/sec +23:47:33 | Rank 0 | INFO | Avg Reward: 0.3485 +23:47:33 | Rank 0 | INFO | ------------------------- +23:47:33 | Rank 0 | INFO | Preparing weights for UCCL broadcast from DDP Master... +23:47:33 | Rank 0 | INFO | UCCL: Starting Broadcast (Sender)... +23:47:33 | Rank 0 | INFO | UCCL Broadcast Complete. Time: 0.019s | BW: 35.25 GB/s +23:47:33 | Rank 0 | INFO | Total Session Time: 182.70 seconds +23:47:33 | Rank 0 | INFO | Waiting for all ranks to complete... +23:47:33 | Rank 7 | INFO | Waiting for all ranks to complete... +23:47:33 | Rank 2 | INFO | Waiting for all ranks to complete... +23:47:33 | Rank 4 | INFO | Waiting for all ranks to complete... +23:47:33 | Rank 6 | INFO | Waiting for all ranks to complete... +23:47:33 | Rank 5 | INFO | Waiting for all ranks to complete... +Destroying Engine... +Destroying Engine... +Destroying Engine... +Destroying Engine... +Destroying Engine... +Destroying Engine...Destroying Engine... +Destroying Engine... + +Engine destroyed +Engine destroyed +Engine destroyed +Engine destroyed +Engine destroyed +Engine destroyed +Engine destroyed +Engine destroyed +chi-mi325x-pod2-101:1948891:1948891 [3] NCCL INFO comm 0x48e1d6c0 rank 3 nranks 16 cudaDev 3 busId 15000 - Destroy COMPLETE +chi-mi325x-pod2-101:1948894:1948894 [6] NCCL INFO comm 0x7ec7cef0 rank 6 nranks 16 cudaDev 6 busId e5000 - Destroy COMPLETE +chi-mi325x-pod2-101:1948890:1948890 [2] NCCL INFO comm 0x7e4c8c40 rank 2 nranks 16 cudaDev 2 busId 65000 - Destroy COMPLETE +chi-mi325x-pod2-101:1948892:1948892 [4] NCCL INFO comm 0x507e6620 rank 4 nranks 16 cudaDev 4 busId f5000 - Destroy COMPLETE +chi-mi325x-pod2-101:1948889:1948889 [1] NCCL INFO comm 0x60eef880 rank 1 nranks 16 cudaDev 1 busId 5000 - Destroy COMPLETE +chi-mi325x-pod2-101:1948893:1948893 [5] NCCL INFO comm 0x648bcbd0 rank 5 nranks 16 cudaDev 5 busId 85000 - Destroy COMPLETE +chi-mi325x-pod2-101:1948895:1948895 [7] NCCL INFO comm 0x4647b6e0 rank 7 nranks 16 cudaDev 7 busId 95000 - Destroy COMPLETE +chi-mi325x-pod2-101:1948888:1948888 [0] NCCL INFO comm 0x80bc0b40 rank 0 nranks 16 cudaDev 0 busId 75000 - Destroy COMPLETE diff --git a/main.py b/junk/main.py similarity index 100% rename from main.py rename to junk/main.py diff --git a/junk/multi_gpu_multi_node_fsdp.py b/junk/multi_gpu_multi_node_fsdp.py new file mode 100644 index 0000000..e65a79f --- /dev/null +++ b/junk/multi_gpu_multi_node_fsdp.py @@ -0,0 +1,559 @@ +from __future__ import annotations +import torch, time, os, sys +import torch.distributed as dist +import logging +from datetime import datetime +import math + +from transformers import GPT2LMHeadModel, GPT2Tokenizer +from datasets import load_dataset +from torch.utils.data import DataLoader +from uccl import p2p +import torch.nn as nn +from torch.distributed.fsdp import FullyShardedDataParallel as FSDP +from torch.distributed.fsdp.wrap import transformer_auto_wrap_policy +from transformers.models.gpt2.modeling_gpt2 import GPT2Block +from torch.utils.data.distributed import DistributedSampler + +# --------------------------- +# Logging +# --------------------------- +def setup_logging(rank): + """Setup logging with timestamps and rank info""" + log_dir = "logs" + os.makedirs(log_dir, exist_ok=True) + + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + log_file = f"{log_dir}/rank_{rank}_{timestamp}.log" + + logging.basicConfig( + level=logging.INFO, + format='%(asctime)s | Rank %(rank)d | %(levelname)s | %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + handlers=[ + logging.FileHandler(log_file), + logging.StreamHandler(sys.stdout) + ] + ) + + # Add rank to all log records + old_factory = logging.getLogRecordFactory() + def record_factory(*args, **kwargs): + record = old_factory(*args, **kwargs) + record.rank = rank + return record + logging.setLogRecordFactory(record_factory) + + logging.info(f"Logging initialized. Log file: {log_file}") + return log_file + + +# --------------------------- +# RDMA send/recv helpers +# --------------------------- +def broadcast_model(ep, conn_ids, model, rank): + """Send model to multiple receivers with detailed logging""" + state_dict = model.state_dict() + items = list(state_dict.items()) + total_tensors = len(items) + total_size_mb = sum(t.numel() * t.element_size() for _, t in items) / 1e6 + + logging.info("="*80) + logging.info(f"BROADCAST START - Sending to {len(conn_ids)} receivers") + logging.info(f"Total tensors: {total_tensors}") + logging.info(f"Total size: {total_size_mb:.2f} MB") + logging.info("="*80) + + broadcast_start = time.perf_counter() + + for idx, (name, tensor) in enumerate(items, 1): + # ensure tensor on GPU for RDMA + if not tensor.is_cuda: + tensor = tensor.cuda() + + size_bytes = tensor.numel() * tensor.element_size() + ptr = tensor.data_ptr() + + # Register memory + ok, mr_id = ep.reg(ptr, size_bytes) + assert ok, f"Failed to register tensor {name}" + + # Send to all receivers + for receiver_idx, conn_id in enumerate(conn_ids, 1): + ok = ep.send(conn_id, mr_id, ptr, size_bytes) + assert ok, f"Send failed for {name} to receiver {receiver_idx}" + + if idx % 20 == 0 or idx == total_tensors: + progress_pct = (idx / total_tensors) * 100 + logging.info(f"Progress: {progress_pct:.1f}% ({idx}/{total_tensors})") + + total_time = time.perf_counter() - broadcast_start + avg_bandwidth = (total_size_mb / 1000) / total_time # GB/s approximation + + logging.info("="*80) + logging.info(f"BROADCAST COMPLETE") + logging.info(f"Total time: {total_time:.2f}s") + logging.info(f"Average bandwidth: {avg_bandwidth:.2f} GB/s") + logging.info("="*80) + + +def recv_model(ep, conn_id, model, rank): + """Receive model from broadcaster with detailed logging""" + state_dict = model.state_dict() + items = list(state_dict.items()) + total_tensors = len(items) + total_size_mb = sum(t.numel() * t.element_size() for _, t in items) / 1e6 + + logging.info("="*80) + logging.info(f"RECEIVE START") + logging.info(f"Total tensors: {total_tensors}") + logging.info(f"Total size: {total_size_mb:.2f} MB") + logging.info("="*80) + + recv_start = time.perf_counter() + + for idx, (name, tensor) in enumerate(items, 1): + # allocate recv tensor on GPU + recv_tensor = torch.empty_like(tensor, device="cuda") + size_bytes = recv_tensor.numel() * recv_tensor.element_size() + ptr = recv_tensor.data_ptr() + + # Register memory + ok, mr_id = ep.reg(ptr, size_bytes) + assert ok, f"Failed to register tensor {name}" + + # Receive tensor + ok = ep.recv(conn_id, mr_id, ptr, size_bytes) + assert ok, f"Receive failed for {name}" + + # Copy into model parameter / buffer (assumes model on GPU) + target = model.state_dict()[name] + # target may be on cuda already if model.cuda() was called + if not target.is_cuda: + # move model param/buffer to cuda if needed (should normally be already) + target = target.cuda() + # use in-place copy + model.state_dict()[name].copy_(recv_tensor) + + if idx % 20 == 0 or idx == total_tensors: + progress_pct = (idx / total_tensors) * 100 + logging.info(f"Progress: {progress_pct:.1f}% ({idx}/{total_tensors})") + + total_time = time.perf_counter() - recv_start + avg_bandwidth = (total_size_mb / 1000) / total_time # GB/s approx + + logging.info("="*80) + logging.info(f"RECEIVE COMPLETE") + logging.info(f"Total time: {total_time:.2f}s") + logging.info(f"Average bandwidth: {avg_bandwidth:.2f} GB/s") + logging.info("="*80) + + +# --------------------------- +# Dataset preparation +# --------------------------- +def prepare_dataset(tokenizer, max_length=128, num_samples=200): + """Load and prepare WikiText-2 dataset""" + logging.info("="*80) + logging.info("LOADING DATASET") + logging.info("="*80) + logging.info("Loading WikiText-2 dataset from Hugging Face...") + + dataset = load_dataset("wikitext", "wikitext-2-raw-v1", split="train") + logging.info(f"Dataset loaded: {len(dataset)} total examples") + + dataset = dataset.filter(lambda x: len(x["text"].strip()) > 0) + logging.info(f"After filtering empty texts: {len(dataset)} examples") + + if len(dataset) > num_samples: + dataset = dataset.select(range(num_samples)) + logging.info(f"Using subset: {num_samples} examples") + + logging.info("Tokenizing dataset...") + + def tokenize_function(examples): + return tokenizer( + examples["text"], + truncation=True, + padding="max_length", + max_length=max_length + ) + + tokenized_dataset = dataset.map( + tokenize_function, + batched=True, + remove_columns=dataset.column_names + ) + + tokenized_dataset.set_format(type='torch', columns=['input_ids', 'attention_mask']) + + logging.info(f"Tokenization complete") + logging.info(f" - Examples: {len(tokenized_dataset)}") + logging.info(f" - Max length: {max_length} tokens") + logging.info("="*80) + + return tokenized_dataset + + +# --------------------------- +# Training (rank 1..N) +# --------------------------- +def run_training(model, tokenizer, num_epochs=2, batch_size=4, lr=5e-5): + """Training loop for trainer ranks""" + logging.info("="*80) + logging.info("TRAINING NODE - Starting training on WikiText-2") + logging.info(f"Visible GPUs on this node: {torch.cuda.device_count()}") + logging.info(f"Model class: {model.__class__.__name__}") + logging.info("="*80) + + train_dataset = prepare_dataset(tokenizer, max_length=128, num_samples=200) + # Use distributed sampler with explicit replicas/rank + sampler = DistributedSampler(train_dataset, num_replicas=dist.get_world_size(), rank=dist.get_rank(), shuffle=True) + train_dataloader = DataLoader( + train_dataset, + batch_size=batch_size, + sampler=sampler + ) + + total_steps = len(train_dataloader) * num_epochs + logging.info(f"Training configuration:") + logging.info(f" - Examples: {len(train_dataset)}") + logging.info(f" - Batch size: {batch_size}") + logging.info(f" - Epochs: {num_epochs}") + logging.info(f" - Steps per epoch: {len(train_dataloader)}") + logging.info(f" - Total steps: {total_steps}") + logging.info(f" - Learning rate: {lr}") + logging.info("="*80) + + model.train() + optimizer = torch.optim.AdamW(model.parameters(), lr=lr) + + # Proper AMP usage + scaler = torch.cuda.amp.GradScaler(enabled=True) + + total_train_start = time.perf_counter() + global_step = 0 + epoch_losses = [] + + for epoch in range(num_epochs): + epoch_start = time.perf_counter() + epoch_loss = 0.0 + epoch_perplexity = 0.0 + + logging.info(f"\n{'='*80}") + logging.info(f"EPOCH {epoch + 1}/{num_epochs}") + logging.info(f"{'='*80}") + + sampler.set_epoch(epoch) # set epoch for DistributedSampler + + for batch_idx, batch in enumerate(train_dataloader): + step_start = time.perf_counter() + global_step += 1 + + # Move batch to GPU + input_ids = batch["input_ids"].cuda(non_blocking=True) + attention_mask = batch["attention_mask"].cuda(non_blocking=True) + + # Forward + backward with autocast + optimizer.zero_grad() + with torch.cuda.amp.autocast(): + outputs = model( + input_ids=input_ids, + attention_mask=attention_mask, + labels=input_ids + ) + loss = outputs.loss + + # reduce if per-GPU vector + if isinstance(loss, torch.Tensor) and loss.dim() > 0: + loss_scalar = loss.mean() + else: + loss_scalar = loss + + scaler.scale(loss_scalar).backward() + scaler.step(optimizer) + scaler.update() + + step_time = time.perf_counter() - step_start + + loss_value = loss_scalar.item() + perplexity = math.exp(loss_value) if loss_value < 100 else float('inf') + + epoch_loss += loss_value + epoch_perplexity += perplexity + + if (batch_idx + 1) % 10 == 0 or (batch_idx + 1) == len(train_dataloader): + logging.info( + f"Step {global_step}/{total_steps} " + f"(Epoch {epoch+1}, Batch {batch_idx+1}/{len(train_dataloader)}) | " + f"Loss: {loss_value:.4f} | Perplexity: {perplexity:.2f} | " + f"Time: {step_time:.3f}s" + ) + + if global_step % 20 == 0: + total_norm_sq = 0.0 + for p in model.parameters(): + if p.grad is not None: + param_norm = p.grad.data.norm(2) + total_norm_sq += param_norm.item() ** 2 + total_norm = total_norm_sq ** 0.5 + logging.info(f" -> Gradient norm: {total_norm:.4f}") + + epoch_time = time.perf_counter() - epoch_start + avg_epoch_loss = epoch_loss / max(1, len(train_dataloader)) + avg_epoch_perplexity = epoch_perplexity / max(1, len(train_dataloader)) + epoch_losses.append(avg_epoch_loss) + + logging.info(f"\n{'='*80}") + logging.info(f"EPOCH {epoch + 1} SUMMARY") + logging.info(f"Average Loss: {avg_epoch_loss:.4f}") + logging.info(f"Average Perplexity: {avg_epoch_perplexity:.2f}") + logging.info(f"Epoch Time: {epoch_time:.2f}s") + logging.info(f"{'='*80}\n") + + total_train_time = time.perf_counter() - total_train_start + avg_step_time = total_train_time / max(1, total_steps) + + logging.info("="*80) + logging.info("TRAINING COMPLETE") + logging.info(f"Total training time: {total_train_time:.2f}s") + logging.info(f"Total steps: {total_steps}") + logging.info(f"Average time per step: {avg_step_time:.3f}s") + logging.info(f"Steps per second: {total_steps/total_train_time:.2f}") + logging.info(f"Final loss: {epoch_losses[-1]:.4f}") + logging.info(f"Loss improvement: {epoch_losses[0] - epoch_losses[-1]:.4f}") + logging.info("="*80) + + checkpoint_dir = "checkpoints" + os.makedirs(checkpoint_dir, exist_ok=True) + + # Save model state depending on FSDP + save_model = model + if isinstance(save_model, FSDP): + # FSDP: use state_dict() (it may return sharded; user can use FSDP.full_state_dict if available) + try: + sd = save_model.state_dict() + except Exception: + sd = save_model.module.state_dict() + else: + sd = save_model.state_dict() + + checkpoint_path = f"{checkpoint_dir}/model_rank{dist.get_rank()}_wikitext2_trained.pt" + torch.save({ + 'model_state_dict': sd, + 'optimizer_state_dict': optimizer.state_dict(), + 'epoch_losses': epoch_losses, + 'total_steps': total_steps, + 'final_loss': epoch_losses[-1], + }, checkpoint_path) + logging.info(f"Model checkpoint saved to: {checkpoint_path}") + + +# --------------------------- +# Inference (rank last) +# --------------------------- +def run_inference(model, tokenizer, num_samples=5): + logging.info("="*80) + logging.info("INFERENCE NODE - Starting inference") + logging.info("="*80) + + model.eval() + prompts = [ + "Once upon a time", + "The future of artificial intelligence", + "In a world where technology", + "Scientists have discovered", + "The most important thing in life", + "Deep learning models", + "Natural language processing", + "Machine learning algorithms", + ] + + logging.info(f"Running inference on {num_samples} prompts") + logging.info("Generation settings: max_length=50, temperature=0.7, top_k=50, top_p=0.95") + + total_inference_start = time.perf_counter() + total_tokens = 0 + + with torch.no_grad(): + for idx, prompt in enumerate(prompts[:num_samples], 1): + inference_start = time.perf_counter() + + logging.info(f"\n{'='*60}") + logging.info(f"SAMPLE {idx}/{num_samples}") + logging.info(f"{'='*60}") + logging.info(f"Prompt: '{prompt}'") + + inputs = tokenizer(prompt, return_tensors="pt") + input_ids = inputs["input_ids"].cuda() + + generation_start = time.perf_counter() + output_ids = model.generate( + input_ids, + max_length=50, + num_return_sequences=1, + temperature=0.7, + do_sample=True, + top_k=50, + top_p=0.95, + pad_token_id=tokenizer.eos_token_id + ) + generation_time = time.perf_counter() - generation_start + + generated_text = tokenizer.decode(output_ids[0], skip_special_tokens=True) + + inference_time = time.perf_counter() - inference_start + tokens_generated = output_ids.shape[1] - input_ids.shape[1] + total_tokens += tokens_generated + tokens_per_sec = tokens_generated / generation_time if generation_time > 0 else 0 + + logging.info(f"Output: '{generated_text}'") + logging.info(f"{'='*60}") + logging.info(f"Metrics:") + logging.info(f" - Total inference time: {inference_time:.3f}s") + logging.info(f" - Generation time: {generation_time:.3f}s") + logging.info(f" - Tokens generated: {tokens_generated}") + logging.info(f" - Tokens/sec: {tokens_per_sec:.1f}") + logging.info(f"{'='*60}") + + total_inference_time = time.perf_counter() - total_inference_start + avg_inference_time = total_inference_time / max(1, num_samples) + avg_tokens_per_sec = total_tokens / max(1e-9, total_inference_time) + + logging.info("\n" + "="*80) + logging.info("INFERENCE COMPLETE") + logging.info(f"Total inference time: {total_inference_time:.2f}s") + logging.info(f"Average time per sample: {avg_inference_time:.3f}s") + logging.info(f"Samples per second: {num_samples/total_inference_time:.2f}") + logging.info(f"Total tokens generated: {total_tokens}") + logging.info(f"Average tokens/sec: {avg_tokens_per_sec:.1f}") + logging.info("="*80) + + +# --------------------------- +# Main (all ranks) +# --------------------------- +def main(): + # init from environment (torchrun) + dist.init_process_group(backend="nccl") + rank = dist.get_rank() + world_size = dist.get_world_size() + + log_file = setup_logging(rank) + logging.info(f"Process started - Rank: {rank}, World size: {world_size}") + assert world_size >= 3, "Need at least 3 ranks: broadcaster + trainer(s) + inference" + + # Setup CUDA device and local_gpu + if torch.cuda.is_available(): + local_rank = int(os.environ.get("LOCAL_RANK", rank)) + torch.cuda.set_device(local_rank) + local_gpu = torch.cuda.current_device() # <-- FIX: define local_gpu + logging.info(f"CUDA device set to GPU {local_gpu}") + else: + local_gpu = None + logging.info("CUDA not available. (RDMA will not work.)") + + logging.info("Initializing P2P endpoint...") + ep = p2p.Endpoint(local_gpu, 4) + local_md = ep.get_metadata() + logging.info(f"Local metadata obtained (size: {len(local_md)} bytes)") + + # Exchange metadata robustly using all_gather_object + logging.info("Starting metadata exchange (all_gather_object)...") + all_metadata = [None] * world_size + dist.all_gather_object(all_metadata, local_md) + logging.info("Metadata exchange complete") + + if rank == 0: + # Broadcaster + logging.info("="*80) + logging.info("BROADCASTER MODE") + logging.info("="*80) + logging.info("Connecting to receivers...") + conn_ids = [] + + # connect to all non-zero ranks except possibly extra ones + receiver_ranks = [r for r in range(1, world_size)] # all other ranks + for receiver_rank in receiver_ranks: + # parse remote metadata + ip, port, r_gpu = p2p.Endpoint.parse_metadata(all_metadata[receiver_rank]) + logging.info(f"Connecting to rank {receiver_rank}: IP={ip}, Port={port}, GPU={r_gpu}") + + ok, conn_id = ep.connect(ip, r_gpu, remote_port=port) + assert ok, f"Connect failed to rank {receiver_rank}" + conn_ids.append((receiver_rank, conn_id)) + node_type = "Training Node" if receiver_rank != world_size - 1 else "Inference Node" + logging.info(f"Connected to {node_type} (rank {receiver_rank}, conn_id={conn_id})") + + logging.info("Loading model on broadcaster...") + model = GPT2LMHeadModel.from_pretrained("gpt2").cuda() + logging.info("Model loaded") + + # pass just list of conn_ids ints to broadcast_model + broadcast_model(ep, [c for _, c in conn_ids], model, rank) + + logging.info("\nBroadcast complete. Other nodes should now process...") + + elif 1 <= rank < world_size - 1: + # Trainer(s) + logging.info("="*80) + logging.info(f"TRAINING NODE (Rank {rank})") + logging.info("="*80) + logging.info("Waiting for broadcaster connection...") + + ok, r_ip, r_gpu, conn_id = ep.accept() + assert ok, "Accept failed" + logging.info("Connected to broadcaster") + + logging.info("Loading model and tokenizer...") + base_model = GPT2LMHeadModel.from_pretrained("gpt2").cuda() + tokenizer = GPT2Tokenizer.from_pretrained("gpt2") + tokenizer.pad_token = tokenizer.eos_token # use eos token as pad + logging.info("Model and tokenizer loaded") + + recv_model(ep, conn_id, base_model, rank) + # FSDP auto wrap policy for GPT-2 transformer blocks + auto_wrap_policy = transformer_auto_wrap_policy({GPT2Block}) + logging.info("Wrapping model with FSDP across nodes...") + model = FSDP( + base_model, + auto_wrap_policy=auto_wrap_policy, + device_id=torch.cuda.current_device() + ) + + run_training(model, tokenizer, num_epochs=2, batch_size=4, lr=5e-5) + + elif rank == world_size - 1: + # Inference node (last rank) + logging.info("="*80) + logging.info("INFERENCE NODE") + logging.info("="*80) + logging.info("Waiting for broadcaster connection...") + + ok, r_ip, r_gpu, conn_id = ep.accept() + assert ok, "Accept failed" + logging.info("Connected to broadcaster") + + logging.info("Loading model and tokenizer...") + model = GPT2LMHeadModel.from_pretrained("gpt2").cuda() + tokenizer = GPT2Tokenizer.from_pretrained("gpt2") + logging.info("Model and tokenizer loaded") + + recv_model(ep, conn_id, model, rank) + run_inference(model, tokenizer, num_samples=5) + + logging.info("Destroying process group...") + dist.destroy_process_group() + logging.info("Process complete. Exiting.") + logging.info(f"Full log saved to: {log_file}") + + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + logging.warning("Interrupted by user") + sys.exit(0) + except Exception as e: + logging.error(f"Fatal error: {e}", exc_info=True) + sys.exit(1) diff --git a/junk/multi_gpu_training_single_node.py b/junk/multi_gpu_training_single_node.py new file mode 100644 index 0000000..aa2f3b4 --- /dev/null +++ b/junk/multi_gpu_training_single_node.py @@ -0,0 +1,548 @@ +from __future__ import annotations +import torch, time, os, sys +import torch.distributed as dist +import logging +from datetime import datetime +import math + +from transformers import GPT2LMHeadModel, GPT2Tokenizer +from datasets import load_dataset +from torch.utils.data import DataLoader +from uccl import p2p +import torch.nn as nn +from torch.distributed.fsdp import FullyShardedDataParallel as FSDP +from torch.distributed.fsdp.wrap import transformer_auto_wrap_policy +from transformers.models.gpt2.modeling_gpt2 import GPT2Block +from torch.utils.data.distributed import DistributedSampler + + +# --------------------------- +# Logging +# --------------------------- +def setup_logging(rank): + """Setup logging with timestamps and rank info""" + log_dir = "logs" + os.makedirs(log_dir, exist_ok=True) + + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + log_file = f"{log_dir}/rank_{rank}_{timestamp}.log" + + logging.basicConfig( + level=logging.INFO, + format='%(asctime)s | Rank %(rank)d | %(levelname)s | %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + handlers=[ + logging.FileHandler(log_file), + logging.StreamHandler(sys.stdout) + ] + ) + + # Add rank to all log records + old_factory = logging.getLogRecordFactory() + def record_factory(*args, **kwargs): + record = old_factory(*args, **kwargs) + record.rank = rank + return record + logging.setLogRecordFactory(record_factory) + + logging.info(f"Logging initialized. Log file: {log_file}") + return log_file + + +# --------------------------- +# RDMA send/recv helpers +# --------------------------- +def broadcast_model(ep, conn_ids, model, rank): + """Send model to multiple receivers with detailed logging""" + state_dict = model.state_dict() + total_tensors = len(list(state_dict.items())) + total_size_mb = sum(t.numel() * t.element_size() for t in state_dict.values()) / 1e6 + + logging.info("="*80) + logging.info(f"BROADCAST START - Sending to {len(conn_ids)} receivers") + logging.info(f"Total tensors: {total_tensors}") + logging.info(f"Total size: {total_size_mb:.2f} MB") + logging.info("="*80) + + broadcast_start = time.perf_counter() + + for idx, (name, tensor) in enumerate(state_dict.items(), 1): + if not tensor.is_cuda: + tensor = tensor.cuda() + + size_bytes = tensor.numel() * tensor.element_size() + ptr = tensor.data_ptr() + + # Register memory + ok, mr_id = ep.reg(ptr, size_bytes) + assert ok, f"Failed to register tensor {name}" + + # Send to all receivers + for receiver_idx, conn_id in enumerate(conn_ids, 1): + ok = ep.send(conn_id, mr_id, ptr, size_bytes) + assert ok, f"Send failed for {name} to receiver {receiver_idx}" + + if idx % 20 == 0 or idx == total_tensors: + progress_pct = (idx / total_tensors) * 100 + logging.info(f"Progress: {progress_pct:.1f}% ({idx}/{total_tensors})") + + total_time = time.perf_counter() - broadcast_start + avg_bandwidth = (total_size_mb / 1000) / total_time # GB/s + + logging.info("="*80) + logging.info(f"BROADCAST COMPLETE") + logging.info(f"Total time: {total_time:.2f}s") + logging.info(f"Average bandwidth: {avg_bandwidth:.2f} GB/s") + logging.info("="*80) + + +def recv_model(ep, conn_id, model, rank): + """Receive model from broadcaster with detailed logging""" + state_dict = model.state_dict() + total_tensors = len(list(state_dict.items())) + total_size_mb = sum(t.numel() * t.element_size() for t in state_dict.values()) / 1e6 + + logging.info("="*80) + logging.info(f"RECEIVE START") + logging.info(f"Total tensors: {total_tensors}") + logging.info(f"Total size: {total_size_mb:.2f} MB") + logging.info("="*80) + + recv_start = time.perf_counter() + + for idx, (name, tensor) in enumerate(state_dict.items(), 1): + recv_tensor = torch.empty_like(tensor, device="cuda") + size_bytes = recv_tensor.numel() * recv_tensor.element_size() + ptr = recv_tensor.data_ptr() + + # Register memory + ok, mr_id = ep.reg(ptr, size_bytes) + assert ok, f"Failed to register tensor {name}" + + # Receive tensor + ok = ep.recv(conn_id, mr_id, ptr, size_bytes) + assert ok, f"Receive failed for {name}" + + model.state_dict()[name].copy_(recv_tensor) + + if idx % 20 == 0 or idx == total_tensors: + progress_pct = (idx / total_tensors) * 100 + logging.info(f"Progress: {progress_pct:.1f}% ({idx}/{total_tensors})") + + total_time = time.perf_counter() - recv_start + avg_bandwidth = (total_size_mb / 1000) / total_time # GB/s + + logging.info("="*80) + logging.info(f"RECEIVE COMPLETE") + logging.info(f"Total time: {total_time:.2f}s") + logging.info(f"Average bandwidth: {avg_bandwidth:.2f} GB/s") + logging.info("="*80) + + +# --------------------------- +# Dataset preparation +# --------------------------- +def prepare_dataset(tokenizer, max_length=128, num_samples=200): + """Load and prepare WikiText-2 dataset""" + logging.info("="*80) + logging.info("LOADING DATASET") + logging.info("="*80) + logging.info("Loading WikiText-2 dataset from Hugging Face...") + + dataset = load_dataset("wikitext", "wikitext-2-raw-v1", split="train") + logging.info(f"Dataset loaded: {len(dataset)} total examples") + + dataset = dataset.filter(lambda x: len(x["text"].strip()) > 0) + logging.info(f"After filtering empty texts: {len(dataset)} examples") + + if len(dataset) > num_samples: + dataset = dataset.select(range(num_samples)) + logging.info(f"Using subset: {num_samples} examples") + + logging.info("Tokenizing dataset...") + + def tokenize_function(examples): + return tokenizer( + examples["text"], + truncation=True, + padding="max_length", + max_length=max_length + ) + + tokenized_dataset = dataset.map( + tokenize_function, + batched=True, + remove_columns=dataset.column_names + ) + + tokenized_dataset.set_format(type='torch', columns=['input_ids', 'attention_mask']) + + logging.info(f"Tokenization complete") + logging.info(f" - Examples: {len(tokenized_dataset)}") + logging.info(f" - Max length: {max_length} tokens") + logging.info("="*80) + + return tokenized_dataset + + +# --------------------------- +# Training (rank 1) +# --------------------------- +def run_training(model, tokenizer, num_epochs=2, batch_size=4, lr=5e-5): + """Training loop for rank 1 (supports DataParallel)""" + logging.info("="*80) + logging.info("TRAINING NODE - Starting training on WikiText-2") + logging.info(f"Visible GPUs on this node: {torch.cuda.device_count()}") + logging.info(f"Model class: {model.__class__.__name__}") + logging.info("="*80) + + train_dataset = prepare_dataset(tokenizer, max_length=128, num_samples=200) + from torch.utils.data.distributed import DistributedSampler + sampler = DistributedSampler(train_dataset, shuffle=True) + train_dataloader = DataLoader( + train_dataset, + batch_size=batch_size, + sampler=sampler + ) + + total_steps = len(train_dataloader) * num_epochs + logging.info(f"Training configuration:") + logging.info(f" - Examples: {len(train_dataset)}") + logging.info(f" - Batch size: {batch_size}") + logging.info(f" - Epochs: {num_epochs}") + logging.info(f" - Steps per epoch: {len(train_dataloader)}") + logging.info(f" - Total steps: {total_steps}") + logging.info(f" - Learning rate: {lr}") + logging.info("="*80) + + model.train() + optimizer = torch.optim.AdamW(model.parameters(), lr=lr) + model.forward = torch.cuda.amp.autocast()(model.forward) + total_train_start = time.perf_counter() + global_step = 0 + epoch_losses = [] + + for epoch in range(num_epochs): + epoch_start = time.perf_counter() + epoch_loss = 0.0 + epoch_perplexity = 0.0 + + logging.info(f"\n{'='*80}") + logging.info(f"EPOCH {epoch + 1}/{num_epochs}") + logging.info(f"{'='*80}") + + for batch_idx, batch in enumerate(train_dataloader): + step_start = time.perf_counter() + global_step += 1 + + # Move batch to GPU 0; DataParallel will scatter internally if enabled + input_ids = batch["input_ids"].cuda() + attention_mask = batch["attention_mask"].cuda() + + # Forward pass + outputs = model( + input_ids=input_ids, + attention_mask=attention_mask, + labels=input_ids + ) + loss = outputs.loss # can be scalar or vector (per-GPU) with DataParallel + + # ---- MULTI-GPU SAFE LOSS REDUCTION ---- + if isinstance(loss, torch.Tensor) and loss.dim() > 0: + loss_scalar = loss.mean() + else: + loss_scalar = loss + + optimizer.zero_grad() + loss_scalar.backward() + optimizer.step() + + step_time = time.perf_counter() - step_start + + # Use scalar loss for metrics + loss_value = loss_scalar.item() + perplexity = math.exp(loss_value) if loss_value < 100 else float('inf') + + epoch_loss += loss_value + epoch_perplexity += perplexity + + if (batch_idx + 1) % 10 == 0 or (batch_idx + 1) == len(train_dataloader): + logging.info( + f"Step {global_step}/{total_steps} " + f"(Epoch {epoch+1}, Batch {batch_idx+1}/{len(train_dataloader)}) | " + f"Loss: {loss_value:.4f} | Perplexity: {perplexity:.2f} | " + f"Time: {step_time:.3f}s" + ) + + if global_step % 20 == 0: + total_norm = 0.0 + for p in model.parameters(): + if p.grad is not None: + param_norm = p.grad.data.norm(2) + total_norm += param_norm.item() ** 2 + total_norm = total_norm ** 0.5 + logging.info(f" -> Gradient norm: {total_norm:.4f}") + + epoch_time = time.perf_counter() - epoch_start + avg_epoch_loss = epoch_loss / len(train_dataloader) + avg_epoch_perplexity = epoch_perplexity / len(train_dataloader) + epoch_losses.append(avg_epoch_loss) + + logging.info(f"\n{'='*80}") + logging.info(f"EPOCH {epoch + 1} SUMMARY") + logging.info(f"Average Loss: {avg_epoch_loss:.4f}") + logging.info(f"Average Perplexity: {avg_epoch_perplexity:.2f}") + logging.info(f"Epoch Time: {epoch_time:.2f}s") + logging.info(f"{'='*80}\n") + + total_train_time = time.perf_counter() - total_train_start + avg_step_time = total_train_time / total_steps + + logging.info("="*80) + logging.info("TRAINING COMPLETE") + logging.info(f"Total training time: {total_train_time:.2f}s") + logging.info(f"Total steps: {total_steps}") + logging.info(f"Average time per step: {avg_step_time:.3f}s") + logging.info(f"Steps per second: {total_steps/total_train_time:.2f}") + logging.info(f"Final loss: {epoch_losses[-1]:.4f}") + logging.info(f"Loss improvement: {epoch_losses[0] - epoch_losses[-1]:.4f}") + logging.info("="*80) + + checkpoint_dir = "checkpoints" + os.makedirs(checkpoint_dir, exist_ok=True) + + # If DataParallel is used, save underlying module + save_model = model + checkpoint_path = f"{checkpoint_dir}/model_rank1_wikitext2_trained.pt" + torch.save({ + 'model_state_dict': FSDP.state_dict(save_model), + 'optimizer_state_dict': optimizer.state_dict(), + 'epoch_losses': epoch_losses, + 'total_steps': total_steps, + 'final_loss': epoch_losses[-1], + }, checkpoint_path) + logging.info(f"Model checkpoint saved to: {checkpoint_path}") + + +# --------------------------- +# Inference (rank 2) +# --------------------------- +def run_inference(model, tokenizer, num_samples=5): + logging.info("="*80) + logging.info("INFERENCE NODE - Starting inference") + logging.info("="*80) + + model.eval() + prompts = [ + "Once upon a time", + "The future of artificial intelligence", + "In a world where technology", + "Scientists have discovered", + "The most important thing in life", + "Deep learning models", + "Natural language processing", + "Machine learning algorithms", + ] + + logging.info(f"Running inference on {num_samples} prompts") + logging.info("Generation settings: max_length=50, temperature=0.7, top_k=50, top_p=0.95") + + total_inference_start = time.perf_counter() + total_tokens = 0 + + with torch.no_grad(): + for idx, prompt in enumerate(prompts[:num_samples], 1): + inference_start = time.perf_counter() + + logging.info(f"\n{'='*60}") + logging.info(f"SAMPLE {idx}/{num_samples}") + logging.info(f"{'='*60}") + logging.info(f"Prompt: '{prompt}'") + + inputs = tokenizer(prompt, return_tensors="pt") + input_ids = inputs["input_ids"].cuda() + + generation_start = time.perf_counter() + output_ids = model.generate( + input_ids, + max_length=50, + num_return_sequences=1, + temperature=0.7, + do_sample=True, + top_k=50, + top_p=0.95, + pad_token_id=tokenizer.eos_token_id + ) + generation_time = time.perf_counter() - generation_start + + generated_text = tokenizer.decode(output_ids[0], skip_special_tokens=True) + + inference_time = time.perf_counter() - inference_start + tokens_generated = output_ids.shape[1] - input_ids.shape[1] + total_tokens += tokens_generated + tokens_per_sec = tokens_generated / generation_time if generation_time > 0 else 0 + + logging.info(f"Output: '{generated_text}'") + logging.info(f"{'='*60}") + logging.info(f"Metrics:") + logging.info(f" - Total inference time: {inference_time:.3f}s") + logging.info(f" - Generation time: {generation_time:.3f}s") + logging.info(f" - Tokens generated: {tokens_generated}") + logging.info(f" - Tokens/sec: {tokens_per_sec:.1f}") + logging.info(f"{'='*60}") + + total_inference_time = time.perf_counter() - total_inference_start + avg_inference_time = total_inference_time / num_samples + avg_tokens_per_sec = total_tokens / total_inference_time + + logging.info("\n" + "="*80) + logging.info("INFERENCE COMPLETE") + logging.info(f"Total inference time: {total_inference_time:.2f}s") + logging.info(f"Average time per sample: {avg_inference_time:.3f}s") + logging.info(f"Samples per second: {num_samples/total_inference_time:.2f}") + logging.info(f"Total tokens generated: {total_tokens}") + logging.info(f"Average tokens/sec: {avg_tokens_per_sec:.1f}") + logging.info("="*80) + + +# --------------------------- +# Main (all ranks) +# --------------------------- +def main(): + dist.init_process_group(backend="nccl") + + rank = dist.get_rank() + world_size = dist.get_world_size() + + log_file = setup_logging(rank) + + logging.info(f"Process started - Rank: {rank}, World size: {world_size}") + # assert world_size == 3, "Run with three ranks (1 broadcaster + 1 training + 1 inference)." + assert world_size >= 3, "Need at least 3 ranks: broadcaster + trainer(s) + inference" + + # IMPORTANT: use GPU 0 on each node; trainer uses DataParallel internally + if torch.cuda.is_available(): + local_rank = int(os.environ.get("LOCAL_RANK", rank)) + torch.cuda.set_device(local_rank) + logging.info(f"CUDA device set to GPU {local_rank}") + else: + local_gpu = None + logging.info("CUDA not available. (RDMA will not work.)") + + logging.info("Initializing P2P endpoint...") + ep = p2p.Endpoint(local_gpu, 4) + local_md = ep.get_metadata() + logging.info(f"Local metadata obtained (size: {len(local_md)} bytes)") + + # Exchange metadata + logging.info("Starting metadata exchange...") + all_metadata = [None] * world_size + all_metadata[rank] = local_md + + metadata_start = time.perf_counter() + for i in range(world_size): + if i == rank: + for j in range(world_size): + if j != rank: + dist.send(torch.ByteTensor(list(local_md)), dst=j) + else: + remote_md = torch.zeros(len(local_md), dtype=torch.uint8) + dist.recv(remote_md, src=i) + all_metadata[i] = bytes(remote_md.tolist()) + metadata_time = time.perf_counter() - metadata_start + logging.info(f"Metadata exchange complete in {metadata_time:.2f}s") + + if rank == 0: + # Broadcaster + logging.info("="*80) + logging.info("BROADCASTER MODE") + logging.info("="*80) + logging.info("Connecting to receivers...") + conn_ids = [] + + for receiver_rank in [1, 2]: + ip, port, r_gpu = p2p.Endpoint.parse_metadata(all_metadata[receiver_rank]) + logging.info(f"Connecting to rank {receiver_rank}: IP={ip}, Port={port}, GPU={r_gpu}") + + ok, conn_id = ep.connect(ip, r_gpu, remote_port=port) + assert ok, f"Connect failed to rank {receiver_rank}" + conn_ids.append(conn_id) + + node_type = "Training Node" if receiver_rank == 1 else "Inference Node" + logging.info(f"Connected to {node_type} (rank {receiver_rank}, conn_id={conn_id})") + + logging.info("Loading model on broadcaster...") + model = GPT2LMHeadModel.from_pretrained("gpt2").cuda() + logging.info("Model loaded") + + broadcast_model(ep, conn_ids, model, rank) + + logging.info("\nBroadcast complete. Training node (rank 1) and Inference node (rank 2) are now processing...") + + elif 1 <= rank < world_size - 1: + # Trainer + logging.info("="*80) + logging.info("TRAINING NODE (Rank 1)") + logging.info("="*80) + logging.info("Waiting for broadcaster connection...") + + ok, r_ip, r_gpu, conn_id = ep.accept() + assert ok, "Accept failed" + logging.info("Connected to broadcaster") + + logging.info("Loading model and tokenizer...") + base_model = GPT2LMHeadModel.from_pretrained("gpt2").cuda() + tokenizer = GPT2Tokenizer.from_pretrained("gpt2") + tokenizer.pad_token = tokenizer.eos_token + logging.info("Model and tokenizer loaded") + + recv_model(ep, conn_id, base_model, rank) + # FSDP auto wrap policy for GPT-2 transformer blocks + auto_wrap_policy = transformer_auto_wrap_policy({ + GPT2Block + }) + + logging.info("Wrapping model with FSDP across nodes...") + + model = FSDP( + base_model, + auto_wrap_policy=auto_wrap_policy, + device_id=torch.cuda.current_device() + ) + + run_training(model, tokenizer, num_epochs=2, batch_size=4, lr=5e-5) + + elif rank == world_size - 1: + # Inference node (rank 2) + logging.info("="*80) + logging.info("INFERENCE NODE (Rank 2)") + logging.info("="*80) + logging.info("Waiting for broadcaster connection...") + + ok, r_ip, r_gpu, conn_id = ep.accept() + assert ok, "Accept failed" + logging.info("Connected to broadcaster") + + logging.info("Loading model and tokenizer...") + model = GPT2LMHeadModel.from_pretrained("gpt2").cuda() + tokenizer = GPT2Tokenizer.from_pretrained("gpt2") + logging.info("Model and tokenizer loaded") + + recv_model(ep, conn_id, model, rank) + run_inference(model, tokenizer, num_samples=5) + + logging.info("Destroying process group...") + dist.destroy_process_group() + logging.info("Process complete. Exiting.") + logging.info(f"Full log saved to: {log_file}") + + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + logging.warning("Interrupted by user") + sys.exit(0) + except Exception as e: + logging.error(f"Fatal error: {e}", exc_info=True) + sys.exit(1) diff --git a/rollout/rollout.py b/rollout/rollout.py index 6cd3220..89823e3 100644 --- a/rollout/rollout.py +++ b/rollout/rollout.py @@ -1,21 +1,43 @@ # rollout/rollout.py -import os, socket, torch, sys +import sys, os sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) +import socket, torch, time from utils.wire import recv_tensor -HOST = os.getenv("HOST", "127.0.0.1") -PORT = int(os.getenv("PORT", "50051")) -DEVICE = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") +CTRL_HOST = os.getenv("CTRL_HOST", "127.0.0.1") +CTRL_PORT = int(os.getenv("CTRL_PORT", "50051")) +GPU = int(os.getenv("GPU", "1")) +DEVICE = torch.device(f"cuda:{GPU}" if torch.cuda.is_available() else "cpu") -print(f"[rollout] connecting to {HOST}:{PORT} on {DEVICE} ...") +print(f"[rollout] connecting to controller {CTRL_HOST}:{CTRL_PORT} on {DEVICE} ...") s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) -s.connect((HOST, PORT)) +s.connect((CTRL_HOST, CTRL_PORT)) +s.sendall(b"ROLE rollout\n") print("[rollout] connected") +recv_count = 0 +t0 = time.time() try: + from transformers import GPT2Model + + model = GPT2Model.from_pretrained("gpt2").to(DEVICE) + params = [p for p in model.parameters()] + while True: - t = recv_tensor(s, device=DEVICE) - print(f"[rollout] got tensor: shape={tuple(t.shape)}, sum={t.sum().item():.3f}, device={t.device}") + flat_tensor = recv_tensor(s, device=DEVICE) + recv_count += 1 + + # copy the received flat weights into model parameters + offset = 0 + for p in params: + numel = p.numel() + p.data.copy_(flat_tensor[offset:offset+numel].view_as(p)) + offset += numel + + if recv_count % 10 == 0: + dt = time.time() - t0 + print(f"[rollout] #{recv_count:04d} received GPT-2 weights (sum={flat_tensor.sum().item():.2f}, rate={recv_count/dt:.1f}/s)") + except (ConnectionError, KeyboardInterrupt): print("[rollout] done") finally: diff --git a/set-up-env.sh b/set-up-env.sh new file mode 100644 index 0000000..538c82d --- /dev/null +++ b/set-up-env.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +# ================================================================= +# AMD GPU + Broadcom RDMA Environment Setup Script +# Usage: source setup_amd_env.sh +# ================================================================= + +echo "Configuring environment for AMD GPUs with Broadcom RDMA..." + +# 0. Activate Conda Environment +if [[ "$CONDA_DEFAULT_ENV" != "rl_rdma_env" ]]; then + echo "Current environment is '$CONDA_DEFAULT_ENV'. Attempting to activate 'rl_rdma_env'..." + conda activate rl_rdma_env + + if [[ "$CONDA_DEFAULT_ENV" != "rl_rdma_env" ]]; then + echo "ERROR: Failed to automatically activate 'rl_rdma_env'." + echo " Please run 'conda activate rl_rdma_env' manually before sourcing this script." + return 1 2>/dev/null || exit 1 + fi + echo " [OK] Activated 'rl_rdma_env'" +else + echo " [OK] Already in 'rl_rdma_env'" +fi + +# 1. GPU Visibility +# Unset this to ensure PyTorch finds all GPUs automatically. +# (If set incorrectly, it causes 'no ROCm-capable device detected') +unset HIP_VISIBLE_DEVICES +echo " [OK] Unset HIP_VISIBLE_DEVICES" + +# 2. Network Selection (CRITICAL) +# Exclude slow ethernet (enp/eth), docker bridges, and loopback. +# NCCL will automatically pick up the remaining high-speed Broadcom (bnxt_re) +# and Mellanox (mlx5) adapters. +export NCCL_IB_HCA="^enp,eth,docker,lo,mlx5_0,mlx5_1" +echo " [OK] NCCL_IB_HCA set to exclude slow interfaces" + +# 3. Control Plane Binding +# Bind the handshake/coordination traffic to the known working interface. +# This prevents hangs during initialization. +export NCCL_SOCKET_IFNAME="enp49s0f1np1" +export GLOO_SOCKET_IFNAME="enp49s0f1np1" +echo " [OK] Socket interfaces bound to enp49s0f1np1" + +# 4. Performance Tuning +# Relaxed Ordering is crucial for high throughput on AMD platforms. +# export NCCL_IB_PCI_RELAXED_ORDERING=1 +# Enable GPU Direct RDMA (GDR) for zero-copy transfer between GPU and NIC. +export NCCL_NET_GDR_LEVEL=2 +# # Optimize Peer-to-Peer chunk sizes +# export NCCL_P2P_NET_CHUNKSIZE=524288 +# export NCCL_BUFFSIZE=8388608 +# # Channel and Queue Pair settings standard for scale-out +export NCCL_MIN_NCHANNELS=4 +export NCCL_MAX_NCHANNELS=8 +# export NCCL_IB_QPS_PER_CONNECTION=4 +# export NCCL_IB_SPLIT_DATA_ON_QPS=1 +export NCCL_IB_DISABLE=0 +echo " [OK] Enabled GPU Direct RDMA (GDR)" +# echo " [OK] Applied performance tuning flags (Relaxed Ordering, GDR)" + +# 5. Debugging +# Set to INFO to verify which interfaces are selected at runtime. +# Change to WARN to reduce clutter once verified. +export NCCL_DEBUG=INFO +echo " [OK] NCCL_DEBUG set to INFO" + + +export NCCL_IB_GID_INDEX=3 +echo " [OK] NCCL_IB_GID_INDEX set to $NCCL_IB_GID_INDEX" + +# 6. Threading +# Prevent CPU contention between PyTorch dataloaders +export OMP_NUM_THREADS=1 +echo " [OK] OMP_NUM_THREADS set to 1" + +echo "=================================================================" +echo "Environment ready. Run your torchrun command now." +echo "=================================================================" \ No newline at end of file diff --git a/trainer/trainer.py b/trainer/trainer.py index f6f653e..a60145c 100644 --- a/trainer/trainer.py +++ b/trainer/trainer.py @@ -1,37 +1,40 @@ -# trainer/trainer.py -import os, time, socket, torch -import sys +import os, time, socket, torch, sys sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) from utils.wire import send_tensor -HOST = "0.0.0.0" -PORT = int(os.getenv("PORT", "50051")) -DEVICE = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") +CTRL_HOST = os.getenv("CTRL_HOST", "127.0.0.1") +CTRL_PORT = int(os.getenv("CTRL_PORT", "50051")) +GPU = int(os.getenv("GPU", "0")) +#GPU =2 +DEVICE = torch.device(f"cuda:{GPU}" if torch.cuda.is_available() else "cpu") -print(f"[trainer] listening on {HOST}:{PORT}, source device={DEVICE}") -server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) -server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) -server.bind((HOST, PORT)) -server.listen(1) +print(f"[trainer] connecting to controller {CTRL_HOST}:{CTRL_PORT}, device={DEVICE}") +s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +s.connect((CTRL_HOST, CTRL_PORT)) +s.sendall(b"ROLE trainer\n") +print("[trainer] connected to controller") -conn, addr = server.accept() -print(f"[trainer] rollout connected from {addr}") +#weights = torch.ones((2048, 2048), device=DEVICE) +from transformers import GPT2Model -# pretend “weights” that change over time -weights = torch.ones((1024, 1024), device=DEVICE) +print("[trainer] loading GPT-2 model on", DEVICE) +model = GPT2Model.from_pretrained("gpt2").to(DEVICE) +weights = torch.cat([p.flatten() for p in model.parameters()]) # flatten all params into one big tensor +print(f"[trainer] total parameters: {weights.numel()}") +step = 0 +t0 = time.time() try: - step = 0 while True: - weights.add_(0.01) # mutate in place - send_tensor(conn, weights) # push update + weights.add_(0.01) + send_tensor(s, weights) step += 1 if step % 10 == 0: - print(f"[trainer] step {step}, sum={weights.sum().item():.2f}") - time.sleep(0.5) # throttle a bit + dt = time.time() - t0 + print(f"[trainer] step={step:04d} sum={weights.sum().item():.2f} rate={step/dt:.1f} upd/s") + time.sleep(0.2) except (BrokenPipeError, KeyboardInterrupt): print("[trainer] done") finally: - conn.close() - server.close() + s.close()