From 2805ab1d620eadaa11b8e0f758b68b105f64d754 Mon Sep 17 00:00:00 2001 From: tastelikefeet Date: Tue, 15 Sep 2026 16:16:59 +0800 Subject: [PATCH] [bugfix] make fp8 scale_inv contiguous before TP/PP collectives `_get_weight` trims the padding that TE adds to `_rowwise_scale_inv`, and slicing the last dim returns a non-contiguous view. Grouped MoE experts keep the weight as a single 3D tensor, so `mg_scale_inv[0]` preserves that view and hands it straight to `_all_gather_tp`, where ProcessGroupNCCL raises `ValueError: Tensors must be contiguous`. This breaks `megatron export --fp8_recipe blockwise --fp8_param_gather true` whenever TP > 1, and would break `dist.broadcast` in `_broadcast_ep_pp` for TP == 1 with PP > 1. Call `.contiguous()` at the slice site so both collectives are covered. It is a no-op when TE did not pad the last dim, so the common path keeps the same storage and adds no copy; when it does copy, the tensor is 1/128 of the fp8 data. Values are unchanged. Multi-shard weights were unaffected because `torch.concat` already returns a contiguous tensor, which is why only grouped-GEMM MoE hit this. Fixes modelscope/ms-swift#10136 --- src/mcore_bridge/bridge/gpt_bridge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mcore_bridge/bridge/gpt_bridge.py b/src/mcore_bridge/bridge/gpt_bridge.py index 41caf7cb..64b473f2 100644 --- a/src/mcore_bridge/bridge/gpt_bridge.py +++ b/src/mcore_bridge/bridge/gpt_bridge.py @@ -405,7 +405,7 @@ def _get_weight( tensor = [tensor] if self._is_fp8_param(tensor[0]): mg_scale_inv = [ - t._rowwise_scale_inv[..., :math.ceil(t._rowwise_data.shape[-1] / self.fp8_block_size)] + t._rowwise_scale_inv[..., :math.ceil(t._rowwise_data.shape[-1] / self.fp8_block_size)].contiguous() for t in tensor ] tensor = [t._rowwise_data for t in tensor]