[Feature][SM75] Run ModelOpt NVFP4 and FP8 linears on Turing through the SM70 QPN kernels

tests/kernels/quantization/test_nvfp4_qpn2_dequant.py

@@ -0,0 +1,138 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""The QPN2 dequantization kernel reproduces the checkpoint weight exactly.
+
+Round trip: random NVFP4 codes and e4m3 block scales -> QPN2 prepack
+(``nvfp4_qpn2_prepare_sm70``) -> Triton dequantization -> compare with the
+direct dequantization of the checkpoint tensors and with the pure-torch
+inverse of the prepack. NVFP4 values times an e4m3 scale times the global
+scale are exact in fp32 and round once to fp16, so the comparison is exact.
+"""
+
+import pytest
+import torch
+
+pytestmark = pytest.mark.skipif(
+    not torch.cuda.is_available(), reason="needs a CUDA device"
+)
+
+E2M1 = torch.tensor([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0])
+
+
+def _checkpoint_dequant(
+    packed: torch.Tensor, scales: torch.Tensor, global_scale: float
+) -> torch.Tensor:
+    n, k_half = packed.shape
+    low = packed & 0xF
+    high = packed >> 4
+    nib = torch.stack([low, high], dim=-1).view(n, k_half * 2)
+    values = E2M1.to(packed.device)[(nib & 7).long()] * torch.where(
+        nib & 8 > 0, -1.0, 1.0
+    )
+    scale = scales.view(torch.float8_e4m3fn).to(torch.float32)
+    values = values.view(n, -1, 16) * scale.unsqueeze(-1) * global_scale
+    return values.view(n, k_half * 2).to(torch.float16)
+
+
+@pytest.mark.parametrize(
+    ("n", "k"),
+    [(32, 64), (64, 256), (3584, 5120), (8704, 5120), (1536, 5120)],
+)
+def test_qpn2_dequant_matches_checkpoint(n: int, k: int):
+    from vllm import _sm70_ops as sm70_ops
+    from vllm.model_executor.layers.quantization.utils.nvfp4_qpn2_dequant import (
+        nvfp4_qpn2_dequant,
+        nvfp4_qpn2_dequant_reference,
+    )
+
+    if not hasattr(torch.ops._C, "nvfp4_qpn2_prepare_sm70"):
+        pytest.skip("build without the SM70 QPN2 extension")
+    generator = torch.Generator(device="cuda").manual_seed(n * 31 + k)
+    packed = torch.randint(
+        0, 256, (n, k // 2), dtype=torch.uint8, device="cuda", generator=generator
+    )
+    # e4m3 scales without the NaN code and without the sign bit, like
+    # ModelOpt block scales.
+    scales = torch.randint(
+        0, 0x7F, (n, k // 16), dtype=torch.uint8, device="cuda", generator=generator
+    )
+    global_scale = 0.0123
+    expected = _checkpoint_dequant(packed, scales, global_scale)
+
+    codes, qpn2_scales = sm70_ops.nvfp4_qpn2_prepare_sm70(
+        packed, scales.view(torch.float8_e4m3fn)
+    )
+    reference = nvfp4_qpn2_dequant_reference(codes, qpn2_scales, global_scale, n, k)
+    kernel = nvfp4_qpn2_dequant(codes, qpn2_scales, global_scale, n, k)
+
+    assert torch.equal(reference, expected)
+    assert torch.equal(kernel, expected)
+
+
+def test_qpn2_dense_linear_matches_fp16_matmul():
+    from vllm import _sm70_ops as sm70_ops
+    from vllm.model_executor.layers.quantization.utils.nvfp4_qpn2_dequant import (
+        nvfp4_qpn2_dense_linear,
+    )
+
+    if not hasattr(torch.ops._C, "nvfp4_qpn2_prepare_sm70"):
+        pytest.skip("build without the SM70 QPN2 extension")
+    n, k, m = 3584, 5120, 64
+    generator = torch.Generator(device="cuda").manual_seed(7)
+    packed = torch.randint(
+        0, 256, (n, k // 2), dtype=torch.uint8, device="cuda", generator=generator
+    )
+    scales = torch.randint(
+        0, 0x7F, (n, k // 16), dtype=torch.uint8, device="cuda", generator=generator
+    )
+    global_scale = 0.0123
+    x = torch.randn(m, k, dtype=torch.float16, device="cuda", generator=generator)
+    codes, qpn2_scales = sm70_ops.nvfp4_qpn2_prepare_sm70(
+        packed, scales.view(torch.float8_e4m3fn)
+    )
+
+    expected = torch.nn.functional.linear(
+        x, _checkpoint_dequant(packed, scales, global_scale)
+    )
+    out = nvfp4_qpn2_dense_linear(x, codes, qpn2_scales, global_scale, n, k)
+    assert out.shape == (m, n)
+    assert torch.equal(out, expected)
+
+
+@pytest.mark.parametrize("m", [1, 4, 32, 33, 64])
+def test_qpn2_dispatch_linear_both_sides_of_the_threshold(m: int):
+    from vllm import _sm70_ops as sm70_ops
+    from vllm.model_executor.layers.quantization import sm70_turbomind as sm70_tm
+    from vllm.model_executor.layers.quantization.utils.nvfp4_qpn2_dequant import (
+        nvfp4_qpn2_dispatch_linear,
+    )
+
+    if not hasattr(torch.ops._C, "nvfp4_qpn2_prepare_sm70"):
+        pytest.skip("build without the SM70 QPN2 extension")
+    n, k = 3584, 5120
+    generator = torch.Generator(device="cuda").manual_seed(11)
+    packed = torch.randint(
+        0, 256, (n, k // 2), dtype=torch.uint8, device="cuda", generator=generator
+    )
+    # Block scales between 0.25 and 1.5 (e4m3 0x28..0x3c): a K=5120 dot
+    # product of such rows stays inside fp16, which the GEMM output is.
+    scales = torch.randint(
+        0x28, 0x3D, (n, k // 16), dtype=torch.uint8, device="cuda", generator=generator
+    )
+    global_scale = 0.01
+    x = torch.randn(m, k, dtype=torch.float16, device="cuda", generator=generator)
+    codes, qpn2_scales = sm70_ops.nvfp4_qpn2_prepare_sm70(
+        packed, scales.view(torch.float8_e4m3fn)
+    )
+    split_k, chains = sm70_tm.qpn2_launch_config(k, n)
+
+    expected = torch.nn.functional.linear(
+        x, _checkpoint_dequant(packed, scales, global_scale)
+    )
+    out = nvfp4_qpn2_dispatch_linear(
+        x, codes, qpn2_scales, global_scale, n, k, split_k, chains
+    )
+    assert out.shape == (m, n)
+    # The QPN2 kernel accumulates in a different order than cuBLAS; both are
+    # fp32 accumulations of exact products, so they agree to fp16 rounding.
+    torch.testing.assert_close(out, expected, rtol=2e-3, atol=2e-2)

tests/quantization/test_sm70_modelopt_mixed_nvfp4.py

@@ -109,20 +109,22 @@ def _qwen4_moe_contract(**overrides):
     return SimpleNamespace(**values)
 
 
-def test_mixed_min_capability_requires_exact_sm70_and_both_turbomind_routes():
+def test_mixed_min_capability_requires_pre_ampere_and_both_turbomind_routes():
+    # Volta takes the TurboMind routes, Turing the QPN routes; both admit the
+    # mixed checkpoint when the FP8 and NVFP4 switches are on.
     with (
-        patch.object(sm70_tm, "is_exact_sm70_cuda_platform", return_value=True),
+        patch.object(sm70_tm, "is_pre_ampere_cuda_platform", return_value=True),
         patch.object(sm70_tm, "use_turbomind", side_effect=[True, True]),
     ):
         assert ModelOptMixedPrecisionConfig.get_min_capability() == 70
 
     with (
-        patch.object(sm70_tm, "is_exact_sm70_cuda_platform", return_value=True),
+        patch.object(sm70_tm, "is_pre_ampere_cuda_platform", return_value=True),
         patch.object(sm70_tm, "use_turbomind", side_effect=[True, False]),
     ):
         assert ModelOptMixedPrecisionConfig.get_min_capability() == 89
 
-    with patch.object(sm70_tm, "is_exact_sm70_cuda_platform", return_value=False):
+    with patch.object(sm70_tm, "is_pre_ampere_cuda_platform", return_value=False):
         assert ModelOptMixedPrecisionConfig.get_min_capability() == 89
 
 

tests/quantization/test_sm70_turbomind_turing_gates.py

@@ -0,0 +1,95 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Turing takes the QPN2 route of the SM70 linear path; Volta is unchanged.
+
+CPU-only: device capabilities are mocked, no CUDA context is created.
+"""
+
+from unittest.mock import patch
+
+import pytest
+import torch
+
+from vllm.model_executor.layers.quantization import sm70_turbomind as sm70_tm
+
+MODULE = "vllm.model_executor.layers.quantization.sm70_turbomind"
+
+
+@pytest.fixture
+def cuda_tensor():
+    tensor = torch.empty(0)
+    with patch.object(
+        torch.Tensor, "is_cuda", new_callable=lambda: property(lambda _: True)
+    ):
+        yield tensor
+
+
+@pytest.mark.parametrize(
+    ("capability", "turbomind", "turing_qpn2"),
+    [
+        ((7, 0), True, False),
+        ((7, 5), False, True),
+        ((8, 0), False, False),
+        ((8, 9), False, False),
+    ],
+)
+def test_prepare_routes_follow_the_tensor_device(
+    cuda_tensor, capability, turbomind, turing_qpn2
+):
+    with (
+        patch(f"{MODULE}.torch.cuda.get_device_capability", return_value=capability),
+        patch(f"{MODULE}.use_turbomind", return_value=True),
+    ):
+        assert sm70_tm.should_prepare_turbomind(cuda_tensor, True) is turbomind
+        assert sm70_tm.should_prepare_turing_qpn2(cuda_tensor, True) is turing_qpn2
+
+
+def test_turing_route_honours_the_backend_switch(cuda_tensor):
+    with (
+        patch(f"{MODULE}.torch.cuda.get_device_capability", return_value=(7, 5)),
+        patch(f"{MODULE}.use_turbomind", return_value=False),
+    ):
+        assert sm70_tm.should_prepare_turing_qpn2(cuda_tensor, True) is False
+
+
+def test_turing_route_needs_a_cuda_tensor():
+    with patch(f"{MODULE}.torch.cuda.get_device_capability", return_value=(7, 5)):
+        assert sm70_tm.should_prepare_turing_qpn2(torch.empty(0), True) is False
+
+
+@pytest.mark.parametrize(
+    ("k", "n", "expected"),
+    [
+        (5120, 8704, (8, 2)),  # table entry
+        (5120, 3584, (16, 2)),  # table entry
+        (4096, 4096, (8, 1)),  # heuristic: 128 tiles x 8 = 1024 warps in flight
+        (2048, 1024, (32, 2)),  # heuristic: needs the largest split to fill the GPU
+        (256, 64, (16, 2)),  # heuristic fallback: 16 groups, split 16
+    ],
+)
+def test_qpn2_launch_config(k, n, expected):
+    assert sm70_tm.qpn2_launch_config(k, n) == expected
+
+
+def test_qpn2_launch_config_rejects_unsplittable_k():
+    with pytest.raises(RuntimeError):
+        sm70_tm.qpn2_launch_config(16 * 3, 64)
+
+
+def test_pad_qpn2_output_rows_pads_to_32():
+    weight = torch.arange(40 * 8, dtype=torch.uint8).view(40, 8)
+    scales = torch.arange(40 * 2, dtype=torch.uint8).view(40, 2)
+    padded_weight, padded_scales, physical_n = sm70_tm.pad_qpn2_output_rows(
+        weight, scales
+    )
+    assert physical_n == 64
+    assert padded_weight.shape == (64, 8) and padded_scales.shape == (64, 2)
+    assert torch.equal(padded_weight[:40], weight)
+    assert torch.equal(padded_scales[:40], scales)
+    assert int(padded_weight[40:].abs().sum()) == 0
+
+    same_weight, same_scales, physical_n = sm70_tm.pad_qpn2_output_rows(
+        weight[:32], scales[:32]
+    )
+    assert physical_n == 32
+    assert same_weight is weight[:32] or torch.equal(same_weight, weight[:32])

vllm/model_executor/layers/quantization/modelopt.py

@@ -461,6 +461,12 @@ def __init__(self, quant_config: ModelOptFp8Config) -> None:
             sm70_tm.is_exact_sm70_cuda_platform()
             and sm70_tm.use_turbomind(envs.VLLM_SM70_FP8_TURBOMIND)
         )
+        # Turing takes the QPN8 kernels with the QPN8 dense prefill; the
+        # TurboMind GEMMs are registered for exact SM70 only.
+        self.use_sm75_fp8_qpn8 = (
+            sm70_tm.is_turing_cuda_platform()
+            and sm70_tm.use_turbomind(envs.VLLM_SM70_FP8_TURBOMIND)
+        )
 
     def create_weights(
         self,
@@ -563,6 +569,26 @@ def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
             logger.info_once("SM70 ModelOpt FP8 TurboMind W8A16 dense path enabled.")
             return
 
+        if self.use_sm75_fp8_qpn8:
+            if self.input_dtype != torch.float16:
+                raise RuntimeError(
+                    "ModelOpt FP8 QPN8 on Turing requires FP16 activations, "
+                    f"got {self.input_dtype}."
+                )
+            sm70_tm.prepare_fp8_qpn8_dense_linear(layer, weight, max_w_scale)
+            replace_parameter(
+                layer,
+                "weight",
+                torch.empty(0, dtype=weight.dtype, device=weight.device),
+            )
+            layer.weight_scale = Parameter(max_w_scale, requires_grad=False)
+            layer.input_scale = None
+            logger.info_once(
+                "SM75 ModelOpt FP8 QPN8 W8A16 dense path enabled "
+                "(QPN8 decode kernels, dense fp16 prefill)."
+            )
+            return
+
         layer.weight = Parameter(weight.t(), requires_grad=False)
         layer.weight_scale = Parameter(max_w_scale, requires_grad=False)
         layer.input_scale = Parameter(layer.input_scale.max(), requires_grad=False)
@@ -599,6 +625,8 @@ def apply(
             if bias is not None:
                 out.add_(bias)
             return out.reshape(*x.shape[:-1], layer.output_size_per_partition)
+        if sm70_tm.has_prepared_fp8_qpn8_linear(layer):
+            return sm70_tm.apply_prepared_fp8_qpn8_linear(layer, x, bias)
         return self.fp8_linear.apply_weights(layer, x, bias)
 
 
@@ -1099,15 +1127,22 @@ def _try_prepare_sm70_modelopt_nvfp4(layer: torch.nn.Module) -> bool:
     ``amax / (6 * 448)`` (the Marlin multiplier). TurboMind combine is
     ``block * global``. Do not infer convention from scale magnitude.
     """
-    if not sm70_tm.should_prepare_turbomind(
+    if sm70_tm.should_prepare_turbomind(layer.weight, envs.VLLM_SM70_NVFP4_TURBOMIND):
+        logger.info_once(
+            "SM70 ModelOpt NVFP4 TurboMind dense path enabled "
+            "(weight-only; activations remain half)."
+        )
+        sm70_tm.prepare_nvfp4_linear(layer)
+    elif sm70_tm.should_prepare_turing_qpn2(
         layer.weight, envs.VLLM_SM70_NVFP4_TURBOMIND
     ):
+        logger.info_once(
+            "SM75 ModelOpt NVFP4 QPN2 path enabled (QPN2 decode kernels, dense "
+            "fp16 prefill; weight-only, activations remain half)."
+        )
+        sm70_tm.prepare_nvfp4_qpn2_dense_linear(layer)
+    else:
         return False
-    logger.info_once(
-        "SM70 ModelOpt NVFP4 TurboMind dense path enabled "
-        "(weight-only; activations remain half)."
-    )
-    sm70_tm.prepare_nvfp4_linear(layer)
     layer.weight = Parameter(
         torch.empty(0, dtype=torch.uint8, device=layer.weight.device),
         requires_grad=False,
@@ -2402,7 +2437,7 @@ def get_supported_act_dtypes(self) -> list[torch.dtype]:
     @classmethod
     def get_min_capability(cls) -> int:
         if (
-            sm70_tm.is_exact_sm70_cuda_platform()
+            sm70_tm.is_pre_ampere_cuda_platform()
             and sm70_tm.use_turbomind(envs.VLLM_SM70_FP8_TURBOMIND)
             and sm70_tm.use_turbomind(envs.VLLM_SM70_NVFP4_TURBOMIND)
         ):