Fix DeepSeek V4 SM10 sparse MLA default

Co-authored-by: OpenAI Codex <codex@openai.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
2026-07-05 01:59:07 +00:00
1 changed files with 10 additions and 2 deletions
@@ -761,8 +761,9 @@ def _select_dsv4_attn_cls(vllm_config: VllmConfig) -> type[DeepseekV4Attention]:

    The generic CUDA backend selector does not instantiate DSv4 layers directly,
    so map generic sparse-MLA choices to the DSv4-specialized attention class.
-    Without an explicit backend, SM12 defaults to FlashInfer while the other
-    CUDA arches keep the FlashMLA path.
+    Without an explicit backend, SM12 defaults to FlashInfer. SM10x also uses
+    FlashInfer for plain FP8 KV cache because the FlashMLA sparse kernel only
+    accepts the older 656-byte row layout, not DeepSeek V4's 584-byte row.
    """
    backend = vllm_config.attention_config.backend
    device_capability = current_platform.get_device_capability()
@@ -785,6 +786,13 @@ def _select_dsv4_attn_cls(vllm_config: VllmConfig) -> type[DeepseekV4Attention]:
    ):
        return DeepseekV4FlashMLAAttention

+    cache_dtype = vllm_config.cache_config.cache_dtype
+    if (
+        device_capability is not None
+        and device_capability.major == 10
+        and cache_dtype in ("fp8", "fp8_e4m3", "fp8_e5m2")
+    ):
+        return DeepseekV4FlashInferMLAAttention
    if device_capability is not None and device_capability.major == 12:
        return DeepseekV4FlashInferSM120Attention
    return DeepseekV4FlashMLAAttention