karylab_agents/vllm
1
0
Fork 0
forked from Karylab-cklius/vllm
Code Pull Requests 2 Actions 1 Packages Activity

[Bugfix] Harden DeepEP low-latency cross-node NVFP4

Browse Source 
Pass use_fabric when DeepEP low-latency MNNVL is enabled so cross-node buffer initialization uses the fabric API, and gracefully fall back when the local DeepEP build does not expose low_latency_dispatch(use_nvfp4=...).

This complements the existing NVFP4 all2all backend selection fix for issue #37931.

Co-authored-by: OpenAI Codex <codex@openai.com>
Signed-off-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
Roger Wang
2026-04-18 18:05:06 -07:00
co-authored by OpenAI Codex
parent 21eaa58f71
commit 8f3ce1492e
2 changed files with 33 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
vllm/distributed/device_communicators/all2all.py
+3 -1
View File
@@ -414,9 +414,11 @@ class DeepEPLLAll2AllManager(DeepEPAll2AllManagerBase):
num_qps_per_rank=num_qps_per_rank,
)
if not current_platform.is_rocm():
use_mnnvl = envs.VLLM_DEEPEP_LOW_LATENCY_USE_MNNVL
kwargs.update(
allow_nvlink_for_low_latency_mode=True,
allow_mnnvl=envs.VLLM_DEEPEP_LOW_LATENCY_USE_MNNVL,
allow_mnnvl=use_mnnvl,
use_fabric=use_mnnvl,
explicitly_destroy=True,
)
return kwargs
vllm/model_executor/layers/fused_moe/prepare_finalize/deepep_ll.py
+30 -11
View File
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import inspect
from collections.abc import Callable
import deep_ep
@@ -120,6 +121,20 @@ class DeepEPLLPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
# time. This setting is handled by post_init_setup.
self.use_ue8m0_dispatch = False
# Check if DeepEP supports use_nvfp4 in low_latency_dispatch.
# This requires the hybrid-ep branch of DeepEP.
self.has_nvfp4_support = (
"use_nvfp4" in inspect.signature(buffer.low_latency_dispatch).parameters
)
if envs.VLLM_DEEPEPLL_NVFP4_DISPATCH and not self.has_nvfp4_support:
logger.warning_once(
"VLLM_DEEPEPLL_NVFP4_DISPATCH=1 but DeepEP does not support "
"use_nvfp4 in low_latency_dispatch. Falling back to FP8/BF16 "
"dispatch. Install DeepEP from the hybrid-ep branch for "
"NvFP4 dispatch support: "
"https://github.com/deepseek-ai/DeepEP/tree/hybrid-ep"
)
def post_init_setup(self, fused_experts: mk.FusedMoEExperts):
if not fused_experts.supports_packed_ue8m0_act_scales():
# Early exit.
@@ -183,27 +198,29 @@ class DeepEPLLPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
assert isinstance(x, (torch.Tensor, tuple))
q_dtype = quant_config.quant_dtype
if q_dtype == "nvfp4" and envs.VLLM_DEEPEPLL_NVFP4_DISPATCH:
nvfp4_quant_path = (
q_dtype == "nvfp4"
and envs.VLLM_DEEPEPLL_NVFP4_DISPATCH
and self.has_nvfp4_support
)
if nvfp4_quant_path:
logger.info_once(
"Since VLLM_DEEPEPLL_NVFP4_DISPATCH==1, make sure "
"using the hybrid-ep branch of DeepEP"
"(https://github.com/deepseek-ai/DeepEP/tree/hybrid-ep)"
"Quantization is fused with DeepEP nvfp4 dispatch (hybrid-ep branch)"
)
assert isinstance(x, tuple)
x_scales = x[1]
x = x[0].permute(2, 0, 1)
num_experts, max_tokens, hidden_dim_by_2 = x.shape
hidden_dim = hidden_dim_by_2 * 2
logger.info_once(
"Quantization is fused with DeepEP nvfp4 dispatch for "
"FlashInfer CUTEDSL as VLLM_DEEPEPLL_NVFP4_DISPATCH==1"
)
else:
if q_dtype == "nvfp4":
q_dtype = None
logger.info_once(
"Using DeepEP bfloat16 dispatch for FlashInfer CUTEDSL as "
"VLLM_DEEPEPLL_NVFP4_DISPATCH==0"
"Using DeepEP bfloat16 dispatch for FlashInfer CUTEDSL "
"(nvfp4 dispatch %s)",
"not supported by this DeepEP build"
if not self.has_nvfp4_support
else "disabled",
)
assert isinstance(x, torch.Tensor)
num_experts, max_tokens, hidden_dim = x.size()
@@ -260,7 +277,9 @@ class DeepEPLLPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
use_nvfp4 = False
nvfp4_dispatch = (
quant_config.quant_dtype == "nvfp4" and envs.VLLM_DEEPEPLL_NVFP4_DISPATCH
quant_config.quant_dtype == "nvfp4"
and envs.VLLM_DEEPEPLL_NVFP4_DISPATCH
and self.has_nvfp4_support
)
if nvfp4_dispatch:
use_nvfp4 = True