From 9fd737badcc5eaeb61fd1e7b894af02ba657f203 Mon Sep 17 00:00:00 2001 From: "Jeff (Junze) Ma" <93145857+majunze2001@users.noreply.github.com> Date: Sun, 14 Jun 2026 00:14:31 -0700 Subject: [PATCH] [Bugfix][DCP] Fix illegal memory access in DCP a2a decode under full CUDA graphs (#45487) --- vllm/v1/attention/ops/dcp_alltoall.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/vllm/v1/attention/ops/dcp_alltoall.py b/vllm/v1/attention/ops/dcp_alltoall.py index 1469a5c754d..5effeea5fb3 100644 --- a/vllm/v1/attention/ops/dcp_alltoall.py +++ b/vllm/v1/attention/ops/dcp_alltoall.py @@ -26,10 +26,6 @@ import torch import torch.distributed as dist from vllm.triton_utils import tl, triton -from vllm.v1.worker.workspace import ( - current_workspace_manager, - is_workspace_manager_initialized, -) if TYPE_CHECKING: from vllm.distributed.parallel_state import GroupCoordinator @@ -117,13 +113,16 @@ def _dcp_a2a_send_recv_buffers( device: torch.device, dtype: torch.dtype, ) -> tuple[torch.Tensor, torch.Tensor]: - if is_workspace_manager_initialized(): - send_buffer, recv_buffer = current_workspace_manager().get_simultaneous( - (shape, dtype), - (shape, dtype), - ) - return send_buffer, recv_buffer - + # Don't use the shared WorkspaceManager here. A FULL cudagraph bakes in the + # buffer address at capture, but the workspace is growable and sized only to + # the largest *captured* batch (the cudagraph capture cap). Any eager a2a + # with a bigger batch regrows it, freeing that address and poisoning every + # captured graph -> illegal memory access on replay. This bites the very + # first request: the post-capture warmup runs an eager decode at + # max_num_seqs (> the cap), so the graphs are already dangling before the + # server is ready. torch.empty buffers instead live in the graph's private + # pool and stay valid for its lifetime (as _dcp_a2a_unpack_combine and the + # AG+RS combine path already rely on). return ( torch.empty(shape, device=device, dtype=dtype), torch.empty(shape, device=device, dtype=dtype),