From c905684cfeaee3b2be2c736eee473b2c6ae7f7bf Mon Sep 17 00:00:00 2001 From: Chenheli Hua Date: Tue, 26 Aug 2025 20:05:34 -0700 Subject: [PATCH] [Core] Asynchronous h2d in merge_multimodal_embeddings via pinned memory. (#23686) Signed-off-by: Chenheli Hua Co-authored-by: Roger Wang --- vllm/model_executor/models/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vllm/model_executor/models/utils.py b/vllm/model_executor/models/utils.py index 6c27fedc61b..11e098f1d7b 100644 --- a/vllm/model_executor/models/utils.py +++ b/vllm/model_executor/models/utils.py @@ -508,7 +508,9 @@ def merge_multimodal_embeddings( """ if isinstance(placeholder_token_id, list): placeholder_token_id = torch.tensor(placeholder_token_id, - device=input_ids.device) + pin_memory=True).to( + device=input_ids.device, + non_blocking=True) return _merge_multimodal_embeddings( inputs_embeds, torch.isin(input_ids, placeholder_token_id),