From 5963c194787d30ed4a49c1e2e01010d8dffe1e79 Mon Sep 17 00:00:00 2001 From: Dakai An <77474977+andakai@users.noreply.github.com> Date: Thu, 28 May 2026 06:34:08 +0800 Subject: [PATCH] Fix Qwen3-VL and Qwen3-omni-thinker accuracy degradation from deepstack inputs under torch.compile (#43617) Signed-off-by: Dakai An --- .../models/qwen3_omni_moe_thinker.py | 25 +++++++++++-------- vllm/model_executor/models/qwen3_vl.py | 25 +++++++++++-------- 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/vllm/model_executor/models/qwen3_omni_moe_thinker.py b/vllm/model_executor/models/qwen3_omni_moe_thinker.py index 6d5123effa5..bd8a87b7dce 100755 --- a/vllm/model_executor/models/qwen3_omni_moe_thinker.py +++ b/vllm/model_executor/models/qwen3_omni_moe_thinker.py @@ -1778,8 +1778,8 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( ) -> IntermediateTensors | None: if not getattr(self, "deepstack_input_embeds", None): return None # If vision tower is skipped - if getattr(self, "deepstack_input_embeds_num_tokens", 0) == 0: - return None + if num_tokens > self.deepstack_input_embeds[0].size(0): + self._resize_deepstack_input_embeds(num_tokens) # get deepstack_input_embeds from buffer, and clear the buffer return IntermediateTensors( @@ -1791,6 +1791,17 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( } ) + def _resize_deepstack_input_embeds(self, num_tokens: int) -> None: + self.deepstack_input_embeds = [ + torch.zeros( + num_tokens, + self.config.text_config.hidden_size, + device=self.deepstack_input_embeds[0].device, + dtype=self.deepstack_input_embeds[0].dtype, + ) + for _ in range(self.deepstack_num_level) + ] + def _set_deepstack_input_embeds(self, deepstack_input_embeds: torch.Tensor) -> None: if not getattr(self, "deepstack_input_embeds", None): return @@ -1798,15 +1809,7 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( # set deepstack_input_embeds to buffer num_tokens = deepstack_input_embeds.size(1) if num_tokens > self.deepstack_input_embeds[0].size(0): - self.deepstack_input_embeds = [ - torch.zeros( - num_tokens, - self.config.text_config.hidden_size, - device=self.deepstack_input_embeds[0].device, - dtype=self.deepstack_input_embeds[0].dtype, - ) - for _ in range(self.deepstack_num_level) - ] + self._resize_deepstack_input_embeds(num_tokens) for idx in range(self.deepstack_num_level): self.deepstack_input_embeds[idx][:num_tokens].copy_( deepstack_input_embeds[idx] diff --git a/vllm/model_executor/models/qwen3_vl.py b/vllm/model_executor/models/qwen3_vl.py index d7765351e6d..a474649cc93 100644 --- a/vllm/model_executor/models/qwen3_vl.py +++ b/vllm/model_executor/models/qwen3_vl.py @@ -1715,8 +1715,8 @@ class Qwen3VLForConditionalGeneration( ) -> IntermediateTensors | None: if not getattr(self, "deepstack_input_embeds", None): return None # If vision tower is skipped - if getattr(self, "deepstack_input_embeds_num_tokens", 0) == 0: - return None + if num_tokens > self.deepstack_input_embeds[0].size(0): + self._resize_deepstack_input_embeds(num_tokens) # get deepstack_input_embeds from buffer, and clear the buffer return IntermediateTensors( @@ -1728,6 +1728,17 @@ class Qwen3VLForConditionalGeneration( } ) + def _resize_deepstack_input_embeds(self, num_tokens: int) -> None: + self.deepstack_input_embeds = [ + torch.zeros( + num_tokens, + self.config.text_config.hidden_size, + device=self.deepstack_input_embeds[0].device, + dtype=self.deepstack_input_embeds[0].dtype, + ) + for _ in range(self.deepstack_num_level) + ] + def _set_deepstack_input_embeds(self, deepstack_input_embeds: torch.Tensor) -> None: if not getattr(self, "deepstack_input_embeds", None): return @@ -1735,15 +1746,7 @@ class Qwen3VLForConditionalGeneration( # set deepstack_input_embeds to buffer num_tokens = deepstack_input_embeds.size(1) if num_tokens > self.deepstack_input_embeds[0].size(0): - self.deepstack_input_embeds = [ - torch.zeros( - num_tokens, - self.config.text_config.hidden_size, - device=self.deepstack_input_embeds[0].device, - dtype=self.deepstack_input_embeds[0].dtype, - ) - for _ in range(self.deepstack_num_level) - ] + self._resize_deepstack_input_embeds(num_tokens) for idx in range(self.deepstack_num_level): self.deepstack_input_embeds[idx][:num_tokens].copy_( deepstack_input_embeds[idx]