diff --git a/tests/models/registry.py b/tests/models/registry.py index 00a84783bf1..7ed8d6225f8 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -497,6 +497,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = { "Qwen2MoeForCausalLM": _HfExamplesInfo("Qwen/Qwen1.5-MoE-A2.7B-Chat"), "Qwen3ForCausalLM": _HfExamplesInfo("Qwen/Qwen3-8B"), "Qwen3MoeForCausalLM": _HfExamplesInfo("Qwen/Qwen3-30B-A3B"), + "Qwen3_5ForCausalLM": _HfExamplesInfo("codecho/Qwen3.5-0.8B-text-only"), + "Qwen3_5MoeForCausalLM": _HfExamplesInfo("codecho/Qwen3.5-35B-A3B-text-only"), "MellumForCausalLM": _HfExamplesInfo("JetBrains/Mellum2-12B-A2.5B-Base"), "Qwen3NextForCausalLM": _HfExamplesInfo( "Qwen/Qwen3-Next-80B-A3B-Instruct", diff --git a/vllm/model_executor/models/config.py b/vllm/model_executor/models/config.py index 3219a5edcda..ebcd26ac3fa 100644 --- a/vllm/model_executor/models/config.py +++ b/vllm/model_executor/models/config.py @@ -768,6 +768,20 @@ class Qwen3_5ForConditionalGenerationConfig(VerifyAndUpdateConfig): ) +class Qwen3_5ForCausalLMConfig(Qwen3_5ForConditionalGenerationConfig): + @staticmethod + def verify_and_update_config(vllm_config: "VllmConfig") -> None: + Qwen3_5ForConditionalGenerationConfig.verify_and_update_config(vllm_config) + + # Text-only Qwen3.5 models use one-dimensional positions. Remove the + # M-RoPE fields inherited from the multimodal configuration. + hf_text_config = vllm_config.model_config.hf_text_config + rope_parameters = getattr(hf_text_config, "rope_parameters", None) + if rope_parameters is not None: + rope_parameters.pop("mrope_section", None) + rope_parameters.pop("mrope_interleaved", None) + + class ColQwen3_5Config(Qwen3_5ForConditionalGenerationConfig): """Apply the attention contract declared by a ColQwen3.5 checkpoint.""" @@ -884,7 +898,9 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = { "Qwen2ForRewardModel": Qwen2ForRewardModelConfig, "Qwen3ForSequenceClassification": Qwen3ForSequenceClassificationConfig, "Qwen3VLForSequenceClassification": Qwen3VLForSequenceClassificationConfig, + "Qwen3_5ForCausalLM": Qwen3_5ForCausalLMConfig, "Qwen3_5ForConditionalGeneration": Qwen3_5ForConditionalGenerationConfig, + "Qwen3_5MoeForCausalLM": Qwen3_5ForCausalLMConfig, "Qwen3_5MoeForConditionalGeneration": Qwen3_5ForConditionalGenerationConfig, "UnlimitedOCRForCausalLM": UnlimitedOCRForCausalLMConfig, "VoyageQwen3BidirectionalEmbedModel": VoyageQwen3BidirectionalEmbedModelConfig, diff --git a/vllm/model_executor/models/qwen3_5.py b/vllm/model_executor/models/qwen3_5.py index 47076337531..de18e36ca1c 100644 --- a/vllm/model_executor/models/qwen3_5.py +++ b/vllm/model_executor/models/qwen3_5.py @@ -282,6 +282,7 @@ class Qwen3_5Model(Qwen3NextModel): class Qwen3_5ForCausalLMBase( nn.Module, HasInnerState, + IsHybrid, SupportsEagle3, SupportsLoRA, SupportsPP, @@ -361,6 +362,45 @@ class Qwen3_5ForCausalLMBase( return hidden_states + @classmethod + def get_mamba_state_dtype_from_config( + cls, + vllm_config: "VllmConfig", + ) -> tuple[torch.dtype, torch.dtype]: + return MambaStateDtypeCalculator.gated_delta_net_state_dtype( + vllm_config.model_config.dtype, + vllm_config.cache_config.mamba_cache_dtype, + vllm_config.cache_config.mamba_ssm_cache_dtype, + ) + + @classmethod + def get_mamba_state_shape_from_config( + cls, vllm_config: "VllmConfig" + ) -> tuple[tuple[int, int], tuple[int, int]]: + parallel_config = vllm_config.parallel_config + hf_config = vllm_config.model_config.hf_text_config + tp_size = parallel_config.tensor_parallel_size + num_spec = ( + vllm_config.speculative_config.num_speculative_tokens + if vllm_config.speculative_config + else 0 + ) + return MambaStateShapeCalculator.gated_delta_net_state_shape( + tp_size, + hf_config.linear_num_key_heads, + hf_config.linear_num_value_heads, + hf_config.linear_key_head_dim, + hf_config.linear_value_head_dim, + hf_config.linear_conv_kernel_dim, + num_spec, + ) + + @classmethod + def get_mamba_state_copy_func( + cls, + ) -> tuple[MambaStateCopyFunc, MambaStateCopyFunc]: + return MambaStateCopyFuncCalculator.gated_delta_net_state_copy_func() + def compute_logits( self, hidden_states: torch.Tensor, diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index 697f55b3727..ca093e59684 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -195,6 +195,8 @@ _TEXT_GENERATION_MODELS = { "Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"), "Qwen3ForCausalLM": ("qwen3", "Qwen3ForCausalLM"), "Qwen3MoeForCausalLM": ("qwen3_moe", "Qwen3MoeForCausalLM"), + "Qwen3_5ForCausalLM": ("qwen3_5", "Qwen3_5ForCausalLM"), + "Qwen3_5MoeForCausalLM": ("qwen3_5", "Qwen3_5MoeForCausalLM"), "RWForCausalLM": ("falcon", "FalconForCausalLM"), "SarvamMoEForCausalLM": ("sarvam", "SarvamMoEForCausalLM"), "SarvamMLAForCausalLM": ("sarvam", "SarvamMLAForCausalLM"), diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index 6f8c25769ef..b1aaa294c7a 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -124,7 +124,9 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict( qwen3_asr="Qwen3ASRConfig", qwen3_next="Qwen3NextConfig", qwen3_5="Qwen3_5Config", + qwen3_5_text="Qwen3_5TextConfig", qwen3_5_moe="Qwen3_5MoeConfig", + qwen3_5_moe_text="Qwen3_5MoeTextConfig", laguna="LagunaConfig", lfm2_moe="Lfm2MoeConfig", **{"unlimited-ocr": "UnlimitedOCRConfig"},