diff --git a/vllm/model_executor/models/AXK1.py b/vllm/model_executor/models/AXK1.py index 3084506282a..dbccc1f80ba 100644 --- a/vllm/model_executor/models/AXK1.py +++ b/vllm/model_executor/models/AXK1.py @@ -1096,7 +1096,7 @@ class AXK1ForCausalLM( else: self.lm_head = PPMissingLayer() self.logits_processor = LogitsProcessor(config.vocab_size) - self.make_empty_intermediate_tensors = ( # type: ignore[method-assign] + self.make_empty_intermediate_tensors = ( self.model.make_empty_intermediate_tensors ) # Set MoE hyperparameters diff --git a/vllm/model_executor/models/adapters.py b/vllm/model_executor/models/adapters.py index 676f6096ad8..20913b671a7 100644 --- a/vllm/model_executor/models/adapters.py +++ b/vllm/model_executor/models/adapters.py @@ -25,7 +25,7 @@ if TYPE_CHECKING: from vllm.config import ModelConfig, VllmConfig from vllm.model_executor.layers.pooler import Pooler -_T = TypeVar("_T", bound=type[nn.Module]) +_T = TypeVar("_T", bound=nn.Module) logger = init_logger(__name__) @@ -126,7 +126,7 @@ def _get_pooling_model_name(orig_model_name: str, pooling_suffix: str) -> str: return model_name + pooling_suffix -def _create_pooling_model_cls(orig_cls: _T) -> _T: +def _create_pooling_model_cls(orig_cls: type[_T]) -> type[_T]: # Lazy import from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead @@ -157,9 +157,10 @@ def _create_pooling_model_cls(orig_cls: _T) -> _T: # If the model already defines a pooler instance, don't overwrite it pooler = getattr(self, "pooler", None) - if not pooler and supports_multimodal(self): + multimodal_model: object = self + if not pooler and supports_multimodal(multimodal_model): # Try to get the pooler from the LM backbone - language_model = self.get_language_model() # type: ignore[call-arg] + language_model = multimodal_model.get_language_model() if hasattr(language_model, "pooler"): pooler = language_model.pooler @@ -175,7 +176,9 @@ def _create_pooling_model_cls(orig_cls: _T) -> _T: ) -> "Pooler": raise NotImplementedError - def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]): + def _load_pooling_model_weights( + self, weights: Iterable[tuple[str, torch.Tensor]] + ): params_dict = dict(self.named_parameters()) # We support loading from both `*ForCausalLM` and `*Model` @@ -226,10 +229,13 @@ def _create_pooling_model_cls(orig_cls: _T) -> _T: load_weights = getattr(super(), "load_weights", default_load_weights) return load_weights(mapped_weights) - return ModelForPooling # type: ignore + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]): + return self._load_pooling_model_weights(weights) + + return ModelForPooling -def as_embedding_model(cls: _T) -> _T: +def as_embedding_model(cls: type[_T]) -> type[_T]: """ Subclass an existing vLLM model to support embeddings. @@ -260,10 +266,10 @@ def as_embedding_model(cls: _T) -> _T: ModelForEmbedding.__name__ = _get_pooling_model_name(cls.__name__, "ForEmbedding") - return ModelForEmbedding # type: ignore + return ModelForEmbedding -def as_seq_cls_model(cls: _T) -> _T: +def as_seq_cls_model(cls: type[_T]) -> type[_T]: """ Subclass an existing vLLM model to support classify and score tasks. @@ -369,7 +375,7 @@ def as_seq_cls_model(cls: _T) -> _T: cls.__name__, "ForSequenceClassification" ) - return ModelForSequenceClassification # type: ignore + return ModelForSequenceClassification class SequenceClassificationConfig(VerifyAndUpdateConfig): @@ -403,14 +409,15 @@ class SequenceClassificationConfig(VerifyAndUpdateConfig): text_config.use_sep_token = use_sep_token -def _get_language_model_for_seq_cls(model) -> nn.Module: +def _get_language_model_for_seq_cls(model: nn.Module) -> nn.Module: """ Get the language model component for sequence classification conversion. For VLMs, returns the inner language model. For standard LLMs, returns model itself. """ - if supports_multimodal(model): + multimodal_model: object = model + if supports_multimodal(multimodal_model): try: - lm = model.get_language_model() # type: ignore[call-arg] + lm = multimodal_model.get_language_model() if lm is not model: return lm except Exception: @@ -512,14 +519,7 @@ def load_weights_using_from_2_way_softmax( language_model.lm_head = language_model.lm_head.tie_weights(embed_tokens) with _disable_seq_cls_loading_on_inner_model(language_model, is_vlm): - # ModelForPooling is dynamically defined inside the _create_pooling_model_cls - # function, so we need use this hacky method to obtain it. - pooling_model_cls = next( - x for x in type(model).__mro__ if x.__name__ == "ModelForPooling" - ) - loaded_weights = pooling_model_cls.load_weights( # type: ignore[attr-defined] - model, weights - ) + loaded_weights = model._load_pooling_model_weights(weights) from vllm.tokenizers import get_tokenizer @@ -587,13 +587,8 @@ def load_weights_no_post_processing(model, weights: Iterable[tuple[str, torch.Te language_model.lm_head = language_model.lm_head.tie_weights(embed_tokens) with _disable_seq_cls_loading_on_inner_model(language_model, is_vlm): - pooling_model_cls = next( - x for x in type(model).__mro__ if x.__name__ == "ModelForPooling" - ) - # Skip ModelForSequenceClassification in MRO to avoid infinite recursion - loaded_weights = pooling_model_cls.load_weights( # type: ignore[attr-defined] - model, weights - ) + # Bypass ModelForSequenceClassification to avoid infinite recursion + loaded_weights = model._load_pooling_model_weights(weights) from vllm.tokenizers import get_tokenizer diff --git a/vllm/model_executor/models/afmoe.py b/vllm/model_executor/models/afmoe.py index f456229b2f7..aefdf082869 100644 --- a/vllm/model_executor/models/afmoe.py +++ b/vllm/model_executor/models/afmoe.py @@ -47,7 +47,6 @@ from vllm.model_executor.models.utils import ( PPMissingLayer, WeightsMapper, extract_layer_index, - make_empty_intermediate_tensors_factory, make_layers, maybe_prefix, ) @@ -409,12 +408,6 @@ class AfmoeModel(nn.Module, EagleModelMixin): else: self.norm = PPMissingLayer() - self.make_empty_intermediate_tensors = ( # type: ignore[method-assign] - make_empty_intermediate_tensors_factory( - ["hidden_states", "residual"], config.hidden_size - ) - ) - def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: return self.embed_tokens(input_ids) @@ -527,7 +520,7 @@ class AfmoeForCausalLM( else: self.lm_head = PPMissingLayer() self.logits_processor = LogitsProcessor(config.vocab_size) - self.make_empty_intermediate_tensors = ( # type: ignore[method-assign] + self.make_empty_intermediate_tensors = ( self.model.make_empty_intermediate_tensors ) # Set MoE hyperparameters @@ -578,7 +571,7 @@ class AfmoeForCausalLM( def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: return self.model.embed_input_ids(input_ids) - def forward( # type: ignore[override] + def forward( self, input_ids: torch.Tensor | None, positions: torch.Tensor, diff --git a/vllm/model_executor/models/apertus.py b/vllm/model_executor/models/apertus.py index ffda816bb47..b997e153a99 100644 --- a/vllm/model_executor/models/apertus.py +++ b/vllm/model_executor/models/apertus.py @@ -454,7 +454,7 @@ class ApertusForCausalLM( else: self.lm_head = PPMissingLayer() - self.make_empty_intermediate_tensors = ( # type: ignore[method-assign] + self.make_empty_intermediate_tensors = ( self.model.make_empty_intermediate_tensors ) diff --git a/vllm/model_executor/models/arcee.py b/vllm/model_executor/models/arcee.py index 185509eb7bc..c32a903bba8 100644 --- a/vllm/model_executor/models/arcee.py +++ b/vllm/model_executor/models/arcee.py @@ -326,7 +326,7 @@ class ArceeForCausalLM( # Placeholder for lm_head on non-last ranks self.lm_head = PPMissingLayer() - self.make_empty_intermediate_tensors = ( # type: ignore[method-assign] + self.make_empty_intermediate_tensors = ( self.model.make_empty_intermediate_tensors ) diff --git a/vllm/model_executor/models/arctic.py b/vllm/model_executor/models/arctic.py index 3922c1025a4..b7e8e6796f3 100644 --- a/vllm/model_executor/models/arctic.py +++ b/vllm/model_executor/models/arctic.py @@ -562,7 +562,7 @@ class ArcticForCausalLM(nn.Module, SupportsPP, SupportsQuant): self.num_experts_per_tok = config.num_experts_per_tok self.logits_processor = LogitsProcessor(config.vocab_size) - self.make_empty_intermediate_tensors = ( # type: ignore[method-assign] + self.make_empty_intermediate_tensors = ( self.model.make_empty_intermediate_tensors ) diff --git a/vllm/model_executor/models/audioflamingo3.py b/vllm/model_executor/models/audioflamingo3.py index 5a471bfaf4c..e64d9e551b1 100644 --- a/vllm/model_executor/models/audioflamingo3.py +++ b/vllm/model_executor/models/audioflamingo3.py @@ -17,7 +17,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from collections.abc import Iterable, Mapping, Sequence +from collections.abc import Iterable, Mapping, Sequence, Sized from typing import Annotated, Any, Literal, TypeAlias import torch @@ -30,7 +30,7 @@ from transformers.models.audioflamingo3 import ( from transformers.models.qwen2_audio import Qwen2AudioEncoder from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions from vllm.inputs import ModalityData, MultiModalDataDict from vllm.model_executor.layers.activation import get_act_fn from vllm.model_executor.models.module_mapping import MultiModelKeys @@ -226,12 +226,13 @@ class AudioFlamingo3DummyInputsBuilder( audio_len = int(hf_processor.max_audio_len * sampling_rate) num_audios = mm_counts.get("audio", 0) audio_overrides = mm_options.get("audio") + assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions) return { "audio": self._get_dummy_audios( length=audio_len, num_audios=num_audios, - overrides=audio_overrides, # type: ignore[arg-type] + overrides=audio_overrides, ) } @@ -412,7 +413,8 @@ class AudioFlamingo3MultiModalProcessor( chunk_counts = [] for audio in audio_list: # audio is numpy array or list - n_samples = len(audio) # type: ignore[arg-type] + assert isinstance(audio, Sized) + n_samples = len(audio) n_win = max(1, (n_samples + window_size - 1) // window_size) if n_win > max_windows: @@ -524,7 +526,7 @@ class AudioFlamingo3ForConditionalGeneration( architectures=["Qwen2ForCausalLM"], ) - self.make_empty_intermediate_tensors = ( # type: ignore[method-assign] + self.make_empty_intermediate_tensors = ( self.language_model.make_empty_intermediate_tensors ) @@ -557,17 +559,15 @@ class AudioFlamingo3ForConditionalGeneration( def _process_audio_input( self, audio_input: AudioFlamingo3Inputs ) -> torch.Tensor | tuple[torch.Tensor, ...]: - if audio_input["type"] == "audio_embeds": - audio_embeds = audio_input["audio_embeds"] + if audio_input.type == "audio_embeds": + audio_embeds = audio_input.audio_embeds return tuple(audio_embeds) ( input_features, feature_attention_mask, chunk_counts, - ) = self._normalize_audio_feature_inputs( - audio_input # type: ignore[arg-type] - ) + ) = self._normalize_audio_feature_inputs(audio_input) audio_hidden_states = self._encode_audio_features( input_features, feature_attention_mask, diff --git a/vllm/model_executor/models/bagel.py b/vllm/model_executor/models/bagel.py index fc18751ec86..e592dd3728c 100644 --- a/vllm/model_executor/models/bagel.py +++ b/vllm/model_executor/models/bagel.py @@ -14,7 +14,7 @@ import torch import torch.nn as nn from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions from vllm.inputs import MultiModalDataDict from vllm.logger import init_logger from vllm.model_executor.layers.activation import get_act_fn @@ -258,13 +258,14 @@ class BagelDummyInputsBuilder(BaseDummyInputsBuilder[BagelProcessingInfo]): # Use the configured image size image_size = vit_config.image_size image_overrides = mm_options.get("image") + assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions) return { "image": self._get_dummy_images( width=image_size, height=image_size, num_images=num_images, - overrides=image_overrides, # type: ignore[arg-type] + overrides=image_overrides, ), } @@ -430,7 +431,7 @@ class BagelForConditionalGeneration( self.connector = StageMissingLayer("image_tower") self.vit_pos_embed = StageMissingLayer("image_tower") - self.make_empty_intermediate_tensors = ( # type: ignore[method-assign] + self.make_empty_intermediate_tensors = ( self.language_model.make_empty_intermediate_tensors ) diff --git a/vllm/model_executor/models/bailing_moe.py b/vllm/model_executor/models/bailing_moe.py index a5d583f1256..c2ba3c33883 100644 --- a/vllm/model_executor/models/bailing_moe.py +++ b/vllm/model_executor/models/bailing_moe.py @@ -522,7 +522,7 @@ class BailingMoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA): else: self.lm_head = PPMissingLayer() - self.make_empty_intermediate_tensors = ( # type: ignore[method-assign] + self.make_empty_intermediate_tensors = ( self.model.make_empty_intermediate_tensors ) diff --git a/vllm/model_executor/models/bailing_moe_linear.py b/vllm/model_executor/models/bailing_moe_linear.py index c7658294e81..36e91129bae 100644 --- a/vllm/model_executor/models/bailing_moe_linear.py +++ b/vllm/model_executor/models/bailing_moe_linear.py @@ -790,7 +790,7 @@ class BailingMoeV25ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsPP): ) @classmethod - def get_mamba_state_shape_from_config( # type: ignore[override] + def get_mamba_state_shape_from_config( cls, vllm_config: VllmConfig, ) -> tuple[tuple[int, ...], ...]: diff --git a/vllm/model_executor/models/bee.py b/vllm/model_executor/models/bee.py index 06e90d486d1..4a57413b6e8 100644 --- a/vllm/model_executor/models/bee.py +++ b/vllm/model_executor/models/bee.py @@ -8,7 +8,7 @@ import torch.nn as nn from transformers.activations import GELUActivation from vllm.config import VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions from vllm.inputs import MultiModalDataDict from vllm.multimodal import MULTIMODAL_REGISTRY @@ -97,13 +97,14 @@ class BeeDummyInputsBuilder(LlavaDummyInputsBuilder[BeeProcessingInfo]): target_width, target_height = self.info.get_image_size_with_most_features() image_overrides = mm_options.get("image") + assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions) return { "image": self._get_dummy_images( width=target_width, height=target_height, num_images=num_images, - overrides=image_overrides, # type: ignore[arg-type] + overrides=image_overrides, ), } @@ -133,7 +134,7 @@ class BeeMultiModalProjector(nn.Module): return hidden_states -@MULTIMODAL_REGISTRY.register_processor( # type: ignore[misc] +@MULTIMODAL_REGISTRY.register_processor( LlavaNextMultiModalProcessor, info=BeeProcessingInfo, dummy_inputs=BeeDummyInputsBuilder, diff --git a/vllm/model_executor/models/blip2.py b/vllm/model_executor/models/blip2.py index b14c73f4d16..baa1a16a59b 100644 --- a/vllm/model_executor/models/blip2.py +++ b/vllm/model_executor/models/blip2.py @@ -14,7 +14,7 @@ from transformers import ( ) from vllm.config import CacheConfig, VllmConfig -from vllm.config.multimodal import BaseDummyOptions +from vllm.config.multimodal import BaseDummyOptions, ImageDummyOptions from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.activation import get_act_fn from vllm.model_executor.layers.quantization import QuantizationConfig @@ -454,13 +454,14 @@ class Blip2DummyInputsBuilder(BaseDummyInputsBuilder[Blip2ProcessingInfo]): num_images = mm_counts.get("image", 0) image_overrides = mm_options.get("image") + assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions) return { "image": self._get_dummy_images( width=max_image_size, height=max_image_size, num_images=num_images, - overrides=image_overrides, # type: ignore[arg-type] + overrides=image_overrides, ) } @@ -523,7 +524,7 @@ class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]): info=Blip2ProcessingInfo, dummy_inputs=Blip2DummyInputsBuilder, ) -class Blip2ForConditionalGeneration( # type: ignore[misc] +class Blip2ForConditionalGeneration( nn.Module, SupportsLoRA, SupportsMultiModal, SupportsPP, SupportsQuant ): @classmethod @@ -576,7 +577,7 @@ class Blip2ForConditionalGeneration( # type: ignore[misc] prefix=maybe_prefix(prefix, "language_model"), ) - self.make_empty_intermediate_tensors = ( # type: ignore[method-assign] + self.make_empty_intermediate_tensors = ( self.language_model.make_empty_intermediate_tensors ) @@ -620,10 +621,10 @@ class Blip2ForConditionalGeneration( # type: ignore[misc] return self._image_pixels_to_features(self.vision_model, pixel_values) def _process_image_input(self, image_input: Blip2ImageInputs) -> torch.Tensor: - if image_input["type"] == "image_embeds": - return image_input["data"] + if image_input.type == "image_embeds": + return image_input.data - image_features = self._process_image_pixels(image_input) # type: ignore[arg-type] + image_features = self._process_image_pixels(image_input) query_tokens = self.query_tokens.expand(image_features.shape[0], -1, -1) query_output = self.qformer( diff --git a/vllm/model_executor/models/bloom.py b/vllm/model_executor/models/bloom.py index 0b0750bd52b..cdcf82f385f 100644 --- a/vllm/model_executor/models/bloom.py +++ b/vllm/model_executor/models/bloom.py @@ -334,7 +334,7 @@ class BloomForCausalLM(nn.Module, SupportsPP, SupportsQuant): ) self.logits_processor = LogitsProcessor(config.vocab_size) - self.make_empty_intermediate_tensors = ( # type: ignore[method-assign] + self.make_empty_intermediate_tensors = ( self.transformer.make_empty_intermediate_tensors ) diff --git a/vllm/model_executor/models/interfaces.py b/vllm/model_executor/models/interfaces.py index 32534f8fbb7..da16f3031a6 100644 --- a/vllm/model_executor/models/interfaces.py +++ b/vllm/model_executor/models/interfaces.py @@ -558,7 +558,7 @@ class SupportsLoRA(Protocol): # The `embedding_module` and `embedding_padding_modules` # are empty by default. embedding_modules: ClassVar[dict[str, str]] = {} - packed_modules_mapping: dict[str, list[str]] = {} + packed_modules_mapping: ClassVar[dict[str, list[str]]] = {} # Module prefixes to skip during LoRA loading (e.g., ["mtp."] for MTP layers) lora_skip_prefixes: ClassVar[list[str]] = [] lora_manager: "LoRAModelManager | None" @@ -633,14 +633,10 @@ class SupportsPP(Protocol): MRO of your model class. """ - def make_empty_intermediate_tensors( - self, - batch_size: int, - dtype: torch.dtype, - device: torch.device, - ) -> "IntermediateTensors": - """Called when PP rank > 0 for profiling purposes.""" - ... + make_empty_intermediate_tensors: Callable[ + [int, torch.dtype, torch.device], "IntermediateTensors" + ] + """Called when PP rank > 0 for profiling purposes.""" def forward( self, @@ -648,7 +644,7 @@ class SupportsPP(Protocol): positions: Tensor, *, intermediate_tensors: "IntermediateTensors | None", - ) -> "IntermediateTensors | None": + ) -> "Tensor | IntermediateTensors | tuple[Tensor, list[Tensor]]": """ Accept [`IntermediateTensors`][vllm.sequence.IntermediateTensors] when PP rank > 0. @@ -665,12 +661,9 @@ class SupportsPP(Protocol): class _SupportsPPType(Protocol): supports_pp: Literal[True] - def make_empty_intermediate_tensors( - self, - batch_size: int, - dtype: torch.dtype, - device: torch.device, - ) -> "IntermediateTensors": ... + make_empty_intermediate_tensors: Callable[ + [int, torch.dtype, torch.device], "IntermediateTensors" + ] def forward( self, @@ -678,7 +671,7 @@ class _SupportsPPType(Protocol): positions: Tensor, *, intermediate_tensors: "IntermediateTensors | None", - ) -> "Tensor | IntermediateTensors": ... + ) -> "Tensor | IntermediateTensors | tuple[Tensor, list[Tensor]]": ... @overload @@ -809,16 +802,14 @@ class IsHybrid(Protocol): def get_mamba_state_shape_from_config( cls, vllm_config: "VllmConfig", - ) -> tuple[tuple[int, int], tuple[int, int, int]]: + ) -> tuple[tuple[int, ...], ...]: """Calculate shapes for Mamba's convolutional and state caches. Args: vllm_config: vLLM config Returns: - Tuple containing: - - conv_state_shape: Shape for convolutional state cache - - temporal_state_shape: Shape for state space model cache + Shapes for each state cache used by the model. """ ... @@ -1007,7 +998,7 @@ class SupportsQuant: """The interface required for all models that support quantization.""" hf_to_vllm_mapper: ClassVar["WeightsMapper | None"] = None - packed_modules_mapping: ClassVar[dict[str, list[str]] | None] = None + packed_modules_mapping: ClassVar[dict[str, list[str]]] = {} quant_config: QuantizationConfig | None = None def __new__(cls, *args, **kwargs) -> Self: @@ -1042,8 +1033,7 @@ class SupportsQuant: if (hf_to_vllm_mapper := self.hf_to_vllm_mapper) is not None: unstacked_mapper = hf_to_vllm_mapper.get_unstacked_mapper() self.quant_config.apply_vllm_mapper(unstacked_mapper) - if self.packed_modules_mapping is not None: - self.quant_config.packed_modules_mapping.update(self.packed_modules_mapping) + self.quant_config.packed_modules_mapping.update(self.packed_modules_mapping) @runtime_checkable diff --git a/vllm/model_executor/models/llava_next.py b/vllm/model_executor/models/llava_next.py index fdd1d41af5a..44d50f434a4 100644 --- a/vllm/model_executor/models/llava_next.py +++ b/vllm/model_executor/models/llava_next.py @@ -204,9 +204,7 @@ class BaseLlavaNextMultiModalProcessor(BaseLlavaMultiModalProcessor[_I]): raise NotImplementedError -class LlavaNextMultiModalProcessor( - BaseLlavaNextMultiModalProcessor[LlavaNextProcessingInfo] -): +class LlavaNextMultiModalProcessor(BaseLlavaNextMultiModalProcessor[_I]): def _get_mm_fields_config( self, hf_inputs: BatchFeature,