diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index 09786fdcdb0..d3ab7fea121 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -695,7 +695,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen | `KeyeForConditionalGeneration` | Keye-VL-8B-Preview | T + IE+ + VE+ | `Kwai-Keye/Keye-VL-8B-Preview` | ✅︎ | ✅︎ | | `KeyeVL1_5ForConditionalGeneration` | Keye-VL-1_5-8B | T + IE+ + VE+ | `Kwai-Keye/Keye-VL-1_5-8B` | ✅︎ | ✅︎ | | `KimiVLForConditionalGeneration` | Kimi-VL-A3B-Instruct, Kimi-VL-A3B-Thinking | T + I+ | `moonshotai/Kimi-VL-A3B-Instruct`, `moonshotai/Kimi-VL-A3B-Thinking` | | ✅︎ | -| `K2VLForConditionalGeneration` | Kimi-K2.5-VL | T + I+ + V+ | `moonshotai/Kimi-K2.5-VL-Preview` | | ✅︎ | +| `KimiK25ForConditionalGeneration` | Kimi-K2.5-VL | T + I+ + V+ | `moonshotai/Kimi-K2.5-VL-Preview` | | ✅︎ | | `LightOnOCRForConditionalGeneration` | LightOnOCR-1B | T + I+ | `lightonai/LightOnOCR-1B`, etc | ✅︎ | ✅︎ | | `Llama4ForConditionalGeneration` | Llama 4 | T + I+ | `meta-llama/Llama-4-Scout-17B-16E-Instruct`, `meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8`, `meta-llama/Llama-4-Maverick-17B-128E-Instruct`, etc. | ✅︎ | ✅︎ | | `Llama_Nemotron_Nano_VL` | Llama Nemotron Nano VL | T + IE+ | `nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1` | ✅︎ | ✅︎ | diff --git a/tests/models/registry.py b/tests/models/registry.py index 86c3e645ae4..7bb76875521 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -716,7 +716,7 @@ _MULTIMODAL_EXAMPLE_MODELS = { ) }, ), - "K2VLForConditionalGeneration": _HfExamplesInfo( + "KimiK25ForConditionalGeneration": _HfExamplesInfo( "moonshotai/Kimi-K2.5-VL-Preview", trust_remote_code=True, ), diff --git a/vllm/entrypoints/openai/serving_engine.py b/vllm/entrypoints/openai/serving_engine.py index 3d9e8715281..e0ebeee9fc5 100644 --- a/vllm/entrypoints/openai/serving_engine.py +++ b/vllm/entrypoints/openai/serving_engine.py @@ -1214,7 +1214,7 @@ class OpenAIServing: mm_data = await mm_data_future - # NOTE: use_unified_vision_chunk is currently specific to Kimi2VL (K2VL) + # NOTE: use_unified_vision_chunk is currently specific to Kimi-K2.5 # model which uses unified vision chunks for both images and videos. if ( getattr(model_config.hf_config, "use_unified_vision_chunk", False) diff --git a/vllm/model_executor/models/k2vl.py b/vllm/model_executor/models/kimi_k25.py similarity index 92% rename from vllm/model_executor/models/k2vl.py rename to vllm/model_executor/models/kimi_k25.py index 0fd321273a8..1f5fb6b9565 100644 --- a/vllm/model_executor/models/k2vl.py +++ b/vllm/model_executor/models/kimi_k25.py @@ -2,15 +2,13 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project # ruff: noqa: E501 """ -K2-VL Model Implementation for vLLM. +Kimi-K2.5 Model Implementation for vLLM. -K2-VL extends Kimi-VL with video support using video-chunks. -A video-chunk is the smallest independently processable unit of video, -typically consisting of 4 consecutive frames (temporal_merge_kernel_size=4). +Kimi-K2.5 extends Kimi-K2 with vision support This module defines: -- K2VLProcessingInfo/K2VLMultiModalProcessor: Processing logic -- K2VLForConditionalGeneration: Main model class +- KimiK25ProcessingInfo/KimiK25MultiModalProcessor: Processing logic +- KimiK25ForConditionalGeneration: Main model class """ import copy @@ -36,8 +34,8 @@ from vllm.model_executor.model_loader.weight_utils import ( ) from vllm.model_executor.models.deepseek_v2 import DeepseekV2Model from vllm.model_executor.models.interfaces import SupportsMultiModal, SupportsPP -from vllm.model_executor.models.k2vl_vit import ( - K2VLMultiModalProjector, +from vllm.model_executor.models.kimi_k25_vit import ( + KimiK25MultiModalProjector, MoonViT3dPretrainedModel, vision_tower_forward, ) @@ -61,7 +59,7 @@ from vllm.multimodal.processing import ( ) from vllm.multimodal.profiling import BaseDummyInputsBuilder, BaseDummyOptions from vllm.sequence import IntermediateTensors -from vllm.transformers_utils.configs import K2VLConfig +from vllm.transformers_utils.configs import KimiK25Config from vllm.transformers_utils.processor import cached_get_image_processor from vllm.utils.tensor_schema import TensorSchema, TensorShape @@ -77,7 +75,7 @@ class MaxImageTokenMeta: height: int = 3000 -class K2VLMediaPixelInputs(TensorSchema): +class KimiK25MediaPixelInputs(TensorSchema): """ Media input schema for K2-VL model. @@ -139,8 +137,8 @@ class MoonshotKimiVAutoProcessor(ProcessorMixin): ) -class K2VLProcessingInfo(BaseProcessingInfo): - """Processing information for K2-VL model. +class KimiK25ProcessingInfo(BaseProcessingInfo): + """Processing information for Kimi-K2.5 model. Provides configuration and utilities for processing both images and video-chunks. @@ -164,17 +162,17 @@ class K2VLProcessingInfo(BaseProcessingInfo): return self.hf_processor def get_hf_config(self): - return self.ctx.get_hf_config(K2VLConfig) + return self.ctx.get_hf_config(KimiK25Config) def get_supported_mm_limits(self) -> Mapping[str, int | None]: # None means unlimited return {"vision_chunk": None} -class K2VLDummyInputsBuilder(BaseDummyInputsBuilder[K2VLProcessingInfo]): - """Builds dummy inputs for K2-VL model profiling.""" +class KimiK25DummyInputsBuilder(BaseDummyInputsBuilder[KimiK25ProcessingInfo]): + """Builds dummy inputs for Kimi-K2.5 model profiling.""" - def __init__(self, info: K2VLProcessingInfo) -> None: + def __init__(self, info: KimiK25ProcessingInfo) -> None: super().__init__(info) self.media_token_id = self.info.media_token_id self.frame_per_chunk = self.info.media_processor.num_frames_per_chunk @@ -232,8 +230,8 @@ class K2VLDummyInputsBuilder(BaseDummyInputsBuilder[K2VLProcessingInfo]): return {"vision_chunk": dummy_items} -class K2VLMultiModalProcessor(BaseMultiModalProcessor[K2VLProcessingInfo]): - """Multi-modal processor for K2-VL. +class KimiK25MultiModalProcessor(BaseMultiModalProcessor[KimiK25ProcessingInfo]): + """Multi-modal processor for Kimi-K2.5. Handles both image and video-chunk modalities. """ @@ -290,12 +288,12 @@ class K2VLMultiModalProcessor(BaseMultiModalProcessor[K2VLProcessingInfo]): @MULTIMODAL_REGISTRY.register_processor( - K2VLMultiModalProcessor, - info=K2VLProcessingInfo, - dummy_inputs=K2VLDummyInputsBuilder, + KimiK25MultiModalProcessor, + info=KimiK25ProcessingInfo, + dummy_inputs=KimiK25DummyInputsBuilder, ) -class K2VLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP): - """K2-VL model for conditional generation. +class KimiK25ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP): + """Kimi-K2.5 model for conditional generation. Supports both image and video-chunk modalities. Video-chunks are temporal segments (typically 4 frames) that are @@ -306,12 +304,12 @@ class K2VLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP): @classmethod def get_placeholder_str(cls, modality: str, i: int) -> str | None: - # K2-VL uses video_chunk for all media types + # Kimi-K2.5 uses video_chunk for all media types if modality == "image": return "<|media_begin|>image<|media_content|><|media_pad|><|media_end|>" elif modality == "video": # return a placeholder, to be replaced in the future. - return "<|k2vl_video_placeholder|>" + return "<|kimi_k25_video_placeholder|>" raise ValueError(f"Unsupported modality: {modality}") @@ -322,7 +320,7 @@ class K2VLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP): ) -> None: super().__init__() model_config = vllm_config.model_config - config: K2VLConfig = model_config.hf_config + config: KimiK25Config = model_config.hf_config self.config = config quant_config = vllm_config.quant_config @@ -332,7 +330,7 @@ class K2VLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP): ) self.hidden_size = config.text_config.hidden_size self.device = torch.cuda.current_device() - # Build vision tower directly with K2VLVisionConfig + # Build vision tower directly with KimiK25VisionConfig self.vision_tower = MoonViT3dPretrainedModel( config.vision_config, multimodal_config=model_config.multimodal_config, @@ -342,8 +340,8 @@ class K2VLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP): device=self.device, dtype=model_config.dtype ) - self.mm_projector = K2VLMultiModalProjector( - config=config, + self.mm_projector = KimiK25MultiModalProjector( + config=config.vision_config, use_data_parallel=self.use_data_parallel, prefix=maybe_prefix(prefix, "mm_projector"), ) @@ -377,7 +375,7 @@ class K2VLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP): def _parse_and_validate_media_input( self, **kwargs: object - ) -> K2VLMediaPixelInputs | None: + ) -> KimiK25MediaPixelInputs | None: pixel_values = kwargs.pop("pixel_values", None) grid_thws = kwargs.pop("grid_thws", None) if pixel_values is None: @@ -400,14 +398,14 @@ class K2VLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP): grid_thws = grid_thws.reshape(-1, grid_thws.shape[-1]) assert grid_thws.ndim == 2, f"unexpected shape for grid_thws: {grid_thws.shape}" - return K2VLMediaPixelInputs( + return KimiK25MediaPixelInputs( type="pixel_values", pixel_values=pixel_values, grid_thws=grid_thws, ) def _process_media_input( - self, media_input: K2VLMediaPixelInputs + self, media_input: KimiK25MediaPixelInputs ) -> list[torch.Tensor]: # NOTE(moyan): This forward will automatically batch the forward pass internally media_features = vision_tower_forward( @@ -569,7 +567,7 @@ class K2VLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP): def get_spec_layer_idx_from_weight_name( - config: K2VLConfig, weight_name: str + config: KimiK25Config, weight_name: str ) -> int | None: if hasattr(config, "num_nextn_predict_layers") and ( config.num_nextn_predict_layers > 0 diff --git a/vllm/model_executor/models/k2vl_vit.py b/vllm/model_executor/models/kimi_k25_vit.py similarity index 97% rename from vllm/model_executor/models/k2vl_vit.py rename to vllm/model_executor/models/kimi_k25_vit.py index ff5c5214b9c..a0b608d8f34 100644 --- a/vllm/model_executor/models/k2vl_vit.py +++ b/vllm/model_executor/models/kimi_k25_vit.py @@ -1,14 +1,13 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """ -Vision tower implementation for K2-VL model. +Vision tower implementation for Kimi-K2.5 model. -This module provides the vision encoder components for K2-VL, +This module provides the vision encoder components for Kimi-K2.5, including 3D patch embedding, RoPE position embedding, and temporal pooling for video chunks. """ -import math from collections.abc import Sequence from copy import deepcopy from typing import Any @@ -32,7 +31,7 @@ from vllm.model_executor.layers.linear import ( RowParallelLinear, ) from vllm.model_executor.models.utils import maybe_prefix -from vllm.transformers_utils.configs.k2vl import K2VLConfig, K2VLVisionConfig +from vllm.transformers_utils.configs.kimi_k25 import KimiK25VisionConfig KIMIV_VT_INFER_MAX_PATCH_NUM = 16328 logger = init_logger(__name__) @@ -557,11 +556,11 @@ def tpool_patch_merger( class MoonViT3dPretrainedModel(PreTrainedModel): """Main vision tower model. - Uses K2VLVisionConfig directly from transformers_utils/configs/k2vl.py. + Uses KimiK25VisionConfig directly from transformers_utils/configs/kimi_k25.py. """ - config_class = K2VLVisionConfig - model_type = "k2_vl_vision" + config_class = KimiK25VisionConfig + model_type = "kimi_k25_vision" _no_split_modules = ["PackingTransformer"] _supports_flash_attn_2 = True _supports_sdpa = True @@ -695,24 +694,23 @@ def vision_tower_forward( return tensors -class K2VLMultiModalProjector(nn.Module): - """Multi-modal projector with patch merging for K2-VL.""" +class KimiK25MultiModalProjector(nn.Module): + """Multi-modal projector with patch merging for Kimi-K2.5.""" def __init__( self, - config: K2VLConfig, + config: KimiK25VisionConfig, use_data_parallel: bool = False, prefix: str = "", ): super().__init__() self.use_data_parallel = use_data_parallel - vc = config.vision_config # Hidden size after patch merging - merge_h, merge_w = vc.merge_kernel_size - self.hidden_size = vc.hidden_size * merge_h * merge_w + merge_h, merge_w = config.merge_kernel_size + self.hidden_size = config.hidden_size * merge_h * merge_w - self.pre_norm = torch.nn.LayerNorm(vc.hidden_size, eps=1e-5) + self.pre_norm = torch.nn.LayerNorm(config.hidden_size, eps=1e-5) self.linear_1 = ReplicatedLinear( self.hidden_size, self.hidden_size, @@ -721,7 +719,7 @@ class K2VLMultiModalProjector(nn.Module): ) self.linear_2 = ReplicatedLinear( self.hidden_size, - config.text_config.hidden_size, + config.mm_hidden_size, bias=True, prefix=maybe_prefix(prefix, "linear_2"), ) diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index e2767807ec4..deafe20101b 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -342,7 +342,7 @@ _MULTIMODAL_MODELS = { ), "RForConditionalGeneration": ("rvl", "RForConditionalGeneration"), "KimiVLForConditionalGeneration": ("kimi_vl", "KimiVLForConditionalGeneration"), # noqa: E501 - "K2VLForConditionalGeneration": ("k2vl", "K2VLForConditionalGeneration"), # noqa: E501 + "KimiK25ForConditionalGeneration": ("kimi_k25", "KimiK25ForConditionalGeneration"), # noqa: E501 "LightOnOCRForConditionalGeneration": ( "lightonocr", "LightOnOCRForConditionalGeneration", diff --git a/vllm/multimodal/video.py b/vllm/multimodal/video.py index dcfcaf7fba7..93a7919b658 100644 --- a/vllm/multimodal/video.py +++ b/vllm/multimodal/video.py @@ -124,9 +124,9 @@ class IdentityVideoLoader(VideoLoader): """IdentityVideoLoader returns raw video bytes without decoding. This allows the model processor to handle video decoding and - is required for models like K2VL that need custom video chunk splitting. + is required for models like Kimi-K2.5 that need custom video chunk splitting. - NOTE: This is temporary for K2VL testing. Remember to change back + NOTE: This is temporary for Kimi-K2.5 testing. Remember to change back to opencv before release if needed. """ diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index b25cf4f3511..4c21aadcfbb 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -84,7 +84,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict( isaac="IsaacConfig", kimi_linear="KimiLinearConfig", kimi_vl="KimiVLConfig", - k2_vl="K2VLConfig", + kimi_k25="KimiK25Config", RefinedWeb="RWConfig", # For tiiuae/falcon-40b(-instruct) RefinedWebModel="RWConfig", # For tiiuae/falcon-7b(-instruct) jais="JAISConfig", diff --git a/vllm/transformers_utils/configs/__init__.py b/vllm/transformers_utils/configs/__init__.py index e72ba23ccd2..bfb9c1758df 100644 --- a/vllm/transformers_utils/configs/__init__.py +++ b/vllm/transformers_utils/configs/__init__.py @@ -38,8 +38,7 @@ _CLASS_TO_MODULE: dict[str, str] = { "MoonViTConfig": "vllm.transformers_utils.configs.moonvit", "KimiLinearConfig": "vllm.transformers_utils.configs.kimi_linear", "KimiVLConfig": "vllm.transformers_utils.configs.kimi_vl", - "K2VLConfig": "vllm.transformers_utils.configs.k2vl", - "K2VLVisionConfig": "vllm.transformers_utils.configs.k2vl", + "KimiK25Config": "vllm.transformers_utils.configs.kimi_k25", "NemotronConfig": "vllm.transformers_utils.configs.nemotron", "NemotronHConfig": "vllm.transformers_utils.configs.nemotron_h", "Olmo3Config": "vllm.transformers_utils.configs.olmo3", @@ -79,8 +78,7 @@ __all__ = [ "MoonViTConfig", "KimiLinearConfig", "KimiVLConfig", - "K2VLConfig", - "K2VLVisionConfig", + "KimiK25Config", "NemotronConfig", "NemotronHConfig", "Olmo3Config", diff --git a/vllm/transformers_utils/configs/k2vl.py b/vllm/transformers_utils/configs/kimi_k25.py similarity index 85% rename from vllm/transformers_utils/configs/k2vl.py rename to vllm/transformers_utils/configs/kimi_k25.py index 34e316c55ca..aa26cb86ae0 100644 --- a/vllm/transformers_utils/configs/k2vl.py +++ b/vllm/transformers_utils/configs/kimi_k25.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """ -K2-VL Model Configuration. +Kimi-K2.5 Model Configuration. This configuration supports video-chunk as an internal modality type. A video-chunk is the smallest independently processable unit of video. @@ -11,8 +11,8 @@ from transformers import DeepseekV3Config from transformers.configuration_utils import PretrainedConfig -class K2VLVisionConfig(PretrainedConfig): - """Vision configuration for K2-VL (vision tower + mm projector). +class KimiK25VisionConfig(PretrainedConfig): + """Vision configuration for Kimi-K2.5 (vision tower + mm projector). Args: Vision Tower Parameters: @@ -36,7 +36,7 @@ class K2VLVisionConfig(PretrainedConfig): projector_ln_eps: Layer norm epsilon for projector. """ - model_type = "k2_vl_vision" + model_type = "kimi_k25_vision" def __init__( self, @@ -84,10 +84,10 @@ class K2VLVisionConfig(PretrainedConfig): self.projector_ln_eps = projector_ln_eps -class K2VLConfig(PretrainedConfig): - """K2-VL model configuration. +class KimiK25Config(PretrainedConfig): + """Kimi-K2.5 model configuration. - K2-VL extends Kimi-VL with video support using video-chunks. + Kimi-K2.5 extends Kimi-K2 with vision support using video-chunks. A video-chunk consists of multiple consecutive frames that are processed together with temporal pooling. @@ -99,25 +99,25 @@ class K2VLConfig(PretrainedConfig): pad_token_id: The token ID for padding. """ - model_type = "k2_vl" + model_type = "kimi_k25" def __init__( self, - vision_config: dict | K2VLVisionConfig | None = None, + vision_config: dict | KimiK25VisionConfig | None = None, text_config: dict | DeepseekV3Config | None = None, ignore_index: int = -100, media_placeholder_token_id: int = 163605, pad_token_id: int = 0, use_unified_vision_chunk: bool = False, - video_placeholder: str = "<|k2vl_video_placeholder|>", + video_placeholder: str = "<|kimi_k25_video_placeholder|>", **kwargs, ): # Vision config if vision_config is None: - vision_config = K2VLVisionConfig() + vision_config = KimiK25VisionConfig() elif isinstance(vision_config, dict): - vision_config = K2VLVisionConfig(**vision_config) - self.vision_config: K2VLVisionConfig = vision_config + vision_config = KimiK25VisionConfig(**vision_config) + self.vision_config: KimiK25VisionConfig = vision_config # Text config if text_config is None: @@ -126,6 +126,10 @@ class K2VLConfig(PretrainedConfig): text_config = DeepseekV3Config(**text_config) self.text_config: DeepseekV3Config = text_config + # Set mm_hidden_size to text hidden size if not explicitly set + if self.vision_config.mm_hidden_size == self.vision_config.hidden_size: + self.vision_config.mm_hidden_size = self.text_config.hidden_size + # Other config self.ignore_index = ignore_index self.media_placeholder_token_id = media_placeholder_token_id