karylab_agents/vllm
1
0
Fork 0
forked from Karylab-cklius/vllm
Code Pull Requests 2 Actions 1 Packages Activity

Rename K2VL to KimiK25

Browse Source 
- Rename files: k2vl.py -> kimi_k25.py, k2vl_vit.py -> kimi_k25_vit.py
- Rename classes: K2VLConfig -> KimiK25Config, K2VLForConditionalGeneration -> KimiK25ForConditionalGeneration, etc.
- Update registry, configs/__init__.py, config.py mappings
- Update docs and comments to reference Kimi-K2.5 instead of K2VL
- Pass vision_config to KimiK25MultiModalProjector per reference diff

Signed-off-by: wanglinian <wanglinian@stu.pku.edu.cn>
This commit is contained in:
wanglinian
2026-01-07 08:53:22 +00:00
parent 875d970dd0
commit 5f54da5e7e
10 changed files with 70 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/models/supported_models.md
+1 -1
View File
@@ -695,7 +695,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
| `KeyeForConditionalGeneration` | Keye-VL-8B-Preview | T + I<sup>E+</sup> + V<sup>E+</sup> | `Kwai-Keye/Keye-VL-8B-Preview` | ✅︎ | ✅︎ |
| `KeyeVL1_5ForConditionalGeneration` | Keye-VL-1_5-8B | T + I<sup>E+</sup> + V<sup>E+</sup> | `Kwai-Keye/Keye-VL-1_5-8B` | ✅︎ | ✅︎ |
| `KimiVLForConditionalGeneration` | Kimi-VL-A3B-Instruct, Kimi-VL-A3B-Thinking | T + I<sup>+</sup> | `moonshotai/Kimi-VL-A3B-Instruct`, `moonshotai/Kimi-VL-A3B-Thinking` | | ✅︎ |
| `K2VLForConditionalGeneration` | Kimi-K2.5-VL | T + I<sup>+</sup> + V<sup>+</sup> | `moonshotai/Kimi-K2.5-VL-Preview` | | ✅︎ |
| `KimiK25ForConditionalGeneration` | Kimi-K2.5-VL | T + I<sup>+</sup> + V<sup>+</sup> | `moonshotai/Kimi-K2.5-VL-Preview` | | ✅︎ |
| `LightOnOCRForConditionalGeneration` | LightOnOCR-1B | T + I<sup>+</sup> | `lightonai/LightOnOCR-1B`, etc | ✅︎ | ✅︎ |
| `Llama4ForConditionalGeneration` | Llama 4 | T + I<sup>+</sup> | `meta-llama/Llama-4-Scout-17B-16E-Instruct`, `meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8`, `meta-llama/Llama-4-Maverick-17B-128E-Instruct`, etc. | ✅︎ | ✅︎ |
| `Llama_Nemotron_Nano_VL` | Llama Nemotron Nano VL | T + I<sup>E+</sup> | `nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1` | ✅︎ | ✅︎ |
tests/models/registry.py
+1 -1
View File
@@ -716,7 +716,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {
)
},
),
"K2VLForConditionalGeneration": _HfExamplesInfo(
"KimiK25ForConditionalGeneration": _HfExamplesInfo(
"moonshotai/Kimi-K2.5-VL-Preview",
trust_remote_code=True,
),
vllm/entrypoints/openai/serving_engine.py
+1 -1
View File
@@ -1214,7 +1214,7 @@ class OpenAIServing:
mm_data = await mm_data_future
# NOTE: use_unified_vision_chunk is currently specific to Kimi2VL (K2VL)
# NOTE: use_unified_vision_chunk is currently specific to Kimi-K2.5
# model which uses unified vision chunks for both images and videos.
if (
getattr(model_config.hf_config, "use_unified_vision_chunk", False)
vllm/model_executor/models/k2vl.py → vllm/model_executor/models/kimi_k25.py
+31 -33
View File
@@ -2,15 +2,13 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# ruff: noqa: E501
"""
K2-VL Model Implementation for vLLM.
Kimi-K2.5 Model Implementation for vLLM.
K2-VL extends Kimi-VL with video support using video-chunks.
A video-chunk is the smallest independently processable unit of video,
typically consisting of 4 consecutive frames (temporal_merge_kernel_size=4).
Kimi-K2.5 extends Kimi-K2 with vision support
This module defines:
- K2VLProcessingInfo/K2VLMultiModalProcessor: Processing logic
- K2VLForConditionalGeneration: Main model class
- KimiK25ProcessingInfo/KimiK25MultiModalProcessor: Processing logic
- KimiK25ForConditionalGeneration: Main model class
"""
import copy
@@ -36,8 +34,8 @@ from vllm.model_executor.model_loader.weight_utils import (
)
from vllm.model_executor.models.deepseek_v2 import DeepseekV2Model
from vllm.model_executor.models.interfaces import SupportsMultiModal, SupportsPP
from vllm.model_executor.models.k2vl_vit import (
K2VLMultiModalProjector,
from vllm.model_executor.models.kimi_k25_vit import (
KimiK25MultiModalProjector,
MoonViT3dPretrainedModel,
vision_tower_forward,
)
@@ -61,7 +59,7 @@ from vllm.multimodal.processing import (
)
from vllm.multimodal.profiling import BaseDummyInputsBuilder, BaseDummyOptions
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs import K2VLConfig
from vllm.transformers_utils.configs import KimiK25Config
from vllm.transformers_utils.processor import cached_get_image_processor
from vllm.utils.tensor_schema import TensorSchema, TensorShape
@@ -77,7 +75,7 @@ class MaxImageTokenMeta:
height: int = 3000
class K2VLMediaPixelInputs(TensorSchema):
class KimiK25MediaPixelInputs(TensorSchema):
"""
Media input schema for K2-VL model.
@@ -139,8 +137,8 @@ class MoonshotKimiVAutoProcessor(ProcessorMixin):
)
class K2VLProcessingInfo(BaseProcessingInfo):
"""Processing information for K2-VL model.
class KimiK25ProcessingInfo(BaseProcessingInfo):
"""Processing information for Kimi-K2.5 model.
Provides configuration and utilities for processing both
images and video-chunks.
@@ -164,17 +162,17 @@ class K2VLProcessingInfo(BaseProcessingInfo):
return self.hf_processor
def get_hf_config(self):
return self.ctx.get_hf_config(K2VLConfig)
return self.ctx.get_hf_config(KimiK25Config)
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
# None means unlimited
return {"vision_chunk": None}
class K2VLDummyInputsBuilder(BaseDummyInputsBuilder[K2VLProcessingInfo]):
"""Builds dummy inputs for K2-VL model profiling."""
class KimiK25DummyInputsBuilder(BaseDummyInputsBuilder[KimiK25ProcessingInfo]):
"""Builds dummy inputs for Kimi-K2.5 model profiling."""
def __init__(self, info: K2VLProcessingInfo) -> None:
def __init__(self, info: KimiK25ProcessingInfo) -> None:
super().__init__(info)
self.media_token_id = self.info.media_token_id
self.frame_per_chunk = self.info.media_processor.num_frames_per_chunk
@@ -232,8 +230,8 @@ class K2VLDummyInputsBuilder(BaseDummyInputsBuilder[K2VLProcessingInfo]):
return {"vision_chunk": dummy_items}
class K2VLMultiModalProcessor(BaseMultiModalProcessor[K2VLProcessingInfo]):
"""Multi-modal processor for K2-VL.
class KimiK25MultiModalProcessor(BaseMultiModalProcessor[KimiK25ProcessingInfo]):
"""Multi-modal processor for Kimi-K2.5.
Handles both image and video-chunk modalities.
"""
@@ -290,12 +288,12 @@ class K2VLMultiModalProcessor(BaseMultiModalProcessor[K2VLProcessingInfo]):
@MULTIMODAL_REGISTRY.register_processor(
K2VLMultiModalProcessor,
info=K2VLProcessingInfo,
dummy_inputs=K2VLDummyInputsBuilder,
KimiK25MultiModalProcessor,
info=KimiK25ProcessingInfo,
dummy_inputs=KimiK25DummyInputsBuilder,
)
class K2VLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
"""K2-VL model for conditional generation.
class KimiK25ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
"""Kimi-K2.5 model for conditional generation.
Supports both image and video-chunk modalities.
Video-chunks are temporal segments (typically 4 frames) that are
@@ -306,12 +304,12 @@ class K2VLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
# K2-VL uses video_chunk for all media types
# Kimi-K2.5 uses video_chunk for all media types
if modality == "image":
return "<|media_begin|>image<|media_content|><|media_pad|><|media_end|>"
elif modality == "video":
# return a placeholder, to be replaced in the future.
return "<|k2vl_video_placeholder|>"
return "<|kimi_k25_video_placeholder|>"
raise ValueError(f"Unsupported modality: {modality}")
@@ -322,7 +320,7 @@ class K2VLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
) -> None:
super().__init__()
model_config = vllm_config.model_config
config: K2VLConfig = model_config.hf_config
config: KimiK25Config = model_config.hf_config
self.config = config
quant_config = vllm_config.quant_config
@@ -332,7 +330,7 @@ class K2VLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
)
self.hidden_size = config.text_config.hidden_size
self.device = torch.cuda.current_device()
# Build vision tower directly with K2VLVisionConfig
# Build vision tower directly with KimiK25VisionConfig
self.vision_tower = MoonViT3dPretrainedModel(
config.vision_config,
multimodal_config=model_config.multimodal_config,
@@ -342,8 +340,8 @@ class K2VLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
device=self.device, dtype=model_config.dtype
)
self.mm_projector = K2VLMultiModalProjector(
config=config,
self.mm_projector = KimiK25MultiModalProjector(
config=config.vision_config,
use_data_parallel=self.use_data_parallel,
prefix=maybe_prefix(prefix, "mm_projector"),
)
@@ -377,7 +375,7 @@ class K2VLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
def _parse_and_validate_media_input(
self, **kwargs: object
) -> K2VLMediaPixelInputs | None:
) -> KimiK25MediaPixelInputs | None:
pixel_values = kwargs.pop("pixel_values", None)
grid_thws = kwargs.pop("grid_thws", None)
if pixel_values is None:
@@ -400,14 +398,14 @@ class K2VLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
grid_thws = grid_thws.reshape(-1, grid_thws.shape[-1])
assert grid_thws.ndim == 2, f"unexpected shape for grid_thws: {grid_thws.shape}"
return K2VLMediaPixelInputs(
return KimiK25MediaPixelInputs(
type="pixel_values",
pixel_values=pixel_values,
grid_thws=grid_thws,
)
def _process_media_input(
self, media_input: K2VLMediaPixelInputs
self, media_input: KimiK25MediaPixelInputs
) -> list[torch.Tensor]:
# NOTE(moyan): This forward will automatically batch the forward pass internally
media_features = vision_tower_forward(
@@ -569,7 +567,7 @@ class K2VLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
def get_spec_layer_idx_from_weight_name(
config: K2VLConfig, weight_name: str
config: KimiK25Config, weight_name: str
) -> int | None:
if hasattr(config, "num_nextn_predict_layers") and (
config.num_nextn_predict_layers > 0
vllm/model_executor/models/k2vl_vit.py → vllm/model_executor/models/kimi_k25_vit.py
+13 -15
View File
@@ -1,14 +1,13 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Vision tower implementation for K2-VL model.
Vision tower implementation for Kimi-K2.5 model.
This module provides the vision encoder components for K2-VL,
This module provides the vision encoder components for Kimi-K2.5,
including 3D patch embedding, RoPE position embedding, and
temporal pooling for video chunks.
"""
import math
from collections.abc import Sequence
from copy import deepcopy
from typing import Any
@@ -32,7 +31,7 @@ from vllm.model_executor.layers.linear import (
RowParallelLinear,
)
from vllm.model_executor.models.utils import maybe_prefix
from vllm.transformers_utils.configs.k2vl import K2VLConfig, K2VLVisionConfig
from vllm.transformers_utils.configs.kimi_k25 import KimiK25VisionConfig
KIMIV_VT_INFER_MAX_PATCH_NUM = 16328
logger = init_logger(__name__)
@@ -557,11 +556,11 @@ def tpool_patch_merger(
class MoonViT3dPretrainedModel(PreTrainedModel):
"""Main vision tower model.
Uses K2VLVisionConfig directly from transformers_utils/configs/k2vl.py.
Uses KimiK25VisionConfig directly from transformers_utils/configs/kimi_k25.py.
"""
config_class = K2VLVisionConfig
model_type = "k2_vl_vision"
config_class = KimiK25VisionConfig
model_type = "kimi_k25_vision"
_no_split_modules = ["PackingTransformer"]
_supports_flash_attn_2 = True
_supports_sdpa = True
@@ -695,24 +694,23 @@ def vision_tower_forward(
return tensors
class K2VLMultiModalProjector(nn.Module):
"""Multi-modal projector with patch merging for K2-VL."""
class KimiK25MultiModalProjector(nn.Module):
"""Multi-modal projector with patch merging for Kimi-K2.5."""
def __init__(
self,
config: K2VLConfig,
config: KimiK25VisionConfig,
use_data_parallel: bool = False,
prefix: str = "",
):
super().__init__()
self.use_data_parallel = use_data_parallel
vc = config.vision_config
# Hidden size after patch merging
merge_h, merge_w = vc.merge_kernel_size
self.hidden_size = vc.hidden_size * merge_h * merge_w
merge_h, merge_w = config.merge_kernel_size
self.hidden_size = config.hidden_size * merge_h * merge_w
self.pre_norm = torch.nn.LayerNorm(vc.hidden_size, eps=1e-5)
self.pre_norm = torch.nn.LayerNorm(config.hidden_size, eps=1e-5)
self.linear_1 = ReplicatedLinear(
self.hidden_size,
self.hidden_size,
@@ -721,7 +719,7 @@ class K2VLMultiModalProjector(nn.Module):
)
self.linear_2 = ReplicatedLinear(
self.hidden_size,
config.text_config.hidden_size,
config.mm_hidden_size,
bias=True,
prefix=maybe_prefix(prefix, "linear_2"),
)
vllm/model_executor/models/registry.py
+1 -1
View File
@@ -342,7 +342,7 @@ _MULTIMODAL_MODELS = {
),
"RForConditionalGeneration": ("rvl", "RForConditionalGeneration"),
"KimiVLForConditionalGeneration": ("kimi_vl", "KimiVLForConditionalGeneration"), # noqa: E501
"K2VLForConditionalGeneration": ("k2vl", "K2VLForConditionalGeneration"), # noqa: E501
"KimiK25ForConditionalGeneration": ("kimi_k25", "KimiK25ForConditionalGeneration"), # noqa: E501
"LightOnOCRForConditionalGeneration": (
"lightonocr",
"LightOnOCRForConditionalGeneration",
vllm/multimodal/video.py
+2 -2
View File
@@ -124,9 +124,9 @@ class IdentityVideoLoader(VideoLoader):
"""IdentityVideoLoader returns raw video bytes without decoding.
This allows the model processor to handle video decoding and
is required for models like K2VL that need custom video chunk splitting.
is required for models like Kimi-K2.5 that need custom video chunk splitting.
NOTE: This is temporary for K2VL testing. Remember to change back
NOTE: This is temporary for Kimi-K2.5 testing. Remember to change back
to opencv before release if needed.
"""
vllm/transformers_utils/config.py
+1 -1
View File
@@ -84,7 +84,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
isaac="IsaacConfig",
kimi_linear="KimiLinearConfig",
kimi_vl="KimiVLConfig",
k2_vl="K2VLConfig",
kimi_k25="KimiK25Config",
RefinedWeb="RWConfig", # For tiiuae/falcon-40b(-instruct)
RefinedWebModel="RWConfig", # For tiiuae/falcon-7b(-instruct)
jais="JAISConfig",
vllm/transformers_utils/configs/__init__.py
+2 -4
View File
@@ -38,8 +38,7 @@ _CLASS_TO_MODULE: dict[str, str] = {
"MoonViTConfig": "vllm.transformers_utils.configs.moonvit",
"KimiLinearConfig": "vllm.transformers_utils.configs.kimi_linear",
"KimiVLConfig": "vllm.transformers_utils.configs.kimi_vl",
"K2VLConfig": "vllm.transformers_utils.configs.k2vl",
"K2VLVisionConfig": "vllm.transformers_utils.configs.k2vl",
"KimiK25Config": "vllm.transformers_utils.configs.kimi_k25",
"NemotronConfig": "vllm.transformers_utils.configs.nemotron",
"NemotronHConfig": "vllm.transformers_utils.configs.nemotron_h",
"Olmo3Config": "vllm.transformers_utils.configs.olmo3",
@@ -79,8 +78,7 @@ __all__ = [
"MoonViTConfig",
"KimiLinearConfig",
"KimiVLConfig",
"K2VLConfig",
"K2VLVisionConfig",
"KimiK25Config",
"NemotronConfig",
"NemotronHConfig",
"Olmo3Config",
vllm/transformers_utils/configs/k2vl.py → vllm/transformers_utils/configs/kimi_k25.py
+17 -13
View File
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
K2-VL Model Configuration.
Kimi-K2.5 Model Configuration.
This configuration supports video-chunk as an internal modality type.
A video-chunk is the smallest independently processable unit of video.
@@ -11,8 +11,8 @@ from transformers import DeepseekV3Config
from transformers.configuration_utils import PretrainedConfig
class K2VLVisionConfig(PretrainedConfig):
"""Vision configuration for K2-VL (vision tower + mm projector).
class KimiK25VisionConfig(PretrainedConfig):
"""Vision configuration for Kimi-K2.5 (vision tower + mm projector).
Args:
Vision Tower Parameters:
@@ -36,7 +36,7 @@ class K2VLVisionConfig(PretrainedConfig):
projector_ln_eps: Layer norm epsilon for projector.
"""
model_type = "k2_vl_vision"
model_type = "kimi_k25_vision"
def __init__(
self,
@@ -84,10 +84,10 @@ class K2VLVisionConfig(PretrainedConfig):
self.projector_ln_eps = projector_ln_eps
class K2VLConfig(PretrainedConfig):
"""K2-VL model configuration.
class KimiK25Config(PretrainedConfig):
"""Kimi-K2.5 model configuration.
K2-VL extends Kimi-VL with video support using video-chunks.
Kimi-K2.5 extends Kimi-K2 with vision support using video-chunks.
A video-chunk consists of multiple consecutive frames
that are processed together with temporal pooling.
@@ -99,25 +99,25 @@ class K2VLConfig(PretrainedConfig):
pad_token_id: The token ID for padding.
"""
model_type = "k2_vl"
model_type = "kimi_k25"
def __init__(
self,
vision_config: dict | K2VLVisionConfig | None = None,
vision_config: dict | KimiK25VisionConfig | None = None,
text_config: dict | DeepseekV3Config | None = None,
ignore_index: int = -100,
media_placeholder_token_id: int = 163605,
pad_token_id: int = 0,
use_unified_vision_chunk: bool = False,
video_placeholder: str = "<|k2vl_video_placeholder|>",
video_placeholder: str = "<|kimi_k25_video_placeholder|>",
**kwargs,
):
# Vision config
if vision_config is None:
vision_config = K2VLVisionConfig()
vision_config = KimiK25VisionConfig()
elif isinstance(vision_config, dict):
vision_config = K2VLVisionConfig(**vision_config)
self.vision_config: K2VLVisionConfig = vision_config
vision_config = KimiK25VisionConfig(**vision_config)
self.vision_config: KimiK25VisionConfig = vision_config
# Text config
if text_config is None:
@@ -126,6 +126,10 @@ class K2VLConfig(PretrainedConfig):
text_config = DeepseekV3Config(**text_config)
self.text_config: DeepseekV3Config = text_config
# Set mm_hidden_size to text hidden size if not explicitly set
if self.vision_config.mm_hidden_size == self.vision_config.hidden_size:
self.vision_config.mm_hidden_size = self.text_config.hidden_size
# Other config
self.ignore_index = ignore_index
self.media_placeholder_token_id = media_placeholder_token_id