From 0a201b60cf70283dddf1fd09e0922613095e60f9 Mon Sep 17 00:00:00 2001 From: bairongz Date: Tue, 5 May 2026 18:51:25 +0800 Subject: [PATCH] [Model] support Qianfan-OCR model (#40136) Signed-off-by: bairongz Signed-off-by: zhuangbairong Co-authored-by: zhuangbairong Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- docs/models/supported_models.md | 1 + .../multimodal/generation/test_common.py | 10 ++ .../generation/vlm_utils/model_utils.py | 91 +++++++++++++++ tests/models/registry.py | 4 + vllm/model_executor/models/qianfan_ocr.py | 92 +++++++++++++++ vllm/model_executor/models/registry.py | 4 + vllm/transformers_utils/config.py | 1 + vllm/transformers_utils/configs/__init__.py | 4 + .../transformers_utils/configs/qianfan_ocr.py | 105 ++++++++++++++++++ 9 files changed, 312 insertions(+) create mode 100644 vllm/model_executor/models/qianfan_ocr.py create mode 100644 vllm/transformers_utils/configs/qianfan_ocr.py diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index 97c08a8b62a..e79fec8169f 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -614,6 +614,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen | `Phi4MMForCausalLM` | Phi-4-multimodal | T + I+ / T + A+ / I+ + A+ | `microsoft/Phi-4-multimodal-instruct`, etc. | ✅︎ | ✅︎ | | `Phi4ForCausalLMV` | Phi-4-reasoning-vision | T + I+ | `microsoft/Phi-4-reasoning-vision-15B`, etc. | | ✅︎ | | `PixtralForConditionalGeneration` | Ministral 3 (Mistral format), Mistral 3 (Mistral format), Mistral Large 3 (Mistral format), Pixtral (Mistral format) | T + I+ | `mistralai/Ministral-3-3B-Instruct-2512`, `mistralai/Mistral-Small-3.1-24B-Instruct-2503`, `mistralai/Mistral-Large-3-675B-Instruct-2512` `mistralai/Pixtral-12B-2409` etc. | ✅︎ | ✅︎ | +| `QianfanOCRForConditionalGeneration` | QianfanOCR | T + IE+ | `baidu/Qianfan-OCR`, etc. | ✅︎ | ✅︎ | | `QwenVLForConditionalGeneration`^ | Qwen-VL | T + IE+ | `Qwen/Qwen-VL`, `Qwen/Qwen-VL-Chat`, etc. | ✅︎ | ✅︎ | | `Qwen2AudioForConditionalGeneration` | Qwen2-Audio | T + A+ | `Qwen/Qwen2-Audio-7B-Instruct` | | ✅︎ | | `Qwen2VLForConditionalGeneration` | QVQ, Qwen2-VL | T + IE+ + VE+ | `Qwen/QVQ-72B-Preview`, `Qwen/Qwen2-VL-7B-Instruct`, `Qwen/Qwen2-VL-72B-Instruct`, etc. | ✅︎ | ✅︎ | diff --git a/tests/models/multimodal/generation/test_common.py b/tests/models/multimodal/generation/test_common.py index fdd306633e3..45e693598f1 100644 --- a/tests/models/multimodal/generation/test_common.py +++ b/tests/models/multimodal/generation/test_common.py @@ -928,6 +928,16 @@ VLM_TEST_SETTINGS = { ), ], ), + "qianfan_ocr": VLMTestInfo( + models=["baidu/Qianfan-OCR"], + test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE), + prompt_formatter=lambda img_prompt: f"<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501 + img_idx_to_prompt=lambda idx: "", + max_model_len=4096, + use_tokenizer_eos=True, + auto_cls=AutoModelForImageTextToText, + hf_model_kwargs=model_utils.qianfan_ocr_hf_model_kwargs("baidu/Qianfan-OCR"), + ), "qwen_vl": VLMTestInfo( models=["Qwen/Qwen-VL"], test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE), diff --git a/tests/models/multimodal/generation/vlm_utils/model_utils.py b/tests/models/multimodal/generation/vlm_utils/model_utils.py index 66307de7312..62ea36061c9 100644 --- a/tests/models/multimodal/generation/vlm_utils/model_utils.py +++ b/tests/models/multimodal/generation/vlm_utils/model_utils.py @@ -1554,3 +1554,94 @@ def moondream3_patch_hf_runner(hf_model: HfRunner) -> HfRunner: hf_model.model.generate = types.MethodType(_generate, hf_model.model) return hf_model + + +def qianfan_ocr_hf_model_kwargs(model_name: str) -> dict: + """Return hf_model_kwargs with a patched config for QianfanOCR.""" + from vllm.transformers_utils.configs.qianfan_ocr import QianfanOCRConfig + + config = QianfanOCRConfig.from_pretrained(model_name) + vc = config.vision_config + if isinstance(vc.image_size, int): + vc.image_size = (vc.image_size, vc.image_size) + if isinstance(vc.patch_size, int): + vc.patch_size = (vc.patch_size, vc.patch_size) + return {"config": config} + + +def qianfan_ocr_patch_hf_runner(hf_model: HfRunner) -> HfRunner: + """Patches an HfRunner instance to run QianfanOCR model inference. + + QianfanOCR shares the same architecture as InternVLChatModel, so the + patching logic mirrors ``internvl_patch_hf_runner``. The only difference + is that we load the config via vllm's registered ``QianfanOCRConfig`` + instead of relying on ``trust_remote_code``. + """ + + class QianfanOCRProcessor: + def __init__(self, hf_runner: HfRunner): + self.tokenizer = hf_runner.tokenizer + + from vllm.transformers_utils.configs.qianfan_ocr import QianfanOCRConfig + + self.config = QianfanOCRConfig.from_pretrained(hf_runner.model_name) + self.vision_config = self.config.vision_config + self.use_thumbnail = self.config.use_thumbnail + self.min_num = self.config.min_dynamic_patch + self.max_num = self.config.max_dynamic_patch + self.image_size = self.vision_config.image_size + + # Compute num_image_token from config instead of model attribute, + # since the transformers-native model doesn't expose it. + image_size = self.config.force_image_size or self.vision_config.image_size + patch_size = self.vision_config.patch_size + downsample_ratio = self.config.downsample_ratio + self.num_image_token = int( + (image_size // patch_size) ** 2 * (downsample_ratio**2) + ) + + def __call__( + self, + text: str, + images: PIL.Image.Image | list[PIL.Image.Image] = None, + **kwargs, + ): + from vllm.transformers_utils.processors.internvl import ( + image_to_pixel_values_internvl, + ) + + IMG_START = "" + IMG_END = "" + IMG_CONTEXT = "" + + images = [images] if isinstance(images, PIL.Image.Image) else images + pixel_values_list = [ + image_to_pixel_values_internvl( + image, + input_size=self.image_size, + min_num=self.min_num, + max_num=self.max_num, + use_thumbnail=self.use_thumbnail, + ) + for image in images + ] + num_patches_list = [pv.shape[0] for pv in pixel_values_list] + pixel_values = torch.cat(pixel_values_list, dim=0) + + for num_patches in num_patches_list: + context_tokens = IMG_CONTEXT * self.num_image_token * num_patches + image_tokens = IMG_START + context_tokens + IMG_END + text = text.replace("", image_tokens, 1) + + prompt = self.tokenizer(text, return_tensors="pt") + prompt.update({"pixel_values": pixel_values}) + return prompt + + img_context_token_id = hf_model.tokenizer.convert_tokens_to_ids("") + hf_model.model.img_context_token_id = img_context_token_id + hf_model.processor = QianfanOCRProcessor(hf_model) + hf_model.model.get_output_embeddings = ( + lambda: hf_model.model.language_model.get_output_embeddings() + ) + hf_model.model.generate = types.MethodType(_internvl_generate, hf_model.model) + return hf_model diff --git a/tests/models/registry.py b/tests/models/registry.py index ec6c3473a78..1833c85a8cb 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -1264,6 +1264,10 @@ _MULTIMODAL_EXAMPLE_MODELS = { }, tokenizer_mode="mistral", ), + "QianfanOCRForConditionalGeneration": _HfExamplesInfo( + "baidu/Qianfan-OCR", + min_transformers_version="5.6.0", + ), "QwenVLForConditionalGeneration": _HfExamplesInfo( "Qwen/Qwen-VL", extras={"chat": "Qwen/Qwen-VL-Chat"}, diff --git a/vllm/model_executor/models/qianfan_ocr.py b/vllm/model_executor/models/qianfan_ocr.py new file mode 100644 index 00000000000..ef2bec1e290 --- /dev/null +++ b/vllm/model_executor/models/qianfan_ocr.py @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +# QianfanOCR is built on InternVL with a Qwen3 language backbone. +# The model architecture and weights are fully compatible with InternVLChatModel, +# only the config model_type / architectures strings differ. + +from transformers import PretrainedConfig + +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.quantization.fp8 import Fp8Config +from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.transformers_utils.processors.internvl import ( + InternVLImageProcessor, + InternVLProcessor, +) + +from .internvl import ( + BaseInternVLDummyInputsBuilder, + BaseInternVLMultiModalProcessor, + BaseInternVLProcessingInfo, + InternVLChatModel, +) + + +class QianfanOCRProcessingInfo(BaseInternVLProcessingInfo): + """Image-only ProcessingInfo for QianfanOCR (no video support).""" + + def get_hf_processor(self, **kwargs: object) -> InternVLProcessor: + config = self.get_hf_config() + vision_config = config.vision_config + + kwargs = self.ctx.get_merged_mm_kwargs(kwargs) + kwargs.setdefault("image_size", vision_config.image_size) + kwargs.setdefault("min_dynamic_patch", config.min_dynamic_patch) + kwargs.setdefault("max_dynamic_patch", config.max_dynamic_patch) + kwargs.setdefault("dynamic_image_size", config.dynamic_image_size) + kwargs.setdefault("use_thumbnail", config.use_thumbnail) + + image_processor = InternVLImageProcessor(**kwargs) + image_size = image_processor.image_size + patch_size = vision_config.patch_size + downsample_ratio = config.downsample_ratio + image_seq_length = int((image_size // patch_size) ** 2 * (downsample_ratio**2)) + + return InternVLProcessor( + tokenizer=self.get_tokenizer(), + image_processor=image_processor, + video_processor=None, + image_seq_length=image_seq_length, + ctx_video_token=None, + ) + + +@MULTIMODAL_REGISTRY.register_processor( + BaseInternVLMultiModalProcessor, + info=QianfanOCRProcessingInfo, + dummy_inputs=BaseInternVLDummyInputsBuilder, +) +class QianfanOCRForConditionalGeneration(InternVLChatModel): + """QianfanOCR multimodal model. + + Identical in structure to InternVLChatModel (InternViT vision encoder + + pixel-shuffle MLP connector + Qwen3 language model). This class exists + solely to register the ``QianfanOCRForConditionalGeneration`` architecture + name that appears in the model's config.json. + """ + + def _patch_quant_config( + self, config: PretrainedConfig, quant_config: QuantizationConfig + ) -> None: + super()._patch_quant_config(config, quant_config) + # ignore vit layers to preserve model performance + if isinstance(quant_config, Fp8Config): + _FP8_IGNORED_LAYERS = [ + *( + layer + for i in range(config.vision_config.num_hidden_layers) + for layer in [ + f"vision_model.encoder.layers.{i}.attn.qkv", + f"vision_model.encoder.layers.{i}.attn.proj", + f"vision_model.encoder.layers.{i}.mlp.fc1", + f"vision_model.encoder.layers.{i}.mlp.fc2", + ] + ), + "language_model.lm_head", + "mlp1.1", + "mlp1.3", + ] + for layer in _FP8_IGNORED_LAYERS: + if layer not in quant_config.ignored_layers: + quant_config.ignored_layers.append(layer) diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index 3cce0733bf5..f82f7d00555 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -511,6 +511,10 @@ _MULTIMODAL_MODELS = { "Phi4ForCausalLMV": ("phi4siglip", "Phi4ForCausalLMV"), "Phi4MMForCausalLM": ("phi4mm", "Phi4MMForCausalLM"), "PixtralForConditionalGeneration": ("pixtral", "PixtralForConditionalGeneration"), + "QianfanOCRForConditionalGeneration": ( + "qianfan_ocr", + "QianfanOCRForConditionalGeneration", + ), "QwenVLForConditionalGeneration": ("qwen_vl", "QwenVLForConditionalGeneration"), "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"), "Qwen2_5_VLForConditionalGeneration": ( diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index c95df9c1077..4a1471b2ddc 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -125,6 +125,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict( step3_vl="Step3VLConfig", step3_text="Step3TextConfig", step3p5="Step3p5Config", + qianfan_ocr="QianfanOCRConfig", qwen3_asr="Qwen3ASRConfig", qwen3_next="Qwen3NextConfig", qwen3_5="Qwen3_5Config", diff --git a/vllm/transformers_utils/configs/__init__.py b/vllm/transformers_utils/configs/__init__.py index 99f099adc78..c3466fddd65 100644 --- a/vllm/transformers_utils/configs/__init__.py +++ b/vllm/transformers_utils/configs/__init__.py @@ -70,6 +70,8 @@ _CLASS_TO_MODULE: dict[str, str] = { "Step3VisionEncoderConfig": "vllm.transformers_utils.configs.step3_vl", "Step3TextConfig": "vllm.transformers_utils.configs.step3_vl", "Step3p5Config": "vllm.transformers_utils.configs.step3p5", + "QianfanOCRConfig": "vllm.transformers_utils.configs.qianfan_ocr", + "QianfanOCRVisionConfig": "vllm.transformers_utils.configs.qianfan_ocr", "Qwen3ASRConfig": "vllm.transformers_utils.configs.qwen3_asr", "Qwen3NextConfig": "vllm.transformers_utils.configs.qwen3_next", "Qwen3_5Config": "vllm.transformers_utils.configs.qwen3_5", @@ -135,6 +137,8 @@ __all__ = [ "Step3VisionEncoderConfig", "Step3TextConfig", "Step3p5Config", + "QianfanOCRConfig", + "QianfanOCRVisionConfig", "Qwen3ASRConfig", "Qwen3NextConfig", "Qwen3_5Config", diff --git a/vllm/transformers_utils/configs/qianfan_ocr.py b/vllm/transformers_utils/configs/qianfan_ocr.py new file mode 100644 index 00000000000..da004bb90f4 --- /dev/null +++ b/vllm/transformers_utils/configs/qianfan_ocr.py @@ -0,0 +1,105 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from typing import Any + +from transformers import PretrainedConfig +from transformers.models.auto import CONFIG_MAPPING + + +class QianfanOCRVisionConfig(PretrainedConfig): + model_type = "qianfan_ocr_vision" + + def __init__( + self, + hidden_size: int = 1024, + intermediate_size: int = 4096, + num_hidden_layers: int = 24, + num_attention_heads: int = 16, + num_channels: int = 3, + image_size: int = 448, + patch_size: int = 14, + hidden_act: str = "gelu", + layer_norm_eps: float = 1e-6, + attention_dropout: float = 0.0, + drop_path_rate: float = 0.1, + qkv_bias: bool = True, + qk_normalization: bool = False, + norm_type: str = "layer_norm", + initializer_range: float = 0.02, + initializer_factor: float = 0.1, + use_mask_token: bool = False, + use_mean_pooling: bool = True, + **kwargs: Any, + ): + super().__init__(**kwargs) + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.num_channels = num_channels + self.image_size = image_size + self.patch_size = patch_size + self.hidden_act = hidden_act + self.layer_norm_eps = layer_norm_eps + self.attention_dropout = attention_dropout + self.drop_path_rate = drop_path_rate + self.qkv_bias = qkv_bias + self.qk_normalization = qk_normalization + self.norm_type = norm_type + self.initializer_range = initializer_range + self.initializer_factor = initializer_factor + self.use_mask_token = use_mask_token + self.use_mean_pooling = use_mean_pooling + + +class QianfanOCRConfig(PretrainedConfig): + model_type = "qianfan_ocr" + + def __init__( + self, + vision_config: dict | None = None, + text_config: dict | None = None, + downsample_ratio: float = 0.5, + dynamic_image_size: bool = True, + force_image_size: int = 448, + image_token_id: int = 151671, + max_dynamic_patch: int = 12, + min_dynamic_patch: int = 1, + pad2square: bool = False, + ps_version: str = "v2", + select_layer: int = -1, + template: str = "internvl2_5", + use_thumbnail: bool = True, + tie_word_embeddings: bool = False, + **kwargs: Any, + ): + super().__init__(**kwargs) + + if isinstance(vision_config, dict): + self.vision_config = QianfanOCRVisionConfig(**vision_config) + elif vision_config is None: + self.vision_config = QianfanOCRVisionConfig() + else: + self.vision_config = vision_config + + if isinstance(text_config, dict): + model_type = text_config.get("model_type", "qwen3") + self.text_config = CONFIG_MAPPING[model_type](**text_config) + elif text_config is None: + self.text_config = CONFIG_MAPPING["qwen3"]() + else: + self.text_config = text_config + + self.downsample_ratio = downsample_ratio + self.dynamic_image_size = dynamic_image_size + self.force_image_size = force_image_size + self.image_token_id = image_token_id + self.max_dynamic_patch = max_dynamic_patch + self.min_dynamic_patch = min_dynamic_patch + self.pad2square = pad2square + self.ps_version = ps_version + self.select_layer = select_layer + self.template = template + self.use_thumbnail = use_thumbnail + self.tie_word_embeddings = tie_word_embeddings