forked from Karylab-cklius/vllm
b9b6306ebe
Signed-off-by: Harshal Janjani <harshaljanjani@gmail.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
80 lines
2.3 KiB
Python
80 lines
2.3 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
import pytest
|
|
|
|
from vllm.assets.image import ImageAsset
|
|
from vllm.config import ModelConfig
|
|
from vllm.multimodal import MULTIMODAL_REGISTRY
|
|
|
|
|
|
@pytest.mark.parametrize("model_id", ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
|
|
def test_multimodal_processor(model_id):
|
|
model_config = ModelConfig(
|
|
model=model_id,
|
|
model_impl="transformers",
|
|
)
|
|
|
|
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
|
|
|
image_pil = ImageAsset("cherry_blossom").pil_image
|
|
mm_data = {"image": image_pil}
|
|
str_prompt = "<|im_start|>user <image>\nWhat is the content of this image?<|im_end|><|im_start|>assistant\n" # noqa: E501
|
|
str_processed_inputs = mm_processor(
|
|
prompt=str_prompt,
|
|
mm_items=mm_processor.info.parse_mm_data(mm_data),
|
|
hf_processor_mm_kwargs={},
|
|
)
|
|
|
|
ids_prompt = [
|
|
151644,
|
|
872,
|
|
220,
|
|
151646,
|
|
198,
|
|
3838,
|
|
374,
|
|
279,
|
|
2213,
|
|
315,
|
|
419,
|
|
2168,
|
|
30,
|
|
151645,
|
|
151644,
|
|
77091,
|
|
198,
|
|
]
|
|
ids_processed_inputs = mm_processor(
|
|
prompt=ids_prompt,
|
|
mm_items=mm_processor.info.parse_mm_data(mm_data),
|
|
hf_processor_mm_kwargs={},
|
|
)
|
|
|
|
assert (
|
|
str_processed_inputs["prompt_token_ids"]
|
|
== ids_processed_inputs["prompt_token_ids"]
|
|
)
|
|
|
|
|
|
def test_image_multiple_inputs():
|
|
"""Multiple images per prompt are each detected as a separate placeholder
|
|
and multi-modal item by the Transformers backend."""
|
|
model_id = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
|
|
model_config = ModelConfig(model=model_id, model_impl="transformers")
|
|
mm_processor = MULTIMODAL_REGISTRY.create_processor(model_config)
|
|
|
|
image = ImageAsset("cherry_blossom").pil_image
|
|
prompt = (
|
|
"<|im_start|>user <image>\n and <image>\n"
|
|
"What do these images show?<|im_end|><|im_start|>assistant\n"
|
|
)
|
|
|
|
result = mm_processor(
|
|
prompt=prompt,
|
|
mm_items=mm_processor.info.parse_mm_data({"image": [image, image]}),
|
|
hf_processor_mm_kwargs={},
|
|
)
|
|
|
|
assert len(result["mm_placeholders"]["image"]) == 2
|
|
assert len(result["mm_kwargs"]["image"]) == 2
|