diff --git a/tests/multimodal/test_hasher.py b/tests/multimodal/test_hasher.py index f1ddc378753..d177fb55e84 100644 --- a/tests/multimodal/test_hasher.py +++ b/tests/multimodal/test_hasher.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import uuid +from io import BytesIO from pathlib import Path import numpy as np @@ -10,6 +11,7 @@ from PIL import Image, ImageDraw from vllm.multimodal.hasher import MultiModalHasher from vllm.multimodal.media.base import MediaWithBytes +from vllm.multimodal.media.image import ImageMediaIO from vllm.multimodal.parse import MultiModalDataParser pytestmark = pytest.mark.cpu_test @@ -134,3 +136,37 @@ def test_hash_image_exif_id(): assert hasher.hash_kwargs(image=image1) == hasher.hash_kwargs(image=id.bytes) # second image has non-UUID in ImageID, so it should hash to the image data assert hasher.hash_kwargs(image=image2) == hasher.hash_kwargs(image=image2a) + + +def _rgba_png_bytes() -> bytes: + image = Image.new("RGBA", (8, 8), (255, 0, 0, 128)) + buf = BytesIO() + image.save(buf, format="PNG") + return buf.getvalue() + + +def test_hash_collision_media_io_config(): + data = _rgba_png_bytes() + white = ImageMediaIO(rgba_background_color=(255, 255, 255)).load_bytes(data) + black = ImageMediaIO(rgba_background_color=(0, 0, 0)).load_bytes(data) + white2 = ImageMediaIO(rgba_background_color=(255, 255, 255)).load_bytes(data) + keep = ImageMediaIO(image_mode=None).load_bytes(data) + + hasher = MultiModalHasher + assert hasher.hash_kwargs(image=white) != hasher.hash_kwargs(image=black) + assert hasher.hash_kwargs(image=white) != hasher.hash_kwargs(image=keep) + assert hasher.hash_kwargs(image=white) == hasher.hash_kwargs(image=white2) + + +def test_hash_media_io_noop_config_preserves_hash(): + image = Image.new("RGB", (8, 8), (0, 128, 255)) + buf = BytesIO() + image.save(buf, format="PNG") + data = buf.getvalue() + + loaded = ImageMediaIO().load_bytes(data) + assert loaded.io_config is None + + plain = MediaWithBytes(loaded.media, data) + hasher = MultiModalHasher + assert hasher.hash_kwargs(image=loaded) == hasher.hash_kwargs(image=plain) diff --git a/vllm/multimodal/hasher.py b/vllm/multimodal/hasher.py index ba9bcd0ea81..2025f51133a 100644 --- a/vllm/multimodal/hasher.py +++ b/vllm/multimodal/hasher.py @@ -81,6 +81,11 @@ class MultiModalHasher: ): return (exif[Image.ExifTags.Base.ImageID].bytes,) + if obj.io_config: + return cls.iter_item_to_bytes( + "image", + {"io_config": obj.io_config, "data": obj.original_bytes}, + ) return cls.iter_item_to_bytes("image", obj.original_bytes) if isinstance(obj, MediaWithBytes) and isinstance(obj.media, np.ndarray): diff --git a/vllm/multimodal/media/base.py b/vllm/multimodal/media/base.py index 3c072a09a54..2927914e645 100644 --- a/vllm/multimodal/media/base.py +++ b/vllm/multimodal/media/base.py @@ -29,6 +29,9 @@ class MediaWithBytes(Generic[_T]): media: _T original_bytes: bytes = field(repr=False) + io_config: dict[str, Any] | None = None + """Decode settings that altered the media relative to `original_bytes` + (e.g. `image_mode` conversion), so they participate in cache hashing.""" def __array__(self, *args, **kwargs) -> np.ndarray: """Allow np.array(obj) to return np.array(obj.media).""" diff --git a/vllm/multimodal/media/image.py b/vllm/multimodal/media/image.py index c98504fdecd..d9fc2132ed5 100644 --- a/vllm/multimodal/media/image.py +++ b/vllm/multimodal/media/image.py @@ -86,10 +86,17 @@ class ImageMediaIO(MediaIO[Image.Image]): ) image = normalize_image(image) image.load() - image = self._convert_image_mode(image) + converted = self._convert_image_mode(image) except (OSError, Image.UnidentifiedImageError) as e: raise ValueError(f"Failed to load image: {e}") from e - return MediaWithBytes(image, data) + + io_config = None + if converted is not image: + io_config = { + "image_mode": self.image_mode, + "rgba_background_color": self.rgba_background_color, + } + return MediaWithBytes(converted, data, io_config) def load_base64(self, media_type: str, data: str) -> MediaWithBytes[Image.Image]: return self.load_bytes(pybase64.b64decode(data, validate=True)) diff --git a/vllm/multimodal/processing/inputs.py b/vllm/multimodal/processing/inputs.py index ae12f4e51b8..35a37d2377a 100644 --- a/vllm/multimodal/processing/inputs.py +++ b/vllm/multimodal/processing/inputs.py @@ -65,7 +65,7 @@ class ProcessorInputs: **{modality: item}, **hf_processor_mm_kwargs, ) - for item in data_items + for item in data_items.get_all_items_for_hash() ] return mm_hashes