[Bugfix][Multimodal] Include media IO config in MM cache hash (#49975)

Signed-off-by: Guan-Ming (Wesley) Chiu <105915352+guan404ming@users.noreply.github.com>

tests/multimodal/test_hasher.py

@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 import uuid
+from io import BytesIO
 from pathlib import Path
 
 import numpy as np
@@ -10,6 +11,7 @@ from PIL import Image, ImageDraw
 
 from vllm.multimodal.hasher import MultiModalHasher
 from vllm.multimodal.media.base import MediaWithBytes
+from vllm.multimodal.media.image import ImageMediaIO
 from vllm.multimodal.parse import MultiModalDataParser
 
 pytestmark = pytest.mark.cpu_test
@@ -134,3 +136,37 @@ def test_hash_image_exif_id():
     assert hasher.hash_kwargs(image=image1) == hasher.hash_kwargs(image=id.bytes)
     # second image has non-UUID in ImageID, so it should hash to the image data
     assert hasher.hash_kwargs(image=image2) == hasher.hash_kwargs(image=image2a)
+
+
+def _rgba_png_bytes() -> bytes:
+    image = Image.new("RGBA", (8, 8), (255, 0, 0, 128))
+    buf = BytesIO()
+    image.save(buf, format="PNG")
+    return buf.getvalue()
+
+
+def test_hash_collision_media_io_config():
+    data = _rgba_png_bytes()
+    white = ImageMediaIO(rgba_background_color=(255, 255, 255)).load_bytes(data)
+    black = ImageMediaIO(rgba_background_color=(0, 0, 0)).load_bytes(data)
+    white2 = ImageMediaIO(rgba_background_color=(255, 255, 255)).load_bytes(data)
+    keep = ImageMediaIO(image_mode=None).load_bytes(data)
+
+    hasher = MultiModalHasher
+    assert hasher.hash_kwargs(image=white) != hasher.hash_kwargs(image=black)
+    assert hasher.hash_kwargs(image=white) != hasher.hash_kwargs(image=keep)
+    assert hasher.hash_kwargs(image=white) == hasher.hash_kwargs(image=white2)
+
+
+def test_hash_media_io_noop_config_preserves_hash():
+    image = Image.new("RGB", (8, 8), (0, 128, 255))
+    buf = BytesIO()
+    image.save(buf, format="PNG")
+    data = buf.getvalue()
+
+    loaded = ImageMediaIO().load_bytes(data)
+    assert loaded.io_config is None
+
+    plain = MediaWithBytes(loaded.media, data)
+    hasher = MultiModalHasher
+    assert hasher.hash_kwargs(image=loaded) == hasher.hash_kwargs(image=plain)

vllm/multimodal/hasher.py

@@ -81,6 +81,11 @@ class MultiModalHasher:
             ):
                 return (exif[Image.ExifTags.Base.ImageID].bytes,)
 
+            if obj.io_config:
+                return cls.iter_item_to_bytes(
+                    "image",
+                    {"io_config": obj.io_config, "data": obj.original_bytes},
+                )
             return cls.iter_item_to_bytes("image", obj.original_bytes)
 
         if isinstance(obj, MediaWithBytes) and isinstance(obj.media, np.ndarray):

vllm/multimodal/media/base.py

@@ -29,6 +29,9 @@ class MediaWithBytes(Generic[_T]):
 
     media: _T
     original_bytes: bytes = field(repr=False)
+    io_config: dict[str, Any] | None = None
+    """Decode settings that altered the media relative to `original_bytes`
+    (e.g. `image_mode` conversion), so they participate in cache hashing."""
 
     def __array__(self, *args, **kwargs) -> np.ndarray:
         """Allow np.array(obj) to return np.array(obj.media)."""

vllm/multimodal/media/image.py

@@ -86,10 +86,17 @@ class ImageMediaIO(MediaIO[Image.Image]):
                 )
             image = normalize_image(image)
             image.load()
-            image = self._convert_image_mode(image)
+            converted = self._convert_image_mode(image)
         except (OSError, Image.UnidentifiedImageError) as e:
             raise ValueError(f"Failed to load image: {e}") from e
-        return MediaWithBytes(image, data)
+
+        io_config = None
+        if converted is not image:
+            io_config = {
+                "image_mode": self.image_mode,
+                "rgba_background_color": self.rgba_background_color,
+            }
+        return MediaWithBytes(converted, data, io_config)
 
     def load_base64(self, media_type: str, data: str) -> MediaWithBytes[Image.Image]:
         return self.load_bytes(pybase64.b64decode(data, validate=True))

vllm/multimodal/processing/inputs.py

@@ -65,7 +65,7 @@ class ProcessorInputs:
                         **{modality: item},
                         **hf_processor_mm_kwargs,
                     )
-                    for item in data_items
+                    for item in data_items.get_all_items_for_hash()
                 ]
 
         return mm_hashes