diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 17b2932e13b..628b677bd0d 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -1270,6 +1270,20 @@ steps: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s entrypoints/speech_to_text +- label: Entrypoints Integration (Multimodal) + timeout_in_minutes: 180 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + agent_pool: mi300_1 + fast_check: true + torch_nightly: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/ + - tests/entrypoints/multimodal + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -v -s entrypoints/multimodal + - label: Entrypoints Integration (LLM) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] @@ -1327,7 +1341,7 @@ steps: - vllm/platforms/rocm.py commands: - pytest -v -s entrypoints/openai/tool_parsers - - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/openai --ignore=entrypoints/serve --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling --ignore=entrypoints/speech_to_text --ignore=tests/entrypoints/generate + - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/openai --ignore=entrypoints/serve --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling --ignore=entrypoints/speech_to_text --ignore=entrypoints/pooling --ignore=entrypoints/multimodal --ignore=entrypoints/generate - label: OpenAI API correctness # TBD timeout_in_minutes: 180 @@ -2865,6 +2879,20 @@ steps: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s entrypoints/speech_to_text +- label: Entrypoints Integration (Multimodal) + timeout_in_minutes: 180 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi355] + agent_pool: mi355_1 + fast_check: true + torch_nightly: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/ + - tests/entrypoints/multimodal + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -v -s entrypoints/multimodal + - label: Entrypoints Integration (Pooling) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] diff --git a/.buildkite/test_areas/entrypoints.yaml b/.buildkite/test_areas/entrypoints.yaml index 88b0eae739e..e76b2e1769d 100644 --- a/.buildkite/test_areas/entrypoints.yaml +++ b/.buildkite/test_areas/entrypoints.yaml @@ -11,7 +11,7 @@ steps: - tests/entrypoints/ commands: - pytest -v -s entrypoints/openai/tool_parsers - - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/openai --ignore=entrypoints/serve --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling --ignore=entrypoints/speech_to_text --ignore=tests/entrypoints/generate + - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/openai --ignore=entrypoints/serve --ignore=entrypoints/test_chat_utils.py --ignore=entrypoints/pooling --ignore=entrypoints/speech_to_text --ignore=entrypoints/multimodal --ignore=entrypoints/generate - label: Entrypoints Integration (LLM) key: entrypoints-integration-llm @@ -120,6 +120,18 @@ steps: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s entrypoints/speech_to_text +- label: Entrypoints Integration (Multimodal) + device: h200_35gb + key: entrypoints-integration-multimodal + timeout_in_minutes: 50 + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/ + - tests/entrypoints/multimodal + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -v -s entrypoints/multimodal + - label: Entrypoints Integration (Pooling) key: entrypoints-integration-pooling timeout_in_minutes: 50 diff --git a/tests/entrypoints/llm/test_chat.py b/tests/entrypoints/llm/test_chat.py index 7d8a0985279..61cdbd3eee2 100644 --- a/tests/entrypoints/llm/test_chat.py +++ b/tests/entrypoints/llm/test_chat.py @@ -4,7 +4,6 @@ import weakref import pytest -from tests.entrypoints.openai.chat_completion.test_vision import TEST_IMAGE_ASSETS from vllm import LLM from vllm.distributed import cleanup_dist_env_and_memory from vllm.sampling_params import SamplingParams @@ -76,47 +75,6 @@ def test_multi_chat(text_llm): assert len(outputs) == 2 -@pytest.fixture(scope="function") -def vision_llm(): - # pytest caches the fixture so we use weakref.proxy to - # enable garbage collection - llm = LLM( - model="microsoft/Phi-3.5-vision-instruct", - max_model_len=4096, - max_num_seqs=5, - enforce_eager=True, - trust_remote_code=True, - limit_mm_per_prompt={"image": 2}, - seed=0, - ) - - yield weakref.proxy(llm) - - del llm - - cleanup_dist_env_and_memory() - - -@pytest.mark.parametrize( - "image_urls", [[TEST_IMAGE_ASSETS[0], TEST_IMAGE_ASSETS[1]]], indirect=True -) -def test_chat_multi_image(vision_llm, image_urls: list[str]): - messages = [ - { - "role": "user", - "content": [ - *( - {"type": "image_url", "image_url": {"url": image_url}} - for image_url in image_urls - ), - {"type": "text", "text": "What's in this image?"}, - ], - } - ] - outputs = vision_llm.chat(messages) - assert len(outputs) >= 0 - - def test_llm_chat_tokenization_no_double_bos(text_llm): """ LLM.chat() should not add special tokens when using chat templates. diff --git a/tests/entrypoints/multimodal/__init__.py b/tests/entrypoints/multimodal/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/entrypoints/multimodal/conftest.py b/tests/entrypoints/multimodal/conftest.py new file mode 100644 index 00000000000..9c260bc2225 --- /dev/null +++ b/tests/entrypoints/multimodal/conftest.py @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +# Test different image extensions (JPG/PNG) and formats (gray/RGB/RGBA) +TEST_IMAGE_ASSETS = [ + "2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" + "Grayscale_8bits_palette_sample_image.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/Grayscale_8bits_palette_sample_image.png", + "1280px-Venn_diagram_rgb.svg.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/1280px-Venn_diagram_rgb.svg.png", + "RGBA_comp.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/RGBA_comp.png", +] diff --git a/tests/entrypoints/multimodal/llm/__init__.py b/tests/entrypoints/multimodal/llm/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/entrypoints/multimodal/llm/test_chat.py b/tests/entrypoints/multimodal/llm/test_chat.py new file mode 100644 index 00000000000..b670c4c3c4e --- /dev/null +++ b/tests/entrypoints/multimodal/llm/test_chat.py @@ -0,0 +1,50 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import weakref + +import pytest + +from tests.entrypoints.multimodal.conftest import TEST_IMAGE_ASSETS +from vllm import LLM +from vllm.distributed import cleanup_dist_env_and_memory + + +@pytest.fixture(scope="function") +def vision_llm(): + # pytest caches the fixture so we use weakref.proxy to + # enable garbage collection + llm = LLM( + model="microsoft/Phi-3.5-vision-instruct", + max_model_len=4096, + max_num_seqs=5, + enforce_eager=True, + trust_remote_code=True, + limit_mm_per_prompt={"image": 2}, + seed=0, + ) + + yield weakref.proxy(llm) + + del llm + + cleanup_dist_env_and_memory() + + +@pytest.mark.parametrize( + "image_urls", [[TEST_IMAGE_ASSETS[0], TEST_IMAGE_ASSETS[1]]], indirect=True +) +def test_chat_multi_image(vision_llm, image_urls: list[str]): + messages = [ + { + "role": "user", + "content": [ + *( + {"type": "image_url", "image_url": {"url": image_url}} + for image_url in image_urls + ), + {"type": "text", "text": "What's in this image?"}, + ], + } + ] + outputs = vision_llm.chat(messages) + assert len(outputs) >= 0 diff --git a/tests/entrypoints/llm/test_mm_cache_external_injection.py b/tests/entrypoints/multimodal/llm/test_mm_cache_external_injection.py similarity index 98% rename from tests/entrypoints/llm/test_mm_cache_external_injection.py rename to tests/entrypoints/multimodal/llm/test_mm_cache_external_injection.py index 3023457c5fe..f3ae499d635 100644 --- a/tests/entrypoints/llm/test_mm_cache_external_injection.py +++ b/tests/entrypoints/multimodal/llm/test_mm_cache_external_injection.py @@ -15,7 +15,7 @@ import logging import pytest import regex as re -from tests.entrypoints.openai.chat_completion.test_vision import TEST_IMAGE_ASSETS +from tests.entrypoints.multimodal.conftest import TEST_IMAGE_ASSETS from vllm import LLM, SamplingParams from vllm.renderers.params import ChatParams from vllm.v1.metrics import loggers as stat_loggers diff --git a/tests/entrypoints/llm/test_mm_cache_stats.py b/tests/entrypoints/multimodal/llm/test_mm_cache_stats.py similarity index 97% rename from tests/entrypoints/llm/test_mm_cache_stats.py rename to tests/entrypoints/multimodal/llm/test_mm_cache_stats.py index 62c6aa9f7a2..496e98d5ca1 100644 --- a/tests/entrypoints/llm/test_mm_cache_stats.py +++ b/tests/entrypoints/multimodal/llm/test_mm_cache_stats.py @@ -6,7 +6,7 @@ import logging import pytest import regex as re -from tests.entrypoints.openai.chat_completion.test_vision import TEST_IMAGE_ASSETS +from tests.entrypoints.multimodal.conftest import TEST_IMAGE_ASSETS from vllm import LLM from vllm.entrypoints.chat_utils import ChatCompletionMessageParam from vllm.v1.metrics import loggers as stat_loggers diff --git a/tests/entrypoints/llm/test_mm_embeds_only.py b/tests/entrypoints/multimodal/llm/test_mm_embeds_only.py similarity index 100% rename from tests/entrypoints/llm/test_mm_embeds_only.py rename to tests/entrypoints/multimodal/llm/test_mm_embeds_only.py diff --git a/tests/entrypoints/llm/test_mm_processor_kwargs.py b/tests/entrypoints/multimodal/llm/test_mm_processor_kwargs.py similarity index 100% rename from tests/entrypoints/llm/test_mm_processor_kwargs.py rename to tests/entrypoints/multimodal/llm/test_mm_processor_kwargs.py diff --git a/tests/entrypoints/multimodal/openai/__init__.py b/tests/entrypoints/multimodal/openai/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/entrypoints/multimodal/openai/chat_completion/__init__.py b/tests/entrypoints/multimodal/openai/chat_completion/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/entrypoints/openai/chat_completion/test_audio.py b/tests/entrypoints/multimodal/openai/chat_completion/test_audio.py similarity index 100% rename from tests/entrypoints/openai/chat_completion/test_audio.py rename to tests/entrypoints/multimodal/openai/chat_completion/test_audio.py diff --git a/tests/entrypoints/openai/chat_completion/test_audio_in_video.py b/tests/entrypoints/multimodal/openai/chat_completion/test_audio_in_video.py similarity index 100% rename from tests/entrypoints/openai/chat_completion/test_audio_in_video.py rename to tests/entrypoints/multimodal/openai/chat_completion/test_audio_in_video.py diff --git a/tests/entrypoints/openai/chat_completion/test_completion_with_image_embeds.py b/tests/entrypoints/multimodal/openai/chat_completion/test_chat_completion_with_image_embeds.py similarity index 98% rename from tests/entrypoints/openai/chat_completion/test_completion_with_image_embeds.py rename to tests/entrypoints/multimodal/openai/chat_completion/test_chat_completion_with_image_embeds.py index b30556fbc81..4d4aedfb359 100644 --- a/tests/entrypoints/openai/chat_completion/test_completion_with_image_embeds.py +++ b/tests/entrypoints/multimodal/openai/chat_completion/test_chat_completion_with_image_embeds.py @@ -52,7 +52,7 @@ async def client_with_image_embeds(server_with_image_embeds): @pytest.mark.asyncio @pytest.mark.parametrize("model_name", [MODEL_NAME]) @pytest.mark.parametrize("dtype", [torch.half, torch.float16, torch.float32]) -async def test_completions_with_image_embeds( +async def test_chat_completions_with_image_embeds( client_with_image_embeds: openai.AsyncOpenAI, model_name: str, image_assets: ImageTestAssets, diff --git a/tests/entrypoints/openai/chat_completion/test_chat_completion_with_mixed_audio_embeds.py b/tests/entrypoints/multimodal/openai/chat_completion/test_chat_completion_with_mixed_audio_embeds.py similarity index 100% rename from tests/entrypoints/openai/chat_completion/test_chat_completion_with_mixed_audio_embeds.py rename to tests/entrypoints/multimodal/openai/chat_completion/test_chat_completion_with_mixed_audio_embeds.py diff --git a/tests/entrypoints/openai/chat_completion/test_chat_completion_with_mixed_image_embeds.py b/tests/entrypoints/multimodal/openai/chat_completion/test_chat_completion_with_mixed_image_embeds.py similarity index 100% rename from tests/entrypoints/openai/chat_completion/test_chat_completion_with_mixed_image_embeds.py rename to tests/entrypoints/multimodal/openai/chat_completion/test_chat_completion_with_mixed_image_embeds.py diff --git a/tests/entrypoints/openai/chat_completion/test_default_mm_loras.py b/tests/entrypoints/multimodal/openai/chat_completion/test_default_mm_loras.py similarity index 100% rename from tests/entrypoints/openai/chat_completion/test_default_mm_loras.py rename to tests/entrypoints/multimodal/openai/chat_completion/test_default_mm_loras.py diff --git a/tests/entrypoints/openai/chat_completion/test_video.py b/tests/entrypoints/multimodal/openai/chat_completion/test_video.py similarity index 100% rename from tests/entrypoints/openai/chat_completion/test_video.py rename to tests/entrypoints/multimodal/openai/chat_completion/test_video.py diff --git a/tests/entrypoints/openai/chat_completion/test_vision.py b/tests/entrypoints/multimodal/openai/chat_completion/test_vision.py similarity index 96% rename from tests/entrypoints/openai/chat_completion/test_vision.py rename to tests/entrypoints/multimodal/openai/chat_completion/test_vision.py index 6cb8433423b..b33311f8af9 100644 --- a/tests/entrypoints/openai/chat_completion/test_vision.py +++ b/tests/entrypoints/multimodal/openai/chat_completion/test_vision.py @@ -8,6 +8,7 @@ import pytest import pytest_asyncio from transformers import AutoProcessor +from tests.entrypoints.multimodal.conftest import TEST_IMAGE_ASSETS from tests.utils import ROCM_ENV_OVERRIDES, ROCM_EXTRA_ARGS, RemoteOpenAIServer from vllm.multimodal.media import MediaWithBytes from vllm.multimodal.utils import encode_image_url, fetch_image @@ -16,14 +17,6 @@ from vllm.platforms import current_platform MODEL_NAME = "microsoft/Phi-3.5-vision-instruct" MAXIMUM_IMAGES = 2 -# Test different image extensions (JPG/PNG) and formats (gray/RGB/RGBA) -TEST_IMAGE_ASSETS = [ - "2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" - "Grayscale_8bits_palette_sample_image.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/Grayscale_8bits_palette_sample_image.png", - "1280px-Venn_diagram_rgb.svg.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/1280px-Venn_diagram_rgb.svg.png", - "RGBA_comp.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/RGBA_comp.png", -] - # Required terms for beam search validation # Each entry is a list of term groups - ALL groups must match # Each group is a list of alternatives - at least ONE term in the group must appear diff --git a/tests/entrypoints/openai/chat_completion/test_vision_embeds.py b/tests/entrypoints/multimodal/openai/chat_completion/test_vision_embeds.py similarity index 100% rename from tests/entrypoints/openai/chat_completion/test_vision_embeds.py rename to tests/entrypoints/multimodal/openai/chat_completion/test_vision_embeds.py diff --git a/tests/entrypoints/multimodal/openai/responses/__init__.py b/tests/entrypoints/multimodal/openai/responses/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/entrypoints/openai/responses/test_image.py b/tests/entrypoints/multimodal/openai/responses/test_image.py similarity index 86% rename from tests/entrypoints/openai/responses/test_image.py rename to tests/entrypoints/multimodal/openai/responses/test_image.py index 644d8ce0068..36ebdde810c 100644 --- a/tests/entrypoints/openai/responses/test_image.py +++ b/tests/entrypoints/multimodal/openai/responses/test_image.py @@ -7,19 +7,13 @@ import openai import pytest import pytest_asyncio +from tests.entrypoints.multimodal.conftest import TEST_IMAGE_ASSETS from tests.utils import RemoteOpenAIServer from vllm.multimodal.utils import encode_image_url # Use a small vision model for testing MODEL_NAME = "Qwen/Qwen2.5-VL-3B-Instruct" MAXIMUM_IMAGES = 2 -# Test different image extensions (JPG/PNG) and formats (gray/RGB/RGBA) -TEST_IMAGE_ASSETS = [ - "2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg" - "Grayscale_8bits_palette_sample_image.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/Grayscale_8bits_palette_sample_image.png", - "1280px-Venn_diagram_rgb.svg.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/1280px-Venn_diagram_rgb.svg.png", - "RGBA_comp.png", # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/RGBA_comp.png", -] @pytest.fixture(scope="module")