update frontend

Signed-off-by: esmeetu <jasonailu87@gmail.com>

.buildkite/performance-benchmarks/README.md

@@ -83,7 +83,6 @@ We test the throughput by using `vllm bench serve` with request rate = inf to co
         "server_parameters": {
             "model": "meta-llama/Meta-Llama-3-8B",
             "tensor_parallel_size": 1,
-            "swap_space": 16,
             "disable_log_stats": "",
             "load_format": "dummy"
         },

.buildkite/performance-benchmarks/tests/serving-tests-hpu.json

@@ -10,7 +10,6 @@
         "server_parameters": {
             "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
             "tensor_parallel_size": 1,
-            "swap_space": 16,
             "disable_log_stats": "",
             "load_format": "dummy",
             "max-model-len": 2048,
@@ -37,7 +36,6 @@
         "server_parameters": {
             "model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
             "tensor_parallel_size": 4,
-            "swap_space": 16,
             "disable_log_stats": "",
             "load_format": "dummy",
             "max-model-len": 2048,
@@ -64,7 +62,6 @@
         "server_parameters": {
             "model": "mistralai/Mixtral-8x7B-Instruct-v0.1",
             "tensor_parallel_size": 2,
-            "swap_space": 16,
             "disable_log_stats": "",
             "load_format": "dummy",
             "max-model-len": 2048,
@@ -91,7 +88,6 @@
         "server_parameters": {
             "model": "deepseek-ai/DeepSeek-R1",
             "tensor_parallel_size": 8,
-            "swap_space": 16,
             "disable_log_stats": "",
             "load_format": "dummy",
             "max-model-len": 2048,

.buildkite/performance-benchmarks/tests/serving-tests.json

@@ -5,7 +5,6 @@
         "server_parameters": {
             "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
             "tensor_parallel_size": 1,
-            "swap_space": 16,
             "disable_log_stats": "",
             "load_format": "dummy"
         },
@@ -23,7 +22,6 @@
         "server_parameters": {
             "model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
             "tensor_parallel_size": 4,
-            "swap_space": 16,
             "disable_log_stats": "",
             "load_format": "dummy"
         },
@@ -41,7 +39,6 @@
         "server_parameters": {
             "model": "mistralai/Mixtral-8x7B-Instruct-v0.1",
             "tensor_parallel_size": 2,
-            "swap_space": 16,
             "disable_log_stats": "",
             "load_format": "dummy"
         },
@@ -59,7 +56,6 @@
         "server_parameters": {
             "model": "meta-llama/Meta-Llama-3.1-70B-Instruct", 
             "tensor_parallel_size": 4,
-            "swap_space": 16,
             "speculative_config": {
                 "model": "turboderp/Qwama-0.5B-Instruct",
                 "num_speculative_tokens": 4,

.buildkite/test-amd.yaml

@@ -2801,7 +2801,7 @@ steps:
   - vllm/v1/attention/selector.py
   - vllm/platforms/cuda.py
   commands:
-    rocm-smi
+    - rocm-smi
     - python3 examples/offline_inference/basic/chat.py
     # Attention
     # num_heads2 broken by https://github.com/flashinfer-ai/flashinfer/issues/1353
@@ -3283,7 +3283,7 @@ steps:
   commands:
   - bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh 0.25 200 8010
 
-- label: Qwen3-30B-A3B-FP8-block Accuracy (B200/MI355)
+- label: Qwen3-30B-A3B-FP8-block Accuracy (B200-MI355)
   mirror_hardwares: [amdexperimental, amdproduction, amdmi355]
   agent_pool: mi355_2
   timeout_in_minutes: 60
@@ -3305,7 +3305,7 @@ steps:
   commands:
   - bash .buildkite/scripts/scheduled_integration_test/qwen3_next_mtp_async_eplb.sh 0.8 1319 8040
 
-- label: Attention Benchmarks Smoke Test (B200/MI355)
+- label: Attention Benchmarks Smoke Test (B200-MI355)
   device: b200
   mirror_hardwares: [amdexperimental, amdmi355]
   agent_pool: mi355_2

benchmarks/attention_benchmarks/mla_runner.py

@@ -145,7 +145,6 @@ def create_minimal_vllm_config(
     cache_config = CacheConfig(
         block_size=block_size,
         gpu_memory_utilization=0.9,
-        swap_space=0,
         cache_dtype="auto",
         enable_prefix_caching=False,
     )

benchmarks/attention_benchmarks/runner.py

@@ -141,7 +141,6 @@ def _create_vllm_config(
     cache_config = CacheConfig(
         block_size=config.block_size,
         cache_dtype="auto",
-        swap_space=0,
     )
     cache_config.num_gpu_blocks = max_num_blocks
     cache_config.num_cpu_blocks = 0

docs/design/metrics.md

@@ -507,10 +507,10 @@ longer relevant in v1:
 - `vllm:num_requests_swapped`
 - `vllm:cpu_cache_usage_perc`
 
-In this mode, when a request is preempted (e.g. to make room in KV
-cache to complete other requests), we swap kv cache blocks out to CPU
-memory. This is also known as "KV cache offloading" and is configured
-with `--swap-space` and `--preemption-mode`.
+In this mode, when a request was preempted (e.g. to make room in KV
+cache to complete other requests), kv cache blocks were swapped out to
+CPU memory. The `--swap-space` flag has been removed as this feature
+is no longer used in V1.
 
 Historically, [vLLM has long supported beam search](https://github.com/vllm-project/vllm/issues/6226). The
 SequenceGroup encapsulated the idea of N Sequences which

docs/serving/integrations/llamaindex.md

@@ -17,7 +17,7 @@ llm = Vllm(
     model="microsoft/Orca-2-7b",
     tensor_parallel_size=4,
     max_new_tokens=100,
-    vllm_kwargs={"swap_space": 1, "gpu_memory_utilization": 0.5},
+    vllm_kwargs={"gpu_memory_utilization": 0.5},
 )
 ```
 

tests/conftest.py

@@ -794,7 +794,6 @@ class VllmRunner:
         tensor_parallel_size: int = 1,
         block_size: int = 16 if not torch.xpu.is_available() else 64,
         enable_chunked_prefill: bool | None = False,
-        swap_space: int = 4,
         enforce_eager: bool | None = False,
         # Set this to avoid hanging issue
         default_torch_num_threads: int | None = None,
@@ -831,7 +830,6 @@ class VllmRunner:
                 trust_remote_code=trust_remote_code,
                 dtype=dtype,
                 seed=seed,
-                swap_space=swap_space,
                 enforce_eager=enforce_eager,
                 disable_log_stats=disable_log_stats,
                 tensor_parallel_size=tensor_parallel_size,

tests/distributed/test_torchrun_example.py

@@ -22,7 +22,7 @@ prompts = [
 
 sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
 
-# set different `gpu_memory_utilization` and `swap_space` for different ranks,
+# set different `gpu_memory_utilization` for different ranks,
 # to test if all ranks agree on the same kv cache configuration.
 llm = LLM(
     model="facebook/opt-125m",
@@ -30,7 +30,6 @@ llm = LLM(
     pipeline_parallel_size=int(os.getenv("PP_SIZE", 1)),
     distributed_executor_backend="external_launcher",
     gpu_memory_utilization=random.uniform(0.7, 0.9),
-    swap_space=random.randint(1, 4),
     seed=0,
 )
 

tests/distributed/test_torchrun_example_moe.py

@@ -28,7 +28,7 @@ if dp_size > 1:
 
 sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
 
-# set different `gpu_memory_utilization` and `swap_space` for different ranks,
+# set different `gpu_memory_utilization` for different ranks,
 # to test if all ranks agree on the same kv cache configuration.
 llm = LLM(
     model="microsoft/Phi-mini-MoE-instruct",
@@ -37,7 +37,6 @@ llm = LLM(
     enable_expert_parallel=int(os.getenv("ENABLE_EP", "0")) == 1,
     distributed_executor_backend="external_launcher",
     gpu_memory_utilization=random.uniform(0.7, 0.9),
-    swap_space=random.randint(1, 4),
     seed=0,
 )
 

tests/entrypoints/openai/test_dashboard.py

@@ -0,0 +1,292 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Tests for vLLM dashboard functionality."""
+
+import os
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
+
+from vllm.entrypoints.serve.dashboard.api_router import (
+    _get_dashboard_html,
+    _get_explicit_cli_args,
+    _get_vllm_env_vars,
+    attach_router,
+    router,
+)
+from vllm.version import __version__ as VLLM_VERSION
+
+
+class TestDashboardHelpers:
+    """Test helper functions in dashboard api_router."""
+
+    def test_get_vllm_env_vars(self):
+        """Test that _get_vllm_env_vars returns VLLM environment variables."""
+        vllm_envs, explicit_envs = _get_vllm_env_vars()
+
+        # Should return a dict of VLLM_* variables
+        assert isinstance(vllm_envs, dict)
+        assert isinstance(explicit_envs, set)
+
+        # All keys should start with VLLM_
+        for key in vllm_envs:
+            assert key.startswith("VLLM_"), f"Key {key} doesn't start with VLLM_"
+
+        # Should not include keys with "KEY" (secrets)
+        for key in vllm_envs:
+            assert "KEY" not in key, f"Key {key} contains KEY (potential secret)"
+
+    def test_get_vllm_env_vars_explicit_detection(self):
+        """Test that explicitly set env vars are detected."""
+        test_var = "VLLM_TEST_DASHBOARD_VAR"
+        try:
+            os.environ[test_var] = "test_value"
+            # Need to reload envs module to pick up the new var
+            # For this test, we just verify the mechanism works
+            _, explicit_envs = _get_vllm_env_vars()
+            # The test var won't be in vllm.envs, but the mechanism is tested
+            assert isinstance(explicit_envs, set)
+        finally:
+            os.environ.pop(test_var, None)
+
+    def test_get_explicit_cli_args_none(self):
+        """Test _get_explicit_cli_args with None args."""
+        result = _get_explicit_cli_args(None)
+        assert result == set()
+
+    def test_get_explicit_cli_args_with_args(self):
+        """Test _get_explicit_cli_args with mock args."""
+        mock_args = MagicMock()
+        mock_args.model = "test-model"
+        mock_args.served_model_name = "test-served-model"
+        mock_args.dtype = "float16"  # Non-default value
+
+        result = _get_explicit_cli_args(mock_args)
+
+        # model should always be explicit if set
+        assert "model" in result
+        assert "served_model_name" in result
+
+    def test_get_dashboard_html(self):
+        """Test that dashboard HTML is loaded correctly."""
+        html = _get_dashboard_html()
+
+        assert isinstance(html, str)
+        assert len(html) > 0
+        assert "<!DOCTYPE html>" in html
+        assert "vLLM Dashboard" in html
+        assert "</html>" in html
+
+
+class TestDashboardRouter:
+    """Test dashboard router endpoints."""
+
+    @pytest.fixture
+    def app(self):
+        """Create a test FastAPI app with dashboard router."""
+        app = FastAPI()
+        app.include_router(router)
+
+        # Mock app state
+        app.state.openai_serving_models = None
+        app.state.vllm_config = None
+        app.state.args = None
+        app.state.engine_client = None
+
+        return app
+
+    @pytest.fixture
+    def client(self, app):
+        """Create a test client."""
+        return TestClient(app)
+
+    def test_dashboard_index(self, client):
+        """Test GET /dashboard returns HTML page."""
+        response = client.get("/dashboard")
+
+        assert response.status_code == 200
+        assert "text/html" in response.headers["content-type"]
+        assert "vLLM Dashboard" in response.text
+
+    def test_dashboard_api_info(self, client):
+        """Test GET /dashboard/api/info returns server info."""
+        response = client.get("/dashboard/api/info")
+
+        assert response.status_code == 200
+        data = response.json()
+
+        assert "version" in data
+        assert data["version"] == VLLM_VERSION
+        assert "status" in data
+        assert data["status"] == "running"
+
+    def test_dashboard_api_info_with_models(self, app):
+        """Test /dashboard/api/info includes model info when available."""
+        # Mock serving models
+        mock_model = MagicMock()
+        mock_model.id = "test-model"
+        mock_model.root = "test-model-root"
+
+        mock_models_response = MagicMock()
+        mock_models_response.data = [mock_model]
+
+        mock_serving_models = AsyncMock()
+        mock_serving_models.show_available_models = AsyncMock(
+            return_value=mock_models_response
+        )
+
+        app.state.openai_serving_models = mock_serving_models
+
+        client = TestClient(app)
+        response = client.get("/dashboard/api/info")
+
+        assert response.status_code == 200
+        data = response.json()
+
+        assert "models" in data
+        assert len(data["models"]) == 1
+        assert data["models"][0]["id"] == "test-model"
+        assert data["models"][0]["root"] == "test-model-root"
+
+    def test_dashboard_api_info_health_check(self, app):
+        """Test /dashboard/api/info uses engine health check."""
+        # Mock healthy engine client
+        mock_engine_client = AsyncMock()
+        mock_engine_client.check_health = AsyncMock(return_value=None)
+        mock_engine_client.is_sleeping = AsyncMock(return_value=False)
+        mock_engine_client.is_paused = AsyncMock(return_value=False)
+
+        app.state.engine_client = mock_engine_client
+
+        client = TestClient(app)
+        response = client.get("/dashboard/api/info")
+
+        assert response.status_code == 200
+        data = response.json()
+        assert data["status"] == "running"
+        assert data["is_sleeping"] is False
+        assert data["is_paused"] is False
+
+    def test_dashboard_api_info_unhealthy_engine(self, app):
+        """Test /dashboard/api/info reports unhealthy when engine is dead."""
+        from vllm.v1.engine.exceptions import EngineDeadError
+
+        # Mock unhealthy engine client
+        mock_engine_client = AsyncMock()
+        mock_engine_client.check_health = AsyncMock(side_effect=EngineDeadError())
+        mock_engine_client.is_sleeping = AsyncMock(return_value=False)
+        mock_engine_client.is_paused = AsyncMock(return_value=False)
+
+        app.state.engine_client = mock_engine_client
+
+        client = TestClient(app)
+        response = client.get("/dashboard/api/info")
+
+        assert response.status_code == 200
+        data = response.json()
+        assert data["status"] == "unhealthy"
+
+    def test_dashboard_api_info_sleeping_engine(self, app):
+        """Test /dashboard/api/info reports is_sleeping when engine is sleeping."""
+        # Mock sleeping engine client
+        mock_engine_client = AsyncMock()
+        mock_engine_client.check_health = AsyncMock(return_value=None)
+        mock_engine_client.is_sleeping = AsyncMock(return_value=True)
+        mock_engine_client.is_paused = AsyncMock(return_value=False)
+
+        app.state.engine_client = mock_engine_client
+
+        client = TestClient(app)
+        response = client.get("/dashboard/api/info")
+
+        assert response.status_code == 200
+        data = response.json()
+        assert data["status"] == "running"
+        assert data["is_sleeping"] is True
+
+    def test_dashboard_api_info_server_load(self, app):
+        """Test /dashboard/api/info includes server_load when available."""
+        app.state.server_load_metrics = 5
+
+        client = TestClient(app)
+        response = client.get("/dashboard/api/info")
+
+        assert response.status_code == 200
+        data = response.json()
+        assert data["server_load"] == 5
+
+    def test_dashboard_api_info_paused_engine(self, app):
+        """Test /dashboard/api/info reports is_paused when engine is paused."""
+        # Mock paused engine client
+        mock_engine_client = AsyncMock()
+        mock_engine_client.check_health = AsyncMock(return_value=None)
+        mock_engine_client.is_sleeping = AsyncMock(return_value=False)
+        mock_engine_client.is_paused = AsyncMock(return_value=True)
+
+        app.state.engine_client = mock_engine_client
+
+        client = TestClient(app)
+        response = client.get("/dashboard/api/info")
+
+        assert response.status_code == 200
+        data = response.json()
+        assert data["status"] == "running"
+        assert data["is_paused"] is True
+
+    def test_dashboard_api_metrics(self, client):
+        """Test GET /dashboard/api/metrics returns metrics."""
+        response = client.get("/dashboard/api/metrics")
+
+        assert response.status_code == 200
+        data = response.json()
+
+        # Should return a dict (may be empty if no metrics available)
+        assert isinstance(data, dict)
+
+
+class TestAttachRouter:
+    """Test attach_router function."""
+
+    def test_attach_router_disabled(self):
+        """Test router is not attached when dashboard is disabled."""
+        app = FastAPI()
+        mock_args = MagicMock()
+        mock_args.enable_dashboard = False
+        app.state.args = mock_args
+
+        attach_router(app)
+
+        # Dashboard routes should not be available
+        client = TestClient(app)
+        response = client.get("/dashboard")
+        assert response.status_code == 404
+
+    def test_attach_router_enabled(self):
+        """Test router is attached when dashboard is enabled."""
+        app = FastAPI()
+        mock_args = MagicMock()
+        mock_args.enable_dashboard = True
+        app.state.args = mock_args
+        app.state.openai_serving_models = None
+        app.state.vllm_config = None
+
+        attach_router(app)
+
+        # Dashboard routes should be available
+        client = TestClient(app)
+        response = client.get("/dashboard")
+        assert response.status_code == 200
+
+    def test_attach_router_no_args(self):
+        """Test router is not attached when args is None."""
+        app = FastAPI()
+        app.state.args = None
+
+        attach_router(app)
+
+        # Dashboard routes should not be available
+        client = TestClient(app)
+        response = client.get("/dashboard")
+        assert response.status_code == 404

tests/entrypoints/openai/test_realtime_validation.py

@@ -13,7 +13,7 @@ import websockets
 
 from vllm.assets.audio import AudioAsset
 
-from ...utils import RemoteOpenAIServer
+from ...utils import ROCM_ENV_OVERRIDES, ROCM_EXTRA_ARGS, RemoteOpenAIServer
 from .conftest import add_attention_backend
 
 MISTRAL_FORMAT_ARGS = [
@@ -23,7 +23,7 @@ MISTRAL_FORMAT_ARGS = [
     "mistral",
     "--load_format",
     "mistral",
-]
+] + ROCM_EXTRA_ARGS
 
 MODEL_NAME = "mistralai/Voxtral-Mini-4B-Realtime-2602"
 
@@ -77,7 +77,9 @@ async def test_multi_chunk_streaming(
 
     add_attention_backend(server_args, rocm_aiter_fa_attention)
 
-    with RemoteOpenAIServer(model_name, server_args) as remote_server:
+    with RemoteOpenAIServer(
+        model_name, server_args, env_dict=ROCM_ENV_OVERRIDES
+    ) as remote_server:
         ws_url = _get_websocket_url(remote_server)
         async with websockets.connect(ws_url) as ws:
             # Receive session.created
@@ -178,7 +180,9 @@ async def test_empty_commit_does_not_crash_engine(
 
     add_attention_backend(server_args, rocm_aiter_fa_attention)
 
-    with RemoteOpenAIServer(model_name, server_args) as remote_server:
+    with RemoteOpenAIServer(
+        model_name, server_args, env_dict=ROCM_ENV_OVERRIDES
+    ) as remote_server:
         ws_url = _get_websocket_url(remote_server)
 
         # --- First connection: empty commit (no audio appended) ----------

tests/entrypoints/openai/test_video.py

@@ -35,6 +35,8 @@ def server():
         "--trust-remote-code",
         "--limit-mm-per-prompt",
         json.dumps({"video": MAXIMUM_VIDEOS}),
+        "--media-io-kwargs",
+        json.dumps({"video": {"num_frames": 32}}),
     ]
 
     # ROCm: Increase timeouts to handle potential network delays and slower
@@ -127,6 +129,73 @@ async def test_single_chat_session_video(
     assert message.content is not None and len(message.content) >= 0
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
+@pytest.mark.parametrize("video_url", [TEST_VIDEO_URLS[0]])
+async def test_request_media_io_kwargs_override_uses_fewer_video_frames(
+    client: openai.AsyncOpenAI, model_name: str, video_url: str
+):
+    messages = dummy_messages_from_video_url(video_url)
+
+    default_resp = await client.chat.completions.create(
+        model=model_name,
+        messages=messages,
+        max_completion_tokens=1,
+        temperature=0.0,
+    )
+    override_resp = await client.chat.completions.create(
+        model=model_name,
+        messages=messages,
+        max_completion_tokens=1,
+        temperature=0.0,
+        extra_body={
+            "media_io_kwargs": {
+                "video": {
+                    "num_frames": 4,
+                }
+            }
+        },
+    )
+
+    assert default_resp.usage is not None
+    assert override_resp.usage is not None
+    assert override_resp.usage.prompt_tokens < default_resp.usage.prompt_tokens
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
+@pytest.mark.parametrize("video_url", [TEST_VIDEO_URLS[0]])
+async def test_invalid_num_frames_request_recoverable(
+    client: openai.AsyncOpenAI, model_name: str, video_url: str
+):
+    messages = dummy_messages_from_video_url(video_url)
+
+    with pytest.raises((openai.BadRequestError, openai.APIStatusError)):
+        await client.chat.completions.create(
+            model=model_name,
+            messages=messages,
+            max_completion_tokens=1,
+            temperature=0.0,
+            extra_body={
+                "media_io_kwargs": {
+                    "video": {
+                        "num_frames": "invalid",
+                    }
+                }
+            },
+        )
+
+    # Server should still handle subsequent requests after the failed one.
+    recovery_resp = await client.chat.completions.create(
+        model=model_name,
+        messages=messages,
+        max_completion_tokens=1,
+        temperature=0.0,
+    )
+    recovery_msg = recovery_resp.choices[0].message
+    assert recovery_msg.content is not None and len(recovery_msg.content) >= 0
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize("model_name", [MODEL_NAME])
 @pytest.mark.parametrize("video_url", TEST_VIDEO_URLS)

tests/entrypoints/pooling/embed/test_online_vision.py

@@ -127,6 +127,39 @@ def test_chat_image_base64_request(server: RemoteOpenAIServer, model_name: str):
     assert output.usage.prompt_tokens == 767
 
 
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
+def test_chat_image_with_media_io_kwargs(server: RemoteOpenAIServer, model_name: str):
+    rgba_image_url = (
+        "https://vllm-public-assets.s3.us-west-2.amazonaws.com"
+        "/vision_model_images/RGBA_comp.png"
+    )
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "Represent the user's input."},
+                {"type": "image_url", "image_url": {"url": rgba_image_url}},
+            ],
+        }
+    ]
+
+    response = requests.post(
+        server.url_for("v1/embeddings"),
+        json={
+            "model": model_name,
+            "messages": messages,
+            "media_io_kwargs": {
+                "image": {"rgba_background_color": [0, 0, 0]},
+            },
+        },
+    )
+    response.raise_for_status()
+
+    output = EmbeddingResponse.model_validate(response.json())
+    assert len(output.data) == 1
+    assert len(output.data[0].embedding) == 3072
+
+
 def get_hf_prompt_tokens(model_name, content, image_url):
     processor = AutoProcessor.from_pretrained(
         model_name, trust_remote_code=True, num_crops=4

tests/lora/test_worker.py

@@ -64,7 +64,6 @@ def test_worker_apply_lora(qwen3_lora_files):
         device_config=DeviceConfig("cuda"),
         cache_config=CacheConfig(
             block_size=16,
-            swap_space=0,
             cache_dtype="auto",
         ),
         lora_config=LoRAConfig(

tests/v1/attention/utils.py

@@ -182,7 +182,6 @@ def create_vllm_config(
     cache_config = CacheConfig(
         block_size=block_size,
         cache_dtype="auto",
-        swap_space=0,
     )
     # Set cache blocks for testing
     #   (these may be set during initialization normally)

tests/v1/core/test_scheduler.py

@@ -1776,7 +1776,6 @@ def create_scheduler_with_priority(
     cache_config = CacheConfig(
         block_size=block_size,
         gpu_memory_utilization=0.9,
-        swap_space=0,
         cache_dtype="auto",
         enable_prefix_caching=enable_prefix_caching,
     )
@@ -3726,7 +3725,6 @@ def _create_encoder_decoder_scheduler(
     cache_config = CacheConfig(
         block_size=block_size,
         gpu_memory_utilization=0.9,
-        swap_space=0,
         cache_dtype="auto",
         enable_prefix_caching=False,
     )

tests/v1/core/utils.py

@@ -94,7 +94,6 @@ def create_scheduler(
     cache_config = CacheConfig(
         block_size=block_size,
         gpu_memory_utilization=0.9,
-        swap_space=0,
         cache_dtype="auto",
         enable_prefix_caching=enable_prefix_caching,
     )

tests/v1/engine/test_engine_core.py

@@ -506,7 +506,6 @@ def test_encoder_instance_zero_kv_cache(
     cache_config = CacheConfig(
         block_size=16,
         gpu_memory_utilization=gpu_memory_utilization,
-        swap_space=0,
         cache_dtype="auto",
         enable_prefix_caching=enable_prefix_caching,
     )

tests/v1/kv_connector/unit/test_moriio_connector.py

@@ -206,7 +206,6 @@ def create_vllm_config(
     cache_config = CacheConfig(
         block_size=block_size,
         gpu_memory_utilization=0.9,
-        swap_space=0,
         cache_dtype="auto",
         enable_prefix_caching=True,
     )

tests/v1/kv_connector/unit/utils.py

@@ -118,7 +118,6 @@ def create_vllm_config(
     cache_config = CacheConfig(
         block_size=block_size,
         gpu_memory_utilization=0.9,
-        swap_space=0,
         cache_dtype=cache_dtype,
         enable_prefix_caching=True,
     )

tests/v1/worker/test_gpu_model_runner.py

@@ -96,7 +96,6 @@ def get_vllm_config():
     cache_config = CacheConfig(
         block_size=BLOCK_SIZE,
         gpu_memory_utilization=0.9,
-        swap_space=0,
         cache_dtype="auto",
     )
     parallel_config = ParallelConfig()
@@ -809,7 +808,6 @@ def test_hybrid_attention_mamba_tensor_shapes():
     cache_config = CacheConfig(
         block_size=BLOCK_SIZE,
         gpu_memory_utilization=0.9,
-        swap_space=0,
         cache_dtype="auto",
     )
     parallel_config = ParallelConfig()
@@ -1242,7 +1240,6 @@ def test_cudagraph_sizes_capped_for_mamba_cache():
     cache_config = CacheConfig(
         block_size=BLOCK_SIZE,
         gpu_memory_utilization=0.9,
-        swap_space=0,
         cache_dtype="auto",
     )
     parallel_config = ParallelConfig()

vllm/benchmarks/throughput.py

@@ -38,6 +38,7 @@ from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
 from vllm.inputs import TextPrompt, TokensPrompt
 from vllm.lora.request import LoRARequest
 from vllm.outputs import RequestOutput
+from vllm.platforms import current_platform
 from vllm.sampling_params import BeamSearchParams
 from vllm.tokenizers import TokenizerLike, get_tokenizer
 from vllm.utils.async_utils import merge_async_iterators
@@ -256,17 +257,21 @@ def run_hf(
     max_batch_size: int,
     trust_remote_code: bool,
     disable_detokenize: bool = False,
+    dtype: torch.dtype | None = torch.float16,
+    enable_torch_compile: bool = False,
 ) -> float:
     assert isinstance(tokenizer, PreTrainedTokenizerBase), (
         "the hf backend only supports HF tokenizers"
     )
     llm = AutoModelForCausalLM.from_pretrained(
-        model, dtype=torch.float16, trust_remote_code=trust_remote_code
+        model, dtype=dtype, trust_remote_code=trust_remote_code
     )
     if llm.config.model_type == "llama":
         # To enable padding in the HF backend.
         tokenizer.pad_token = tokenizer.eos_token
-    llm = llm.cuda()
+    llm = llm.to(current_platform.device_type)
+    if enable_torch_compile:
+        llm = torch.compile(llm)
 
     pbar = tqdm(total=len(requests))
     start = time.perf_counter()
@@ -295,7 +300,7 @@ def run_hf(
         # Generate the sequences.
         input_ids = tokenizer(batch, return_tensors="pt", padding=True).input_ids
         llm_outputs = llm.generate(
-            input_ids=input_ids.cuda(),
+            input_ids=input_ids.to(current_platform.device_type),
             do_sample=True,
             num_return_sequences=n,
             temperature=1.0,
@@ -733,6 +738,12 @@ def add_cli_args(parser: argparse.ArgumentParser):
         default=None,
         help="Maximum batch size for HF backend.",
     )
+    parser.add_argument(
+        "--hf-enable-torch-compile",
+        action="store_true",
+        default=False,
+        help="Enable Torch compile for HF backend.",
+    )
     parser.add_argument(
         "--output-json",
         type=str,
@@ -884,6 +895,8 @@ def main(args: argparse.Namespace):
             args.hf_max_batch_size,
             args.trust_remote_code,
             args.disable_detokenize,
+            dtype=args.dtype,
+            enable_torch_compile=args.hf_enable_torch_compile,
         )
     elif args.backend == "vllm-chat":
         elapsed_time, request_outputs = run_vllm_chat(

vllm/config/cache.py

@@ -1,21 +1,13 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 
-import math
 from dataclasses import field
-from typing import TYPE_CHECKING, Any, Literal
+from typing import Literal
 
 from pydantic import Field, SkipValidation, field_validator
 
 from vllm.config.utils import config
 from vllm.logger import init_logger
-from vllm.utils.mem_constants import GiB_bytes
-from vllm.utils.mem_utils import format_gib, get_cpu_memory
-
-if TYPE_CHECKING:
-    from vllm.config.parallel import ParallelConfig
-else:
-    ParallelConfig = Any
 
 logger = init_logger(__name__)
 
@@ -53,8 +45,6 @@ class CacheConfig:
     not matter if you have another vLLM instance running on the same GPU. For
     example, if you have two vLLM instances running on the same GPU, you can
     set the GPU memory utilization to 0.5 for each instance."""
-    swap_space: float = Field(default=4, ge=0)
-    """Size of the CPU swap space per GPU (in GiB)."""
     cache_dtype: CacheDType = "auto"
     """Data type for kv cache storage. If "auto", will use model data type.
     CUDA 11.8+ supports fp8 (=fp8_e4m3) and fp8_e5m2. ROCm (AMD GPU) supports
@@ -173,7 +163,6 @@ class CacheConfig:
         ignored_factors = {
             # Runtime/derived knobs that don't affect compiled graph shape
             "gpu_memory_utilization",
-            "swap_space",
             "is_attention_free",
             "num_gpu_blocks_override",
             "enable_prefix_caching",
@@ -208,24 +197,3 @@ class CacheConfig:
                 "scaling factor."
             )
         return cache_dtype
-
-    def verify_with_parallel_config(
-        self,
-        parallel_config: ParallelConfig,
-    ) -> None:
-        swap_space_bytes = math.ceil(self.swap_space * GiB_bytes)
-        total_cpu_memory = get_cpu_memory()
-        # FIXME(woosuk): Here, it is assumed that the GPUs in a tensor parallel
-        # group are in the same node. However, the GPUs may span multiple nodes.
-        num_gpus_per_node = parallel_config.tensor_parallel_size
-        cpu_memory_usage = swap_space_bytes * num_gpus_per_node
-
-        msg = (
-            f"{format_gib(cpu_memory_usage)} GiB out of the "
-            f"{format_gib(total_cpu_memory)} GiB total CPU memory "
-            "is allocated for the swap space."
-        )
-        if cpu_memory_usage > 0.7 * total_cpu_memory:
-            raise ValueError("Too large swap space. " + msg)
-        elif cpu_memory_usage > 0.4 * total_cpu_memory:
-            logger.warning("Possibly too large swap space. %s", msg)

vllm/config/vllm.py

@@ -674,8 +674,6 @@ class VllmConfig:
 
             self.parallel_config.is_moe_model = self.model_config.is_moe
 
-        self.cache_config.verify_with_parallel_config(self.parallel_config)
-
         if self.lora_config is not None:
             self.lora_config.verify_with_model_config(self.model_config)
 

vllm/distributed/eplb/eplb_state.py

@@ -571,22 +571,18 @@ class EplbState:
                     .float()
                 )
 
-                # Compute per-layer balancedness ratio:
-                #   for each layer: (mean across ranks) / (max across ranks)
-                #   then average across layers.
-                # dim=-1 is the rank dimension.
-                avg_per_layer = num_tokens_per_rank.mean(dim=-1)
-                max_per_layer = num_tokens_per_rank.max(dim=-1).values
-                per_layer_balance = torch.where(
-                    max_per_layer > 0,
-                    avg_per_layer / max_per_layer,
-                    torch.ones_like(max_per_layer),
-                )
-                balancedness = float(per_layer_balance.mean().item())
+                # Compute balancedness ratio:
+                # for each layer:
+                #   (mean load across ranks) / (max load across ranks)
+                avg_tokens_tensor = num_tokens_per_rank.mean(dim=0).sum(dim=0)
+                max_tokens_tensor = num_tokens_per_rank.max(dim=0).values.sum(dim=0)
 
-                # Summary stats for logging
-                avg_tokens = float(avg_per_layer.sum().item())
-                max_tokens = float(max_per_layer.sum().item())
+                # Just to make type checker happy
+                tokens_tensors: list[float] = torch.stack(
+                    [avg_tokens_tensor, max_tokens_tensor]
+                ).tolist()
+                avg_tokens, max_tokens = tokens_tensors
+                balancedness = avg_tokens / max_tokens if max_tokens > 0 else 0.0
 
                 if ep_group.rank() == 0:
                     logger.info(
@@ -602,71 +598,6 @@ class EplbState:
                         - self.expert_rearrangement_step,
                     )
 
-                    # Per-layer breakdown: worst/best layers,
-                    # per-rank token counts for the worst layer
-                    worst_layer = int(per_layer_balance.argmin().item())
-                    best_layer = int(per_layer_balance.argmax().item())
-                    worst_balance = float(
-                        per_layer_balance[worst_layer].item()
-                    )
-                    best_balance = float(
-                        per_layer_balance[best_layer].item()
-                    )
-
-                    worst_layer_ranks = num_tokens_per_rank[worst_layer]
-                    worst_min_rank = int(
-                        worst_layer_ranks.argmin().item()
-                    )
-                    worst_max_rank = int(
-                        worst_layer_ranks.argmax().item()
-                    )
-
-                    logger.info(
-                        "EPLB balance breakdown: "
-                        "worst_layer=%d (balance=%.4f, "
-                        "min_rank=%d[%.0f], max_rank=%d[%.0f]), "
-                        "best_layer=%d (balance=%.4f), "
-                        "num_layers=%d",
-                        worst_layer,
-                        worst_balance,
-                        worst_min_rank,
-                        float(
-                            worst_layer_ranks[worst_min_rank].item()
-                        ),
-                        worst_max_rank,
-                        float(
-                            worst_layer_ranks[worst_max_rank].item()
-                        ),
-                        best_layer,
-                        best_balance,
-                        num_tokens_per_rank.shape[0],
-                    )
-
-                    # Log replica distribution for debug
-                    replica_count = (
-                        eplb_model_state.logical_replica_count
-                    )
-                    if (
-                        replica_count is not None
-                        and replica_count.numel() > 0
-                    ):
-                        rc_float = replica_count.float()
-                        logger.debug(
-                            "EPLB replica stats (layer avg): "
-                            "min=%.1f, max=%.1f, mean=%.2f, "
-                            "num_with_replicas=%d/%d",
-                            float(rc_float.min().item()),
-                            float(rc_float.max().item()),
-                            float(rc_float.mean().item()),
-                            int(
-                                (rc_float > 1)
-                                .any(dim=0)
-                                .sum()
-                                .item()
-                            ),
-                            replica_count.shape[-1],
-                        )
-
         # Update the expert load sliding window
         if not is_dummy:
             for eplb_model_state in self.model_states.values():
@@ -743,40 +674,6 @@ class EplbState:
             )
 
         # Map the physical expert load to global logical experts
-        if is_main_rank:
-            # Log window utilization diagnostics
-            nonzero_slots = sum(
-                int(
-                    (ms.expert_load_window.sum(dim=(1, 2)) > 0)
-                    .sum()
-                    .item()
-                )
-                for ms in self.model_states.values()
-            )
-            logger.info(
-                "EPLB window state: window_step=%d/%d, "
-                "rearrangement_step=%d/%d, "
-                "nonzero_window_slots=%d/%d",
-                self.expert_load_window_step,
-                self.expert_load_window_size,
-                self.expert_rearrangement_step,
-                self.expert_rearrangement_step_interval,
-                nonzero_slots,
-                self.expert_load_window_size,
-            )
-            if (
-                self.expert_load_window_size
-                > self.expert_rearrangement_step_interval
-            ):
-                logger.warning(
-                    "EPLB: window_size (%d) > step_interval (%d). "
-                    "Stale window entries from before the last "
-                    "rearrangement will be converted with the current "
-                    "physical->logical mapping, which may be incorrect. "
-                    "Consider setting window_size <= step_interval.",
-                    self.expert_load_window_size,
-                    self.expert_rearrangement_step_interval,
-                )
         global_expert_load_windows = []
         for eplb_model_state in self.model_states.values():
             expert_load_window = eplb_model_state.expert_load_window[
@@ -839,40 +736,6 @@ class EplbState:
         for eplb_model_state, global_expert_load_window in zip(
             self.model_states.values(), global_expert_load_windows
         ):
-            if is_main_rank:
-                # Log load statistics the algorithm will use
-                load = global_expert_load_window.float()
-                load_per_layer = load.sum(dim=-1)
-                logger.info(
-                    "EPLB rearrange input: "
-                    "num_replicas=%d, num_groups=%d, "
-                    "num_nodes=%d, num_gpus=%d, "
-                    "total_load_per_layer: "
-                    "min=%.0f, max=%.0f, mean=%.0f",
-                    num_replicas,
-                    num_groups,
-                    num_nodes,
-                    num_gpus,
-                    float(load_per_layer.min().item()),
-                    float(load_per_layer.max().item()),
-                    float(load_per_layer.mean().item()),
-                )
-                # Top-5 hottest experts (averaged across layers)
-                avg_load = load.mean(dim=0)
-                top5_vals, top5_ids = avg_load.topk(
-                    min(5, avg_load.shape[0])
-                )
-                logger.info(
-                    "EPLB top-5 hottest logical experts "
-                    "(avg across layers): %s",
-                    ", ".join(
-                        f"e{int(eid)}={float(val):.0f}"
-                        for eid, val in zip(
-                            top5_ids.tolist(), top5_vals.tolist()
-                        )
-                    ),
-                )
-
             if not self.is_async or is_profile:
                 # Get new expert mappings for the model
                 (
@@ -888,66 +751,6 @@ class EplbState:
                     eplb_model_state.physical_to_logical_map,
                 )
 
-                if is_main_rank and not is_profile:
-                    # Log what the algorithm decided
-                    old_p2l = eplb_model_state.physical_to_logical_map
-                    new_p2l = new_physical_to_logical_map.to(
-                        old_p2l.device
-                    )
-                    changed_slots = int(
-                        (old_p2l != new_p2l).sum().item()
-                    )
-                    total_slots = old_p2l.numel()
-                    rc = new_logical_replica_count.float()
-                    logger.info(
-                        "EPLB rearrange result: "
-                        "changed_slots=%d/%d (%.1f%%), "
-                        "replica_count: "
-                        "min=%.0f, max=%.0f, mean=%.2f",
-                        changed_slots,
-                        total_slots,
-                        100.0 * changed_slots / max(total_slots, 1),
-                        float(rc.min().item()),
-                        float(rc.max().item()),
-                        float(rc.mean().item()),
-                    )
-
-                    # Simulate new per-rank load to preview
-                    # balancedness
-                    new_rc = new_logical_replica_count.to(
-                        load.device
-                    ).float()
-                    per_expert_load = load / new_rc.clamp(min=1)
-                    phys_load = per_expert_load.gather(
-                        dim=-1,
-                        index=new_p2l.to(load.device).long(),
-                    )
-                    per_rank_load = phys_load.reshape(
-                        phys_load.shape[0], num_gpus, -1
-                    ).sum(dim=-1)
-                    avg_rl = per_rank_load.mean(dim=-1)
-                    max_rl = per_rank_load.max(dim=-1).values
-                    predicted_balance = torch.where(
-                        max_rl > 0,
-                        avg_rl / max_rl,
-                        torch.ones_like(max_rl),
-                    )
-                    logger.info(
-                        "EPLB predicted post-rearrange "
-                        "balancedness: mean=%.4f, "
-                        "min=%.4f (layer %d), max=%.4f",
-                        float(
-                            predicted_balance.mean().item()
-                        ),
-                        float(predicted_balance.min().item()),
-                        int(
-                            predicted_balance.argmin().item()
-                        ),
-                        float(
-                            predicted_balance.max().item()
-                        ),
-                    )
-
                 # Update expert weights
                 rearrange_expert_weights_inplace(
                     eplb_model_state.physical_to_logical_map,

vllm/engine/arg_utils.py

@@ -447,7 +447,6 @@ class EngineArgs:
     )
     disable_sliding_window: bool = ModelConfig.disable_sliding_window
     disable_cascade_attn: bool = ModelConfig.disable_cascade_attn
-    swap_space: float = CacheConfig.swap_space
     offload_backend: str = OffloadConfig.offload_backend
     cpu_offload_gb: float = UVAOffloadConfig.cpu_offload_gb
     cpu_offload_params: set[str] = get_field(UVAOffloadConfig, "cpu_offload_params")
@@ -961,7 +960,6 @@ class EngineArgs:
         cache_group.add_argument(
             "--kv-cache-memory-bytes", **cache_kwargs["kv_cache_memory_bytes"]
         )
-        cache_group.add_argument("--swap-space", **cache_kwargs["swap_space"])
         cache_group.add_argument("--kv-cache-dtype", **cache_kwargs["cache_dtype"])
         cache_group.add_argument(
             "--num-gpu-blocks-override", **cache_kwargs["num_gpu_blocks_override"]
@@ -1526,7 +1524,6 @@ class EngineArgs:
             block_size=self.block_size,
             gpu_memory_utilization=self.gpu_memory_utilization,
             kv_cache_memory_bytes=self.kv_cache_memory_bytes,
-            swap_space=self.swap_space,
             cache_dtype=resolved_cache_dtype,  # type: ignore[arg-type]
             is_attention_free=model_config.is_attention_free,
             num_gpu_blocks_override=self.num_gpu_blocks_override,

vllm/entrypoints/chat_utils.py

@@ -462,10 +462,15 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
     maximum per prompt.
     """
 
-    def __init__(self, model_config: ModelConfig):
+    def __init__(
+        self,
+        model_config: ModelConfig,
+        media_io_kwargs: dict[str, dict[str, Any]] | None = None,
+    ):
         super().__init__()
 
         self._model_config = model_config
+        self._media_io_kwargs = media_io_kwargs
 
         self._items_by_modality = defaultdict[str, list[_T]](list)
         # Track original modality for each vision_chunk item (image or video)
@@ -487,6 +492,14 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
         model_cls = get_model_cls(self.model_config)
         return cast(type[SupportsMultiModal], model_cls)
 
+    @property
+    def media_io_kwargs(self) -> dict[str, dict[str, Any]] | None:
+        return self._media_io_kwargs or (
+            self._model_config.multimodal_config.media_io_kwargs
+            if self._model_config.multimodal_config
+            else None
+        )
+
     @property
     def allowed_local_media_path(self):
         return self._model_config.allowed_local_media_path
@@ -769,12 +782,10 @@ class MultiModalContentParser(BaseMultiModalContentParser):
         super().__init__()
 
         self._tracker = tracker
-        multimodal_config = self._tracker.model_config.multimodal_config
-        media_io_kwargs = getattr(multimodal_config, "media_io_kwargs", None)
 
         self._connector: MediaConnector = MEDIA_CONNECTOR_REGISTRY.load(
             envs.VLLM_MEDIA_CONNECTOR,
-            media_io_kwargs=media_io_kwargs,
+            media_io_kwargs=tracker.media_io_kwargs,
             allowed_local_media_path=tracker.allowed_local_media_path,
             allowed_media_domains=tracker.allowed_media_domains,
         )
@@ -881,11 +892,9 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
         super().__init__()
 
         self._tracker = tracker
-        multimodal_config = self._tracker.model_config.multimodal_config
-        media_io_kwargs = getattr(multimodal_config, "media_io_kwargs", None)
         self._connector: MediaConnector = MEDIA_CONNECTOR_REGISTRY.load(
             envs.VLLM_MEDIA_CONNECTOR,
-            media_io_kwargs=media_io_kwargs,
+            media_io_kwargs=tracker.media_io_kwargs,
             allowed_local_media_path=tracker.allowed_local_media_path,
             allowed_media_domains=tracker.allowed_media_domains,
         )
@@ -1530,13 +1539,14 @@ def parse_chat_messages(
     messages: list[ChatCompletionMessageParam],
     model_config: ModelConfig,
     content_format: ChatTemplateContentFormat,
+    media_io_kwargs: dict[str, dict[str, Any]] | None = None,
 ) -> tuple[
     list[ConversationMessage],
     MultiModalDataDict | None,
     MultiModalUUIDDict | None,
 ]:
     conversation: list[ConversationMessage] = []
-    mm_tracker = MultiModalItemTracker(model_config)
+    mm_tracker = MultiModalItemTracker(model_config, media_io_kwargs=media_io_kwargs)
 
     for msg in messages:
         sub_messages = _parse_chat_message_content(
@@ -1563,13 +1573,16 @@ async def parse_chat_messages_async(
     messages: list[ChatCompletionMessageParam],
     model_config: ModelConfig,
     content_format: ChatTemplateContentFormat,
+    media_io_kwargs: dict[str, dict[str, Any]] | None = None,
 ) -> tuple[
     list[ConversationMessage],
     MultiModalDataDict | None,
     MultiModalUUIDDict | None,
 ]:
     conversation: list[ConversationMessage] = []
-    mm_tracker = AsyncMultiModalItemTracker(model_config)
+    mm_tracker = AsyncMultiModalItemTracker(
+        model_config, media_io_kwargs=media_io_kwargs
+    )
 
     for msg in messages:
         sub_messages = _parse_chat_message_content(

vllm/entrypoints/llm.py

@@ -164,12 +164,6 @@ class LLM:
             compared with using gpu_memory_utilization. Note that
             kv_cache_memory_bytes (when not-None) ignores
             gpu_memory_utilization
-        swap_space: The size (GiB) of CPU memory per GPU to use as swap space.
-            This can be used for temporarily storing the states of the requests
-            when their `best_of` sampling parameters are larger than 1. If all
-            requests will have `best_of=1`, you can safely set this to 0.
-            Noting that `best_of` is only supported in V0. Otherwise, too small
-            values may cause out-of-memory (OOM) errors.
         cpu_offload_gb: The size (GiB) of CPU memory to use for offloading
             the model weights. This virtually increases the GPU memory space
             you can use to hold the model weights, at the cost of CPU-GPU data
@@ -240,7 +234,6 @@ class LLM:
         chat_template: Path | str | None = None,
         seed: int = 0,
         gpu_memory_utilization: float = 0.9,
-        swap_space: float = 4,
         cpu_offload_gb: float = 0,
         offload_group_size: int = 0,
         offload_num_in_group: int = 1,
@@ -265,6 +258,17 @@ class LLM:
     ) -> None:
         """LLM constructor."""
 
+        if "swap_space" in kwargs:
+            kwargs.pop("swap_space")
+            import warnings
+
+            warnings.warn(
+                "The 'swap_space' parameter is deprecated and ignored. "
+                "It will be removed in a future version.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+
         if "disable_log_stats" not in kwargs:
             kwargs["disable_log_stats"] = True
 
@@ -353,7 +357,6 @@ class LLM:
             seed=seed,
             gpu_memory_utilization=gpu_memory_utilization,
             kv_cache_memory_bytes=kv_cache_memory_bytes,
-            swap_space=swap_space,
             cpu_offload_gb=cpu_offload_gb,
             offload_group_size=offload_group_size,
             offload_num_in_group=offload_num_in_group,

vllm/entrypoints/openai/chat_completion/protocol.py

@@ -268,6 +268,13 @@ class ChatCompletionRequest(OpenAIBaseModel):
             "Will be accessible by the chat template."
         ),
     )
+    media_io_kwargs: dict[str, dict[str, Any]] | None = Field(
+        default=None,
+        description=(
+            "Additional kwargs to pass to the media IO connectors, "
+            "keyed by modality. Merged with engine-level media_io_kwargs."
+        ),
+    )
     mm_processor_kwargs: dict[str, Any] | None = Field(
         default=None,
         description=("Additional kwargs to pass to the HF processor."),
@@ -366,6 +373,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
                     reasoning_effort=self.reasoning_effort,
                 ),
             ),
+            media_io_kwargs=self.media_io_kwargs,
         )
 
     def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:

vllm/entrypoints/openai/cli_args.py

@@ -281,6 +281,11 @@ class FrontendArgs(BaseFrontendArgs):
     Enable offline FastAPI documentation for air-gapped environments.
     Uses vendored static assets bundled with vLLM.
     """
+    enable_dashboard: bool = False
+    """
+    Enable the vLLM web dashboard at /dashboard for monitoring and testing.
+    Provides server info, metrics display, and a chat interface for testing.
+    """
     use_gpu_for_pooling_score: bool = False
     """If set, run pooling score MaxSim on GPU in the API server process.
     Can significantly improve late-interaction scoring performance.

vllm/entrypoints/openai/engine/serving.py

@@ -900,10 +900,15 @@ class OpenAIServing:
             ),
         )
 
+        mm_config = self.model_config.multimodal_config
+
         tok_params = request.build_tok_params(self.model_config)
         chat_params = request.build_chat_params(
             default_template, default_template_content_format
-        ).with_defaults(default_template_kwargs)
+        ).with_defaults(
+            default_template_kwargs,
+            default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
+        )
 
         (conversation,), (engine_prompt,) = await renderer.render_chat_async(
             [messages],

vllm/entrypoints/openai/responses/protocol.py

@@ -197,6 +197,13 @@ class ResponsesRequest(OpenAIBaseModel):
             "through out the inference process and return in response."
         ),
     )
+    media_io_kwargs: dict[str, dict[str, Any]] | None = Field(
+        default=None,
+        description=(
+            "Additional kwargs to pass to the media IO connectors, "
+            "keyed by modality. Merged with engine-level media_io_kwargs."
+        ),
+    )
     mm_processor_kwargs: dict[str, Any] | None = Field(
         default=None,
         description=("Additional kwargs to pass to the HF processor."),
@@ -276,6 +283,7 @@ class ResponsesRequest(OpenAIBaseModel):
                     reasoning_effort=None if reasoning is None else reasoning.effort,
                 ),
             ),
+            media_io_kwargs=self.media_io_kwargs,
         )
 
     def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:

vllm/entrypoints/openai/server_utils.py

@@ -129,8 +129,8 @@ def get_uvicorn_log_config(args: Namespace) -> dict | None:
 
     Priority:
     1. If log_config_file is specified, use it
-    2. If disable_access_log_for_endpoints is specified, create a config with
-       the access log filter
+    2. If disable_access_log_for_endpoints is specified or dashboard is enabled,
+       create a config with the access log filter
     3. Otherwise, return None (use uvicorn defaults)
     """
     # First, try to load from file if specified
@@ -138,16 +138,23 @@ def get_uvicorn_log_config(args: Namespace) -> dict | None:
     if log_config is not None:
         return log_config
 
-    # If endpoints to filter are specified, create a config with the filter
-    if args.disable_access_log_for_endpoints:
-        from vllm.logging_utils import create_uvicorn_log_config
+    excluded_paths: list[str] = []
 
-        # Parse comma-separated string into list
+    # User-specified endpoints to filter
+    if args.disable_access_log_for_endpoints:
         excluded_paths = [
             p.strip()
             for p in args.disable_access_log_for_endpoints.split(",")
             if p.strip()
         ]
+
+    # Dashboard polling endpoints create noise every 5s; suppress them
+    if getattr(args, "enable_dashboard", False):
+        excluded_paths.extend(["/dashboard/api/info", "/dashboard/api/metrics", "/dashboard"])
+
+    if excluded_paths:
+        from vllm.logging_utils import create_uvicorn_log_config
+
         return create_uvicorn_log_config(
             excluded_paths=excluded_paths,
             log_level=args.uvicorn_log_level,

vllm/entrypoints/pooling/base/io_processor.py

@@ -123,10 +123,15 @@ class PoolingIOProcessor:
             ),
         )
 
+        mm_config = self.model_config.multimodal_config
+
         tok_params = request.build_tok_params(self.model_config)
         chat_params = request.build_chat_params(
             default_template, default_template_content_format
-        ).with_defaults(default_template_kwargs)
+        ).with_defaults(
+            default_template_kwargs,
+            default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
+        )
 
         (conversation,), (engine_prompt,) = renderer.render_chat(
             [messages],

vllm/entrypoints/pooling/base/protocol.py

@@ -124,6 +124,13 @@ class ChatRequestMixin(OpenAIBaseModel):
             "Will be accessible by the chat template."
         ),
     )
+    media_io_kwargs: dict[str, dict[str, Any]] | None = Field(
+        default=None,
+        description=(
+            "Additional kwargs to pass to the media IO connectors, "
+            "keyed by modality. Merged with engine-level media_io_kwargs."
+        ),
+    )
     # --8<-- [end:chat-extra-params]
 
     @model_validator(mode="before")
@@ -151,6 +158,7 @@ class ChatRequestMixin(OpenAIBaseModel):
                     continue_final_message=self.continue_final_message,
                 ),
             ),
+            media_io_kwargs=self.media_io_kwargs,
         )
 
 

vllm/entrypoints/serve/__init__.py

@@ -52,6 +52,12 @@ def register_vllm_serve_api_routers(app: FastAPI):
 
     attach_tokenize_router(app)
 
+    from vllm.entrypoints.serve.dashboard.api_router import (
+        attach_router as attach_dashboard_router,
+    )
+
+    attach_dashboard_router(app)
+
     from .instrumentator import register_instrumentator_api_routers
 
     register_instrumentator_api_routers(app)

vllm/entrypoints/serve/dashboard/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

vllm/entrypoints/serve/dashboard/api_router.py

@@ -0,0 +1,458 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""
+Dashboard router for vLLM web UI.
+
+Provides endpoints for:
+- /dashboard - Main dashboard HTML page
+- /dashboard/api/info - Server information JSON (config, env, load, status)
+- /dashboard/api/metrics - Metrics JSON
+"""
+
+import asyncio
+import functools
+import pathlib
+
+import pydantic
+from fastapi import APIRouter, FastAPI, Request
+from fastapi.responses import HTMLResponse, JSONResponse
+
+import vllm.envs as envs
+from vllm.config import VllmConfig
+from vllm.logger import init_logger
+from vllm.version import __version__ as VLLM_VERSION
+
+logger = init_logger(__name__)
+
+router = APIRouter()
+PydanticVllmConfig = pydantic.TypeAdapter(VllmConfig)
+
+
+def _get_vllm_env_vars() -> tuple[dict, set]:
+    """Get all VLLM environment variables and which ones are explicitly set."""
+    import os
+
+    from vllm.config.utils import normalize_value
+
+    vllm_envs = {}
+    explicit_envs = set()
+
+    for key in dir(envs):
+        if key.startswith("VLLM_") and "KEY" not in key:
+            value = getattr(envs, key, None)
+            if value is not None:
+                value = normalize_value(value)
+                vllm_envs[key] = value
+                # Check if explicitly set in environment
+                if key in os.environ:
+                    explicit_envs.add(key)
+
+    return vllm_envs, explicit_envs
+
+
+def _get_explicit_cli_args(args) -> tuple[set, dict]:
+    """Get CLI args that were explicitly set by user (non-default values).
+
+    Returns a tuple of (set of explicit arg names, dict of arg name -> value).
+    """
+    from vllm.config.utils import normalize_value
+    from vllm.engine.arg_utils import AsyncEngineArgs
+    from vllm.entrypoints.openai.cli_args import FrontendArgs
+
+    explicit_args: set[str] = set()
+    non_default_args: dict = {}
+    if args is None:
+        return explicit_args, non_default_args
+
+    # Get default values from dataclasses
+    try:
+        frontend_defaults = FrontendArgs()
+        engine_defaults = AsyncEngineArgs(model="")
+
+        for key, default_val in vars(frontend_defaults).items():
+            if hasattr(args, key):
+                current_val = getattr(args, key)
+                if current_val != default_val:
+                    explicit_args.add(key)
+                    non_default_args[key] = normalize_value(current_val)
+
+        for key, default_val in vars(engine_defaults).items():
+            if hasattr(args, key) and key != "model":
+                current_val = getattr(args, key)
+                if current_val != default_val:
+                    explicit_args.add(key)
+                    non_default_args[key] = normalize_value(current_val)
+
+        # model is always explicit if set
+        if hasattr(args, "model") and args.model:
+            explicit_args.add("model")
+            non_default_args["model"] = args.model
+        if hasattr(args, "served_model_name") and args.served_model_name:
+            explicit_args.add("served_model_name")
+            non_default_args["served_model_name"] = normalize_value(
+                args.served_model_name
+            )
+
+    except Exception as e:
+        logger.debug("Failed to get explicit CLI args: %s", e)
+
+    return explicit_args, non_default_args
+
+
+_resolved_attn_backend: str | None = None
+
+
+def _resolve_attn_backend_name(vllm_config, configured_backend) -> str | None:
+    """Best-effort resolution of the actual attention backend.
+
+    When the user leaves the backend on auto (None), we call the platform's
+    selection logic with the model's head_size / dtype so the dashboard can
+    show the concrete backend (e.g. FLASH_ATTN) instead of "auto".
+    Returns the resolved backend name string or None on failure.
+    """
+    global _resolved_attn_backend
+    if _resolved_attn_backend is not None:
+        return _resolved_attn_backend
+
+    try:
+        from vllm.platforms import current_platform
+        from vllm.v1.attention.backends.registry import AttentionBackendEnum
+        from vllm.v1.attention.selector import AttentionSelectorConfig
+
+        mc = vllm_config.model_config
+        cc = vllm_config.cache_config
+
+        selector_cfg = AttentionSelectorConfig(
+            head_size=mc.get_head_size(),
+            dtype=mc.dtype,
+            kv_cache_dtype=cc.cache_dtype if cc.cache_dtype != "auto" else None,
+            block_size=getattr(cc, "block_size", None),
+            use_mla=mc.use_mla,
+        )
+        cls_path = current_platform.get_attn_backend_cls(
+            configured_backend,
+            attn_selector_config=selector_cfg,
+        )
+        if not cls_path:
+            return None
+        for member in AttentionBackendEnum:
+            if member.value and member.value == cls_path:
+                _resolved_attn_backend = member.name
+                return _resolved_attn_backend
+        _resolved_attn_backend = cls_path.rsplit(".", 1)[-1].replace(
+            "Backend", ""
+        )
+        return _resolved_attn_backend
+    except Exception as e:
+        logger.debug("Failed to resolve attention backend: %s", e)
+        return None
+
+
+def _get_startup_info(vllm_config) -> dict:
+    """Derive startup info from vllm_config available in the API server.
+
+    Exposes values that are computed properties or enum-typed in the config
+    and would otherwise be missing or opaque in the JSON-serialized output.
+    Also computes KV cache capacity and max concurrency from cache_config
+    (num_gpu_blocks is set via the engine READY handshake).
+    """
+    info: dict = {}
+    if vllm_config is None:
+        return info
+
+    # Architecture is a @property, not serialized by Pydantic
+    try:
+        mc = vllm_config.model_config
+        info["architecture"] = mc.architecture
+    except Exception:
+        pass
+
+    # Attention backend — resolve the actual backend even when config is auto
+    try:
+        ac = vllm_config.attention_config
+        configured = getattr(ac, "backend", None)
+        resolved_name = _resolve_attn_backend_name(vllm_config, configured)
+        if resolved_name:
+            info["attention_backend"] = resolved_name
+        elif configured is not None:
+            info["attention_backend"] = configured.name
+    except Exception:
+        pass
+
+    # Optimization level
+    try:
+        opt = vllm_config.optimization_level
+        if opt is not None:
+            info["optimization_level"] = f"O{int(opt)}"
+    except Exception:
+        pass
+
+    # Compilation config: mode, cudagraph, inductor partition, fusion passes
+    try:
+        cpc = vllm_config.compilation_config
+        mode = getattr(cpc, "mode", None)
+        if mode is not None:
+            info["compilation_mode"] = mode.name
+        cgm = getattr(cpc, "cudagraph_mode", None)
+        if cgm is not None:
+            info["cudagraph_mode"] = cgm.name
+        igp = getattr(cpc, "use_inductor_graph_partition", None)
+        if igp is not None:
+            info["inductor_graph_partition"] = igp
+
+        pc = getattr(cpc, "pass_config", None)
+        if pc is not None:
+            fusions = {}
+            for attr in (
+                "fuse_norm_quant",
+                "fuse_act_quant",
+                "fuse_attn_quant",
+                "enable_sp",
+                "fuse_gemm_comms",
+                "fuse_allreduce_rms",
+                "enable_qk_norm_rope_fusion",
+            ):
+                val = getattr(pc, attr, None)
+                if val is True:
+                    fusions[attr] = True
+            if fusions:
+                info["pass_config_fusions"] = fusions
+    except Exception:
+        pass
+
+    # Kernel config: flashinfer autotune, MoE backend
+    try:
+        kc = vllm_config.kernel_config
+        fi = getattr(kc, "enable_flashinfer_autotune", None)
+        if fi is not None:
+            info["flashinfer_autotune"] = fi
+        moe = getattr(kc, "moe_backend", None)
+        if moe is not None:
+            info["moe_backend"] = moe
+    except Exception:
+        pass
+
+    # KV cache capacity and max concurrency
+    try:
+        cc = vllm_config.cache_config
+        mc = vllm_config.model_config
+
+        num_gpu_blocks = getattr(cc, "num_gpu_blocks", None)
+        block_size = getattr(cc, "block_size", None)
+        if num_gpu_blocks and block_size:
+            kv_cache_tokens = num_gpu_blocks * block_size
+            info["kv_cache_tokens"] = kv_cache_tokens
+
+            max_model_len = getattr(mc, "max_model_len", None)
+            if max_model_len and max_model_len > 0:
+                info["max_concurrency"] = round(
+                    kv_cache_tokens / max_model_len, 2
+                )
+    except Exception:
+        pass
+
+    return info
+
+
+@functools.lru_cache(maxsize=1)
+def _get_system_env_info_cached() -> dict:
+    """Get system environment info (GPU, CUDA, torch, OS, etc.).
+
+    Cached since this info never changes during the lifetime of the server.
+    """
+    from vllm.collect_env import get_env_info
+
+    return get_env_info()._asdict()
+
+
+def _get_dashboard_html() -> str:
+    """Load the dashboard HTML from static file."""
+    static_dir = pathlib.Path(__file__).parent / "static"
+    html_file = static_dir / "index.html"
+    if html_file.exists():
+        return html_file.read_text(encoding="utf-8")
+    return "<html><body><h1>Dashboard HTML not found</h1></body></html>"
+
+
+@router.get("/dashboard", response_class=HTMLResponse)
+async def dashboard_index() -> HTMLResponse:
+    """Serve the main dashboard page."""
+    html_content = _get_dashboard_html()
+    return HTMLResponse(content=html_content)
+
+
+@router.get("/dashboard/api/info")
+async def dashboard_info(request: Request) -> JSONResponse:
+    """Get server information for dashboard display."""
+    from vllm.v1.engine.exceptions import EngineDeadError
+
+    info: dict = {
+        "version": VLLM_VERSION,
+        "status": "running",
+    }
+
+    # Check engine health
+    try:
+        engine_client = getattr(request.app.state, "engine_client", None)
+        if engine_client is not None:
+            await engine_client.check_health()
+            info["status"] = "running"
+    except EngineDeadError:
+        info["status"] = "unhealthy"
+    except Exception as e:
+        logger.debug("Failed to check engine health: %s", e)
+
+    # Get model information
+    try:
+        serving_models = request.app.state.openai_serving_models
+        if serving_models is not None:
+            models_response = await serving_models.show_available_models()
+            info["models"] = [
+                {"id": model.id, "root": model.root} for model in models_response.data
+            ]
+    except Exception as e:
+        logger.warning("Failed to get model info for dashboard: %s", e)
+        info["models"] = []
+
+    # Get full engine config and derive startup info
+    vllm_config = getattr(request.app.state, "vllm_config", None)
+    try:
+        if vllm_config is not None:
+            info["vllm_config"] = PydanticVllmConfig.dump_python(
+                vllm_config, mode="json", fallback=str
+            )
+    except Exception as e:
+        logger.warning("Failed to get vllm_config for dashboard: %s", e)
+
+    try:
+        startup_info = _get_startup_info(vllm_config)
+        if startup_info:
+            info["startup_info"] = startup_info
+    except Exception as e:
+        logger.debug("Failed to get startup_info for dashboard: %s", e)
+
+    # Get environment variables (with explicit markers)
+    try:
+        vllm_env, explicit_envs = _get_vllm_env_vars()
+        info["vllm_env"] = vllm_env
+        info["explicit_envs"] = list(explicit_envs)
+    except Exception as e:
+        logger.warning("Failed to get vllm_env for dashboard: %s", e)
+
+    # Get system environment info (GPU, CUDA, torch, OS, etc.)
+    try:
+        info["system_env"] = await asyncio.to_thread(
+            _get_system_env_info_cached
+        )
+    except Exception as e:
+        logger.debug("Failed to get system_env for dashboard: %s", e)
+
+    # Get explicit CLI args and non-default args dict
+    try:
+        args = getattr(request.app.state, "args", None)
+        explicit_args, non_default_args = _get_explicit_cli_args(args)
+        info["explicit_args"] = list(explicit_args)
+        info["non_default_args"] = non_default_args
+    except Exception as e:
+        logger.warning("Failed to get explicit_args for dashboard: %s", e)
+
+    # Get server load metrics
+    try:
+        server_load = getattr(request.app.state, "server_load_metrics", None)
+        if server_load is not None:
+            info["server_load"] = server_load
+    except Exception as e:
+        logger.debug("Failed to get server_load for dashboard: %s", e)
+
+    # Check if engine is sleeping (only available in dev mode)
+    try:
+        engine_client = getattr(request.app.state, "engine_client", None)
+        if engine_client is not None:
+            is_sleeping_method = getattr(engine_client, "is_sleeping", None)
+            if is_sleeping_method is not None:
+                info["is_sleeping"] = await is_sleeping_method()
+    except Exception as e:
+        logger.debug("Failed to check is_sleeping for dashboard: %s", e)
+
+    # Check if engine is paused (for RLHF workflows)
+    try:
+        engine_client = getattr(request.app.state, "engine_client", None)
+        if engine_client is not None:
+            is_paused_method = getattr(engine_client, "is_paused", None)
+            if is_paused_method is not None:
+                info["is_paused"] = await is_paused_method()
+    except Exception as e:
+        logger.debug("Failed to check is_paused for dashboard: %s", e)
+
+    return JSONResponse(content=info)
+
+
+@router.get("/dashboard/api/metrics")
+async def dashboard_metrics(request: Request) -> JSONResponse:
+    """Get metrics for dashboard display.
+
+    Returns both Prometheus metrics and internal engine stats that are only
+    accessible in-process (not available via external /metrics endpoint).
+    """
+    metrics: dict = {}
+
+    # Try to get metrics from prometheus registry
+    try:
+        from vllm.v1.metrics.reader import (
+            Counter,
+            Gauge,
+            Histogram,
+            get_metrics_snapshot,
+        )
+
+        snapshot = get_metrics_snapshot()
+        for metric in snapshot:
+            name = metric.name
+            # For labeled metrics, create unique keys to avoid overwriting
+            # e.g. vllm:request_success with finished_reason label
+            if metric.labels:
+                # Filter out common labels (model_name, engine) for key
+                key_labels = {
+                    k: v
+                    for k, v in metric.labels.items()
+                    if k not in ("model_name", "engine")
+                }
+                if key_labels:
+                    # Create key like "vllm:request_success:stop"
+                    label_suffix = ":".join(str(v) for v in key_labels.values())
+                    name = f"{metric.name}:{label_suffix}"
+            if isinstance(metric, (Counter, Gauge)):
+                metrics[name] = {
+                    "value": metric.value,
+                    "labels": metric.labels,
+                }
+            elif isinstance(metric, Histogram):
+                metrics[name] = {
+                    "count": metric.count,
+                    "sum": metric.sum,
+                    "labels": metric.labels,
+                }
+    except ImportError:
+        logger.debug("Metrics reader not available")
+    except Exception as e:
+        logger.warning("Failed to get metrics for dashboard: %s", e)
+
+    # Get server load if tracking is enabled
+    try:
+        server_load = getattr(request.app.state, "server_load_metrics", None)
+        if server_load is not None:
+            metrics["server_load"] = {"value": server_load, "labels": {}}
+    except Exception:
+        pass
+
+    return JSONResponse(content=metrics)
+
+
+def attach_router(app: FastAPI) -> None:
+    """Attach dashboard router if enabled via args."""
+    args = getattr(app.state, "args", None)
+    if args is None or not getattr(args, "enable_dashboard", False):
+        return
+
+    logger.info("Enabling vLLM dashboard at /dashboard")
+    app.include_router(router)

vllm/entrypoints/serve/tokenize/protocol.py

@@ -100,6 +100,13 @@ class TokenizeChatRequest(OpenAIBaseModel):
             "Will be accessible by the chat template."
         ),
     )
+    media_io_kwargs: dict[str, dict[str, Any]] | None = Field(
+        default=None,
+        description=(
+            "Additional kwargs to pass to the media IO connectors, "
+            "keyed by modality. Merged with engine-level media_io_kwargs."
+        ),
+    )
     mm_processor_kwargs: dict[str, Any] | None = Field(
         default=None,
         description="Additional kwargs to pass to the HF processor.",
@@ -134,6 +141,7 @@ class TokenizeChatRequest(OpenAIBaseModel):
                     continue_final_message=self.continue_final_message,
                 ),
             ),
+            media_io_kwargs=self.media_io_kwargs,
         )
 
     def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:

vllm/model_executor/layers/attention/mla_attention.py

@@ -905,6 +905,10 @@ def unified_mla_kv_cache_update(
     the data dependency between them to ensure torch.compile preserves ordering.
     """
     forward_context = get_forward_context()
+    if forward_context.attn_metadata is None:
+        # Dummy/profile forwards should not update live KV cache pages.
+        return torch.empty(0, device=kv_c_normed.device, dtype=kv_c_normed.dtype)
+
     attn_layer = forward_context.no_compile_layers[layer_name]
     kv_cache = attn_layer.kv_cache[forward_context.virtual_engine]
 

vllm/model_executor/models/deepseek_v2.py

@@ -756,7 +756,7 @@ direct_register_custom_op(
 )
 
 
-class DeepSeekV2FusedQkvAProj(MergedColumnParallelLinear):
+class DeepSeekV2FusedQkvAProjLinear(MergedColumnParallelLinear):
     def __init__(
         self,
         input_size: int,
@@ -848,7 +848,7 @@ class DeepseekV2MLAAttention(nn.Module):
         self.max_position_embeddings = max_position_embeddings
 
         if self.q_lora_rank is not None:
-            self.fused_qkv_a_proj = DeepSeekV2FusedQkvAProj(
+            self.fused_qkv_a_proj = DeepSeekV2FusedQkvAProjLinear(
                 self.hidden_size,
                 [self.q_lora_rank, self.kv_lora_rank + self.qk_rope_head_dim],
                 quant_config=quant_config,

vllm/multimodal/media/audio.py

@@ -83,11 +83,17 @@ def extract_audio_from_video_bytes(
 
 
 class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
+    """Configuration values can be user-provided either by --media-io-kwargs or
+    by the runtime API field "media_io_kwargs". Ensure proper validation and
+    error handling.
+    """
+
     def __init__(self, **kwargs) -> None:
         super().__init__()
 
         # `kwargs` contains custom arguments from
-        # --media-io-kwargs for this modality.
+        # --media-io-kwargs for this modality, merged with
+        # per-request runtime media_io_kwargs via merge_kwargs().
         # They can be passed to the underlying
         # media loaders (e.g. custom implementations)
         # for flexible control.
@@ -122,6 +128,11 @@ class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
 
 
 class AudioEmbeddingMediaIO(MediaIO[torch.Tensor]):
+    """Configuration values can be user-provided either by --media-io-kwargs or
+    by the runtime API field "media_io_kwargs". Ensure proper validation and
+    error handling.
+    """
+
     def __init__(self) -> None:
         super().__init__()
 

vllm/multimodal/media/base.py

@@ -44,6 +44,28 @@ class MediaWithBytes(Generic[_T]):
 
 
 class MediaIO(ABC, Generic[_T]):
+    """Configuration values can be user-provided either by --media-io-kwargs or
+    by the runtime API field "media_io_kwargs". Ensure proper validation and
+    error handling.
+    """
+
+    @classmethod
+    def merge_kwargs(
+        cls,
+        default_kwargs: dict[str, Any] | None,
+        runtime_kwargs: dict[str, Any] | None,
+    ) -> dict[str, Any]:
+        """Merge config-level kwargs and request-level kwargs.
+
+        By default this performs a shallow merge where runtime kwargs override
+        keys in default kwargs. Subclasses may override to apply modality-
+        specific behavior.
+        """
+        merged = dict(default_kwargs or {})
+        if runtime_kwargs:
+            merged.update(runtime_kwargs)
+        return merged
+
     @abstractmethod
     def load_bytes(self, data: bytes) -> _T:
         raise NotImplementedError

vllm/multimodal/media/connector.py

@@ -32,9 +32,43 @@ atexit.register(global_thread_pool.shutdown)
 
 MEDIA_CONNECTOR_REGISTRY = ExtensionManager()
 
+MODALITY_IO_MAP: dict[str, type[MediaIO]] = {
+    "audio": AudioMediaIO,
+    "image": ImageMediaIO,
+    "video": VideoMediaIO,
+}
+
+
+def merge_media_io_kwargs(
+    defaults: dict[str, dict[str, Any]] | None,
+    overrides: dict[str, dict[str, Any]] | None,
+) -> dict[str, dict[str, Any]] | None:
+    """Merge config-level and per-request media_io_kwargs per modality.
+
+    Each modality key is merged using the corresponding MediaIO subclass's
+    ``merge_kwargs``, which may apply modality-specific logic (e.g.
+    VideoMediaIO clears cross-dependent fps/num_frames fields).
+    """
+    if not defaults and not overrides:
+        return None
+    all_keys = set(defaults or {}) | set(overrides or {})
+    merged = {}
+    for key in all_keys:
+        io_cls = MODALITY_IO_MAP.get(key, MediaIO)
+        merged[key] = io_cls.merge_kwargs(
+            (defaults or {}).get(key),
+            (overrides or {}).get(key),
+        )
+    return merged or None
+
 
 @MEDIA_CONNECTOR_REGISTRY.register("http")
 class MediaConnector:
+    """Configuration values can be user-provided either by --media-io-kwargs or
+    by the runtime API field "media_io_kwargs". Ensure proper validation and
+    error handling.
+    """
+
     def __init__(
         self,
         media_io_kwargs: dict[str, dict[str, Any]] | None = None,

vllm/multimodal/media/image.py

@@ -15,12 +15,18 @@ from .base import MediaIO, MediaWithBytes
 
 
 class ImageMediaIO(MediaIO[Image.Image]):
+    """Configuration values can be user-provided either by --media-io-kwargs or
+    by the runtime API field "media_io_kwargs". Ensure proper validation and
+    error handling.
+    """
+
     def __init__(self, image_mode: str = "RGB", **kwargs) -> None:
         super().__init__()
 
         self.image_mode = image_mode
         # `kwargs` contains custom arguments from
-        # --media-io-kwargs for this modality.
+        # --media-io-kwargs for this modality, merged with
+        # per-request runtime media_io_kwargs via merge_kwargs().
         # They can be passed to the underlying
         # media loaders (e.g. custom implementations)
         # for flexible control.
@@ -88,6 +94,13 @@ class ImageMediaIO(MediaIO[Image.Image]):
 
 
 class ImageEmbeddingMediaIO(MediaIO[torch.Tensor]):
+    """Image embedding MediaIO implementation.
+
+    Configuration values can be user-provided either by --media-io-kwargs or
+    by the runtime API field "media_io_kwargs". Ensure proper validation and
+    error handling.
+    """
+
     def __init__(self) -> None:
         super().__init__()
 

vllm/multimodal/media/video.py

@@ -17,6 +17,28 @@ from .image import ImageMediaIO
 
 
 class VideoMediaIO(MediaIO[tuple[npt.NDArray, dict[str, Any]]]):
+    """Configuration values can be user-provided either by --media-io-kwargs or
+    by the runtime API field "media_io_kwargs". Ensure proper validation and
+    error handling.
+    """
+
+    @classmethod
+    def merge_kwargs(
+        cls,
+        default_kwargs: dict[str, Any] | None,
+        runtime_kwargs: dict[str, Any] | None,
+    ) -> dict[str, Any]:
+        merged = super().merge_kwargs(default_kwargs, runtime_kwargs)
+        # fps and num_frames interact with each other, so if either is
+        # overridden at request time, wipe the other from defaults to
+        # avoid unintuitive cross-field interactions.
+        if runtime_kwargs:
+            if "num_frames" in runtime_kwargs and "fps" not in runtime_kwargs:
+                merged.pop("fps", None)
+            elif "fps" in runtime_kwargs and "num_frames" not in runtime_kwargs:
+                merged.pop("num_frames", None)
+        return merged
+
     def __init__(
         self,
         image_io: ImageMediaIO,
@@ -28,7 +50,8 @@ class VideoMediaIO(MediaIO[tuple[npt.NDArray, dict[str, Any]]]):
         self.image_io = image_io
         self.num_frames = num_frames
         # `kwargs` contains custom arguments from
-        # --media-io-kwargs for this modality.
+        # --media-io-kwargs for this modality, merged with
+        # per-request runtime media_io_kwargs via merge_kwargs().
         # They can be passed to the underlying
         # media loaders (e.g. custom implementations)
         # for flexible control.

vllm/parser/__init__.py

@@ -22,13 +22,6 @@ _PARSERS_TO_REGISTER = {
     ),
 }
 
-# Register lazy parsers
-ParserManager.register_lazy_module(
-    name="minimax_m2",
-    module_path="vllm.parser.minimax_m2_parser",
-    class_name="MiniMaxM2Parser",
-)
-
 
 def register_lazy_parsers():
     for name, (file_name, class_name) in _PARSERS_TO_REGISTER.items():

vllm/renderers/deepseek_v32.py

@@ -49,6 +49,7 @@ class DeepseekV32Renderer(BaseRenderer[DeepseekV32Tokenizer]):
             messages,
             self.model_config,
             content_format="string",
+            media_io_kwargs=params.media_io_kwargs,
         )
 
         prompt_raw = tokenizer.apply_chat_template(
@@ -75,6 +76,7 @@ class DeepseekV32Renderer(BaseRenderer[DeepseekV32Tokenizer]):
             messages,
             self.model_config,
             content_format="string",
+            media_io_kwargs=params.media_io_kwargs,
         )
 
         prompt_raw = tokenizer.apply_chat_template(

vllm/renderers/grok2.py

@@ -49,6 +49,7 @@ class Grok2Renderer(BaseRenderer[Grok2Tokenizer]):
             messages,
             self.model_config,
             content_format="string",
+            media_io_kwargs=params.media_io_kwargs,
         )
 
         prompt_raw = tokenizer.apply_chat_template(
@@ -75,6 +76,7 @@ class Grok2Renderer(BaseRenderer[Grok2Tokenizer]):
             messages,
             self.model_config,
             content_format="string",
+            media_io_kwargs=params.media_io_kwargs,
         )
 
         prompt_raw = tokenizer.apply_chat_template(

vllm/renderers/hf.py

@@ -635,6 +635,7 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
                 tokenizer=tokenizer,
                 model_config=model_config,
             ),
+            media_io_kwargs=params.media_io_kwargs,
         )
 
         prompt_raw = safe_apply_chat_template(
@@ -689,6 +690,7 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
                 tokenizer=tokenizer,
                 model_config=model_config,
             ),
+            media_io_kwargs=params.media_io_kwargs,
         )
 
         prompt_raw = safe_apply_chat_template(

vllm/renderers/mistral.py

@@ -90,6 +90,7 @@ class MistralRenderer(BaseRenderer[MistralTokenizer]):
             messages,
             self.model_config,
             content_format="string",
+            media_io_kwargs=params.media_io_kwargs,
         )
 
         prompt_raw = safe_apply_chat_template(
@@ -116,6 +117,7 @@ class MistralRenderer(BaseRenderer[MistralTokenizer]):
             messages,
             self.model_config,
             content_format="string",
+            media_io_kwargs=params.media_io_kwargs,
         )
 
         prompt_raw = await self._apply_chat_template_async(

vllm/renderers/params.py

@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Any, TypeVar
 from vllm.exceptions import VLLMValidationError
 from vllm.inputs import EmbedsPrompt, TextPrompt, TokensPrompt
 from vllm.logger import init_logger
+from vllm.multimodal.media.connector import merge_media_io_kwargs
 from vllm.tokenizers import TokenizerLike
 from vllm.utils.import_utils import LazyLoader
 
@@ -52,8 +53,15 @@ class ChatParams:
     chat_template_kwargs: dict[str, Any] = field(default_factory=dict)
     """The kwargs to pass to the chat template."""
 
-    def with_defaults(self, default_chat_template_kwargs: dict[str, Any] | None):
-        if not default_chat_template_kwargs:
+    media_io_kwargs: dict[str, dict[str, Any]] | None = None
+    """Per-modality kwargs for media I/O (loading/decoding images, videos, etc.)."""
+
+    def with_defaults(
+        self,
+        default_chat_template_kwargs: dict[str, Any] | None = None,
+        default_media_io_kwargs: dict[str, dict[str, Any]] | None = None,
+    ):
+        if not default_chat_template_kwargs and not default_media_io_kwargs:
             return self
 
         return ChatParams(
@@ -63,6 +71,10 @@ class ChatParams:
                 default_chat_template_kwargs,
                 self.chat_template_kwargs,
             ),
+            media_io_kwargs=merge_media_io_kwargs(
+                default_media_io_kwargs,
+                self.media_io_kwargs,
+            ),
         )
 
     def get_apply_chat_template_kwargs(self) -> dict[str, Any]:

vllm/renderers/terratorch.py

@@ -43,6 +43,7 @@ class TerratorchRenderer(BaseRenderer):
             messages,
             model_config,
             content_format="string",
+            media_io_kwargs=params.media_io_kwargs,
         )
 
         prompt = parse_dec_only_prompt([1])  # Dummy token IDs
@@ -64,6 +65,7 @@ class TerratorchRenderer(BaseRenderer):
             messages,
             model_config,
             content_format="string",
+            media_io_kwargs=params.media_io_kwargs,
         )
 
         prompt = parse_dec_only_prompt([1])  # Dummy token IDs

vllm/transformers_utils/config.py

@@ -24,7 +24,10 @@ from transformers.utils import CONFIG_NAME as HF_CONFIG_NAME
 from vllm import envs
 from vllm.logger import init_logger
 from vllm.transformers_utils.repo_utils import is_mistral_model_repo
-from vllm.transformers_utils.utils import parse_safetensors_file_metadata
+from vllm.transformers_utils.utils import (
+    parse_safetensors_file_metadata,
+    without_trust_remote_code,
+)
 
 from .config_parser_base import ConfigParserBase
 from .gguf_utils import (
@@ -140,11 +143,12 @@ class HFConfigParser(ConfigParserBase):
         **kwargs,
     ) -> tuple[dict, PretrainedConfig]:
         kwargs["local_files_only"] = huggingface_hub.constants.HF_HUB_OFFLINE
+        trust_remote_code |= kwargs.get("trust_remote_code", False)
+        kwargs = without_trust_remote_code(kwargs)
         config_dict, _ = PretrainedConfig.get_config_dict(
             model,
             revision=revision,
             code_revision=code_revision,
-            trust_remote_code=trust_remote_code,
             **kwargs,
         )
         # Use custom model class if it's in our registry
@@ -225,7 +229,7 @@ class MistralConfigParser(ConfigParserBase):
                 model,
                 revision=revision,
                 code_revision=code_revision,
-                **kwargs,
+                **without_trust_remote_code(kwargs),
             )
         except OSError:  # Not found
             hf_config_dict = {}
@@ -521,8 +525,7 @@ def maybe_override_with_speculators(
     config_dict, _ = PretrainedConfig.get_config_dict(
         model if gguf_model_repo is None else gguf_model_repo,
         revision=revision,
-        trust_remote_code=trust_remote_code,
-        **kwargs,
+        **without_trust_remote_code(kwargs),
     )
     speculators_config = config_dict.get("speculators_config")
 

vllm/transformers_utils/configs/eagle.py

@@ -5,6 +5,8 @@ import os
 
 from transformers import AutoConfig, DeepseekV2Config, PretrainedConfig
 
+from vllm.transformers_utils.utils import without_trust_remote_code
+
 
 class EAGLEConfig(PretrainedConfig):
     model_type = "eagle"
@@ -79,7 +81,7 @@ class EAGLEConfig(PretrainedConfig):
         **kwargs,
     ) -> "EAGLEConfig":
         config_dict, kwargs = cls.get_config_dict(
-            pretrained_model_name_or_path, **kwargs
+            pretrained_model_name_or_path, **without_trust_remote_code(kwargs)
         )
         return cls.from_dict(config_dict, **kwargs)
 

vllm/transformers_utils/configs/extract_hidden_states.py

@@ -7,6 +7,8 @@ import os
 
 from transformers import PretrainedConfig
 
+from vllm.transformers_utils.utils import without_trust_remote_code
+
 
 class ExtractHiddenStatesConfig(PretrainedConfig):
     model_type = "extract_hidden_states"
@@ -42,7 +44,7 @@ class ExtractHiddenStatesConfig(PretrainedConfig):
         **kwargs,
     ) -> "ExtractHiddenStatesConfig":
         config_dict, kwargs = cls.get_config_dict(
-            pretrained_model_name_or_path, **kwargs
+            pretrained_model_name_or_path, **without_trust_remote_code(kwargs)
         )
         return cls.from_dict(config_dict, **kwargs)
 

vllm/transformers_utils/configs/medusa.py

@@ -5,6 +5,8 @@ import os
 
 from transformers import PretrainedConfig
 
+from vllm.transformers_utils.utils import without_trust_remote_code
+
 
 class MedusaConfig(PretrainedConfig):
     model_type = "medusa"
@@ -42,7 +44,7 @@ class MedusaConfig(PretrainedConfig):
         **kwargs,
     ) -> "MedusaConfig":
         config_dict, kwargs = cls.get_config_dict(
-            pretrained_model_name_or_path, **kwargs
+            pretrained_model_name_or_path, **without_trust_remote_code(kwargs)
         )
         for k in list(config_dict.keys()):
             if "num" in k:

vllm/transformers_utils/configs/speculators/base.py

@@ -11,6 +11,8 @@ from vllm.transformers_utils.configs.speculators.algos import (
 
 __all__ = ["SpeculatorsConfig"]
 
+from vllm.transformers_utils.utils import without_trust_remote_code
+
 
 class SpeculatorsConfig(PretrainedConfig):
     model_type = "speculators"
@@ -22,7 +24,9 @@ class SpeculatorsConfig(PretrainedConfig):
         **kwargs,
     ) -> "SpeculatorsConfig":
         """Load speculators Eagle config and convert to vLLM format."""
-        config_dict, _ = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
+        config_dict, _ = cls.get_config_dict(
+            pretrained_model_name_or_path, **without_trust_remote_code(kwargs)
+        )
 
         vllm_config = cls.extract_transformers_pre_trained_config(config_dict)
         return cls(**vllm_config)

vllm/transformers_utils/utils.py

@@ -27,6 +27,13 @@ def is_cloud_storage(model_or_path: str) -> bool:
     return is_s3(model_or_path) or is_gcs(model_or_path)
 
 
+def without_trust_remote_code(kwargs: dict[str, Any]) -> dict[str, Any]:
+    """Return kwargs without trust_remote_code without modifying original dict."""
+    if "trust_remote_code" not in kwargs:
+        return kwargs
+    return {k: v for k, v in kwargs.items() if k != "trust_remote_code"}
+
+
 def modelscope_list_repo_files(
     repo_id: str,
     revision: str | None = None,

vllm/v1/attention/backends/rocm_aiter_fa.py

@@ -1152,11 +1152,10 @@ class AiterFlashAttentionImpl(AttentionImpl):
                 decode_max_query_len = attn_metadata.decode_metadata.max_query_len
 
                 # Use unified_attention for speculative decoding (multi-token)
-                # or when sliding window is enabled
-                if self.sliding_window[0] != -1 or decode_max_query_len > 1:
+                if decode_max_query_len > 1:
                     assert not rocm_aiter_ops.is_shuffle_kv_cache_enabled(), (
-                        "Shuffle KV cache layout is not supported with sliding "
-                        "window or speculative decoding (multi-token decode)."
+                        "Shuffle KV cache layout is not supported with "
+                        "speculative decoding (multi-token decode)."
                     )
                     from aiter.ops.triton.unified_attention import (
                         unified_attention,

vllm/v1/engine/core_client.py

@@ -567,10 +567,7 @@ class MPClient(EngineCoreClient):
                 )
 
                 with launch_core_engines(
-                    vllm_config,
-                    executor_class,
-                    log_stats,
-                    addresses,
+                    vllm_config, executor_class, log_stats, addresses
                 ) as (engine_manager, coordinator, addresses):
                     self.resources.coordinator = coordinator
                     self.resources.engine_manager = engine_manager
@@ -638,10 +635,10 @@ class MPClient(EngineCoreClient):
 
     def shutdown(self, timeout: float | None = None) -> None:
         """Shutdown engine manager under timeout and clean up resources."""
-        self._finalizer.detach()
-        if self.resources.engine_manager is not None:
-            self.resources.engine_manager.shutdown(timeout=timeout)
-        self.resources()
+        if self._finalizer.detach() is not None:
+            if self.resources.engine_manager is not None:
+                self.resources.engine_manager.shutdown(timeout=timeout)
+            self.resources()
 
     def _format_exception(self, e: Exception) -> Exception:
         """If errored, use EngineDeadError so root cause is clear."""
@@ -685,7 +682,7 @@ class MPClient(EngineCoreClient):
             sentinels = [proc.sentinel for proc in engine_processes]
             died = multiprocessing.connection.wait(sentinels)
             _self = self_ref()
-            if not _self or _self.resources.engine_dead:
+            if not _self or not _self._finalizer.alive or _self.resources.engine_dead:
                 return
             _self.resources.engine_dead = True
             proc_name = next(

vllm/v1/worker/gpu/warmup.py

@@ -1,6 +1,9 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 
+from collections.abc import Callable
+from typing import Any
+
 import numpy as np
 import torch
 
@@ -17,9 +20,14 @@ from vllm.v1.worker.gpu.model_runner import GPUModelRunner
 
 
 @torch.inference_mode()
-def warmup_kernels(model_runner: GPUModelRunner) -> None:
+def warmup_kernels(
+    model_runner: GPUModelRunner,
+    worker_execute_model: Callable[[SchedulerOutput], Any],
+    worker_sample_tokens: Callable[[GrammarOutput | None], Any],
+) -> None:
     """Run two execute_model + sample_tokens iterations to JIT compile
-    triton kernels.
+    triton kernels. We must call the provided worker's execute_model for
+    pipeline parallel coordination.
 
     The first iteration simulates a prefill with requests of 2 prompt
     tokens each. The second iteration simulates a decode step with all
@@ -83,7 +91,7 @@ def warmup_kernels(model_runner: GPUModelRunner) -> None:
 
     # Disable KV connector for warmup run.
     model_runner.kv_connector.set_disabled(True)
-    model_runner.execute_model(prefill_output)
+    worker_execute_model(prefill_output)
 
     if not model_runner.is_pooling_model:
         # Warm up sampler and perform a decode step for non-pooling models.
@@ -101,7 +109,7 @@ def warmup_kernels(model_runner: GPUModelRunner) -> None:
                 structured_output_request_ids=req_ids, grammar_bitmask=grammar_bitmask
             )
 
-        model_runner.sample_tokens(grammar_output)
+        worker_sample_tokens(grammar_output)
 
         # Step 2: Decode all requests with 1 token each.
         cached_req_data = CachedRequestData.make_empty()
@@ -120,12 +128,12 @@ def warmup_kernels(model_runner: GPUModelRunner) -> None:
         decode_output.total_num_scheduled_tokens = num_reqs
         decode_output.num_common_prefix_blocks = [0] * num_kv_cache_groups
 
-        model_runner.execute_model(decode_output)
-        model_runner.sample_tokens(None)
+        worker_execute_model(decode_output)
+        worker_sample_tokens(None)
 
     # Clean up - process finish_req_ids.
     cleanup_output = SchedulerOutput.make_empty()
     cleanup_output.finished_req_ids = set(req_ids)
-    model_runner.execute_model(cleanup_output)
+    worker_execute_model(cleanup_output)
     model_runner.kv_connector.set_disabled(False)
     torch.accelerator.synchronize()

vllm/v1/worker/gpu_worker.py

@@ -584,7 +584,7 @@ class Worker(WorkerBase):
 
         if self.use_v2_model_runner:
             # V2: Run full execute_model + sample_tokens to JIT compile triton kernels.
-            warmup_kernels(self.model_runner)
+            warmup_kernels(self.model_runner, self.execute_model, self.sample_tokens)
         elif get_pp_group().is_last_rank:
             # V1: Warm up sampler and preallocate memory buffer for logits and other
             # sampling related tensors of max possible shape to avoid memory