[Frontend] Consolidate scale out entrypoints (#44512)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>

.buildkite/test-amd.yaml

@@ -845,10 +845,12 @@ steps:
   source_file_dependencies:
   - vllm/
   - tests/entrypoints/serve
+  - tests/entrypoints/scale_out
   commands:
   - export VLLM_WORKER_MULTIPROC_METHOD=spawn
   - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc
   - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc
+  - pytest -v -s entrypoints/scale_out
 
 - label: Entrypoints Integration (API Server OpenAI - Part 1) # TBD
   timeout_in_minutes: 180
@@ -2559,10 +2561,12 @@ steps:
   source_file_dependencies:
   - vllm/
   - tests/entrypoints/serve
+  - tests/entrypoints/scale_out
   commands:
   - export VLLM_WORKER_MULTIPROC_METHOD=spawn
   - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc
   - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc
+  - pytest -v -s entrypoints/scale_out
 
 - label: Entrypoints Integration (API Server OpenAI - Part 1) # TBD
   timeout_in_minutes: 180

.buildkite/test_areas/entrypoints.yaml

@@ -40,10 +40,12 @@ steps:
   source_file_dependencies:
   - vllm/
   - tests/entrypoints/serve
+  - tests/entrypoints/scale_out
   commands:
   - export VLLM_WORKER_MULTIPROC_METHOD=spawn
   - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc
   - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc
+  - pytest -v -s entrypoints/scale_out
   mirror:
     amd:
       device: mi325_1

.buildkite/test_areas/rust_frontend.yaml

@@ -46,7 +46,7 @@ steps:
   - vllm/v1/engine/
   - tests/utils.py
   # - tests/entrypoints/serve/dev/rpc/test_collective_rpc.py
-  - tests/entrypoints/serve/disagg/test_serving_tokens.py
+  - tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py
   - tests/entrypoints/serve/instrumentator/test_basic.py
   - tests/entrypoints/serve/instrumentator/test_metrics.py
   # - tests/entrypoints/serve/dev/test_sleep.py
@@ -55,7 +55,7 @@ steps:
   - export VLLM_WORKER_MULTIPROC_METHOD=spawn
   # - pytest -v -s entrypoints/serve/dev/rpc/test_collective_rpc.py
   - pytest -v -s entrypoints/serve/instrumentator/test_basic.py -k "not show_version and not server_load"
-  - pytest -v -s entrypoints/serve/disagg/test_serving_tokens.py -k "not stream and not lora and not test_generate_logprobs and not stop_string_workflow"
+  - pytest -v -s entrypoints/scale_out/token_in_token_out/test_serving_tokens.py -k "not stream and not lora and not test_generate_logprobs and not stop_string_workflow"
   - pytest -v -s entrypoints/serve/instrumentator/test_metrics.py -k "text and not show and not run_batch and not test_metrics_counts and not test_metrics_exist"
   # - pytest -v -s entrypoints/serve/dev/test_sleep.py
 

docs/examples/README.md

@@ -13,5 +13,6 @@ vLLM's examples are organized into the following categories:
 - **[`rl/`](../../examples/rl)** – Reinforcement learning examples.
 - **[`deployment/`](../../examples/deployment)** – Examples for deploying vLLM in production.
 - **[`ray_serving/`](../../examples/ray_serving)** – Scalable serving using Ray.
-- **[`disaggregated/`](../../examples/disaggregated)** – Examples for disaggregated serving (separate prefill and decode), including various kv cache connectors (LMCache, Mooncake, FlexKV, P2P NCCL) and failure recovery.
+- **[`disaggregated/`](../../examples/disaggregated)** – Examples for Disaggregated P/D (Prefill/Decoding) inference, including various kv cache connectors (LMCache, Mooncake, FlexKV, P2P NCCL) and failure recovery.
+- **[`scale_out/`](../../examples/scale_out)** – Examples for Token In <> Token Out API Server.
 - **[`observability/`](../../examples/observability)** – Metrics, logging, tracing (OpenTelemetry), and dashboards (Grafana, Perses).

docs/serving/online_serving/README.md

@@ -119,9 +119,9 @@ For further details on profiling vLLM, please refer to [this page](../../contrib
 - `/ping` - SageMaker health check
 - `/invocations` - SageMaker-compatible endpoint (routes to the same inference functions as `/v1` endpoints)
 
-## Disaggregated Everything
+## Scale-Out APIs
 
-### Tokens IN <> Tokens OUT
+### Tokens IN <> Tokens OUT APIs
 
 - `/inference/v1/generate` - Generate completions
 - `/abort_requests` - Abort in-flight requests (only when `--tokens-only` is also set)

tests/entrypoints/openai/chat_completion/test_chat_error.py

@@ -17,10 +17,9 @@ from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
 from vllm.entrypoints.openai.engine.protocol import GenerationError
 from vllm.entrypoints.openai.models.protocol import BaseModelPath
 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
-from vllm.entrypoints.serve.render.serving import ServingRender
+from vllm.entrypoints.scale_out.render.serving import ServingRender
 from vllm.outputs import CompletionOutput, RequestOutput
 from vllm.renderers.hf import HfRenderer
-from vllm.renderers.online_derenderer import OnlineDerenderer
 from vllm.renderers.online_renderer import OnlineRenderer
 from vllm.tokenizers.registry import cached_tokenizer_from_config
 from vllm.v1.engine.async_llm import AsyncLLM
@@ -206,15 +205,8 @@ def _build_serving_render(engine: AsyncLLM) -> ServingRender:
         chat_template=None,
         chat_template_content_format="auto",
     )
-    online_derenderer = OnlineDerenderer(
-        model_config=engine.model_config,
-        renderer=engine.renderer,
-        request_logger=None,
-        chat_template=None,
-        chat_template_content_format="auto",
-    )
 
-    serving_render = ServingRender(models, online_renderer, online_derenderer)
+    serving_render = ServingRender(models, online_renderer)
 
     async def _fake_preprocess_chat(*args, **kwargs):
         # return conversation, engine_inputs

tests/entrypoints/openai/completion/test_completion_error.py

@@ -14,10 +14,9 @@ from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
 from vllm.entrypoints.openai.engine.protocol import GenerationError
 from vllm.entrypoints.openai.models.protocol import BaseModelPath
 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
-from vllm.entrypoints.serve.render.serving import ServingRender
+from vllm.entrypoints.scale_out.render.serving import ServingRender
 from vllm.outputs import CompletionOutput, RequestOutput
 from vllm.renderers.hf import HfRenderer
-from vllm.renderers.online_derenderer import OnlineDerenderer
 from vllm.renderers.online_renderer import OnlineRenderer
 from vllm.tokenizers.registry import cached_tokenizer_from_config
 from vllm.v1.engine.async_llm import AsyncLLM
@@ -191,15 +190,8 @@ def _build_serving_render(engine: AsyncLLM) -> ServingRender:
         chat_template=None,
         chat_template_content_format="auto",
     )
-    online_derenderer = OnlineDerenderer(
-        model_config=engine.model_config,
-        renderer=engine.renderer,
-        request_logger=None,
-        chat_template=None,
-        chat_template_content_format="auto",
-    )
 
-    serving_render = ServingRender(models, online_renderer, online_derenderer)
+    serving_render = ServingRender(models, online_renderer)
 
     async def _fake_preprocess_chat(*args, **kwargs):
         # return conversation, engine_inputs

tests/entrypoints/openai/test_render_token_offsets.py

@@ -3,7 +3,7 @@
 """Unit tests for the token-offsets request/response protocol wiring:
 the request flag flowing into ``TokenizeParams`` and the ``GenerateRequest``
 serialization boundary. End-to-end behavior is covered by
-``tests/entrypoints/serve/render/test_render.py``; plain Pydantic field
+``tests/entrypoints/scale_out/render/test_render.py``; plain Pydantic field
 storage is not retested here.
 """
 
@@ -12,7 +12,7 @@ from unittest.mock import Mock
 from vllm.config import ModelConfig
 from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
 from vllm.entrypoints.openai.completion.protocol import CompletionRequest
-from vllm.entrypoints.serve.disagg.protocol import GenerateRequest
+from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest
 from vllm.sampling_params import SamplingParams
 
 

tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py

@@ -12,11 +12,11 @@ from vllm.config.multimodal import MultiModalConfig
 from vllm.entrypoints.openai.engine.protocol import StreamOptions
 from vllm.entrypoints.openai.models.protocol import BaseModelPath
 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
-from vllm.entrypoints.serve.disagg.protocol import (
+from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
     GenerateRequest,
     GenerateResponse,
 )
-from vllm.entrypoints.serve.disagg.serving import ServingTokens
+from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
 from vllm.logprobs import Logprob
 from vllm.outputs import CompletionOutput, RequestOutput
 from vllm.renderers import renderer_from_config

tests/entrypoints/scale_out/token_in_token_out/test_mm_serde.py

@@ -1,14 +1,17 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""Roundtrip tests for multimodal serde used by the disagg generate endpoint."""
+"""
+Roundtrip tests for multimodal serde used by the
+token_in_token_out generate endpoint.
+"""
 
 import torch
 
-from vllm.entrypoints.serve.disagg.mm_serde import (
+from vllm.entrypoints.scale_out.token_in_token_out.mm_serde import (
     decode_mm_kwargs_item,
     encode_mm_kwargs_item,
 )
-from vllm.entrypoints.serve.disagg.protocol import (
+from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
     MultiModalFeatures,
     PlaceholderRangeInfo,
 )

tests/entrypoints/scale_out/token_in_token_out/test_protocol.py

@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""Unit tests for the disagg request/response protocol.
+"""Unit tests for the token_in_token_out request/response protocol.
 
 These tests intentionally avoid spinning up a server — they exercise the
 pydantic validators on ``GenerateRequest`` directly so they run fast and
@@ -9,7 +9,7 @@ fail loudly if the validator semantics ever drift.
 
 import json
 
-from vllm.entrypoints.serve.disagg.protocol import GenerateRequest
+from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest
 from vllm.sampling_params import SamplingParams
 
 

tests/entrypoints/scale_out/token_in_token_out/test_tokens_logprobs.py

@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 
-from vllm.entrypoints.serve.disagg.serving import ServingTokens
+from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
 from vllm.logprobs import Logprob
 
 

tests/renderers/test_token_offsets.py

@@ -5,7 +5,7 @@
 These exercise ``_tokenize_prompt`` (offset extraction + capability/MM
 gating) and the ``_tokenize_prompt -> _process_tokens -> TokensInput``
 forwarding chain. Endpoint-level coverage lives in
-``tests/entrypoints/serve/render/test_render.py``.
+``tests/entrypoints/scale_out/render/test_render.py``.
 """
 
 import pytest

vllm/entrypoints/generate/api_router.py

@@ -66,7 +66,6 @@ async def init_generate_state(
     from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
     from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
     from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
-    from vllm.entrypoints.serve.disagg.serving import ServingTokens
     from vllm.entrypoints.serve.utils.fingerprint import set_default_fingerprint_mode
 
     # Applied before any serving class is constructed so that each one picks
@@ -175,20 +174,6 @@ async def init_generate_state(
         if "generate" in supported_tasks
         else None
     )
-    state.serving_tokens = (
-        ServingTokens(
-            engine_client,
-            state.openai_serving_models,
-            state.online_renderer,
-            request_logger=request_logger,
-            return_tokens_as_token_ids=args.return_tokens_as_token_ids,
-            enable_prompt_tokens_details=args.enable_prompt_tokens_details,
-            enable_log_outputs=args.enable_log_outputs,
-            force_no_detokenize=args.tokens_only,
-        )
-        if "generate" in supported_tasks
-        else None
-    )
 
     from .generative_scoring.serving import ServingGenerativeScoring
 

vllm/entrypoints/openai/api_server.py

@@ -32,7 +32,6 @@ from vllm.entrypoints.openai.engine.protocol import GenerationError
 from vllm.entrypoints.openai.models.protocol import BaseModelPath
 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
 from vllm.entrypoints.serve.elastic_ep.middleware import ScalingMiddleware
-from vllm.entrypoints.serve.render.serving import ServingRender
 from vllm.entrypoints.serve.sagemaker.api_router import sagemaker_standards_bootstrap
 from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
 from vllm.entrypoints.serve.utils.api_utils import (
@@ -208,12 +207,6 @@ def build_app(
 
         register_generate_api_routers(app)
 
-        from vllm.entrypoints.serve.disagg.api_router import (
-            attach_router as attach_disagg_router,
-        )
-
-        attach_disagg_router(app)
-
         from vllm.entrypoints.serve.elastic_ep.api_router import (
             attach_router as elastic_ep_attach_router,
         )
@@ -221,11 +214,9 @@ def build_app(
         elastic_ep_attach_router(app)
 
     if "generate" in supported_tasks or "render" in supported_tasks:
-        from vllm.entrypoints.serve.render.api_router import (
-            attach_router as attach_render_router,
-        )
+        from vllm.entrypoints.scale_out.factories import register_scale_out_api_routers
 
-        attach_render_router(app)
+        register_scale_out_api_routers(app, supported_tasks)
 
     if "transcription" in supported_tasks or "realtime" in supported_tasks:
         from vllm.entrypoints.speech_to_text.factories import (
@@ -401,12 +392,6 @@ async def init_app_state(
         default_chat_template_kwargs=args.default_chat_template_kwargs,
         trust_request_chat_template=args.trust_request_chat_template,
     )
-    state.serving_render = ServingRender(
-        state.openai_serving_models,
-        state.online_renderer,
-        state.online_derenderer,
-        request_logger=request_logger,
-    )
 
     if "generate" in supported_tasks:
         from vllm.entrypoints.generate.api_router import init_generate_state
@@ -415,6 +400,10 @@ async def init_app_state(
             engine_client, state, args, request_logger, supported_tasks
         )
 
+        from vllm.entrypoints.scale_out.factories import init_scale_out_state
+
+        init_scale_out_state(state, args, engine_client, request_logger)
+
     if "transcription" in supported_tasks or "realtime" in supported_tasks:
         from vllm.entrypoints.speech_to_text.factories import init_speech_to_text_state
 
@@ -505,12 +494,10 @@ async def init_render_app_state(
         default_chat_template_kwargs=args.default_chat_template_kwargs,
         trust_request_chat_template=args.trust_request_chat_template,
     )
-    state.serving_render = ServingRender(
-        model_registry,
-        state.online_renderer,
-        state.online_derenderer,
-        request_logger=request_logger,
-    )
+
+    from vllm.entrypoints.scale_out.factories import init_render_state
+
+    init_render_state(state, request_logger)
 
     state.vllm_config = vllm_config
     # Disable stats logging — there is no engine to poll.

vllm/entrypoints/scale_out/derender/api_router.py

@@ -0,0 +1,74 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+from http import HTTPStatus
+
+from fastapi import APIRouter, Depends, Request
+from fastapi.responses import JSONResponse
+
+from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionResponse
+from vllm.entrypoints.openai.completion.protocol import CompletionResponse
+from vllm.entrypoints.openai.engine.protocol import ErrorResponse
+from vllm.entrypoints.serve.utils.api_utils import validate_json_request
+from vllm.logger import init_logger
+
+from ..token_in_token_out.protocol import (
+    DerenderChatRequest,
+    DerenderCompletionRequest,
+)
+from .serving import ServingDerender
+
+logger = init_logger(__name__)
+
+router = APIRouter()
+
+
+def derender(request: Request) -> ServingDerender | None:
+    return getattr(request.app.state, "serving_derender", None)
+
+
+@router.post(
+    "/v1/chat/completions/derender",
+    dependencies=[Depends(validate_json_request)],
+    response_model=ChatCompletionResponse,
+    responses={
+        HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
+        HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
+        HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
+    },
+)
+async def derender_chat_completion(request: DerenderChatRequest, raw_request: Request):
+    handler = derender(raw_request)
+    if handler is None:
+        raise NotImplementedError(
+            "The model does not support Chat Completions Derender API"
+        )
+
+    result = await handler.derender_chat_response(request)
+
+    if isinstance(result, ErrorResponse):
+        return JSONResponse(content=result.model_dump(), status_code=result.error.code)
+
+    return JSONResponse(content=result.model_dump())
+
+
+@router.post(
+    "/v1/completions/derender",
+    dependencies=[Depends(validate_json_request)],
+    response_model=CompletionResponse,
+    responses={
+        HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
+        HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
+        HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
+    },
+)
+async def derender_completion(request: DerenderCompletionRequest, raw_request: Request):
+    handler = derender(raw_request)
+    if handler is None:
+        raise NotImplementedError("The model does not support Completions Derender API")
+
+    result = await handler.derender_completion_response(request)
+
+    if isinstance(result, ErrorResponse):
+        return JSONResponse(content=result.model_dump(), status_code=result.error.code)
+
+    return JSONResponse(content=result.model_dump())

vllm/entrypoints/scale_out/derender/serving.py

@@ -0,0 +1,202 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import time
+from typing import cast
+
+from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionResponse
+from vllm.entrypoints.openai.completion.protocol import CompletionResponse
+from vllm.entrypoints.openai.engine.protocol import (
+    ErrorResponse,
+    UsageInfo,
+)
+from vllm.entrypoints.openai.models.serving import (
+    OpenAIModelRegistry,
+    OpenAIServingModels,
+)
+from vllm.entrypoints.serve.engine.serving import BaseServing
+from vllm.entrypoints.serve.utils.request_logger import RequestLogger
+from vllm.inputs import (
+    EngineInput,
+    MultiModalHashes,
+    MultiModalInput,
+    MultiModalPlaceholders,
+)
+from vllm.logger import init_logger
+from vllm.renderers.online_derenderer import OnlineDerenderer
+
+from ..token_in_token_out.mm_serde import encode_mm_kwargs_item
+from ..token_in_token_out.protocol import (
+    DerenderChatRequest,
+    DerenderCompletionRequest,
+    MultiModalFeatures,
+    PlaceholderRangeInfo,
+)
+
+logger = init_logger(__name__)
+
+
+class ServingDerender(BaseServing):
+    def __init__(
+        self,
+        models: OpenAIServingModels | OpenAIModelRegistry,
+        online_derenderer: "OnlineDerenderer",
+        *,
+        request_logger: RequestLogger | None = None,
+    ) -> None:
+        super().__init__(
+            models=models,
+            model_config=models.model_config,
+            request_logger=request_logger,
+        )
+
+        self.online_derenderer = online_derenderer
+
+    async def derender_chat_response(
+        self,
+        request: DerenderChatRequest,
+    ) -> ChatCompletionResponse | ErrorResponse:
+        """Postprocess a GenerateResponse into a ChatCompletionResponse.
+
+        Non-streaming only: expects the complete GenerateResponse with all
+        token IDs present.  Uses ``parser.parse()`` for one-shot extraction.
+
+        When ``request.chat_request`` is provided, the parser splits the
+        output into (reasoning, content, tool_calls).  Otherwise falls
+        back to plain detokenization.
+        """
+        error_check_ret = await self._check_model(request)
+        if error_check_ret is not None:
+            return error_check_ret
+
+        try:
+            choices = await self.online_derenderer.derender_chat(
+                request.generate_response, request.chat_request
+            )
+        except ValueError as exc:
+            return self.create_error_response(str(exc))
+
+        prompt_tokens = (
+            request.prompt_tokens if request.prompt_tokens is not None else 0
+        )
+        gen = request.generate_response
+        completion_tokens = sum(len(ch.token_ids) for ch in gen.choices if ch.token_ids)
+        usage = UsageInfo(
+            prompt_tokens=prompt_tokens,
+            completion_tokens=completion_tokens,
+            total_tokens=prompt_tokens + completion_tokens,
+        )
+
+        logger.debug(
+            "derender_chat request_id=%s model=%s choices=%d completion_tokens=%d",
+            gen.request_id,
+            request.model,
+            len(choices),
+            completion_tokens,
+        )
+        return ChatCompletionResponse(
+            id=gen.request_id,
+            model=request.model,
+            created=int(time.time()),
+            choices=choices,
+            usage=usage,
+            prompt_logprobs=gen.prompt_logprobs,
+            kv_transfer_params=gen.kv_transfer_params,
+        )
+
+    async def derender_completion_response(
+        self,
+        request: DerenderCompletionRequest,
+    ) -> CompletionResponse | ErrorResponse:
+        """Postprocess a list of GenerateResponses into a CompletionResponse.
+
+        Non-streaming only.  Mirrors the multi-prompt completions case: one
+        GenerateResponse per prompt, parallel to the list[GenerateRequest]
+        from /v1/completions/render.
+        """
+        error_check_ret = await self._check_model(request)
+        if error_check_ret is not None:
+            return error_check_ret
+
+        (
+            choices,
+            total_prompt_tokens,
+            total_completion_tokens,
+        ) = await self.online_derenderer.derender_completion(
+            request.generate_responses, request.prompt_tokens
+        )
+
+        if not request.generate_responses:
+            return self.create_error_response("generate_responses must not be empty")
+
+        first = request.generate_responses[0]
+        kv_params = first.kv_transfer_params
+        if any(
+            r.kv_transfer_params != kv_params for r in request.generate_responses[1:]
+        ):
+            logger.warning(
+                "derender_completion: kv_transfer_params differ across responses; "
+                "setting to None on the aggregated response"
+            )
+            kv_params = None
+
+        usage = UsageInfo(
+            prompt_tokens=total_prompt_tokens,
+            completion_tokens=total_completion_tokens,
+            total_tokens=total_prompt_tokens + total_completion_tokens,
+        )
+
+        logger.debug(
+            "derender_completion request_id=%s model=%s choices=%d"
+            " completion_tokens=%d",
+            first.request_id,
+            request.model,
+            len(choices),
+            total_completion_tokens,
+        )
+        return CompletionResponse(
+            id=first.request_id,
+            model=request.model,
+            created=int(time.time()),
+            choices=choices,
+            usage=usage,
+            kv_transfer_params=kv_params,
+        )
+
+    @staticmethod
+    def _extract_mm_features(
+        engine_input: EngineInput,
+    ) -> MultiModalFeatures | None:
+        """Extract multimodal metadata from a rendered engine prompt.
+
+        Returns ``None`` for text-only prompts.
+        """
+        if engine_input.get("type") != "multimodal":
+            return None
+
+        # At this point engine_input is a MultiModalInput TypedDict.
+        mm_engine_input = cast(MultiModalInput, engine_input)
+        mm_hashes: MultiModalHashes = mm_engine_input["mm_hashes"]
+        raw_placeholders: MultiModalPlaceholders = mm_engine_input["mm_placeholders"]
+
+        mm_placeholders = {
+            modality: [
+                PlaceholderRangeInfo(offset=p.offset, length=p.length) for p in ranges
+            ]
+            for modality, ranges in raw_placeholders.items()
+        }
+
+        # Serialize tensor data per modality.
+        kwargs_data: dict[str, list[str | None]] | None = None
+        if raw_mm_kwargs := mm_engine_input.get("mm_kwargs"):
+            kwargs_data = {}
+            for modality, items in raw_mm_kwargs.items():
+                kwargs_data[modality] = [
+                    encode_mm_kwargs_item(item) if item is not None else None
+                    for item in items
+                ]
+
+        return MultiModalFeatures(
+            mm_hashes=mm_hashes,
+            mm_placeholders=mm_placeholders,
+            kwargs_data=kwargs_data,
+        )

vllm/entrypoints/scale_out/factories.py

@@ -0,0 +1,78 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+from argparse import Namespace
+from typing import TYPE_CHECKING
+
+from fastapi import FastAPI
+
+from vllm.engine.protocol import EngineClient
+from vllm.tasks import SupportedTask
+
+if TYPE_CHECKING:
+    from starlette.datastructures import State
+
+    from vllm.entrypoints.serve.utils.request_logger import RequestLogger
+else:
+    RequestLogger = object
+
+
+def init_render_state(
+    state: "State",
+    request_logger: RequestLogger | None,
+):
+    from .derender.serving import ServingDerender
+    from .render.serving import ServingRender
+
+    state.serving_render = ServingRender(
+        state.openai_serving_models,
+        state.online_renderer,
+        request_logger=request_logger,
+    )
+
+    state.serving_derender = ServingDerender(
+        state.openai_serving_models,
+        state.online_derenderer,
+        request_logger=request_logger,
+    )
+
+
+def init_scale_out_state(
+    state: "State",
+    args: "Namespace",
+    engine_client: "EngineClient",
+    request_logger: RequestLogger | None,
+):
+    init_render_state(state, request_logger)
+
+    from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
+
+    state.serving_tokens = ServingTokens(
+        engine_client,
+        state.openai_serving_models,
+        state.online_renderer,
+        request_logger=request_logger,
+        return_tokens_as_token_ids=args.return_tokens_as_token_ids,
+        enable_prompt_tokens_details=args.enable_prompt_tokens_details,
+        enable_log_outputs=args.enable_log_outputs,
+        force_no_detokenize=args.tokens_only,
+    )
+
+
+def register_scale_out_api_routers(
+    app: FastAPI,
+    supported_tasks: tuple["SupportedTask", ...],
+):
+    from .render.api_router import router as render_render
+
+    app.include_router(render_render)
+
+    from .derender.api_router import router as derender_render
+
+    app.include_router(derender_render)
+
+    if "generate" in supported_tasks:
+        from .token_in_token_out.api_router import (
+            attach_router as attach_disagg_router,
+        )
+
+        attach_disagg_router(app)

vllm/entrypoints/scale_out/render/api_router.py

@@ -2,27 +2,18 @@
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 from http import HTTPStatus
 
-from fastapi import APIRouter, Depends, FastAPI, Request
+from fastapi import APIRouter, Depends, Request
 from fastapi.responses import JSONResponse
 
-from vllm.entrypoints.openai.chat_completion.protocol import (
-    ChatCompletionRequest,
-    ChatCompletionResponse,
-)
-from vllm.entrypoints.openai.completion.protocol import (
-    CompletionRequest,
-    CompletionResponse,
-)
+from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
+from vllm.entrypoints.openai.completion.protocol import CompletionRequest
 from vllm.entrypoints.openai.engine.protocol import ErrorResponse
-from vllm.entrypoints.serve.disagg.protocol import (
-    DerenderChatRequest,
-    DerenderCompletionRequest,
-    GenerateRequest,
-)
-from vllm.entrypoints.serve.render.serving import ServingRender
 from vllm.entrypoints.serve.utils.api_utils import validate_json_request
 from vllm.logger import init_logger
 
+from ..token_in_token_out.protocol import GenerateRequest
+from .serving import ServingRender
+
 logger = init_logger(__name__)
 
 router = APIRouter()
@@ -79,55 +70,3 @@ async def render_completion(request: CompletionRequest, raw_request: Request):
         return JSONResponse(content=result.model_dump(), status_code=result.error.code)
 
     return JSONResponse(content=[item.model_dump() for item in result])
-
-
-@router.post(
-    "/v1/chat/completions/derender",
-    dependencies=[Depends(validate_json_request)],
-    response_model=ChatCompletionResponse,
-    responses={
-        HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
-        HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
-        HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
-    },
-)
-async def derender_chat_completion(request: DerenderChatRequest, raw_request: Request):
-    handler = render(raw_request)
-    if handler is None:
-        raise NotImplementedError(
-            "The model does not support Chat Completions Derender API"
-        )
-
-    result = await handler.derender_chat_response(request)
-
-    if isinstance(result, ErrorResponse):
-        return JSONResponse(content=result.model_dump(), status_code=result.error.code)
-
-    return JSONResponse(content=result.model_dump())
-
-
-@router.post(
-    "/v1/completions/derender",
-    dependencies=[Depends(validate_json_request)],
-    response_model=CompletionResponse,
-    responses={
-        HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
-        HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
-        HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
-    },
-)
-async def derender_completion(request: DerenderCompletionRequest, raw_request: Request):
-    handler = render(raw_request)
-    if handler is None:
-        raise NotImplementedError("The model does not support Completions Derender API")
-
-    result = await handler.derender_completion_response(request)
-
-    if isinstance(result, ErrorResponse):
-        return JSONResponse(content=result.model_dump(), status_code=result.error.code)
-
-    return JSONResponse(content=result.model_dump())
-
-
-def attach_router(app: FastAPI) -> None:
-    app.include_router(router)

vllm/entrypoints/scale_out/render/serving.py

@@ -1,28 +1,16 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-import time
 from typing import cast
 
-from vllm.entrypoints.openai.chat_completion.protocol import (
-    ChatCompletionRequest,
-    ChatCompletionResponse,
-)
-from vllm.entrypoints.openai.completion.protocol import (
-    CompletionRequest,
-    CompletionResponse,
-)
-from vllm.entrypoints.openai.engine.protocol import (
-    ErrorResponse,
-    UsageInfo,
-)
+from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
+from vllm.entrypoints.openai.completion.protocol import CompletionRequest
+from vllm.entrypoints.openai.engine.protocol import ErrorResponse
 from vllm.entrypoints.openai.models.serving import (
     OpenAIModelRegistry,
     OpenAIServingModels,
 )
-from vllm.entrypoints.serve.disagg.mm_serde import encode_mm_kwargs_item
-from vllm.entrypoints.serve.disagg.protocol import (
-    DerenderChatRequest,
-    DerenderCompletionRequest,
+from vllm.entrypoints.scale_out.token_in_token_out.mm_serde import encode_mm_kwargs_item
+from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
     GenerateRequest,
     MultiModalFeatures,
     PlaceholderRangeInfo,
@@ -41,7 +29,6 @@ from vllm.renderers.inputs.preprocess import (
     extract_prompt_components,
     extract_prompt_len,
 )
-from vllm.renderers.online_derenderer import OnlineDerenderer
 from vllm.renderers.online_renderer import OnlineRenderer
 from vllm.utils import random_uuid
 
@@ -53,7 +40,6 @@ class ServingRender(BaseServing):
         self,
         models: OpenAIServingModels | OpenAIModelRegistry,
         online_renderer: "OnlineRenderer",
-        online_derenderer: "OnlineDerenderer",
         *,
         request_logger: RequestLogger | None = None,
     ) -> None:
@@ -64,7 +50,6 @@ class ServingRender(BaseServing):
         )
 
         self.online_renderer = online_renderer
-        self.online_derenderer = online_derenderer
 
         self.default_sampling_params = (
             online_renderer.model_config.get_diff_sampling_param()
@@ -223,117 +208,6 @@ class ServingRender(BaseServing):
 
         return generate_requests
 
-    async def derender_chat_response(
-        self,
-        request: DerenderChatRequest,
-    ) -> ChatCompletionResponse | ErrorResponse:
-        """Postprocess a GenerateResponse into a ChatCompletionResponse.
-
-        Non-streaming only: expects the complete GenerateResponse with all
-        token IDs present.  Uses ``parser.parse()`` for one-shot extraction.
-
-        When ``request.chat_request`` is provided, the parser splits the
-        output into (reasoning, content, tool_calls).  Otherwise falls
-        back to plain detokenization.
-        """
-        error_check_ret = await self._check_model(request)
-        if error_check_ret is not None:
-            return error_check_ret
-
-        try:
-            choices = await self.online_derenderer.derender_chat(
-                request.generate_response, request.chat_request
-            )
-        except ValueError as exc:
-            return self.create_error_response(str(exc))
-
-        prompt_tokens = (
-            request.prompt_tokens if request.prompt_tokens is not None else 0
-        )
-        gen = request.generate_response
-        completion_tokens = sum(len(ch.token_ids) for ch in gen.choices if ch.token_ids)
-        usage = UsageInfo(
-            prompt_tokens=prompt_tokens,
-            completion_tokens=completion_tokens,
-            total_tokens=prompt_tokens + completion_tokens,
-        )
-
-        logger.debug(
-            "derender_chat request_id=%s model=%s choices=%d completion_tokens=%d",
-            gen.request_id,
-            request.model,
-            len(choices),
-            completion_tokens,
-        )
-        return ChatCompletionResponse(
-            id=gen.request_id,
-            model=request.model,
-            created=int(time.time()),
-            choices=choices,
-            usage=usage,
-            prompt_logprobs=gen.prompt_logprobs,
-            kv_transfer_params=gen.kv_transfer_params,
-        )
-
-    async def derender_completion_response(
-        self,
-        request: DerenderCompletionRequest,
-    ) -> CompletionResponse | ErrorResponse:
-        """Postprocess a list of GenerateResponses into a CompletionResponse.
-
-        Non-streaming only.  Mirrors the multi-prompt completions case: one
-        GenerateResponse per prompt, parallel to the list[GenerateRequest]
-        from /v1/completions/render.
-        """
-        error_check_ret = await self._check_model(request)
-        if error_check_ret is not None:
-            return error_check_ret
-
-        (
-            choices,
-            total_prompt_tokens,
-            total_completion_tokens,
-        ) = await self.online_derenderer.derender_completion(
-            request.generate_responses, request.prompt_tokens
-        )
-
-        if not request.generate_responses:
-            return self.create_error_response("generate_responses must not be empty")
-
-        first = request.generate_responses[0]
-        kv_params = first.kv_transfer_params
-        if any(
-            r.kv_transfer_params != kv_params for r in request.generate_responses[1:]
-        ):
-            logger.warning(
-                "derender_completion: kv_transfer_params differ across responses; "
-                "setting to None on the aggregated response"
-            )
-            kv_params = None
-
-        usage = UsageInfo(
-            prompt_tokens=total_prompt_tokens,
-            completion_tokens=total_completion_tokens,
-            total_tokens=total_prompt_tokens + total_completion_tokens,
-        )
-
-        logger.debug(
-            "derender_completion request_id=%s model=%s choices=%d"
-            " completion_tokens=%d",
-            first.request_id,
-            request.model,
-            len(choices),
-            total_completion_tokens,
-        )
-        return CompletionResponse(
-            id=first.request_id,
-            model=request.model,
-            created=int(time.time()),
-            choices=choices,
-            usage=usage,
-            kv_transfer_params=kv_params,
-        )
-
     @staticmethod
     def _extract_mm_features(
         engine_input: EngineInput,

vllm/entrypoints/scale_out/token_in_token_out/api_router.py

@@ -11,13 +11,6 @@ from fastapi.responses import JSONResponse, StreamingResponse
 
 from vllm.engine.protocol import EngineClient
 from vllm.entrypoints.openai.engine.protocol import ErrorResponse
-from vllm.entrypoints.serve.disagg.protocol import (
-    GenerateRequest,
-    GenerateResponse,
-)
-from vllm.entrypoints.serve.disagg.serving import (
-    ServingTokens,
-)
 from vllm.entrypoints.serve.tokenize.serving import ServingTokenization
 from vllm.entrypoints.serve.utils.api_utils import (
     load_aware_call,
@@ -26,6 +19,12 @@ from vllm.entrypoints.serve.utils.api_utils import (
 )
 from vllm.logger import init_logger
 
+from .protocol import (
+    GenerateRequest,
+    GenerateResponse,
+)
+from .serving import ServingTokens
+
 logger = init_logger(__name__)
 
 

vllm/entrypoints/scale_out/token_in_token_out/serving.py

@@ -28,14 +28,6 @@ from vllm.entrypoints.openai.engine.protocol import (
 )
 from vllm.entrypoints.openai.engine.serving import OpenAIServing, clamp_prompt_logprobs
 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
-from vllm.entrypoints.serve.disagg.mm_serde import decode_mm_kwargs_item
-from vllm.entrypoints.serve.disagg.protocol import (
-    GenerateRequest,
-    GenerateResponse,
-    GenerateResponseChoice,
-    GenerateResponseStreamChoice,
-    GenerateStreamResponse,
-)
 from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
 from vllm.entrypoints.serve.utils.request_logger import RequestLogger
 from vllm.inputs import EngineInput, mm_input
@@ -51,6 +43,15 @@ from vllm.renderers.online_renderer import OnlineRenderer
 from vllm.sampling_params import RequestOutputKind, SamplingParams
 from vllm.utils.collection_utils import as_list
 
+from .mm_serde import decode_mm_kwargs_item
+from .protocol import (
+    GenerateRequest,
+    GenerateResponse,
+    GenerateResponseChoice,
+    GenerateResponseStreamChoice,
+    GenerateStreamResponse,
+)
+
 logger = init_logger(__name__)
 
 

vllm/entrypoints/serve/engine/typing.py

@@ -15,7 +15,7 @@ from vllm.entrypoints.openai.completion.protocol import (
     CompletionResponse,
 )
 from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
-from vllm.entrypoints.serve.disagg.protocol import (
+from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
     DerenderChatRequest,
     DerenderCompletionRequest,
     GenerateRequest,

vllm/renderers/online_derenderer.py

@@ -16,7 +16,7 @@ from vllm.entrypoints.openai.completion.protocol import (
 )
 from vllm.entrypoints.openai.engine.protocol import ToolCall
 from vllm.entrypoints.openai.engine.serving import resolve_token_id_placeholder
-from vllm.entrypoints.serve.disagg.protocol import GenerateResponse
+from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateResponse
 from vllm.entrypoints.serve.utils.request_logger import RequestLogger
 from vllm.logger import init_logger
 from vllm.parser import Parser, ParserManager