From 3483240b7ea3d4372b6c79369ea36617f8b1fbb2 Mon Sep 17 00:00:00 2001 From: "wang.yuqi" Date: Mon, 29 Jun 2026 18:18:53 +0800 Subject: [PATCH] [Frontend] Consolidate scale out entrypoints (#44512) Signed-off-by: wang.yuqi --- .buildkite/test-amd.yaml | 4 + .buildkite/test_areas/entrypoints.yaml | 2 + .buildkite/test_areas/rust_frontend.yaml | 4 +- docs/examples/README.md | 3 +- docs/serving/online_serving/README.md | 4 +- .../serve/disagg => examples}/__init__.py | 0 .../render => examples/scale_out}/__init__.py | 0 .../example_mm_serve.py | 0 .../token_generation_client.py | 0 .../openai/chat_completion/test_chat_error.py | 12 +- .../completion/test_completion_error.py | 12 +- .../openai/test_render_token_offsets.py | 4 +- .../entrypoints/scale_out}/__init__.py | 0 .../scale_out/derender/__init__.py | 0 .../derender}/test_derender.py | 0 .../entrypoints/scale_out/render/__init__.py | 0 .../render/test_launch_render.py | 0 .../render/test_render.py | 0 .../render/test_render_multimodal.py | 0 .../scale_out/token_in_token_out/__init__.py | 0 .../test_generate_stream.py | 4 +- .../token_in_token_out}/test_mm_serde.py | 9 +- .../token_in_token_out}/test_protocol.py | 4 +- .../test_return_routed_experts.py | 0 .../test_serving_multimodal_tokens.py | 0 .../test_serving_tokens.py | 0 .../test_tokens_logprobs.py | 2 +- tests/renderers/test_token_offsets.py | 2 +- vllm/entrypoints/generate/api_router.py | 15 -- vllm/entrypoints/openai/api_server.py | 33 +-- vllm/entrypoints/scale_out/__init__.py | 0 .../scale_out/derender/__init__.py | 0 .../scale_out/derender/api_router.py | 74 +++++++ .../entrypoints/scale_out/derender/serving.py | 202 ++++++++++++++++++ vllm/entrypoints/scale_out/factories.py | 78 +++++++ .../{serve => scale_out}/render/__init__.py | 0 .../{serve => scale_out}/render/api_router.py | 73 +------ .../{serve => scale_out}/render/serving.py | 136 +----------- .../scale_out/token_in_token_out/__init__.py | 0 .../token_in_token_out}/api_router.py | 13 +- .../token_in_token_out}/mm_serde.py | 0 .../token_in_token_out}/protocol.py | 0 .../token_in_token_out}/serving.py | 17 +- vllm/entrypoints/serve/engine/typing.py | 2 +- vllm/renderers/online_derenderer.py | 2 +- 45 files changed, 422 insertions(+), 289 deletions(-) rename {tests/entrypoints/serve/disagg => examples}/__init__.py (100%) rename {tests/entrypoints/serve/render => examples/scale_out}/__init__.py (100%) rename examples/{disaggregated/disaggregated_serving => scale_out}/example_mm_serve.py (100%) rename examples/{generate => scale_out}/token_generation_client.py (100%) rename {vllm/entrypoints/serve/disagg => tests/entrypoints/scale_out}/__init__.py (100%) create mode 100644 tests/entrypoints/scale_out/derender/__init__.py rename tests/entrypoints/{serve/render => scale_out/derender}/test_derender.py (100%) create mode 100644 tests/entrypoints/scale_out/render/__init__.py rename tests/entrypoints/{serve => scale_out}/render/test_launch_render.py (100%) rename tests/entrypoints/{serve => scale_out}/render/test_render.py (100%) rename tests/entrypoints/{serve => scale_out}/render/test_render_multimodal.py (100%) create mode 100644 tests/entrypoints/scale_out/token_in_token_out/__init__.py rename tests/entrypoints/{serve/disagg => scale_out/token_in_token_out}/test_generate_stream.py (99%) rename tests/entrypoints/{openai => scale_out/token_in_token_out}/test_mm_serde.py (94%) rename tests/entrypoints/{serve/disagg => scale_out/token_in_token_out}/test_protocol.py (95%) rename tests/entrypoints/{serve/disagg => scale_out/token_in_token_out}/test_return_routed_experts.py (100%) rename tests/entrypoints/{serve/disagg => scale_out/token_in_token_out}/test_serving_multimodal_tokens.py (100%) rename tests/entrypoints/{serve/disagg => scale_out/token_in_token_out}/test_serving_tokens.py (100%) rename tests/entrypoints/{serve/disagg => scale_out/token_in_token_out}/test_tokens_logprobs.py (92%) create mode 100644 vllm/entrypoints/scale_out/__init__.py create mode 100644 vllm/entrypoints/scale_out/derender/__init__.py create mode 100644 vllm/entrypoints/scale_out/derender/api_router.py create mode 100644 vllm/entrypoints/scale_out/derender/serving.py create mode 100644 vllm/entrypoints/scale_out/factories.py rename vllm/entrypoints/{serve => scale_out}/render/__init__.py (100%) rename vllm/entrypoints/{serve => scale_out}/render/api_router.py (50%) rename vllm/entrypoints/{serve => scale_out}/render/serving.py (66%) create mode 100644 vllm/entrypoints/scale_out/token_in_token_out/__init__.py rename vllm/entrypoints/{serve/disagg => scale_out/token_in_token_out}/api_router.py (96%) rename vllm/entrypoints/{serve/disagg => scale_out/token_in_token_out}/mm_serde.py (100%) rename vllm/entrypoints/{serve/disagg => scale_out/token_in_token_out}/protocol.py (100%) rename vllm/entrypoints/{serve/disagg => scale_out/token_in_token_out}/serving.py (99%) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 7521901c9a6..ea76ae1c37f 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -845,10 +845,12 @@ steps: source_file_dependencies: - vllm/ - tests/entrypoints/serve + - tests/entrypoints/scale_out commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc + - pytest -v -s entrypoints/scale_out - label: Entrypoints Integration (API Server OpenAI - Part 1) # TBD timeout_in_minutes: 180 @@ -2559,10 +2561,12 @@ steps: source_file_dependencies: - vllm/ - tests/entrypoints/serve + - tests/entrypoints/scale_out commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc + - pytest -v -s entrypoints/scale_out - label: Entrypoints Integration (API Server OpenAI - Part 1) # TBD timeout_in_minutes: 180 diff --git a/.buildkite/test_areas/entrypoints.yaml b/.buildkite/test_areas/entrypoints.yaml index f6307f097d9..d95b7e0d008 100644 --- a/.buildkite/test_areas/entrypoints.yaml +++ b/.buildkite/test_areas/entrypoints.yaml @@ -40,10 +40,12 @@ steps: source_file_dependencies: - vllm/ - tests/entrypoints/serve + - tests/entrypoints/scale_out commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc + - pytest -v -s entrypoints/scale_out mirror: amd: device: mi325_1 diff --git a/.buildkite/test_areas/rust_frontend.yaml b/.buildkite/test_areas/rust_frontend.yaml index 5ea0f7ef77c..adb27c4a049 100644 --- a/.buildkite/test_areas/rust_frontend.yaml +++ b/.buildkite/test_areas/rust_frontend.yaml @@ -46,7 +46,7 @@ steps: - vllm/v1/engine/ - tests/utils.py # - tests/entrypoints/serve/dev/rpc/test_collective_rpc.py - - tests/entrypoints/serve/disagg/test_serving_tokens.py + - tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py - tests/entrypoints/serve/instrumentator/test_basic.py - tests/entrypoints/serve/instrumentator/test_metrics.py # - tests/entrypoints/serve/dev/test_sleep.py @@ -55,7 +55,7 @@ steps: - export VLLM_WORKER_MULTIPROC_METHOD=spawn # - pytest -v -s entrypoints/serve/dev/rpc/test_collective_rpc.py - pytest -v -s entrypoints/serve/instrumentator/test_basic.py -k "not show_version and not server_load" - - pytest -v -s entrypoints/serve/disagg/test_serving_tokens.py -k "not stream and not lora and not test_generate_logprobs and not stop_string_workflow" + - pytest -v -s entrypoints/scale_out/token_in_token_out/test_serving_tokens.py -k "not stream and not lora and not test_generate_logprobs and not stop_string_workflow" - pytest -v -s entrypoints/serve/instrumentator/test_metrics.py -k "text and not show and not run_batch and not test_metrics_counts and not test_metrics_exist" # - pytest -v -s entrypoints/serve/dev/test_sleep.py diff --git a/docs/examples/README.md b/docs/examples/README.md index 5569db9119c..a9a127a4d5d 100644 --- a/docs/examples/README.md +++ b/docs/examples/README.md @@ -13,5 +13,6 @@ vLLM's examples are organized into the following categories: - **[`rl/`](../../examples/rl)** – Reinforcement learning examples. - **[`deployment/`](../../examples/deployment)** – Examples for deploying vLLM in production. - **[`ray_serving/`](../../examples/ray_serving)** – Scalable serving using Ray. -- **[`disaggregated/`](../../examples/disaggregated)** – Examples for disaggregated serving (separate prefill and decode), including various kv cache connectors (LMCache, Mooncake, FlexKV, P2P NCCL) and failure recovery. +- **[`disaggregated/`](../../examples/disaggregated)** – Examples for Disaggregated P/D (Prefill/Decoding) inference, including various kv cache connectors (LMCache, Mooncake, FlexKV, P2P NCCL) and failure recovery. +- **[`scale_out/`](../../examples/scale_out)** – Examples for Token In <> Token Out API Server. - **[`observability/`](../../examples/observability)** – Metrics, logging, tracing (OpenTelemetry), and dashboards (Grafana, Perses). diff --git a/docs/serving/online_serving/README.md b/docs/serving/online_serving/README.md index 40fc8b7c426..60476fa5edb 100644 --- a/docs/serving/online_serving/README.md +++ b/docs/serving/online_serving/README.md @@ -119,9 +119,9 @@ For further details on profiling vLLM, please refer to [this page](../../contrib - `/ping` - SageMaker health check - `/invocations` - SageMaker-compatible endpoint (routes to the same inference functions as `/v1` endpoints) -## Disaggregated Everything +## Scale-Out APIs -### Tokens IN <> Tokens OUT +### Tokens IN <> Tokens OUT APIs - `/inference/v1/generate` - Generate completions - `/abort_requests` - Abort in-flight requests (only when `--tokens-only` is also set) diff --git a/tests/entrypoints/serve/disagg/__init__.py b/examples/__init__.py similarity index 100% rename from tests/entrypoints/serve/disagg/__init__.py rename to examples/__init__.py diff --git a/tests/entrypoints/serve/render/__init__.py b/examples/scale_out/__init__.py similarity index 100% rename from tests/entrypoints/serve/render/__init__.py rename to examples/scale_out/__init__.py diff --git a/examples/disaggregated/disaggregated_serving/example_mm_serve.py b/examples/scale_out/example_mm_serve.py similarity index 100% rename from examples/disaggregated/disaggregated_serving/example_mm_serve.py rename to examples/scale_out/example_mm_serve.py diff --git a/examples/generate/token_generation_client.py b/examples/scale_out/token_generation_client.py similarity index 100% rename from examples/generate/token_generation_client.py rename to examples/scale_out/token_generation_client.py diff --git a/tests/entrypoints/openai/chat_completion/test_chat_error.py b/tests/entrypoints/openai/chat_completion/test_chat_error.py index 3eea57d3f53..4b6be87ae5c 100644 --- a/tests/entrypoints/openai/chat_completion/test_chat_error.py +++ b/tests/entrypoints/openai/chat_completion/test_chat_error.py @@ -17,10 +17,9 @@ from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat from vllm.entrypoints.openai.engine.protocol import GenerationError from vllm.entrypoints.openai.models.protocol import BaseModelPath from vllm.entrypoints.openai.models.serving import OpenAIServingModels -from vllm.entrypoints.serve.render.serving import ServingRender +from vllm.entrypoints.scale_out.render.serving import ServingRender from vllm.outputs import CompletionOutput, RequestOutput from vllm.renderers.hf import HfRenderer -from vllm.renderers.online_derenderer import OnlineDerenderer from vllm.renderers.online_renderer import OnlineRenderer from vllm.tokenizers.registry import cached_tokenizer_from_config from vllm.v1.engine.async_llm import AsyncLLM @@ -206,15 +205,8 @@ def _build_serving_render(engine: AsyncLLM) -> ServingRender: chat_template=None, chat_template_content_format="auto", ) - online_derenderer = OnlineDerenderer( - model_config=engine.model_config, - renderer=engine.renderer, - request_logger=None, - chat_template=None, - chat_template_content_format="auto", - ) - serving_render = ServingRender(models, online_renderer, online_derenderer) + serving_render = ServingRender(models, online_renderer) async def _fake_preprocess_chat(*args, **kwargs): # return conversation, engine_inputs diff --git a/tests/entrypoints/openai/completion/test_completion_error.py b/tests/entrypoints/openai/completion/test_completion_error.py index 9d2fedae361..062c3e7583a 100644 --- a/tests/entrypoints/openai/completion/test_completion_error.py +++ b/tests/entrypoints/openai/completion/test_completion_error.py @@ -14,10 +14,9 @@ from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion from vllm.entrypoints.openai.engine.protocol import GenerationError from vllm.entrypoints.openai.models.protocol import BaseModelPath from vllm.entrypoints.openai.models.serving import OpenAIServingModels -from vllm.entrypoints.serve.render.serving import ServingRender +from vllm.entrypoints.scale_out.render.serving import ServingRender from vllm.outputs import CompletionOutput, RequestOutput from vllm.renderers.hf import HfRenderer -from vllm.renderers.online_derenderer import OnlineDerenderer from vllm.renderers.online_renderer import OnlineRenderer from vllm.tokenizers.registry import cached_tokenizer_from_config from vllm.v1.engine.async_llm import AsyncLLM @@ -191,15 +190,8 @@ def _build_serving_render(engine: AsyncLLM) -> ServingRender: chat_template=None, chat_template_content_format="auto", ) - online_derenderer = OnlineDerenderer( - model_config=engine.model_config, - renderer=engine.renderer, - request_logger=None, - chat_template=None, - chat_template_content_format="auto", - ) - serving_render = ServingRender(models, online_renderer, online_derenderer) + serving_render = ServingRender(models, online_renderer) async def _fake_preprocess_chat(*args, **kwargs): # return conversation, engine_inputs diff --git a/tests/entrypoints/openai/test_render_token_offsets.py b/tests/entrypoints/openai/test_render_token_offsets.py index f7653ab66fc..a2e66b7bd6c 100644 --- a/tests/entrypoints/openai/test_render_token_offsets.py +++ b/tests/entrypoints/openai/test_render_token_offsets.py @@ -3,7 +3,7 @@ """Unit tests for the token-offsets request/response protocol wiring: the request flag flowing into ``TokenizeParams`` and the ``GenerateRequest`` serialization boundary. End-to-end behavior is covered by -``tests/entrypoints/serve/render/test_render.py``; plain Pydantic field +``tests/entrypoints/scale_out/render/test_render.py``; plain Pydantic field storage is not retested here. """ @@ -12,7 +12,7 @@ from unittest.mock import Mock from vllm.config import ModelConfig from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest from vllm.entrypoints.openai.completion.protocol import CompletionRequest -from vllm.entrypoints.serve.disagg.protocol import GenerateRequest +from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest from vllm.sampling_params import SamplingParams diff --git a/vllm/entrypoints/serve/disagg/__init__.py b/tests/entrypoints/scale_out/__init__.py similarity index 100% rename from vllm/entrypoints/serve/disagg/__init__.py rename to tests/entrypoints/scale_out/__init__.py diff --git a/tests/entrypoints/scale_out/derender/__init__.py b/tests/entrypoints/scale_out/derender/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/entrypoints/serve/render/test_derender.py b/tests/entrypoints/scale_out/derender/test_derender.py similarity index 100% rename from tests/entrypoints/serve/render/test_derender.py rename to tests/entrypoints/scale_out/derender/test_derender.py diff --git a/tests/entrypoints/scale_out/render/__init__.py b/tests/entrypoints/scale_out/render/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/entrypoints/serve/render/test_launch_render.py b/tests/entrypoints/scale_out/render/test_launch_render.py similarity index 100% rename from tests/entrypoints/serve/render/test_launch_render.py rename to tests/entrypoints/scale_out/render/test_launch_render.py diff --git a/tests/entrypoints/serve/render/test_render.py b/tests/entrypoints/scale_out/render/test_render.py similarity index 100% rename from tests/entrypoints/serve/render/test_render.py rename to tests/entrypoints/scale_out/render/test_render.py diff --git a/tests/entrypoints/serve/render/test_render_multimodal.py b/tests/entrypoints/scale_out/render/test_render_multimodal.py similarity index 100% rename from tests/entrypoints/serve/render/test_render_multimodal.py rename to tests/entrypoints/scale_out/render/test_render_multimodal.py diff --git a/tests/entrypoints/scale_out/token_in_token_out/__init__.py b/tests/entrypoints/scale_out/token_in_token_out/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/entrypoints/serve/disagg/test_generate_stream.py b/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py similarity index 99% rename from tests/entrypoints/serve/disagg/test_generate_stream.py rename to tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py index a31655e4307..ce3100f196c 100644 --- a/tests/entrypoints/serve/disagg/test_generate_stream.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py @@ -12,11 +12,11 @@ from vllm.config.multimodal import MultiModalConfig from vllm.entrypoints.openai.engine.protocol import StreamOptions from vllm.entrypoints.openai.models.protocol import BaseModelPath from vllm.entrypoints.openai.models.serving import OpenAIServingModels -from vllm.entrypoints.serve.disagg.protocol import ( +from vllm.entrypoints.scale_out.token_in_token_out.protocol import ( GenerateRequest, GenerateResponse, ) -from vllm.entrypoints.serve.disagg.serving import ServingTokens +from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens from vllm.logprobs import Logprob from vllm.outputs import CompletionOutput, RequestOutput from vllm.renderers import renderer_from_config diff --git a/tests/entrypoints/openai/test_mm_serde.py b/tests/entrypoints/scale_out/token_in_token_out/test_mm_serde.py similarity index 94% rename from tests/entrypoints/openai/test_mm_serde.py rename to tests/entrypoints/scale_out/token_in_token_out/test_mm_serde.py index c568d822e1c..d24436bbd4b 100644 --- a/tests/entrypoints/openai/test_mm_serde.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_mm_serde.py @@ -1,14 +1,17 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Roundtrip tests for multimodal serde used by the disagg generate endpoint.""" +""" +Roundtrip tests for multimodal serde used by the +token_in_token_out generate endpoint. +""" import torch -from vllm.entrypoints.serve.disagg.mm_serde import ( +from vllm.entrypoints.scale_out.token_in_token_out.mm_serde import ( decode_mm_kwargs_item, encode_mm_kwargs_item, ) -from vllm.entrypoints.serve.disagg.protocol import ( +from vllm.entrypoints.scale_out.token_in_token_out.protocol import ( MultiModalFeatures, PlaceholderRangeInfo, ) diff --git a/tests/entrypoints/serve/disagg/test_protocol.py b/tests/entrypoints/scale_out/token_in_token_out/test_protocol.py similarity index 95% rename from tests/entrypoints/serve/disagg/test_protocol.py rename to tests/entrypoints/scale_out/token_in_token_out/test_protocol.py index 414fc2a2612..674ce18b7f3 100644 --- a/tests/entrypoints/serve/disagg/test_protocol.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_protocol.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Unit tests for the disagg request/response protocol. +"""Unit tests for the token_in_token_out request/response protocol. These tests intentionally avoid spinning up a server — they exercise the pydantic validators on ``GenerateRequest`` directly so they run fast and @@ -9,7 +9,7 @@ fail loudly if the validator semantics ever drift. import json -from vllm.entrypoints.serve.disagg.protocol import GenerateRequest +from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest from vllm.sampling_params import SamplingParams diff --git a/tests/entrypoints/serve/disagg/test_return_routed_experts.py b/tests/entrypoints/scale_out/token_in_token_out/test_return_routed_experts.py similarity index 100% rename from tests/entrypoints/serve/disagg/test_return_routed_experts.py rename to tests/entrypoints/scale_out/token_in_token_out/test_return_routed_experts.py diff --git a/tests/entrypoints/serve/disagg/test_serving_multimodal_tokens.py b/tests/entrypoints/scale_out/token_in_token_out/test_serving_multimodal_tokens.py similarity index 100% rename from tests/entrypoints/serve/disagg/test_serving_multimodal_tokens.py rename to tests/entrypoints/scale_out/token_in_token_out/test_serving_multimodal_tokens.py diff --git a/tests/entrypoints/serve/disagg/test_serving_tokens.py b/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py similarity index 100% rename from tests/entrypoints/serve/disagg/test_serving_tokens.py rename to tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py diff --git a/tests/entrypoints/serve/disagg/test_tokens_logprobs.py b/tests/entrypoints/scale_out/token_in_token_out/test_tokens_logprobs.py similarity index 92% rename from tests/entrypoints/serve/disagg/test_tokens_logprobs.py rename to tests/entrypoints/scale_out/token_in_token_out/test_tokens_logprobs.py index 844dd24d541..80f08078da2 100644 --- a/tests/entrypoints/serve/disagg/test_tokens_logprobs.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_tokens_logprobs.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from vllm.entrypoints.serve.disagg.serving import ServingTokens +from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens from vllm.logprobs import Logprob diff --git a/tests/renderers/test_token_offsets.py b/tests/renderers/test_token_offsets.py index ab881782659..f973e8610bc 100644 --- a/tests/renderers/test_token_offsets.py +++ b/tests/renderers/test_token_offsets.py @@ -5,7 +5,7 @@ These exercise ``_tokenize_prompt`` (offset extraction + capability/MM gating) and the ``_tokenize_prompt -> _process_tokens -> TokensInput`` forwarding chain. Endpoint-level coverage lives in -``tests/entrypoints/serve/render/test_render.py``. +``tests/entrypoints/scale_out/render/test_render.py``. """ import pytest diff --git a/vllm/entrypoints/generate/api_router.py b/vllm/entrypoints/generate/api_router.py index 38ecdec5ce2..5a26e475b06 100644 --- a/vllm/entrypoints/generate/api_router.py +++ b/vllm/entrypoints/generate/api_router.py @@ -66,7 +66,6 @@ async def init_generate_state( from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses - from vllm.entrypoints.serve.disagg.serving import ServingTokens from vllm.entrypoints.serve.utils.fingerprint import set_default_fingerprint_mode # Applied before any serving class is constructed so that each one picks @@ -175,20 +174,6 @@ async def init_generate_state( if "generate" in supported_tasks else None ) - state.serving_tokens = ( - ServingTokens( - engine_client, - state.openai_serving_models, - state.online_renderer, - request_logger=request_logger, - return_tokens_as_token_ids=args.return_tokens_as_token_ids, - enable_prompt_tokens_details=args.enable_prompt_tokens_details, - enable_log_outputs=args.enable_log_outputs, - force_no_detokenize=args.tokens_only, - ) - if "generate" in supported_tasks - else None - ) from .generative_scoring.serving import ServingGenerativeScoring diff --git a/vllm/entrypoints/openai/api_server.py b/vllm/entrypoints/openai/api_server.py index 9fc4560adbe..6ae6dd70abd 100644 --- a/vllm/entrypoints/openai/api_server.py +++ b/vllm/entrypoints/openai/api_server.py @@ -32,7 +32,6 @@ from vllm.entrypoints.openai.engine.protocol import GenerationError from vllm.entrypoints.openai.models.protocol import BaseModelPath from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.serve.elastic_ep.middleware import ScalingMiddleware -from vllm.entrypoints.serve.render.serving import ServingRender from vllm.entrypoints.serve.sagemaker.api_router import sagemaker_standards_bootstrap from vllm.entrypoints.serve.tokenize.serving import ServingTokenization from vllm.entrypoints.serve.utils.api_utils import ( @@ -208,12 +207,6 @@ def build_app( register_generate_api_routers(app) - from vllm.entrypoints.serve.disagg.api_router import ( - attach_router as attach_disagg_router, - ) - - attach_disagg_router(app) - from vllm.entrypoints.serve.elastic_ep.api_router import ( attach_router as elastic_ep_attach_router, ) @@ -221,11 +214,9 @@ def build_app( elastic_ep_attach_router(app) if "generate" in supported_tasks or "render" in supported_tasks: - from vllm.entrypoints.serve.render.api_router import ( - attach_router as attach_render_router, - ) + from vllm.entrypoints.scale_out.factories import register_scale_out_api_routers - attach_render_router(app) + register_scale_out_api_routers(app, supported_tasks) if "transcription" in supported_tasks or "realtime" in supported_tasks: from vllm.entrypoints.speech_to_text.factories import ( @@ -401,12 +392,6 @@ async def init_app_state( default_chat_template_kwargs=args.default_chat_template_kwargs, trust_request_chat_template=args.trust_request_chat_template, ) - state.serving_render = ServingRender( - state.openai_serving_models, - state.online_renderer, - state.online_derenderer, - request_logger=request_logger, - ) if "generate" in supported_tasks: from vllm.entrypoints.generate.api_router import init_generate_state @@ -415,6 +400,10 @@ async def init_app_state( engine_client, state, args, request_logger, supported_tasks ) + from vllm.entrypoints.scale_out.factories import init_scale_out_state + + init_scale_out_state(state, args, engine_client, request_logger) + if "transcription" in supported_tasks or "realtime" in supported_tasks: from vllm.entrypoints.speech_to_text.factories import init_speech_to_text_state @@ -505,12 +494,10 @@ async def init_render_app_state( default_chat_template_kwargs=args.default_chat_template_kwargs, trust_request_chat_template=args.trust_request_chat_template, ) - state.serving_render = ServingRender( - model_registry, - state.online_renderer, - state.online_derenderer, - request_logger=request_logger, - ) + + from vllm.entrypoints.scale_out.factories import init_render_state + + init_render_state(state, request_logger) state.vllm_config = vllm_config # Disable stats logging — there is no engine to poll. diff --git a/vllm/entrypoints/scale_out/__init__.py b/vllm/entrypoints/scale_out/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/vllm/entrypoints/scale_out/derender/__init__.py b/vllm/entrypoints/scale_out/derender/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/vllm/entrypoints/scale_out/derender/api_router.py b/vllm/entrypoints/scale_out/derender/api_router.py new file mode 100644 index 00000000000..3f88d51f0a9 --- /dev/null +++ b/vllm/entrypoints/scale_out/derender/api_router.py @@ -0,0 +1,74 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from http import HTTPStatus + +from fastapi import APIRouter, Depends, Request +from fastapi.responses import JSONResponse + +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionResponse +from vllm.entrypoints.openai.completion.protocol import CompletionResponse +from vllm.entrypoints.openai.engine.protocol import ErrorResponse +from vllm.entrypoints.serve.utils.api_utils import validate_json_request +from vllm.logger import init_logger + +from ..token_in_token_out.protocol import ( + DerenderChatRequest, + DerenderCompletionRequest, +) +from .serving import ServingDerender + +logger = init_logger(__name__) + +router = APIRouter() + + +def derender(request: Request) -> ServingDerender | None: + return getattr(request.app.state, "serving_derender", None) + + +@router.post( + "/v1/chat/completions/derender", + dependencies=[Depends(validate_json_request)], + response_model=ChatCompletionResponse, + responses={ + HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse}, + HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse}, + HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse}, + }, +) +async def derender_chat_completion(request: DerenderChatRequest, raw_request: Request): + handler = derender(raw_request) + if handler is None: + raise NotImplementedError( + "The model does not support Chat Completions Derender API" + ) + + result = await handler.derender_chat_response(request) + + if isinstance(result, ErrorResponse): + return JSONResponse(content=result.model_dump(), status_code=result.error.code) + + return JSONResponse(content=result.model_dump()) + + +@router.post( + "/v1/completions/derender", + dependencies=[Depends(validate_json_request)], + response_model=CompletionResponse, + responses={ + HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse}, + HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse}, + HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse}, + }, +) +async def derender_completion(request: DerenderCompletionRequest, raw_request: Request): + handler = derender(raw_request) + if handler is None: + raise NotImplementedError("The model does not support Completions Derender API") + + result = await handler.derender_completion_response(request) + + if isinstance(result, ErrorResponse): + return JSONResponse(content=result.model_dump(), status_code=result.error.code) + + return JSONResponse(content=result.model_dump()) diff --git a/vllm/entrypoints/scale_out/derender/serving.py b/vllm/entrypoints/scale_out/derender/serving.py new file mode 100644 index 00000000000..e125007a549 --- /dev/null +++ b/vllm/entrypoints/scale_out/derender/serving.py @@ -0,0 +1,202 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import time +from typing import cast + +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionResponse +from vllm.entrypoints.openai.completion.protocol import CompletionResponse +from vllm.entrypoints.openai.engine.protocol import ( + ErrorResponse, + UsageInfo, +) +from vllm.entrypoints.openai.models.serving import ( + OpenAIModelRegistry, + OpenAIServingModels, +) +from vllm.entrypoints.serve.engine.serving import BaseServing +from vllm.entrypoints.serve.utils.request_logger import RequestLogger +from vllm.inputs import ( + EngineInput, + MultiModalHashes, + MultiModalInput, + MultiModalPlaceholders, +) +from vllm.logger import init_logger +from vllm.renderers.online_derenderer import OnlineDerenderer + +from ..token_in_token_out.mm_serde import encode_mm_kwargs_item +from ..token_in_token_out.protocol import ( + DerenderChatRequest, + DerenderCompletionRequest, + MultiModalFeatures, + PlaceholderRangeInfo, +) + +logger = init_logger(__name__) + + +class ServingDerender(BaseServing): + def __init__( + self, + models: OpenAIServingModels | OpenAIModelRegistry, + online_derenderer: "OnlineDerenderer", + *, + request_logger: RequestLogger | None = None, + ) -> None: + super().__init__( + models=models, + model_config=models.model_config, + request_logger=request_logger, + ) + + self.online_derenderer = online_derenderer + + async def derender_chat_response( + self, + request: DerenderChatRequest, + ) -> ChatCompletionResponse | ErrorResponse: + """Postprocess a GenerateResponse into a ChatCompletionResponse. + + Non-streaming only: expects the complete GenerateResponse with all + token IDs present. Uses ``parser.parse()`` for one-shot extraction. + + When ``request.chat_request`` is provided, the parser splits the + output into (reasoning, content, tool_calls). Otherwise falls + back to plain detokenization. + """ + error_check_ret = await self._check_model(request) + if error_check_ret is not None: + return error_check_ret + + try: + choices = await self.online_derenderer.derender_chat( + request.generate_response, request.chat_request + ) + except ValueError as exc: + return self.create_error_response(str(exc)) + + prompt_tokens = ( + request.prompt_tokens if request.prompt_tokens is not None else 0 + ) + gen = request.generate_response + completion_tokens = sum(len(ch.token_ids) for ch in gen.choices if ch.token_ids) + usage = UsageInfo( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + ) + + logger.debug( + "derender_chat request_id=%s model=%s choices=%d completion_tokens=%d", + gen.request_id, + request.model, + len(choices), + completion_tokens, + ) + return ChatCompletionResponse( + id=gen.request_id, + model=request.model, + created=int(time.time()), + choices=choices, + usage=usage, + prompt_logprobs=gen.prompt_logprobs, + kv_transfer_params=gen.kv_transfer_params, + ) + + async def derender_completion_response( + self, + request: DerenderCompletionRequest, + ) -> CompletionResponse | ErrorResponse: + """Postprocess a list of GenerateResponses into a CompletionResponse. + + Non-streaming only. Mirrors the multi-prompt completions case: one + GenerateResponse per prompt, parallel to the list[GenerateRequest] + from /v1/completions/render. + """ + error_check_ret = await self._check_model(request) + if error_check_ret is not None: + return error_check_ret + + ( + choices, + total_prompt_tokens, + total_completion_tokens, + ) = await self.online_derenderer.derender_completion( + request.generate_responses, request.prompt_tokens + ) + + if not request.generate_responses: + return self.create_error_response("generate_responses must not be empty") + + first = request.generate_responses[0] + kv_params = first.kv_transfer_params + if any( + r.kv_transfer_params != kv_params for r in request.generate_responses[1:] + ): + logger.warning( + "derender_completion: kv_transfer_params differ across responses; " + "setting to None on the aggregated response" + ) + kv_params = None + + usage = UsageInfo( + prompt_tokens=total_prompt_tokens, + completion_tokens=total_completion_tokens, + total_tokens=total_prompt_tokens + total_completion_tokens, + ) + + logger.debug( + "derender_completion request_id=%s model=%s choices=%d" + " completion_tokens=%d", + first.request_id, + request.model, + len(choices), + total_completion_tokens, + ) + return CompletionResponse( + id=first.request_id, + model=request.model, + created=int(time.time()), + choices=choices, + usage=usage, + kv_transfer_params=kv_params, + ) + + @staticmethod + def _extract_mm_features( + engine_input: EngineInput, + ) -> MultiModalFeatures | None: + """Extract multimodal metadata from a rendered engine prompt. + + Returns ``None`` for text-only prompts. + """ + if engine_input.get("type") != "multimodal": + return None + + # At this point engine_input is a MultiModalInput TypedDict. + mm_engine_input = cast(MultiModalInput, engine_input) + mm_hashes: MultiModalHashes = mm_engine_input["mm_hashes"] + raw_placeholders: MultiModalPlaceholders = mm_engine_input["mm_placeholders"] + + mm_placeholders = { + modality: [ + PlaceholderRangeInfo(offset=p.offset, length=p.length) for p in ranges + ] + for modality, ranges in raw_placeholders.items() + } + + # Serialize tensor data per modality. + kwargs_data: dict[str, list[str | None]] | None = None + if raw_mm_kwargs := mm_engine_input.get("mm_kwargs"): + kwargs_data = {} + for modality, items in raw_mm_kwargs.items(): + kwargs_data[modality] = [ + encode_mm_kwargs_item(item) if item is not None else None + for item in items + ] + + return MultiModalFeatures( + mm_hashes=mm_hashes, + mm_placeholders=mm_placeholders, + kwargs_data=kwargs_data, + ) diff --git a/vllm/entrypoints/scale_out/factories.py b/vllm/entrypoints/scale_out/factories.py new file mode 100644 index 00000000000..341dad13a86 --- /dev/null +++ b/vllm/entrypoints/scale_out/factories.py @@ -0,0 +1,78 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from argparse import Namespace +from typing import TYPE_CHECKING + +from fastapi import FastAPI + +from vllm.engine.protocol import EngineClient +from vllm.tasks import SupportedTask + +if TYPE_CHECKING: + from starlette.datastructures import State + + from vllm.entrypoints.serve.utils.request_logger import RequestLogger +else: + RequestLogger = object + + +def init_render_state( + state: "State", + request_logger: RequestLogger | None, +): + from .derender.serving import ServingDerender + from .render.serving import ServingRender + + state.serving_render = ServingRender( + state.openai_serving_models, + state.online_renderer, + request_logger=request_logger, + ) + + state.serving_derender = ServingDerender( + state.openai_serving_models, + state.online_derenderer, + request_logger=request_logger, + ) + + +def init_scale_out_state( + state: "State", + args: "Namespace", + engine_client: "EngineClient", + request_logger: RequestLogger | None, +): + init_render_state(state, request_logger) + + from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens + + state.serving_tokens = ServingTokens( + engine_client, + state.openai_serving_models, + state.online_renderer, + request_logger=request_logger, + return_tokens_as_token_ids=args.return_tokens_as_token_ids, + enable_prompt_tokens_details=args.enable_prompt_tokens_details, + enable_log_outputs=args.enable_log_outputs, + force_no_detokenize=args.tokens_only, + ) + + +def register_scale_out_api_routers( + app: FastAPI, + supported_tasks: tuple["SupportedTask", ...], +): + from .render.api_router import router as render_render + + app.include_router(render_render) + + from .derender.api_router import router as derender_render + + app.include_router(derender_render) + + if "generate" in supported_tasks: + from .token_in_token_out.api_router import ( + attach_router as attach_disagg_router, + ) + + attach_disagg_router(app) diff --git a/vllm/entrypoints/serve/render/__init__.py b/vllm/entrypoints/scale_out/render/__init__.py similarity index 100% rename from vllm/entrypoints/serve/render/__init__.py rename to vllm/entrypoints/scale_out/render/__init__.py diff --git a/vllm/entrypoints/serve/render/api_router.py b/vllm/entrypoints/scale_out/render/api_router.py similarity index 50% rename from vllm/entrypoints/serve/render/api_router.py rename to vllm/entrypoints/scale_out/render/api_router.py index 3b3ad476124..d2452866446 100644 --- a/vllm/entrypoints/serve/render/api_router.py +++ b/vllm/entrypoints/scale_out/render/api_router.py @@ -2,27 +2,18 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from http import HTTPStatus -from fastapi import APIRouter, Depends, FastAPI, Request +from fastapi import APIRouter, Depends, Request from fastapi.responses import JSONResponse -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, - ChatCompletionResponse, -) -from vllm.entrypoints.openai.completion.protocol import ( - CompletionRequest, - CompletionResponse, -) +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest +from vllm.entrypoints.openai.completion.protocol import CompletionRequest from vllm.entrypoints.openai.engine.protocol import ErrorResponse -from vllm.entrypoints.serve.disagg.protocol import ( - DerenderChatRequest, - DerenderCompletionRequest, - GenerateRequest, -) -from vllm.entrypoints.serve.render.serving import ServingRender from vllm.entrypoints.serve.utils.api_utils import validate_json_request from vllm.logger import init_logger +from ..token_in_token_out.protocol import GenerateRequest +from .serving import ServingRender + logger = init_logger(__name__) router = APIRouter() @@ -79,55 +70,3 @@ async def render_completion(request: CompletionRequest, raw_request: Request): return JSONResponse(content=result.model_dump(), status_code=result.error.code) return JSONResponse(content=[item.model_dump() for item in result]) - - -@router.post( - "/v1/chat/completions/derender", - dependencies=[Depends(validate_json_request)], - response_model=ChatCompletionResponse, - responses={ - HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse}, - HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse}, - HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse}, - }, -) -async def derender_chat_completion(request: DerenderChatRequest, raw_request: Request): - handler = render(raw_request) - if handler is None: - raise NotImplementedError( - "The model does not support Chat Completions Derender API" - ) - - result = await handler.derender_chat_response(request) - - if isinstance(result, ErrorResponse): - return JSONResponse(content=result.model_dump(), status_code=result.error.code) - - return JSONResponse(content=result.model_dump()) - - -@router.post( - "/v1/completions/derender", - dependencies=[Depends(validate_json_request)], - response_model=CompletionResponse, - responses={ - HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse}, - HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse}, - HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse}, - }, -) -async def derender_completion(request: DerenderCompletionRequest, raw_request: Request): - handler = render(raw_request) - if handler is None: - raise NotImplementedError("The model does not support Completions Derender API") - - result = await handler.derender_completion_response(request) - - if isinstance(result, ErrorResponse): - return JSONResponse(content=result.model_dump(), status_code=result.error.code) - - return JSONResponse(content=result.model_dump()) - - -def attach_router(app: FastAPI) -> None: - app.include_router(router) diff --git a/vllm/entrypoints/serve/render/serving.py b/vllm/entrypoints/scale_out/render/serving.py similarity index 66% rename from vllm/entrypoints/serve/render/serving.py rename to vllm/entrypoints/scale_out/render/serving.py index adcaf8af9af..105bd75185d 100644 --- a/vllm/entrypoints/serve/render/serving.py +++ b/vllm/entrypoints/scale_out/render/serving.py @@ -1,28 +1,16 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import time from typing import cast -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, - ChatCompletionResponse, -) -from vllm.entrypoints.openai.completion.protocol import ( - CompletionRequest, - CompletionResponse, -) -from vllm.entrypoints.openai.engine.protocol import ( - ErrorResponse, - UsageInfo, -) +from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest +from vllm.entrypoints.openai.completion.protocol import CompletionRequest +from vllm.entrypoints.openai.engine.protocol import ErrorResponse from vllm.entrypoints.openai.models.serving import ( OpenAIModelRegistry, OpenAIServingModels, ) -from vllm.entrypoints.serve.disagg.mm_serde import encode_mm_kwargs_item -from vllm.entrypoints.serve.disagg.protocol import ( - DerenderChatRequest, - DerenderCompletionRequest, +from vllm.entrypoints.scale_out.token_in_token_out.mm_serde import encode_mm_kwargs_item +from vllm.entrypoints.scale_out.token_in_token_out.protocol import ( GenerateRequest, MultiModalFeatures, PlaceholderRangeInfo, @@ -41,7 +29,6 @@ from vllm.renderers.inputs.preprocess import ( extract_prompt_components, extract_prompt_len, ) -from vllm.renderers.online_derenderer import OnlineDerenderer from vllm.renderers.online_renderer import OnlineRenderer from vllm.utils import random_uuid @@ -53,7 +40,6 @@ class ServingRender(BaseServing): self, models: OpenAIServingModels | OpenAIModelRegistry, online_renderer: "OnlineRenderer", - online_derenderer: "OnlineDerenderer", *, request_logger: RequestLogger | None = None, ) -> None: @@ -64,7 +50,6 @@ class ServingRender(BaseServing): ) self.online_renderer = online_renderer - self.online_derenderer = online_derenderer self.default_sampling_params = ( online_renderer.model_config.get_diff_sampling_param() @@ -223,117 +208,6 @@ class ServingRender(BaseServing): return generate_requests - async def derender_chat_response( - self, - request: DerenderChatRequest, - ) -> ChatCompletionResponse | ErrorResponse: - """Postprocess a GenerateResponse into a ChatCompletionResponse. - - Non-streaming only: expects the complete GenerateResponse with all - token IDs present. Uses ``parser.parse()`` for one-shot extraction. - - When ``request.chat_request`` is provided, the parser splits the - output into (reasoning, content, tool_calls). Otherwise falls - back to plain detokenization. - """ - error_check_ret = await self._check_model(request) - if error_check_ret is not None: - return error_check_ret - - try: - choices = await self.online_derenderer.derender_chat( - request.generate_response, request.chat_request - ) - except ValueError as exc: - return self.create_error_response(str(exc)) - - prompt_tokens = ( - request.prompt_tokens if request.prompt_tokens is not None else 0 - ) - gen = request.generate_response - completion_tokens = sum(len(ch.token_ids) for ch in gen.choices if ch.token_ids) - usage = UsageInfo( - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - total_tokens=prompt_tokens + completion_tokens, - ) - - logger.debug( - "derender_chat request_id=%s model=%s choices=%d completion_tokens=%d", - gen.request_id, - request.model, - len(choices), - completion_tokens, - ) - return ChatCompletionResponse( - id=gen.request_id, - model=request.model, - created=int(time.time()), - choices=choices, - usage=usage, - prompt_logprobs=gen.prompt_logprobs, - kv_transfer_params=gen.kv_transfer_params, - ) - - async def derender_completion_response( - self, - request: DerenderCompletionRequest, - ) -> CompletionResponse | ErrorResponse: - """Postprocess a list of GenerateResponses into a CompletionResponse. - - Non-streaming only. Mirrors the multi-prompt completions case: one - GenerateResponse per prompt, parallel to the list[GenerateRequest] - from /v1/completions/render. - """ - error_check_ret = await self._check_model(request) - if error_check_ret is not None: - return error_check_ret - - ( - choices, - total_prompt_tokens, - total_completion_tokens, - ) = await self.online_derenderer.derender_completion( - request.generate_responses, request.prompt_tokens - ) - - if not request.generate_responses: - return self.create_error_response("generate_responses must not be empty") - - first = request.generate_responses[0] - kv_params = first.kv_transfer_params - if any( - r.kv_transfer_params != kv_params for r in request.generate_responses[1:] - ): - logger.warning( - "derender_completion: kv_transfer_params differ across responses; " - "setting to None on the aggregated response" - ) - kv_params = None - - usage = UsageInfo( - prompt_tokens=total_prompt_tokens, - completion_tokens=total_completion_tokens, - total_tokens=total_prompt_tokens + total_completion_tokens, - ) - - logger.debug( - "derender_completion request_id=%s model=%s choices=%d" - " completion_tokens=%d", - first.request_id, - request.model, - len(choices), - total_completion_tokens, - ) - return CompletionResponse( - id=first.request_id, - model=request.model, - created=int(time.time()), - choices=choices, - usage=usage, - kv_transfer_params=kv_params, - ) - @staticmethod def _extract_mm_features( engine_input: EngineInput, diff --git a/vllm/entrypoints/scale_out/token_in_token_out/__init__.py b/vllm/entrypoints/scale_out/token_in_token_out/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/vllm/entrypoints/serve/disagg/api_router.py b/vllm/entrypoints/scale_out/token_in_token_out/api_router.py similarity index 96% rename from vllm/entrypoints/serve/disagg/api_router.py rename to vllm/entrypoints/scale_out/token_in_token_out/api_router.py index e5bd351e01f..30857e4c1cf 100644 --- a/vllm/entrypoints/serve/disagg/api_router.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/api_router.py @@ -11,13 +11,6 @@ from fastapi.responses import JSONResponse, StreamingResponse from vllm.engine.protocol import EngineClient from vllm.entrypoints.openai.engine.protocol import ErrorResponse -from vllm.entrypoints.serve.disagg.protocol import ( - GenerateRequest, - GenerateResponse, -) -from vllm.entrypoints.serve.disagg.serving import ( - ServingTokens, -) from vllm.entrypoints.serve.tokenize.serving import ServingTokenization from vllm.entrypoints.serve.utils.api_utils import ( load_aware_call, @@ -26,6 +19,12 @@ from vllm.entrypoints.serve.utils.api_utils import ( ) from vllm.logger import init_logger +from .protocol import ( + GenerateRequest, + GenerateResponse, +) +from .serving import ServingTokens + logger = init_logger(__name__) diff --git a/vllm/entrypoints/serve/disagg/mm_serde.py b/vllm/entrypoints/scale_out/token_in_token_out/mm_serde.py similarity index 100% rename from vllm/entrypoints/serve/disagg/mm_serde.py rename to vllm/entrypoints/scale_out/token_in_token_out/mm_serde.py diff --git a/vllm/entrypoints/serve/disagg/protocol.py b/vllm/entrypoints/scale_out/token_in_token_out/protocol.py similarity index 100% rename from vllm/entrypoints/serve/disagg/protocol.py rename to vllm/entrypoints/scale_out/token_in_token_out/protocol.py diff --git a/vllm/entrypoints/serve/disagg/serving.py b/vllm/entrypoints/scale_out/token_in_token_out/serving.py similarity index 99% rename from vllm/entrypoints/serve/disagg/serving.py rename to vllm/entrypoints/scale_out/token_in_token_out/serving.py index cbd6f83f233..70185a85b30 100644 --- a/vllm/entrypoints/serve/disagg/serving.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/serving.py @@ -28,14 +28,6 @@ from vllm.entrypoints.openai.engine.protocol import ( ) from vllm.entrypoints.openai.engine.serving import OpenAIServing, clamp_prompt_logprobs from vllm.entrypoints.openai.models.serving import OpenAIServingModels -from vllm.entrypoints.serve.disagg.mm_serde import decode_mm_kwargs_item -from vllm.entrypoints.serve.disagg.protocol import ( - GenerateRequest, - GenerateResponse, - GenerateResponseChoice, - GenerateResponseStreamChoice, - GenerateStreamResponse, -) from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage from vllm.entrypoints.serve.utils.request_logger import RequestLogger from vllm.inputs import EngineInput, mm_input @@ -51,6 +43,15 @@ from vllm.renderers.online_renderer import OnlineRenderer from vllm.sampling_params import RequestOutputKind, SamplingParams from vllm.utils.collection_utils import as_list +from .mm_serde import decode_mm_kwargs_item +from .protocol import ( + GenerateRequest, + GenerateResponse, + GenerateResponseChoice, + GenerateResponseStreamChoice, + GenerateStreamResponse, +) + logger = init_logger(__name__) diff --git a/vllm/entrypoints/serve/engine/typing.py b/vllm/entrypoints/serve/engine/typing.py index 8f0b7835dab..2e01c092c7b 100644 --- a/vllm/entrypoints/serve/engine/typing.py +++ b/vllm/entrypoints/serve/engine/typing.py @@ -15,7 +15,7 @@ from vllm.entrypoints.openai.completion.protocol import ( CompletionResponse, ) from vllm.entrypoints.openai.responses.protocol import ResponsesRequest -from vllm.entrypoints.serve.disagg.protocol import ( +from vllm.entrypoints.scale_out.token_in_token_out.protocol import ( DerenderChatRequest, DerenderCompletionRequest, GenerateRequest, diff --git a/vllm/renderers/online_derenderer.py b/vllm/renderers/online_derenderer.py index 91d03bbe819..fb4b880c48c 100644 --- a/vllm/renderers/online_derenderer.py +++ b/vllm/renderers/online_derenderer.py @@ -16,7 +16,7 @@ from vllm.entrypoints.openai.completion.protocol import ( ) from vllm.entrypoints.openai.engine.protocol import ToolCall from vllm.entrypoints.openai.engine.serving import resolve_token_id_placeholder -from vllm.entrypoints.serve.disagg.protocol import GenerateResponse +from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateResponse from vllm.entrypoints.serve.utils.request_logger import RequestLogger from vllm.logger import init_logger from vllm.parser import Parser, ParserManager