diff --git a/tests/entrypoints/openai/test_fingerprint.py b/tests/entrypoints/openai/test_fingerprint.py new file mode 100644 index 00000000000..b78ed38636c --- /dev/null +++ b/tests/entrypoints/openai/test_fingerprint.py @@ -0,0 +1,76 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for ``system_fingerprint`` construction.""" + +from types import SimpleNamespace + +import pytest + +from vllm.entrypoints.openai import fingerprint as fp + + +def _cfg(tp=1, pp=1, dp=1, ep=False, digest="a3b21f94deadbeef"): + c = SimpleNamespace( + parallel_config=SimpleNamespace( + tensor_parallel_size=tp, + pipeline_parallel_size=pp, + data_parallel_size=dp, + enable_expert_parallel=ep, + ) + ) + c.compute_hash = lambda: digest # type: ignore[attr-defined] + return c + + +@pytest.fixture(autouse=True) +def _reset(): + fp.set_default_fingerprint_mode("full") + yield + fp.set_default_fingerprint_mode("full") + + +def test_four_modes_produce_expected_shapes(): + from vllm import __version__ as v + + cfg = _cfg(tp=8, ep=True) + + assert fp.build_system_fingerprint(cfg, "full") == (f"vllm-{v}-tp8-ep-a3b21f94") + assert fp.build_system_fingerprint(cfg, "hash") == f"vllm-{v}-a3b21f94" + assert fp.build_system_fingerprint(cfg, "custom", "my-fp") == "my-fp" + assert fp.build_system_fingerprint(cfg, "none") is None + + +def test_full_mode_emits_only_non_trivial_parallelism(): + from vllm import __version__ as v + + # Single-GPU: nothing between version and hash. + assert fp.build_system_fingerprint(_cfg(), "full") == f"vllm-{v}-a3b21f94" + # All parallelism axes. + assert ( + fp.build_system_fingerprint(_cfg(tp=8, pp=2, dp=4, ep=True), "full") + == f"vllm-{v}-tp8-pp2-dp4-ep-a3b21f94" + ) + + +def test_get_respects_set_default(): + cfg = _cfg(tp=8) + full = fp.get_system_fingerprint(cfg) + assert full == fp.get_system_fingerprint(cfg) + + fp.set_default_fingerprint_mode("hash") + hashed = fp.get_system_fingerprint(cfg) + assert hashed != full + assert "tp8" not in hashed + + fp.set_default_fingerprint_mode("custom", "deploy-42") + assert fp.get_system_fingerprint(cfg) == "deploy-42" + + fp.set_default_fingerprint_mode("none") + assert fp.get_system_fingerprint(cfg) is None + + +def test_compute_hash_failure_does_not_raise(): + cfg = _cfg() + cfg.compute_hash = lambda: (_ for _ in ()).throw(RuntimeError("boom")) + assert fp.build_system_fingerprint(cfg, "full").endswith("-nohash") + assert fp.build_system_fingerprint(cfg, "hash").endswith("-nohash") diff --git a/vllm/entrypoints/openai/chat_completion/batch_serving.py b/vllm/entrypoints/openai/chat_completion/batch_serving.py index b8b26d393c7..cc49909b836 100644 --- a/vllm/entrypoints/openai/chat_completion/batch_serving.py +++ b/vllm/entrypoints/openai/chat_completion/batch_serving.py @@ -317,4 +317,5 @@ class OpenAIServingChatBatch(OpenAIServingChat): model=model_name, choices=choices, usage=usage, + system_fingerprint=self.system_fingerprint, ) diff --git a/vllm/entrypoints/openai/chat_completion/protocol.py b/vllm/entrypoints/openai/chat_completion/protocol.py index aacac38e07f..01d2df88d69 100644 --- a/vllm/entrypoints/openai/chat_completion/protocol.py +++ b/vllm/entrypoints/openai/chat_completion/protocol.py @@ -129,6 +129,9 @@ class ChatCompletionStreamResponse(OpenAIBaseModel): model: str choices: list[ChatCompletionResponseStreamChoice] usage: UsageInfo | None = Field(default=None) + # Set only on the final chunk of a stream to mirror non-streaming responses + # without the per-chunk serialization overhead. + system_fingerprint: str | None = None # not part of the OpenAI spec but for tracing the tokens prompt_token_ids: list[int] | None = None diff --git a/vllm/entrypoints/openai/chat_completion/serving.py b/vllm/entrypoints/openai/chat_completion/serving.py index 12dc2cd98e2..b916379f3fb 100644 --- a/vllm/entrypoints/openai/chat_completion/serving.py +++ b/vllm/entrypoints/openai/chat_completion/serving.py @@ -1195,6 +1195,16 @@ class OpenAIServingChat(OpenAIServing): choices=[choice_data], model=model_name, ) + # Stamp the fingerprint on terminal chunks only (those with + # finish_reason set). When ``include_usage`` is on, the + # trailing usage chunk below overrides this as the true + # final message. + if ( + not include_usage + and self.system_fingerprint is not None + and choice_data.finish_reason is not None + ): + chunk.system_fingerprint = self.system_fingerprint # handle usage stats if requested & if continuous if include_continuous_usage: @@ -1229,6 +1239,7 @@ class OpenAIServingChat(OpenAIServing): choices=[], model=model_name, usage=final_usage, + system_fingerprint=self.system_fingerprint, ) final_usage_data = final_usage_chunk.model_dump_json( exclude_unset=True, exclude_none=True @@ -1637,6 +1648,7 @@ class OpenAIServingChat(OpenAIServing): model=model_name, choices=choices, usage=usage, + system_fingerprint=self.system_fingerprint, prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs), prompt_token_ids=( final_res.prompt_token_ids if request.return_token_ids else None diff --git a/vllm/entrypoints/openai/cli_args.py b/vllm/entrypoints/openai/cli_args.py index 898e62f7713..3ebdec3c67f 100644 --- a/vllm/entrypoints/openai/cli_args.py +++ b/vllm/entrypoints/openai/cli_args.py @@ -153,9 +153,21 @@ class BaseFrontendArgs: """If set to True, log the stack trace of error responses""" tokens_only: bool = False """ - If set to True, only enable the Tokens In<>Out endpoint. + If set to True, only enable the Tokens In<>Out endpoint. This is intended for use in a Disaggregated Everything setup. """ + fingerprint_mode: Literal["full", "hash", "custom", "none"] = "full" + """Controls the ``system_fingerprint`` field on responses. + + - ``full`` (default): ``vllm-[-]-``. Encodes + server version, non-trivial parallelism degrees (tp/pp/dp/ep), and an + 8-char config hash. + - ``hash``: ``vllm--``. Parallelism stripped. + - ``custom``: emits the literal string from ``--fingerprint-value``. + - ``none``: the field is omitted (serialized as ``null``). + """ + fingerprint_value: str | None = None + """Literal fingerprint string used when ``--fingerprint-mode=custom``.""" @classmethod def _customize_cli_kwargs( diff --git a/vllm/entrypoints/openai/completion/protocol.py b/vllm/entrypoints/openai/completion/protocol.py index c785d254084..7edea50d73a 100644 --- a/vllm/entrypoints/openai/completion/protocol.py +++ b/vllm/entrypoints/openai/completion/protocol.py @@ -512,3 +512,6 @@ class CompletionStreamResponse(OpenAIBaseModel): model: str choices: list[CompletionResponseStreamChoice] usage: UsageInfo | None = Field(default=None) + # Set only on the final chunk of a stream to mirror non-streaming responses + # without the per-chunk serialization overhead. + system_fingerprint: str | None = None diff --git a/vllm/entrypoints/openai/completion/serving.py b/vllm/entrypoints/openai/completion/serving.py index fb7f253c7ea..454b170a5fa 100644 --- a/vllm/entrypoints/openai/completion/serving.py +++ b/vllm/entrypoints/openai/completion/serving.py @@ -383,6 +383,7 @@ class OpenAIServingCompletion(OpenAIServing): chunk = CompletionStreamResponse( id=request_id, + object="text_completion", created=created_time, model=model_name, choices=[ @@ -401,6 +402,14 @@ class OpenAIServingCompletion(OpenAIServing): ) ], ) + # Stamp on terminal chunk only when no trailing usage chunk + # will follow (that one is the true final message). + if ( + not include_usage + and self.system_fingerprint is not None + and finish_reason is not None + ): + chunk.system_fingerprint = self.system_fingerprint if include_continuous_usage: prompt_tokens = num_prompt_tokens[prompt_idx] completion_tokens = previous_num_tokens[i] @@ -410,7 +419,7 @@ class OpenAIServingCompletion(OpenAIServing): total_tokens=prompt_tokens + completion_tokens, ) - response_json = chunk.model_dump_json(exclude_unset=False) + response_json = chunk.model_dump_json(exclude_unset=True) yield f"data: {response_json}\n\n" total_prompt_tokens = sum(num_prompt_tokens) @@ -433,6 +442,7 @@ class OpenAIServingCompletion(OpenAIServing): model=model_name, choices=[], usage=final_usage_info, + system_fingerprint=self.system_fingerprint, ) final_usage_data = final_usage_chunk.model_dump_json( exclude_unset=False, exclude_none=True @@ -562,6 +572,7 @@ class OpenAIServingCompletion(OpenAIServing): model=model_name, choices=choices, usage=usage, + system_fingerprint=self.system_fingerprint, kv_transfer_params=kv_transfer_params, ) diff --git a/vllm/entrypoints/openai/engine/serving.py b/vllm/entrypoints/openai/engine/serving.py index d5fb5e137ef..77cce6bec5b 100644 --- a/vllm/entrypoints/openai/engine/serving.py +++ b/vllm/entrypoints/openai/engine/serving.py @@ -157,6 +157,19 @@ class OpenAIServing: self.renderer = engine_client.renderer self.input_processor = engine_client.input_processor + # Computed once at startup (cached by ``vllm_config`` identity) and + # stamped on non-streaming responses. Streaming chunks deliberately + # omit it to avoid per-chunk overhead. + from vllm.entrypoints.openai.fingerprint import get_system_fingerprint + + try: + self.system_fingerprint: str | None = get_system_fingerprint( + engine_client.vllm_config + ) + except Exception: + # Never fail server startup over the fingerprint. + self.system_fingerprint = None + async def beam_search( self, prompt: EngineInput, diff --git a/vllm/entrypoints/openai/fingerprint.py b/vllm/entrypoints/openai/fingerprint.py new file mode 100644 index 00000000000..e858e667d06 --- /dev/null +++ b/vllm/entrypoints/openai/fingerprint.py @@ -0,0 +1,84 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Build the ``system_fingerprint`` string returned by the OpenAI-compatible +server. + +Four modes, configured via ``--fingerprint-mode``: + +* ``full`` (default): ``vllm-[-]-`` — encodes + server version, any non-trivial parallelism degree (tp/pp/dp/ep), and an + 8-char prefix of ``vllm_config.compute_hash()`` (covers model identity, + quant config, speculative, attention backend, etc.). +* ``hash``: ``vllm--`` — parallelism stripped. +* ``custom``: user-provided literal via ``--fingerprint-value``. +* ``none``: the field is omitted (serialized as ``null``). + +``get_system_fingerprint`` is only called at serving-class init (a handful +of times per server); each subclass caches the returned string on +``self.system_fingerprint``, so per-request cost is one attribute read. +""" + +from __future__ import annotations + +from typing import Any, Literal + +FingerprintMode = Literal["full", "hash", "custom", "none"] + +_DEFAULT_MODE: FingerprintMode = "full" +_CUSTOM_VALUE: str | None = None + + +def set_default_fingerprint_mode( + mode: FingerprintMode, + custom_value: str | None = None, +) -> None: + """Configure the fingerprint mode for subsequent ``get_system_fingerprint`` + calls. Called once at server startup.""" + global _DEFAULT_MODE, _CUSTOM_VALUE + _DEFAULT_MODE = mode + _CUSTOM_VALUE = custom_value + + +def get_system_fingerprint(vllm_config: Any) -> str | None: + """Return the fingerprint for ``vllm_config`` using the mode configured by + ``set_default_fingerprint_mode``.""" + return build_system_fingerprint(vllm_config, _DEFAULT_MODE, _CUSTOM_VALUE) + + +def build_system_fingerprint( + vllm_config: Any, + mode: FingerprintMode = "full", + custom_value: str | None = None, +) -> str | None: + if mode == "none": + return None + if mode == "custom": + return custom_value + + from vllm import __version__ as vllm_version + + try: + hash8 = vllm_config.compute_hash()[:8] + except Exception: + hash8 = "nohash" + + if mode == "hash": + return f"vllm-{vllm_version}-{hash8}" + + # mode == "full" + parts: list[str] = [f"vllm-{vllm_version}"] + pc = getattr(vllm_config, "parallel_config", None) + if pc is not None: + tp = getattr(pc, "tensor_parallel_size", 1) + if tp > 1: + parts.append(f"tp{tp}") + pp = getattr(pc, "pipeline_parallel_size", 1) + if pp > 1: + parts.append(f"pp{pp}") + dp = getattr(pc, "data_parallel_size", 1) + if dp > 1: + parts.append(f"dp{dp}") + if getattr(pc, "enable_expert_parallel", False): + parts.append("ep") + parts.append(hash8) + return "-".join(parts) diff --git a/vllm/entrypoints/openai/generate/api_router.py b/vllm/entrypoints/openai/generate/api_router.py index 9a64db929e8..4386baa14e1 100644 --- a/vllm/entrypoints/openai/generate/api_router.py +++ b/vllm/entrypoints/openai/generate/api_router.py @@ -61,9 +61,17 @@ async def init_generate_state( ) from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion + from vllm.entrypoints.openai.fingerprint import set_default_fingerprint_mode from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses from vllm.entrypoints.serve.disagg.serving import ServingTokens + # Applied before any serving class is constructed so that each one picks + # up the chosen mode on its first cache miss. + set_default_fingerprint_mode( + getattr(args, "fingerprint_mode", "full"), + getattr(args, "fingerprint_value", None), + ) + if args.tool_server == "demo": tool_server: ToolServer | None = DemoToolServer() assert isinstance(tool_server, DemoToolServer)