From fd3e0cac12e99ced4922ee13562afde46c5cdf2a Mon Sep 17 00:00:00 2001 From: mistral0105 Date: Fri, 19 Jun 2026 04:51:01 +0000 Subject: [PATCH] Address review on Anthropic cache usage reporting - api_router: stop silently overriding --enable-prompt-tokens-details for AnthropicServingMessages; pass through the user's CLI setting like the other serving objects. - _compute_cache_usage: rewrite docstring to document where prompt_tokens_details is attached in vLLM's OpenAI streaming path (terminal include_usage chunk only), why message_start cannot populate cache fields today, and why cache_creation_input_tokens defaults to 0 rather than None when cache info is present. - AnthropicUsage construction: omit cache fields entirely when the underlying cache info is unknown (cache_read is None), rather than emitting null. Applied uniformly to non-streaming responses, message_start, and message_delta so "unknown" is signaled by key absence rather than null, distinguishing it from a real zero. - Tests: add TestStreamingCacheUsageSemantics covering the three usage states (cache hit, cache miss with details, no details at all) for both message_start and message_delta. Signed-off-by: mistral0105 --- .../test_anthropic_messages_conversion.py | 129 ++++++++++++++++++ vllm/entrypoints/anthropic/serving.py | 95 +++++++++---- vllm/entrypoints/generate/api_router.py | 4 +- 3 files changed, 200 insertions(+), 28 deletions(-) diff --git a/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py b/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py index d304c863e24..3743406d1d9 100644 --- a/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py +++ b/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py @@ -1205,6 +1205,135 @@ class TestMessageStartIncludesTypeAndRole: assert message["role"] == "assistant" +class TestStreamingCacheUsageSemantics: + """Locks in the documented streaming behavior of cache usage fields. + + vLLM's OpenAI chat completion streaming only attaches + ``prompt_tokens_details`` to the terminal usage chunk. The Anthropic layer + mirrors that contract: cache fields are omitted on ``message_start`` (key + absence signals "unknown") and populated on ``message_delta`` (the final + cumulative count). This is intentionally consistent with vLLM's OpenAI + behavior, even though Anthropic's upstream API populates cache fields on + ``message_start``; closing that gap requires plumbing cache info into the + first chunk at the OpenAI layer, which is out of scope here. + """ + + @pytest.mark.asyncio + async def test_streaming_cache_fields_absent_then_populated(self): + """First chunk lacks prompt_tokens_details (vLLM contract); + message_start omits cache fields. The final chunk carries + prompt_tokens_details, so message_delta carries resolved values.""" + + async def sse_input(): + yield _make_stream_chunk( + delta=DeltaMessage(role="assistant", content="hi"), + usage=UsageInfo(prompt_tokens=100, total_tokens=100), + ) + yield _make_stream_chunk(finish_reason="stop") + yield _make_stream_chunk( + choices=[], + usage=UsageInfo( + prompt_tokens=100, + completion_tokens=5, + total_tokens=105, + prompt_tokens_details=PromptTokenUsageInfo(cached_tokens=80), + ), + ) + yield "data: [DONE]" + + converter = _make_stream_converter() + output = [] + async for event in converter.message_stream_converter(sse_input()): + output.append(event) + events = _parse_sse_events(output) + + # message_start: cache fields unknown → omitted from JSON entirely. + start_usage = events[0][1]["message"]["usage"] + assert events[0][0] == "message_start" + assert start_usage["input_tokens"] == 100 + assert "cache_read_input_tokens" not in start_usage + assert "cache_creation_input_tokens" not in start_usage + + # message_delta: authoritative usage with cache fields populated. + delta_usage = next( + data["usage"] for ev, data in events if ev == "message_delta" + ) + assert delta_usage["input_tokens"] == 20 # 100 - 80 + assert delta_usage["cache_read_input_tokens"] == 80 + assert delta_usage["cache_creation_input_tokens"] == 0 + + @pytest.mark.asyncio + async def test_streaming_no_cache_hit(self): + """When the final chunk reports cached_tokens=0, message_delta carries + cache fields = 0 (cache miss); message_start still omits them.""" + + async def sse_input(): + yield _make_stream_chunk( + delta=DeltaMessage(role="assistant"), + usage=UsageInfo(prompt_tokens=50, total_tokens=50), + ) + yield _make_stream_chunk(finish_reason="stop") + yield _make_stream_chunk( + choices=[], + usage=UsageInfo( + prompt_tokens=50, + completion_tokens=5, + total_tokens=55, + prompt_tokens_details=PromptTokenUsageInfo(cached_tokens=0), + ), + ) + yield "data: [DONE]" + + converter = _make_stream_converter() + output = [] + async for event in converter.message_stream_converter(sse_input()): + output.append(event) + events = _parse_sse_events(output) + + start_usage = events[0][1]["message"]["usage"] + delta_usage = next( + data["usage"] for ev, data in events if ev == "message_delta" + ) + assert start_usage["input_tokens"] == 50 + assert "cache_read_input_tokens" not in start_usage + assert "cache_creation_input_tokens" not in start_usage + assert delta_usage["input_tokens"] == 50 # 50 - 0 + assert delta_usage["cache_read_input_tokens"] == 0 + assert delta_usage["cache_creation_input_tokens"] == 0 + + @pytest.mark.asyncio + async def test_streaming_no_prompt_tokens_details_at_all(self): + """If --enable-prompt-tokens-details is off, no chunk carries cache + info; both message_start and message_delta omit cache fields.""" + + async def sse_input(): + yield _make_stream_chunk( + delta=DeltaMessage(role="assistant"), + usage=UsageInfo(prompt_tokens=30, total_tokens=30), + ) + yield _make_stream_chunk(finish_reason="stop") + yield _make_stream_chunk( + choices=[], + usage=UsageInfo(prompt_tokens=30, completion_tokens=2, total_tokens=32), + ) + yield "data: [DONE]" + + converter = _make_stream_converter() + output = [] + async for event in converter.message_stream_converter(sse_input()): + output.append(event) + events = _parse_sse_events(output) + + start_usage = events[0][1]["message"]["usage"] + delta_usage = next( + data["usage"] for ev, data in events if ev == "message_delta" + ) + assert "cache_read_input_tokens" not in start_usage + assert "cache_creation_input_tokens" not in start_usage + assert "cache_read_input_tokens" not in delta_usage + assert "cache_creation_input_tokens" not in delta_usage + + # ====================================================================== # Auto-detection of system-first template requirement # ====================================================================== diff --git a/vllm/entrypoints/anthropic/serving.py b/vllm/entrypoints/anthropic/serving.py index 145ef49eaac..6ef32702120 100644 --- a/vllm/entrypoints/anthropic/serving.py +++ b/vllm/entrypoints/anthropic/serving.py @@ -68,12 +68,33 @@ def _compute_cache_usage( ) -> tuple[int, int | None, int | None]: """Compute Anthropic-compatible input_tokens and cache fields. - Anthropic defines: total_input = input_tokens + cache_read + cache_creation - vLLM's prompt_tokens is the total; input_tokens should exclude cached. - vLLM's prefix caching only tracks cache hits (cache_read), not writes, - so cache_creation is always 0 when cache info is available. + The Anthropic Messages API defines the invariant + ``total_input == input_tokens + cache_read + cache_creation``. vLLM's + ``prompt_tokens`` is the total, so ``input_tokens`` is derived as + ``prompt_tokens - cached_tokens``. - Returns (input_tokens, cache_read_input_tokens, cache_creation_input_tokens). + The underlying OpenAI usage protocol carries only ``cached_tokens`` + (cache hits, mapped to ``cache_read_input_tokens``). There is no + ``cached_tokens`` analog for cache creation, so when cache info is + present we report ``cache_creation_input_tokens = 0`` to satisfy the + invariant above. When cache info is absent (the caller didn't enable + ``--enable-prompt-tokens-details``, or vLLM has not attached + ``prompt_tokens_details`` yet), both cache fields return ``None`` and + the caller omits them from the JSON entirely. + + Origin of ``prompt_tokens_details`` in vLLM: + * Non-streaming responses attach it on the final ``UsageInfo`` (see + ``OpenAIServingChat.chat_completion_full_generator``). + * Streaming attaches it only on the terminal ``include_usage`` chunk + (``choices == []``), not on the per-token continuous-usage chunks + (see ``OpenAIServingChat.chat_completion_stream_generator``). + + This is why ``message_start.usage`` cannot populate cache fields today. + The Anthropic layer mirrors vLLM's OpenAI streaming: cache fields appear + on ``message_delta`` (the terminal chunk), not on ``message_start``. + + Returns: + (input_tokens, cache_read_input_tokens, cache_creation_input_tokens). """ cached = _get_cached_tokens(usage) if cached is not None: @@ -608,16 +629,21 @@ class AnthropicServingMessages(OpenAIServingChat): input_tokens, cache_read, cache_creation = _compute_cache_usage( generator.usage.prompt_tokens, generator.usage ) + # Omit cache fields when unknown (--enable-prompt-tokens-details off), + # so "unknown" is signaled by key absence rather than null. Consistent + # with streaming message_start / message_delta below. + usage_kwargs: dict[str, Any] = dict( + input_tokens=input_tokens, + output_tokens=generator.usage.completion_tokens, + ) + if cache_read is not None: + usage_kwargs["cache_read_input_tokens"] = cache_read + usage_kwargs["cache_creation_input_tokens"] = cache_creation result = AnthropicMessagesResponse( id=generator.id, content=[], model=generator.model, - usage=AnthropicUsage( - input_tokens=input_tokens, - output_tokens=generator.usage.completion_tokens, - cache_read_input_tokens=cache_read, - cache_creation_input_tokens=cache_creation, - ), + usage=AnthropicUsage(**usage_kwargs), kv_transfer_params=generator.kv_transfer_params, ) choice = generator.choices[0] @@ -790,6 +816,20 @@ class AnthropicServingMessages(OpenAIServingChat): input_tokens, cache_read, cache_creation = ( _compute_cache_usage(prompt_tokens, origin_chunk.usage) ) + # vLLM does not attach prompt_tokens_details to the + # first stream chunk, so cache fields are unknown + # here. Omit them entirely (rather than emit null) + # so clients can distinguish "unknown" from "zero". + # The authoritative values arrive on message_delta. + usage_kwargs: dict[str, Any] = dict( + input_tokens=input_tokens, + output_tokens=0, + ) + if cache_read is not None: + usage_kwargs["cache_read_input_tokens"] = cache_read + usage_kwargs["cache_creation_input_tokens"] = ( + cache_creation + ) chunk = AnthropicStreamEvent( type="message_start", message=AnthropicMessagesResponse( @@ -805,12 +845,7 @@ class AnthropicServingMessages(OpenAIServingChat): model=origin_chunk.model, stop_reason=None, stop_sequence=None, - usage=AnthropicUsage( - input_tokens=input_tokens, - output_tokens=0, - cache_read_input_tokens=cache_read, - cache_creation_input_tokens=cache_creation, - ), + usage=AnthropicUsage(**usage_kwargs), ), ) first_item = False @@ -833,17 +868,27 @@ class AnthropicServingMessages(OpenAIServingChat): input_tokens, cache_read, cache_creation = ( _compute_cache_usage(prompt_tokens, origin_chunk.usage) ) + # Omit cache fields when unknown + # (--enable-prompt-tokens-details off), same as + # message_start, so "unknown" is signaled by key + # absence rather than null. + delta_usage_kwargs: dict[str, Any] = dict( + input_tokens=input_tokens, + output_tokens=origin_chunk.usage.completion_tokens + if origin_chunk.usage + else 0, + ) + if cache_read is not None: + delta_usage_kwargs["cache_read_input_tokens"] = ( + cache_read + ) + delta_usage_kwargs["cache_creation_input_tokens"] = ( + cache_creation + ) chunk = AnthropicStreamEvent( type="message_delta", delta=AnthropicDelta(stop_reason=stop_reason), - usage=AnthropicUsage( - input_tokens=input_tokens, - output_tokens=origin_chunk.usage.completion_tokens - if origin_chunk.usage - else 0, - cache_read_input_tokens=cache_read, - cache_creation_input_tokens=cache_creation, - ), + usage=AnthropicUsage(**delta_usage_kwargs), ) data = chunk.model_dump_json(exclude_unset=True) yield wrap_data_with_event(data, "message_delta") diff --git a/vllm/entrypoints/generate/api_router.py b/vllm/entrypoints/generate/api_router.py index 6e166d69c71..b1e6cea44fe 100644 --- a/vllm/entrypoints/generate/api_router.py +++ b/vllm/entrypoints/generate/api_router.py @@ -168,9 +168,7 @@ async def init_generate_state( enable_auto_tools=args.enable_auto_tool_choice, tool_parser=args.tool_call_parser, reasoning_parser=args.structured_outputs_config.reasoning_parser, - # Always enable prompt tokens details for Anthropic API - # to populate cache_read_input_tokens / cache_creation_input_tokens. - enable_prompt_tokens_details=True, + enable_prompt_tokens_details=args.enable_prompt_tokens_details, enable_force_include_usage=args.enable_force_include_usage, default_chat_template_kwargs=args.default_chat_template_kwargs, )