karylab_agents/vllm
1
0
Fork 0
forked from Karylab-cklius/vllm
Code Pull Requests 2 Actions 1 Packages Activity

[Refactor] Remove dead states from chat completion serving (#45081)

Browse Source 
Signed-off-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
Flora Feng
2026-06-09 22:20:15 -07:00
committed by GitHub
parent f4966f8b3d
commit 6aec99f030
1 changed files with 1 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files
vllm/entrypoints/openai/chat_completion/serving.py
+1 -73
View File
@@ -376,7 +376,6 @@ class OpenAIServingChat(OpenAIServing):
conversation,
tokenizer,
request_metadata,
reasoning_parser,
chat_template_kwargs=chat_template_kwargs,
)
@@ -405,7 +404,6 @@ class OpenAIServingChat(OpenAIServing):
conversation: list[ConversationMessage],
tokenizer: TokenizerLike,
request_metadata: RequestResponseMetadata,
reasoning_parser: ReasoningParser | None = None,
chat_template_kwargs: dict[str, Any] | None = None,
) -> AsyncGenerator[str, None]:
created_time = int(time.time())
@@ -430,33 +428,13 @@ class OpenAIServingChat(OpenAIServing):
else:
tool_choice_function_name = None
# Determine whether tools are in use with "auto" tool choice
tool_choice_auto = (
not tool_choice_function_name
and self._should_stream_with_auto_tool_parsing(request)
)
all_previous_token_ids: list[list[int]] | None
if self.tool_call_id_type == "kimi_k2":
history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
else:
history_tool_call_cnt = 0
# Always track previous_texts for comprehensive output logging
previous_texts = [""] * num_choices
# Only one of these will be used, thus previous_texts and
# all_previous_token_ids will not be used twice in the same iteration.
if (
tool_choice_auto
or tool_choice_function_name
or request.tool_choice == "required"
or reasoning_parser
):
all_previous_token_ids = [[] for _ in range(num_choices)]
else:
all_previous_token_ids = None
try:
if self.parser_cls is not None:
if tokenizer is None:
@@ -639,26 +617,6 @@ class OpenAIServingChat(OpenAIServing):
delta_message: DeltaMessage | None
# just update previous_texts and previous_token_ids
if (
tool_choice_auto
or tool_choice_function_name
or request.tool_choice == "required"
or reasoning_parser
):
assert previous_texts is not None
assert all_previous_token_ids is not None
previous_text = previous_texts[i]
previous_token_ids = all_previous_token_ids[i]
current_text = previous_text + delta_text
# avoid the None + list error.
if previous_token_ids:
current_token_ids = previous_token_ids + as_list(
output.token_ids
)
else:
current_token_ids = as_list(output.token_ids)
if self.use_harmony:
delta_message, tools_streamed_flag = (
extract_harmony_streaming_delta(
@@ -683,21 +641,7 @@ class OpenAIServingChat(OpenAIServing):
else:
delta_message = DeltaMessage(content=delta_text)
# update the previous values for the next iteration
if (
tool_choice_auto
or tool_choice_function_name
or request.tool_choice == "required"
or reasoning_parser
) and not self.use_harmony:
assert previous_texts is not None
assert all_previous_token_ids is not None
previous_texts[i] = current_text
all_previous_token_ids[i] = current_token_ids
else:
# Update for comprehensive logging even in simple case
assert previous_texts is not None
previous_texts[i] += delta_text
previous_texts[i] += delta_text
# set the previous values for the next iteration
previous_num_tokens[i] += len(output.token_ids)
@@ -1326,19 +1270,3 @@ class OpenAIServingChat(OpenAIServing):
)
return ChatCompletionLogProbs(content=logprobs_content)
def _should_stream_with_auto_tool_parsing(self, request: ChatCompletionRequest):
"""
Utility function to check if streamed tokens should go through the tool
call parser that was configured.
We only want to do this IF user-provided tools are set, a tool parser
is configured, "auto" tool choice is enabled, and the request's tool
choice field indicates that "auto" tool choice should be used.
"""
return (
request.tools
and self.tool_parser
and self.enable_auto_tools
and request.tool_choice in ["auto", None]
)