From cd347298e86c83d43d3d404b05899b1be07489d1 Mon Sep 17 00:00:00 2001 From: Maxwill Lin <0312fs3@gmail.com> Date: Wed, 24 Jun 2026 17:08:42 -0700 Subject: [PATCH] [Frontend] Port seed_oss to the streaming parser engine as a Qwen3 subclass (#46314) Signed-off-by: EazyReal <8047065+EazyReal@users.noreply.github.com> --- tests/parser/engine/test_seed_oss.py | 189 ++++++ tests/parser/engine/trace_builder.py | 57 ++ .../test_seedoss_reasoning_parser.py | 236 ------- .../tool_parsers/test_seed_oss_tool_parser.py | 522 --------------- vllm/parser/engine/registered_adapters.py | 6 + vllm/parser/qwen3.py | 53 +- vllm/parser/seed_oss.py | 28 + vllm/reasoning/__init__.py | 4 +- .../seed_oss_engine_reasoning_parser.py | 6 + vllm/reasoning/seedoss_reasoning_parser.py | 27 - vllm/tool_parsers/__init__.py | 4 +- .../seed_oss_engine_tool_parser.py | 8 + vllm/tool_parsers/seed_oss_tool_parser.py | 633 ------------------ 13 files changed, 338 insertions(+), 1435 deletions(-) create mode 100644 tests/parser/engine/test_seed_oss.py delete mode 100644 tests/reasoning/test_seedoss_reasoning_parser.py delete mode 100644 tests/tool_parsers/test_seed_oss_tool_parser.py create mode 100644 vllm/parser/seed_oss.py create mode 100644 vllm/reasoning/seed_oss_engine_reasoning_parser.py delete mode 100644 vllm/reasoning/seedoss_reasoning_parser.py create mode 100644 vllm/tool_parsers/seed_oss_engine_tool_parser.py delete mode 100644 vllm/tool_parsers/seed_oss_tool_parser.py diff --git a/tests/parser/engine/test_seed_oss.py b/tests/parser/engine/test_seed_oss.py new file mode 100644 index 00000000000..ce118faa443 --- /dev/null +++ b/tests/parser/engine/test_seed_oss.py @@ -0,0 +1,189 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for the engine-based seed_oss parser. + +seed_oss is Qwen3 with four overridden wrapper tokens, so the shared grammar +(arg types, multiline values, parallel calls, streaming mechanics, …) is +already covered by ``test_qwen3.py``/``test_qwen3_reasoning.py``. These tests +cover only what is seed_oss-specific: that the ``seed:`` token overrides are +wired through, the reasoning→tool boundary holds with them, the malformed +header from #46314 no longer drops sibling calls, and the registered adapters +resolve. Seed-specific budget-reflect tags inside reasoning are also covered +here because the old dedicated parser tests exercised them. +""" + +import json + +import pytest + +from tests.parser.engine.conftest import make_mock_tokenizer +from tests.parser.engine.streaming_helpers import ( + collect_function_name, + collect_tool_arguments, + simulate_reasoning_streaming, + simulate_tool_streaming, +) +from vllm.parser.engine.registered_adapters import ( + SeedOssParserReasoningAdapter, + SeedOssParserToolAdapter, +) +from vllm.parser.seed_oss import SeedOssParser + +TOOL_CALL_START = "" +TOOL_CALL_END = "" +THINK_START = "" +THINK_END = "" + +_THINK_END_ID = 51 +_TOOL_CALL_ID = 60 + +_SEED_OSS_VOCAB = { + THINK_START: 50, + THINK_END: _THINK_END_ID, + TOOL_CALL_START: _TOOL_CALL_ID, + TOOL_CALL_END: 61, +} + + +@pytest.fixture +def mock_tokenizer(): + return make_mock_tokenizer(_SEED_OSS_VOCAB) + + +@pytest.fixture +def tool_parser(mock_tokenizer): + return SeedOssParser( + mock_tokenizer, chat_template_kwargs={"enable_thinking": False} + ) + + +@pytest.fixture +def parser(mock_tokenizer): + return SeedOssParser(mock_tokenizer) + + +def test_token_overrides_wired(parser): + assert parser.parser_engine_config.name == "seed_oss" + assert parser.reasoning_start_str == THINK_START + assert parser.reasoning_end_str == THINK_END + + +def test_single_tool_call(tool_parser, mock_request): + text = ( + f"{TOOL_CALL_START}\n\n" + "Tokyo\n" + f"\n{TOOL_CALL_END}" + ) + result = tool_parser.extract_tool_calls(text, mock_request) + + assert result.tools_called is True + assert result.tool_calls[0].function.name == "get_weather" + assert json.loads(result.tool_calls[0].function.arguments) == {"city": "Tokyo"} + + +def test_malformed_function_end_does_not_drop_siblings(tool_parser, mock_request): + """Regression for #46314: a malformed ```` with no closing ``>`` + on the header must not discard the other, well-formed calls.""" + text = ( + f"{TOOL_CALL_START}\n\n{TOOL_CALL_END}" + f"{TOOL_CALL_START}\n\n" + "Tokyo\n" + f"\n{TOOL_CALL_END}" + ) + result = tool_parser.extract_tool_calls(text, mock_request) + + weather = next(tc for tc in result.tool_calls if tc.function.name == "get_weather") + assert json.loads(weather.function.arguments) == {"city": "Tokyo"} + + +def test_basic_streaming(tool_parser, mock_request): + chunks = [ + f"{TOOL_CALL_START}\n", + "\n", + "Tokyo", + "\n", + "\n", + f"{TOOL_CALL_END}", + ] + results = simulate_tool_streaming(tool_parser, mock_request, chunks) + + assert collect_function_name(results) == "get_weather" + assert json.loads(collect_tool_arguments(results)) == {"city": "Tokyo"} + + +def test_reasoning_then_tool_call(parser): + text = ( + f"{THINK_START}I need to read the file.{THINK_END}" + f"{TOOL_CALL_START}\n\n" + "/tmp/x\n" + f"\n{TOOL_CALL_END}" + ) + reasoning, _ = parser.extract_reasoning(text, None) + assert reasoning == "I need to read the file." + assert TOOL_CALL_START not in reasoning + + +def test_streaming_think_end_and_tool_call_same_delta(parser): + """```` and ```` arriving in one delta must + not leak the terminal tokens into the reasoning text.""" + reasoning, content = simulate_reasoning_streaming( + parser, + [ + "Let me list the directory.", + f"{THINK_END}{TOOL_CALL_START}", + "", + ], + [(1,), (_THINK_END_ID, _TOOL_CALL_ID), (2,)], + ) + assert reasoning == "Let me list the directory." + assert THINK_END not in reasoning + assert TOOL_CALL_START not in reasoning + assert content is not None + + +def test_end_to_end_through_registered_adapters(mock_tokenizer, mock_request): + reasoning_parser = SeedOssParserReasoningAdapter(mock_tokenizer) + tool_parser = SeedOssParserToolAdapter(mock_tokenizer) + text = ( + f"{THINK_START}Plan the call.{THINK_END}" + f"{TOOL_CALL_START}\n\n" + "Tokyo\n" + f"\n{TOOL_CALL_END}" + ) + reasoning, remaining = reasoning_parser.extract_reasoning(text, mock_request) + assert reasoning == "Plan the call." + + tool_result = tool_parser.extract_tool_calls(remaining, mock_request) + assert tool_result.tool_calls[0].function.name == "get_weather" + assert json.loads(tool_result.tool_calls[0].function.arguments) == {"city": "Tokyo"} + + +def test_budget_reflect_tags_do_not_break_adapter_pipeline( + mock_tokenizer, + mock_request, +): + reasoning_parser = SeedOssParserReasoningAdapter(mock_tokenizer) + tool_parser = SeedOssParserToolAdapter(mock_tokenizer) + text = ( + f"{THINK_START}" + "The user's current thinking budget is 512.\n" + "I need the weather.\n" + "I have used 131 tokens." + "\n" + f"{THINK_END}" + f"{TOOL_CALL_START}\n\n" + "Barcelona\n" + f"\n{TOOL_CALL_END}" + ) + + reasoning, remaining = reasoning_parser.extract_reasoning(text, mock_request) + assert reasoning is not None + assert "current thinking budget is 512" in reasoning + assert "" in reasoning + assert "" in reasoning + + tool_result = tool_parser.extract_tool_calls(remaining, mock_request) + assert tool_result.tool_calls[0].function.name == "get_weather" + assert json.loads(tool_result.tool_calls[0].function.arguments) == { + "city": "Barcelona" + } diff --git a/tests/parser/engine/trace_builder.py b/tests/parser/engine/trace_builder.py index bee3d5d8b28..7f41b2b9513 100644 --- a/tests/parser/engine/trace_builder.py +++ b/tests/parser/engine/trace_builder.py @@ -34,6 +34,7 @@ from vllm.parser.engine.registered_adapters import ( MinimaxM2Parser, NemotronV3Parser, Qwen3Parser, + SeedOssParser, ) # ── Data structures ────────────────────────────────────────────────── @@ -587,6 +588,61 @@ def _build_nemotron_v3(scenario: Scenario, validate: bool = True) -> Sample: ) +# ── Seed-OSS (Qwen3 XML grammar with Seed wrapper tokens) ──────────── + +_SEED_OSS_VOCAB: dict[str, int] = { + "": 50, + "": 51, + "": 60, + "": 61, +} + + +def _seed_oss_tool_segments(tc: ToolCallSpec) -> list[tuple[str, bool]]: + parts = [f"\n"] + for key, value in tc.arguments.items(): + parts.append(f"\n{_qwen3_arg_value(value)}") + parts.append("\n\n") + return [ + ("", True), + ("".join(parts), False), + ("", True), + ] + + +def _seed_oss_segments(scenario: Scenario) -> list[tuple[str, bool]]: + segs: list[tuple[str, bool]] = [] + if scenario.reasoning is not None: + segs.append((scenario.reasoning, False)) + if scenario.content is not None or scenario.tool_calls is not None: + segs.append(("", True)) + if scenario.tool_calls is not None and not scenario.tool_calls: + segs.append(("", True)) + segs.append(("", True)) + if scenario.content is not None: + segs.append((scenario.content, False)) + if scenario.tool_calls: + for tc in scenario.tool_calls: + segs.extend(_seed_oss_tool_segments(tc)) + return segs + + +def _build_seed_oss(scenario: Scenario, validate: bool = True) -> Sample: + sample = _make_sample( + sample_id=f"seed_oss-{scenario.id}", + description=scenario.description, + vocab=_SEED_OSS_VOCAB, + segments=_seed_oss_segments(scenario), + expected_reasoning=scenario.reasoning if scenario.reasoning is not None else "", + expected_content=_qwen3_expected_content(scenario), + expected_tool_calls=_expected_tc(scenario), + tools=_expected_tools(scenario), + ) + if validate: + _validate_sample(sample, SeedOssParser) + return sample + + # ── GLM-4.7 MoE (XML tool format, starts in REASONING) ────────────── _GLM47_MOE_VOCAB: dict[str, int] = { @@ -668,6 +724,7 @@ _BUILDERS: dict[str, Any] = { "gemma4": _build_gemma4, "minimax_m2": _build_minimax_m2, "nemotron_v3": _build_nemotron_v3, + "seed_oss": _build_seed_oss, "glm47_moe": _build_glm47_moe, } diff --git a/tests/reasoning/test_seedoss_reasoning_parser.py b/tests/reasoning/test_seedoss_reasoning_parser.py deleted file mode 100644 index 33d56d32965..00000000000 --- a/tests/reasoning/test_seedoss_reasoning_parser.py +++ /dev/null @@ -1,236 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from typing import Any, cast - -import pytest -from transformers import AutoTokenizer - -from tests.reasoning.utils import run_reasoning_extraction -from vllm.reasoning import ReasoningParser, ReasoningParserManager - -parser_name = "seed_oss" -start_token = "" -end_token = "" - -# Use a test model that contains our custom tokens -REASONING_MODEL_NAME = "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B" - - -@pytest.fixture(scope="module") -def seedoss_tokenizer(): - tokenizer = AutoTokenizer.from_pretrained(REASONING_MODEL_NAME) - # Add custom SeedOSS tokens if they don't exist - if start_token not in tokenizer.get_vocab(): - tokenizer.add_tokens([start_token, end_token]) - return tokenizer - - -SIMPLE_REASONING: dict[str, Any] = { - "output": "This is a reasoning sectionThis is the rest", - "reasoning": "This is a reasoning section", - "content": "This is the rest", - "is_reasoning_end": True, -} -COMPLETE_REASONING: dict[str, Any] = { - "output": "This is a reasoning section", - "reasoning": "This is a reasoning section", - "content": None, - "is_reasoning_end": True, -} -NO_CONTENT: dict[str, Any] = { - "output": "This is content", - "reasoning": "This is content", - "content": None, - "is_reasoning_end": False, -} -NO_REASONING_STREAMING: dict[str, Any] = { - "output": "This is a reasoning section", - "reasoning": "This is a reasoning section", - "content": None, - "is_reasoning_end": False, -} -MULTIPLE_LINES: dict[str, Any] = { - "output": "This\nThatThis is the rest\nThat", - "reasoning": "This\nThat", - "content": "This is the rest\nThat", - "is_reasoning_end": True, -} -WITH_START_TOKEN: dict[str, Any] = { - "output": ("This is a reasoning sectionThis is the rest"), - "reasoning": "This is a reasoning section", - "content": "This is the rest", - "is_reasoning_end": True, -} -ONLY_END_TOKEN: dict[str, Any] = { - "output": "Some reasoningThis is the rest", - "reasoning": "Some reasoning", - "content": "This is the rest", - "is_reasoning_end": True, -} -NO_TOKENS: dict[str, Any] = { - "output": "This is just content without any reasoning tokens", - "reasoning": "This is just content without any reasoning tokens", - "content": None, - "is_reasoning_end": False, -} - - -def test_seedoss_reasoning_parser_creation(seedoss_tokenizer): - """Test that the SeedOSS reasoning parser can be created and registered.""" - parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name) - parser = parser_cls(seedoss_tokenizer) - assert isinstance(parser, ReasoningParser) - assert parser.start_token == start_token - assert parser.end_token == end_token - - -@pytest.mark.parametrize("streaming", [True, False]) -def test_simple_reasoning(seedoss_tokenizer, streaming): - """Test basic reasoning extraction with both tokens.""" - parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name) - parser = parser_cls(seedoss_tokenizer) - - reasoning, content = run_reasoning_extraction( - parser, [cast(str, SIMPLE_REASONING["output"])], streaming=streaming - ) - - assert reasoning == SIMPLE_REASONING["reasoning"] - assert content == SIMPLE_REASONING["content"] - - -@pytest.mark.parametrize("streaming", [True, False]) -def test_complete_reasoning(seedoss_tokenizer, streaming): - """Test reasoning extraction when there's no content after reasoning.""" - parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name) - parser = parser_cls(seedoss_tokenizer) - - reasoning, content = run_reasoning_extraction( - parser, [cast(str, COMPLETE_REASONING["output"])], streaming=streaming - ) - - assert reasoning == COMPLETE_REASONING["reasoning"] - assert content == COMPLETE_REASONING["content"] - - -@pytest.mark.parametrize("streaming", [True, False]) -def test_no_content(seedoss_tokenizer, streaming): - """Test when there's no end token - everything is reasoning content.""" - parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name) - parser = parser_cls(seedoss_tokenizer) - - reasoning, content = run_reasoning_extraction( - parser, [cast(str, NO_CONTENT["output"])], streaming=streaming - ) - - assert reasoning == NO_CONTENT["reasoning"] - assert content == NO_CONTENT["content"] - - -@pytest.mark.parametrize("streaming", [True, False]) -def test_multiple_lines(seedoss_tokenizer, streaming): - """Test reasoning extraction with multiline content.""" - parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name) - parser = parser_cls(seedoss_tokenizer) - - reasoning, content = run_reasoning_extraction( - parser, [cast(str, MULTIPLE_LINES["output"])], streaming=streaming - ) - - assert reasoning == MULTIPLE_LINES["reasoning"] - assert content == MULTIPLE_LINES["content"] - - -@pytest.mark.parametrize("streaming", [True, False]) -def test_with_start_token(seedoss_tokenizer, streaming): - """Test reasoning extraction with both start and end tokens.""" - parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name) - parser = parser_cls(seedoss_tokenizer) - - reasoning, content = run_reasoning_extraction( - parser, [cast(str, WITH_START_TOKEN["output"])], streaming=streaming - ) - - assert reasoning == WITH_START_TOKEN["reasoning"] - assert content == WITH_START_TOKEN["content"] - - -@pytest.mark.parametrize("streaming", [True, False]) -def test_only_end_token(seedoss_tokenizer, streaming): - """ - Test reasoning extraction with only end token - (SeedOSS typical behavior). - """ - parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name) - parser = parser_cls(seedoss_tokenizer) - - reasoning, content = run_reasoning_extraction( - parser, [cast(str, ONLY_END_TOKEN["output"])], streaming=streaming - ) - - assert reasoning == ONLY_END_TOKEN["reasoning"] - assert content == ONLY_END_TOKEN["content"] - - -@pytest.mark.parametrize("streaming", [True, False]) -def test_no_tokens(seedoss_tokenizer, streaming): - """Test when there are no reasoning tokens at all.""" - parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name) - parser = parser_cls(seedoss_tokenizer) - - reasoning, content = run_reasoning_extraction( - parser, [cast(str, NO_TOKENS["output"])], streaming=streaming - ) - - assert reasoning == NO_TOKENS["reasoning"] - assert content == NO_TOKENS["content"] - - -def test_is_reasoning_end(seedoss_tokenizer): - """Test the is_reasoning_end method.""" - parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name) - parser = parser_cls(seedoss_tokenizer) - - # Test with end token present - end_token_id = parser.end_token_id - assert parser.is_reasoning_end([1, 2, end_token_id, 4]) is True - - # Test without end token - assert parser.is_reasoning_end([1, 2, 3, 4]) is False - - -def test_extract_content_ids(seedoss_tokenizer): - """Test the extract_content_ids method.""" - parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name) - parser = parser_cls(seedoss_tokenizer) - - end_token_id = parser.end_token_id - - # Test with end token in the middle - input_ids = [1, 2, end_token_id, 4, 5] - content_ids = parser.extract_content_ids(input_ids) - assert content_ids == [4, 5] - - # Test with end token at the end - input_ids = [1, 2, 3, end_token_id] - content_ids = parser.extract_content_ids(input_ids) - assert content_ids == [] - - # Test without end token - input_ids = [1, 2, 3, 4] - content_ids = parser.extract_content_ids(input_ids) - assert content_ids == [] - - -def test_streaming_delta_processing(seedoss_tokenizer): - """Test streaming processing with small deltas.""" - parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name) - parser = parser_cls(seedoss_tokenizer) - - # Test streaming with incremental tokens - deltas = ["Some ", "reasoning ", "content", "", "Final ", "answer"] - - reasoning, content = run_reasoning_extraction(parser, deltas, streaming=True) - - assert reasoning == "Some reasoning content" - assert content == "Final answer" diff --git a/tests/tool_parsers/test_seed_oss_tool_parser.py b/tests/tool_parsers/test_seed_oss_tool_parser.py deleted file mode 100644 index 4ff96fb01be..00000000000 --- a/tests/tool_parsers/test_seed_oss_tool_parser.py +++ /dev/null @@ -1,522 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# ruff: noqa: E501 - -import json -from collections.abc import Generator - -import pytest - -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, - ChatCompletionToolsParam, -) -from vllm.entrypoints.openai.engine.protocol import ( - DeltaMessage, - FunctionCall, - ToolCall, -) -from vllm.tokenizers import TokenizerLike, get_tokenizer -from vllm.tokenizers.detokenizer_utils import detokenize_incrementally -from vllm.tool_parsers.seed_oss_tool_parser import SeedOssToolParser - -# Use a common model that is likely to be available -MODEL = "ByteDance-Seed/Seed-OSS-36B-Instruct" - - -@pytest.fixture(scope="module") -def seed_oss_tokenizer(): - return get_tokenizer(tokenizer_name=MODEL, trust_remote_code=True) - - -@pytest.fixture -def seed_oss_tool_parser(seed_oss_tokenizer, sample_tools): - return SeedOssToolParser(seed_oss_tokenizer, tools=sample_tools) - - -@pytest.fixture -def sample_tools(): - return [ - ChatCompletionToolsParam( - type="function", - function={ - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. Bogotá, Colombia", - }, - "unit": { - "type": "string", - "description": "this is the unit of temperature", - }, - }, - "required": ["location"], - "additionalProperties": False, - }, - "returns": { - "type": "object", - "properties": { - "temperature": { - "type": "number", - "description": "temperature in celsius", - } - }, - "required": ["temperature"], - "additionalProperties": False, - }, - "strict": True, - }, - ), - ] - - -def assert_tool_calls( - actual_tool_calls: list[ToolCall], expected_tool_calls: list[ToolCall] -): - assert len(actual_tool_calls) == len(expected_tool_calls) - - for actual_tool_call, expected_tool_call in zip( - actual_tool_calls, expected_tool_calls - ): - # Seed-OSS tool call will not generate id - assert actual_tool_call.type == "function" - assert actual_tool_call.function == expected_tool_call.function - - assert actual_tool_call.function.name == expected_tool_call.function.name - assert ( - actual_tool_call.function.arguments == expected_tool_call.function.arguments - ) - - -def test_extract_tool_calls_no_tools(seed_oss_tool_parser): - model_output = "This is a test response without any tool calls" - extracted_tool_calls = seed_oss_tool_parser.extract_tool_calls( - model_output, request=None - ) # type: ignore[arg-type] - - assert not extracted_tool_calls.tools_called - assert extracted_tool_calls.tool_calls == [] - assert extracted_tool_calls.content == model_output - - -@pytest.mark.parametrize( - ids=[ - "tool_call_0_thinking_budget", - "tool_call_512_thinking_budget", - "tool_call_unlimited_thinking_budget", - ], - argnames=["model_output", "expected_tool_calls", "expected_content"], - argvalues=[ - ( - """\n\n""" - """Barcelona, Spain\n\n""", - [ - ToolCall( - function=FunctionCall( - name="get_weather", - arguments=json.dumps( - { - "location": "Barcelona, Spain", - }, - ), - ), - type="function", - ) - ], - None, - ), - ( - """The user\'s current thinking budget is 512.\nLet me analyze the """ - """question. The user wants to know the weather in Barcelona, Spain. Looking at the functions available, """ - """there\'s a get_weather function that can retrieve the current temperature for a given location. \n\nFirst, """ - """check the parameters required by get_weather: location is mandatory (needs city and country), and unit is """ - """optional. The user provided "Barcelona Spain" as the location, which fits the required format (city, """ - """country). \nI have used 131 tokens, and there are 381 tokens remaining for use.""" - """\n Since the unit isn\'t specified, the function will default to Celsius, which """ - """is fine. \n\nThere\'s no need to ask for more information because the location is clear. So I should call """ - """the get_weather function with location set to "Barcelona, Spain" (adding a comma for clarity, though the """ - """user\'s input has a space, but the function might accept either; to be safe, using the standard format """ - """with a comma).\nI have used 257 tokens, and there are 255 tokens remaining for """ - """use.\n The unit parameter can be omitted since it\'s optional.\n""" - """\n\nBarcelona, Spain\n""" - """\n""", - [ - ToolCall( - function=FunctionCall( - name="get_weather", - arguments=json.dumps( - { - "location": "Barcelona, Spain", - }, - ), - ), - type="function", - ) - ], - """The user\'s current thinking budget is 512.\nLet me analyze the """ - """question. The user wants to know the weather in Barcelona, Spain. Looking at the functions available, """ - """there\'s a get_weather function that can retrieve the current temperature for a given location. \n\nFirst, """ - """check the parameters required by get_weather: location is mandatory (needs city and country), and unit is """ - """optional. The user provided "Barcelona Spain" as the location, which fits the required format (city, """ - """country). \nI have used 131 tokens, and there are 381 tokens remaining for use.""" - """\n Since the unit isn\'t specified, the function will default to Celsius, which """ - """is fine. \n\nThere\'s no need to ask for more information because the location is clear. So I should call """ - """the get_weather function with location set to "Barcelona, Spain" (adding a comma for clarity, though the """ - """user\'s input has a space, but the function might accept either; to be safe, using the standard format """ - """with a comma).\nI have used 257 tokens, and there are 255 tokens remaining for """ - """use.\n The unit parameter can be omitted since it\'s optional.\n""", - ), - ( - """\nGot it, let\'s see. The user asked for the weather in Barcelona, Spain. """ - """First, I need to remember the function I can use: get_weather. The function requires a """ - """location (city and country) which is "Barcelona, Spain" here, and unit is optional. Since """ - """the user didn\'t specify the unit, the default in the function is Celsius, right? Wait, """ - """let me check the function docstring again. Oh, the function says unit is optional, and """ - """returns temperature in Celsius. So I should call get_weather with location "Barcelona, """ - """Spain" and maybe omit unit or set to Celsius. Let me format the function call correctly. """ - """The format is \n\nBarcelona, """ - """Spain\ncelsius\n\n. """ - """Wait, but does the unit parameter accept "celsius"? The docstring says unit is the unit """ - """of temperature, but the return is in Celsius anyway. Maybe even if I don\'t pass unit, """ - """it\'s okay, but to be explicit, maybe pass "celsius". Let me go with that. So the function """ - """call should be as above. Then wait for the result to come back and tell the user the """ - """temperature in Celsius.\n\n""" - """Barcelona, Spain\ncelsius\n\n""", - [ - ToolCall( - function=FunctionCall( - name="get_weather", - arguments=json.dumps( - { - "location": "Barcelona, Spain", - "unit": "celsius", - }, - ), - ), - type="function", - ) - ], - """\nGot it, let\'s see. The user asked for the weather in Barcelona, Spain. """ - """First, I need to remember the function I can use: get_weather. The function requires a """ - """location (city and country) which is "Barcelona, Spain" here, and unit is optional. Since """ - """the user didn\'t specify the unit, the default in the function is Celsius, right? Wait, """ - """let me check the function docstring again. Oh, the function says unit is optional, and """ - """returns temperature in Celsius. So I should call get_weather with location "Barcelona, """ - """Spain" and maybe omit unit or set to Celsius. Let me format the function call correctly. """ - """The format is \n\nBarcelona, """ - """Spain\ncelsius\n\n. """ - """Wait, but does the unit parameter accept "celsius"? The docstring says unit is the unit """ - """of temperature, but the return is in Celsius anyway. Maybe even if I don\'t pass unit, """ - """it\'s okay, but to be explicit, maybe pass "celsius". Let me go with that. So the function """ - """call should be as above. Then wait for the result to come back and tell the user the """ - """temperature in Celsius.""", - ), - ], -) -def test_extract_tool_calls( - seed_oss_tool_parser, - sample_tools, - model_output, - expected_tool_calls, - expected_content, -): - request = ChatCompletionRequest(model=MODEL, messages=[], tools=sample_tools) - extracted_tool_calls = seed_oss_tool_parser.extract_tool_calls( - model_output, request=request - ) # type: ignore[arg-type] - assert extracted_tool_calls.tools_called - - assert_tool_calls(extracted_tool_calls.tool_calls, expected_tool_calls) - - assert extracted_tool_calls.content == expected_content - - -def test_streaming_tool_calls_no_tools(seed_oss_tool_parser): - model_output = "This is a test response without any tool calls" - - result = seed_oss_tool_parser.extract_tool_calls_streaming( - previous_text="his is a test response", - current_text=model_output, - delta_text=" without any tool calls.", - previous_token_ids=[], - current_token_ids=[], - delta_token_ids=[], - request=None, - ) - - # Should return the delta text as content - assert result is not None - assert hasattr(result, "content") - assert result.content == " without any tool calls." - - -def stream_delta_message_generator( - seed_oss_tool_parser: SeedOssToolParser, - seed_oss_tokenizer: TokenizerLike, - model_output: str, - request: ChatCompletionRequest | None = None, -) -> Generator[DeltaMessage, None, None]: - all_token_ids = seed_oss_tokenizer.encode(model_output, add_special_tokens=False) - - previous_text = "" - previous_tokens = None - prefix_offset = 0 - read_offset = 0 - for i, delta_token in enumerate(all_token_ids): - delta_token_ids = [delta_token] - previous_token_ids = all_token_ids[:i] - current_token_ids = all_token_ids[: i + 1] - - (new_tokens, delta_text, new_prefix_offset, new_read_offset) = ( - detokenize_incrementally( - tokenizer=seed_oss_tokenizer, - all_input_ids=current_token_ids, - prev_tokens=previous_tokens, - prefix_offset=prefix_offset, - read_offset=read_offset, - skip_special_tokens=False, - spaces_between_special_tokens=True, - ) - ) - - current_text = previous_text + delta_text - - delta_message = seed_oss_tool_parser.extract_tool_calls_streaming( - previous_text, - current_text, - delta_text, - previous_token_ids, - current_token_ids, - delta_token_ids, - request=request, - ) - if delta_message: - yield delta_message - - previous_text = current_text - previous_tokens = ( - previous_tokens + new_tokens if previous_tokens else new_tokens - ) - prefix_offset = new_prefix_offset - read_offset = new_read_offset - - -@pytest.mark.parametrize( - ids=[ - "tool_call_0_thinking_budget", - "tool_call_512_thinking_budget", - "tool_call_unlimited_thinking_budget", - ], - argnames=["model_output", "expected_tool_calls", "expected_content"], - argvalues=[ - ( - """\n\n\n""" - """The current thinking budget is 0, so I will directly start answering the question.\n\n""" - """\n\n""" - """Barcelona, Spain\n\n""", - [ - ToolCall( - function=FunctionCall( - name="get_weather", - arguments=json.dumps( - { - "location": "Barcelona, Spain", - }, - ), - ), - type="function", - ) - ], - """\n\n\n""" - """The current thinking budget is 0, so I will directly start answering the question.\n\n""", - ), - ( - """The user\'s current thinking budget is 512.\nLet me analyze the """ - """question. The user wants to know the weather in Barcelona, Spain. Looking at the functions available, """ - """there\'s a get_weather function that can retrieve the current temperature for a given location. \n\nFirst, """ - """check the parameters required by get_weather: location is mandatory (needs city and country), and unit is """ - """optional. The user provided "Barcelona Spain" as the location, which fits the required format (city, """ - """country). \nI have used 131 tokens, and there are 381 tokens remaining for use.""" - """\n Since the unit isn\'t specified, the function will default to Celsius, which """ - """is fine. \n\nThere\'s no need to ask for more information because the location is clear. So I should call """ - """the get_weather function with location set to "Barcelona, Spain" (adding a comma for clarity, though the """ - """user\'s input has a space, but the function might accept either; to be safe, using the standard format """ - """with a comma).\nI have used 257 tokens, and there are 255 tokens remaining for """ - """use.\n The unit parameter can be omitted since it\'s optional.\n""" - """\n\nBarcelona, Spain\n""" - """\n""", - [ - ToolCall( - function=FunctionCall( - name="get_weather", - arguments=json.dumps( - { - "location": "Barcelona, Spain", - }, - ), - ), - type="function", - ) - ], - """The user\'s current thinking budget is 512.\nLet me analyze the """ - """question. The user wants to know the weather in Barcelona, Spain. Looking at the functions available, """ - """there\'s a get_weather function that can retrieve the current temperature for a given location. \n\nFirst, """ - """check the parameters required by get_weather: location is mandatory (needs city and country), and unit is """ - """optional. The user provided "Barcelona Spain" as the location, which fits the required format (city, """ - """country). \nI have used 131 tokens, and there are 381 tokens remaining for use.""" - """\n Since the unit isn\'t specified, the function will default to Celsius, which """ - """is fine. \n\nThere\'s no need to ask for more information because the location is clear. So I should call """ - """the get_weather function with location set to "Barcelona, Spain" (adding a comma for clarity, though the """ - """user\'s input has a space, but the function might accept either; to be safe, using the standard format """ - """with a comma).\nI have used 257 tokens, and there are 255 tokens remaining for """ - """use.\n The unit parameter can be omitted since it\'s optional.\n""", - ), - ( - """\nGot it, let\'s see. The user asked for the weather in Barcelona, Spain. """ - """First, I need to remember the function I can use: get_weather. The function requires a """ - """location (city and country) which is "Barcelona, Spain" here, and unit is optional. Since """ - """the user didn\'t specify the unit, the default in the function is Celsius, right? Wait, """ - """let me check the function docstring again. Oh, the function says unit is optional, and """ - """returns temperature in Celsius. So I should call get_weather with location "Barcelona, """ - """Spain" and maybe omit unit or set to Celsius. Let me format the function call correctly. """ - """The format is \n\nBarcelona, """ - """Spain\ncelsius\n\n. """ - """Wait, but does the unit parameter accept "celsius"? The docstring says unit is the unit """ - """of temperature, but the return is in Celsius anyway. Maybe even if I don\'t pass unit, """ - """it\'s okay, but to be explicit, maybe pass "celsius". Let me go with that. So the function """ - """call should be as above. Then wait for the result to come back and tell the user the """ - """temperature in Celsius.\n\n""" - """Barcelona, Spain\ncelsius\n\n""", - [ - ToolCall( - function=FunctionCall( - name="get_weather", - arguments=json.dumps( - { - "location": "Barcelona, Spain", - "unit": "celsius", - }, - ), - ), - type="function", - ) - ], - """\nGot it, let\'s see. The user asked for the weather in Barcelona, Spain. """ - """First, I need to remember the function I can use: get_weather. The function requires a """ - """location (city and country) which is "Barcelona, Spain" here, and unit is optional. Since """ - """the user didn\'t specify the unit, the default in the function is Celsius, right? Wait, """ - """let me check the function docstring again. Oh, the function says unit is optional, and """ - """returns temperature in Celsius. So I should call get_weather with location "Barcelona, """ - """Spain" and maybe omit unit or set to Celsius. Let me format the function call correctly. """ - """The format is \n\nBarcelona, """ - """Spain\ncelsius\n\n. """ - """Wait, but does the unit parameter accept "celsius"? The docstring says unit is the unit """ - """of temperature, but the return is in Celsius anyway. Maybe even if I don\'t pass unit, """ - """it\'s okay, but to be explicit, maybe pass "celsius". Let me go with that. So the function """ - """call should be as above. Then wait for the result to come back and tell the user the """ - """temperature in Celsius.""", - ), - ], -) -def test_streaming_tool_calls( - seed_oss_tool_parser, - seed_oss_tokenizer, - sample_tools, - model_output, - expected_tool_calls, - expected_content, -): - """Test incremental streaming behavior""" - request = ChatCompletionRequest(model=MODEL, messages=[], tools=sample_tools) - - other_content = "" - tool_states = {} # Track state per tool index - - for delta_message in stream_delta_message_generator( - seed_oss_tool_parser, seed_oss_tokenizer, model_output, request - ): - # role should never be streamed from tool parser - assert not delta_message.role - - if delta_message.content: - other_content += delta_message.content - - if delta_message.tool_calls: - for tool_call in delta_message.tool_calls: - idx = tool_call.index - - # Initialize state for new tool - if idx not in tool_states: - tool_states[idx] = { - "id": None, - "name": None, - "arguments": "", - "type": None, - } - - # First chunk should have id, name, and type - if tool_call.id: - tool_states[idx]["id"] = tool_call.id - - if tool_call.type: - assert tool_call.type == "function" - tool_states[idx]["type"] = tool_call.type - - if tool_call.function: - if tool_call.function.name: - # Should only be set once - assert tool_states[idx]["name"] is None - tool_states[idx]["name"] = tool_call.function.name - - if tool_call.function.arguments is not None: - # Accumulate arguments incrementally - tool_states[idx]["arguments"] += tool_call.function.arguments - - # Verify final content - assert other_content == expected_content - - # Verify we got all expected tool calls - assert len(tool_states) == len(expected_tool_calls) - - # Verify each tool call - for idx, expected_tool in enumerate(expected_tool_calls): - state = tool_states[idx] - assert state["id"] is not None - assert state["type"] == "function" - assert state["name"] == expected_tool.function.name - - # Parse accumulated arguments - arguments_str = state["arguments"] - assert arguments_str is not None - actual_args = json.loads(arguments_str) - expected_args = json.loads(expected_tool.function.arguments) - assert actual_args == expected_args - - -def test_streaming_tool_calls_non_ascii( - seed_oss_tool_parser, seed_oss_tokenizer, sample_tools -): - request = ChatCompletionRequest(model=MODEL, messages=[], tools=sample_tools) - model_output = ( - """\n\n\n""" - """The current thinking budget is 0, so I will directly start answering the question.\n\n""" - """\n\n""" - """北京\n\n""" - ) - - args = "".join( - tool_call.function.arguments - for delta_message in stream_delta_message_generator( - seed_oss_tool_parser, seed_oss_tokenizer, model_output, request - ) - if delta_message.tool_calls - for tool_call in delta_message.tool_calls - if tool_call.function and tool_call.function.arguments is not None - ) - - assert "北京" in args - assert "\\u" not in args diff --git a/vllm/parser/engine/registered_adapters.py b/vllm/parser/engine/registered_adapters.py index 9d670f30564..c5250abf82a 100644 --- a/vllm/parser/engine/registered_adapters.py +++ b/vllm/parser/engine/registered_adapters.py @@ -13,6 +13,7 @@ from vllm.parser.glm47_moe import Glm47MoeParser from vllm.parser.minimax_m2 import MinimaxM2Parser from vllm.parser.nemotron_v3 import NemotronV3Parser from vllm.parser.qwen3 import Qwen3Parser +from vllm.parser.seed_oss import SeedOssParser ( MinimaxM2ParserReasoningAdapter, @@ -34,6 +35,11 @@ from vllm.parser.qwen3 import Qwen3Parser Qwen3ParserToolAdapter, ) = make_adapters(Qwen3Parser) +( + SeedOssParserReasoningAdapter, + SeedOssParserToolAdapter, +) = make_adapters(SeedOssParser) + ( Glm47MoeParserReasoningAdapter, Glm47MoeParserToolAdapter, diff --git a/vllm/parser/qwen3.py b/vllm/parser/qwen3.py index f14da8234c5..f80aa6ff7a2 100644 --- a/vllm/parser/qwen3.py +++ b/vllm/parser/qwen3.py @@ -38,6 +38,8 @@ if TYPE_CHECKING: from vllm.tokenizers import TokenizerLike from vllm.tool_parsers.abstract_tool_parser import Tool +THINK_START = "" +THINK_END = "" TOOL_CALL_START = "" TOOL_CALL_END = "" FUNC_PREFIX = " str: @functools.cache -def qwen3_config(thinking: bool = True) -> ParserEngineConfig: +def qwen3_config( + thinking: bool = True, + *, + name: str = "qwen3", + think_start: str = THINK_START, + think_end: str = THINK_END, + tool_start: str = TOOL_CALL_START, + tool_end: str = TOOL_CALL_END, +) -> ParserEngineConfig: return ParserEngineConfig( - name="qwen3", + name=name, initial_state=ParserState.REASONING if thinking else ParserState.CONTENT, terminals={ # Reasoning terminals - "THINK_START": "", - "THINK_END": "", + "THINK_START": think_start, + "THINK_END": think_end, # Tool call terminals - "TOOL_START": TOOL_CALL_START, - "TOOL_END": TOOL_CALL_END, + "TOOL_START": tool_start, + "TOOL_END": tool_end, "FUNC_PREFIX": FUNC_PREFIX, "FUNC_END": FUNC_END, "PARAM_START": PARAM_START, @@ -93,10 +103,10 @@ def qwen3_config(thinking: bool = True) -> ParserEngineConfig: "CLOSE_ANGLE": ">", }, token_id_terminals={ - "THINK_START": "", - "THINK_END": "", - "TOOL_START": TOOL_CALL_START, - "TOOL_END": TOOL_CALL_END, + "THINK_START": think_start, + "THINK_END": think_end, + "TOOL_START": tool_start, + "TOOL_END": tool_end, }, transitions={ # -- Reasoning transitions -- @@ -185,8 +195,18 @@ class Qwen3Parser(ParserEngine): - ```` as implicit reasoning end - Unpaired ```` token ID detection for ``is_reasoning_end`` + + Subclasses that share the grammar but differ only in the four wrapper + token strings (reasoning + tool-call) override the class attributes + below; everything else is inherited unchanged. """ + CONFIG_NAME = "qwen3" + THINK_START = THINK_START + THINK_END = THINK_END + TOOL_START = TOOL_CALL_START + TOOL_END = TOOL_CALL_END + def __init__( self, tokenizer: TokenizerLike, @@ -197,7 +217,14 @@ class Qwen3Parser(ParserEngine): self.thinking_enabled = chat_kwargs.get("enable_thinking", True) kwargs.setdefault( "parser_engine_config", - qwen3_config(thinking=self.thinking_enabled), + qwen3_config( + thinking=self.thinking_enabled, + name=self.CONFIG_NAME, + think_start=self.THINK_START, + think_end=self.THINK_END, + tool_start=self.TOOL_START, + tool_end=self.TOOL_END, + ), ) super().__init__( tokenizer, @@ -205,8 +232,8 @@ class Qwen3Parser(ParserEngine): **kwargs, ) vocab = self.vocab - self._tool_call_token_id: int | None = vocab.get("") - self._tool_call_end_token_id: int | None = vocab.get("") + self._tool_call_token_id: int | None = vocab.get(self.TOOL_START) + self._tool_call_end_token_id: int | None = vocab.get(self.TOOL_END) def extract_reasoning( self, diff --git a/vllm/parser/seed_oss.py b/vllm/parser/seed_oss.py new file mode 100644 index 00000000000..2f709f0ad67 --- /dev/null +++ b/vllm/parser/seed_oss.py @@ -0,0 +1,28 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""seed_oss parser for tool calls and reasoning. + +seed_oss shares the Qwen3 XML grammar exactly; only the four wrapper +token strings differ:: + + -> + -> + -> + -> + +```` and ```` are byte-identical, so the +entire transition table and ``_qwen3_arg_converter`` are inherited from +:class:`Qwen3Parser` unchanged. +""" + +from __future__ import annotations + +from vllm.parser.qwen3 import Qwen3Parser + + +class SeedOssParser(Qwen3Parser): + CONFIG_NAME = "seed_oss" + THINK_START = "" + THINK_END = "" + TOOL_START = "" + TOOL_END = "" diff --git a/vllm/reasoning/__init__.py b/vllm/reasoning/__init__.py index cbb1fa350f5..fc74cf2f3f7 100644 --- a/vllm/reasoning/__init__.py +++ b/vllm/reasoning/__init__.py @@ -117,8 +117,8 @@ _REASONING_PARSERS_TO_REGISTER = { "Qwen3ParserReasoningAdapter", ), "seed_oss": ( - "seedoss_reasoning_parser", - "SeedOSSReasoningParser", + "seed_oss_engine_reasoning_parser", + "SeedOssParserReasoningAdapter", ), "step3": ( "step3_reasoning_parser", diff --git a/vllm/reasoning/seed_oss_engine_reasoning_parser.py b/vllm/reasoning/seed_oss_engine_reasoning_parser.py new file mode 100644 index 00000000000..e651d411f43 --- /dev/null +++ b/vllm/reasoning/seed_oss_engine_reasoning_parser.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm.parser.engine.registered_adapters import SeedOssParserReasoningAdapter + +__all__ = ["SeedOssParserReasoningAdapter"] diff --git a/vllm/reasoning/seedoss_reasoning_parser.py b/vllm/reasoning/seedoss_reasoning_parser.py deleted file mode 100644 index d3d4d8ec074..00000000000 --- a/vllm/reasoning/seedoss_reasoning_parser.py +++ /dev/null @@ -1,27 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - - -from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser - - -class SeedOSSReasoningParser(BaseThinkingReasoningParser): - """ - Reasoning parser for SeedOSS model. - - The SeedOSS model uses ... tokens to - denote reasoning content text. This parser extracts - the reasoning content from the model output. - Similar to DeepSeek R1, it supports cases - where the model doesn't generate the start token. - """ - - @property - def start_token(self) -> str: - """The token that starts reasoning content.""" - return "" - - @property - def end_token(self) -> str: - """The token that ends reasoning content.""" - return "" diff --git a/vllm/tool_parsers/__init__.py b/vllm/tool_parsers/__init__.py index 109189a033a..b9a9c9ad07b 100644 --- a/vllm/tool_parsers/__init__.py +++ b/vllm/tool_parsers/__init__.py @@ -163,8 +163,8 @@ _TOOL_PARSERS_TO_REGISTER = { "Qwen3EngineToolParser", ), "seed_oss": ( - "seed_oss_tool_parser", - "SeedOssToolParser", + "seed_oss_engine_tool_parser", + "SeedOssEngineToolParser", ), "step3": ( "step3_tool_parser", diff --git a/vllm/tool_parsers/seed_oss_engine_tool_parser.py b/vllm/tool_parsers/seed_oss_engine_tool_parser.py new file mode 100644 index 00000000000..e708afd1710 --- /dev/null +++ b/vllm/tool_parsers/seed_oss_engine_tool_parser.py @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm.parser.engine.registered_adapters import SeedOssParserToolAdapter + + +class SeedOssEngineToolParser(SeedOssParserToolAdapter): # type: ignore[valid-type, misc] + structural_tag_model = None diff --git a/vllm/tool_parsers/seed_oss_tool_parser.py b/vllm/tool_parsers/seed_oss_tool_parser.py deleted file mode 100644 index 42e4ba77691..00000000000 --- a/vllm/tool_parsers/seed_oss_tool_parser.py +++ /dev/null @@ -1,633 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -# Adapted from qwen3coder xml parser, All rights reserved. -# ruff: noqa: E501 - -import json -import uuid -from collections.abc import Sequence - -import regex as re - -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, -) -from vllm.entrypoints.openai.engine.protocol import ( - DeltaFunctionCall, - DeltaMessage, - DeltaToolCall, - ExtractedToolCallInformation, - FunctionCall, - ToolCall, -) -from vllm.logger import init_logger -from vllm.tokenizers import TokenizerLike -from vllm.tool_parsers.abstract_tool_parser import ( - Tool, - ToolParser, -) -from vllm.tool_parsers.utils import ( - coerce_to_schema_type, - extract_types_from_schema, - find_tool_properties, -) - -logger = init_logger(__name__) - - -class SeedOssToolParser(ToolParser): - TOOL_CALL_START = "" - TOOL_CALL_END = "" - - def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): - super().__init__(tokenizer, tools) - - # --- streaming state --- - self._reset_streaming_state() - self.prev_tool_call_arr: list[dict] = [] - - self.tool_call_start_token: str = self.TOOL_CALL_START - self.tool_call_end_token: str = self.TOOL_CALL_END - # Sentinel tokens for streaming mode - self.tool_call_prefix: str = " or its closing tag." - ) - - tool_start_re = re.escape(self.tool_call_start_token) - tool_end_re = re.escape(self.tool_call_end_token) - - self.tool_call_complete_regex = re.compile( - rf"{tool_start_re}(.*?){tool_end_re}", re.DOTALL - ) - self.tool_call_regex = re.compile( - rf"{tool_start_re}(.*?){tool_end_re}|{tool_start_re}(.*?)$", re.DOTALL - ) - - self.tool_call_function_regex = re.compile( - r"|| str: - """Generate a unique tool call ID.""" - return f"call_{uuid.uuid4().hex[:24]}" - - def _reset_streaming_state(self): - """Reset all streaming state.""" - self.current_tool_index = 0 - self.is_tool_call_started = False - self.header_sent = False - self.current_tool_id = -1 - self.current_function_name = None - self.current_param_name = None - self.current_param_value = "" - self.param_count = 0 - self.in_param = False - self.in_function = False - self.accumulated_text = "" - self.json_started = False - self.json_closed = False - - def _parse_xml_function_call( - self, function_call_str: str, tools: list[Tool] | None - ) -> ToolCall | None: - # Extract function name - end_index = function_call_str.index(">") - function_name = function_call_str[:end_index] - tool_properties = find_tool_properties(tools, function_name) - parameters = function_call_str[end_index + 1 :] - param_dict = {} - for match in self.tool_call_parameter_regex.findall(parameters): - match_text = match[0] if match[0] else match[1] - idx = match_text.index(">") - param_name = match_text[:idx] - param_value = str(match_text[idx + 1 :]) - # Remove prefix and trailing \n - if param_value.startswith("\n"): - param_value = param_value[1:] - if param_value.endswith("\n"): - param_value = param_value[:-1] - - param_types = extract_types_from_schema(tool_properties.get(param_name, {})) - param_dict[param_name] = coerce_to_schema_type(param_value, param_types) - return ToolCall( - type="function", - function=FunctionCall( - name=function_name, arguments=json.dumps(param_dict, ensure_ascii=False) - ), - ) - - def _get_function_calls(self, model_output: str) -> list[str]: - # Find all tool calls - matched_ranges = self.tool_call_regex.findall(model_output) - raw_tool_calls = [ - match[0] if match[0] else match[1] for match in matched_ranges - ] - - # Back-off strategy if no tool_call tags found - if len(raw_tool_calls) == 0: - raw_tool_calls = [model_output] - - raw_function_calls = [] - for tool_call in raw_tool_calls: - raw_function_calls.extend(self.tool_call_function_regex.findall(tool_call)) - - function_calls = [ - match[0] if match[0] else match[1] for match in raw_function_calls - ] - return function_calls - - def extract_tool_calls( - self, - model_output: str, - request: ChatCompletionRequest, - ) -> ExtractedToolCallInformation: - # Quick check to avoid unnecessary processing - if self.tool_call_prefix not in model_output: - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - # Check if both think start and end tokens are present - if ( - self.think_start_token in model_output - and self.think_end_token in model_output - ): - # Find the position of think end token - think_end_index = model_output.find(self.think_end_token) + len( - self.think_end_token - ) - # Extract content after think end token - result_content = model_output[think_end_index:] - thinking_content = model_output[:think_end_index] - else: - thinking_content = "" - result_content = model_output - - try: - function_calls = self._get_function_calls(result_content) - if len(function_calls) == 0: - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - tool_calls = [ - self._parse_xml_function_call(function_call_str, self.tools) - for function_call_str in function_calls - ] - - # Populate prev_tool_call_arr for serving layer to set finish_reason - self.prev_tool_call_arr.clear() # Clear previous calls - for tool_call in tool_calls: - if tool_call: - self.prev_tool_call_arr.append( - { - "name": tool_call.function.name, - "arguments": tool_call.function.arguments, - } - ) - - # Extract content before tool calls - tool_call_start_index = result_content.find(self.tool_call_start_token) - tool_call_start_index = ( - tool_call_start_index - if tool_call_start_index >= 0 - else result_content.find(self.tool_call_prefix) - ) - content = thinking_content + result_content[:tool_call_start_index] - - return ExtractedToolCallInformation( - tools_called=(len(tool_calls) > 0), - tool_calls=tool_calls, - content=content if content else None, - ) - - except Exception: - logger.exception("Error in extracting tool call from response.") - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - def extract_tool_calls_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - ) -> DeltaMessage | None: - # If no delta text, return None unless - # it's an EOS token after tool calls - if not delta_text: - # Check if this is an EOS token after all tool calls are complete - # We check for tool calls in the text even if is_tool_call_started - # is False because it might have been reset after processing all tools - if delta_token_ids and self.tool_call_end_token_id not in delta_token_ids: - # Count complete tool calls - complete_calls = len( - self.tool_call_complete_regex.findall(current_text) - ) - - # If we have completed tool calls and populated prev_tool_call_arr - if complete_calls > 0 and len(self.prev_tool_call_arr) > 0: - # Check if all tool calls are closed - open_calls = current_text.count( - self.tool_call_start_token - ) - current_text.count(self.tool_call_end_token) - if open_calls == 0: - # Return empty delta message to allow finish_reason processing - return DeltaMessage(content="") - elif not self.is_tool_call_started and current_text: - # This is a regular content response that's now complete - return DeltaMessage(content="") - return None - - # Check if this is the first call (reset state if needed) - if not previous_text: - self._reset_streaming_state() - - # Update accumulated text - self.accumulated_text = current_text - - # Check if we need to advance to next tool - if self.json_closed and not self.in_function: - # Check if this tool call has ended - tool_ends = current_text.count(self.tool_call_end_token) - if tool_ends > self.current_tool_index: - # This tool has ended, advance to next - self.current_tool_index += 1 - self.header_sent = False - self.param_count = 0 - self.json_started = False - self.json_closed = False - - # Check if there are more tool calls - if self.current_tool_index >= current_text.count( - self.tool_call_start_token - ): - # No more tool calls - self.is_tool_call_started = False - # Continue processing next tool - return None - - # Check if end thinking - if not self.is_thinking_end and ( - self.think_end_token_id in delta_token_ids - or self.think_end_token in delta_text - ): - self.is_thinking_end = True - - # If thinking hasn't ended yet, don't process any tool calls - if not self.is_thinking_end: - return DeltaMessage(content=delta_text) - - # Handle normal content before tool calls - if not self.is_tool_call_started: - # Check if tool call is starting - if ( - self.tool_call_start_token_id in delta_token_ids - or self.tool_call_start_token in delta_text - ): - self.is_tool_call_started = True - # Return any content before the tool call - if self.tool_call_start_token in delta_text: - content_before = delta_text[ - : delta_text.index(self.tool_call_start_token) - ] - if content_before: - return DeltaMessage(content=content_before) - return None - else: - # Check if we're between tool calls - skip whitespace - if ( - current_text.rstrip().endswith(self.tool_call_end_token) - and delta_text.strip() == "" - ): - # We just ended a tool call, skip whitespace - return None - # Normal content, no tool call - return DeltaMessage(content=delta_text) - - # Check if we're between tool calls (waiting for next one) - # Count tool calls we've seen vs processed - tool_starts_count = current_text.count(self.tool_call_start_token) - if self.current_tool_index >= tool_starts_count: - # We're past all tool calls, shouldn't be here - return None - - # We're in a tool call, find the current tool call portion - # Need to find the correct tool call based on current_tool_index - # Only process tool calls after think_end_token - think_end_index = ( - current_text.find(self.think_end_token) + len(self.think_end_token) - if self.think_end_token in current_text - else 0 - ) - tool_starts: list[int] = [] - idx = think_end_index - while True: - idx = current_text.find(self.tool_call_start_token, idx) - if idx == -1: - break - tool_starts.append(idx) - idx += len(self.tool_call_start_token) - - if self.current_tool_index >= len(tool_starts): - # No more tool calls to process yet - return None - - tool_start_idx = tool_starts[self.current_tool_index] - # Find where this tool call ends (or current position if not ended yet) - tool_end_idx = current_text.find(self.tool_call_end_token, tool_start_idx) - if tool_end_idx == -1: - tool_text = current_text[tool_start_idx:] - else: - tool_text = current_text[ - tool_start_idx : tool_end_idx + len(self.tool_call_end_token) - ] - - # Looking for function header - if not self.header_sent: - if self.tool_call_prefix in tool_text: - func_start = tool_text.find(self.tool_call_prefix) + len( - self.tool_call_prefix - ) - func_end = tool_text.find(">", func_start) - - if func_end != -1: - # Found complete function name - self.current_function_name = tool_text[func_start:func_end] - self.current_tool_id = self._generate_tool_call_id() # type: ignore - self.header_sent = True - self.in_function = True - - # IMPORTANT: Add to prev_tool_call_arr immediately when we detect a tool call - # This ensures finish_reason="tool_calls" even if parsing isn't complete - already_added = any( - tool.get("name") == self.current_function_name - for tool in self.prev_tool_call_arr - ) - if not already_added: - self.prev_tool_call_arr.append( - { - "name": self.current_function_name, - "arguments": "{}", # Placeholder, will be updated later - } - ) - - # Send header with function info - return DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_index, - id=self.current_tool_id, - function=DeltaFunctionCall( - name=self.current_function_name, arguments="" - ), - type="function", - ) - ] - ) - return None - - # We've sent header, now handle function body - if self.in_function: - # Send opening brace if not sent yet - if not self.json_started and self.parameter_prefix not in delta_text: - self.json_started = True - return DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_index, - function=DeltaFunctionCall(arguments="{"), - ) - ] - ) - - # Make sure json_started is set if we're processing parameters - if not self.json_started: - self.json_started = True - - # Check for function end in accumulated text - if not self.json_closed and self.function_end_token in tool_text: - # Close JSON - self.json_closed = True - - # Extract the complete tool call to update prev_tool_call_arr with final arguments - # Find the function content - func_start = tool_text.find(self.tool_call_prefix) + len( - self.tool_call_prefix - ) - func_content_end = tool_text.find(self.function_end_token, func_start) - if func_content_end != -1: - func_content = tool_text[func_start:func_content_end] - # Parse to get the complete arguments - try: - parsed_tool = self._parse_xml_function_call( - func_content, self.tools - ) - if parsed_tool: - # Update existing entry in prev_tool_call_arr with complete arguments - for i, tool in enumerate(self.prev_tool_call_arr): - if tool.get("name") == parsed_tool.function.name: - self.prev_tool_call_arr[i]["arguments"] = ( - parsed_tool.function.arguments - ) - break - except Exception: - logger.warning( - "Failed to parse tool arguments during streaming.", - exc_info=True, - ) - - result = DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_index, - function=DeltaFunctionCall(arguments="}"), - ) - ] - ) - - # Reset state for next tool - self.in_function = False - self.json_closed = True - - return result - - # Look for parameters - # Count how many complete parameters we have processed - complete_params = tool_text.count(self.parameter_end_token) - - # Check if we should start a new parameter - if not self.in_param and self.param_count < complete_params: - # Find the unprocessed parameter - # Count parameter starts - param_starts = [] - idx = 0 - while True: - idx = tool_text.find(self.parameter_prefix, idx) - if idx == -1: - break - param_starts.append(idx) - idx += len(self.parameter_prefix) - - if len(param_starts) > self.param_count: - # Process the next parameter - param_idx = param_starts[self.param_count] - param_start = param_idx + len(self.parameter_prefix) - remaining = tool_text[param_start:] - - if ">" in remaining: - # We have the complete parameter name - name_end = remaining.find(">") - self.current_param_name = remaining[:name_end] - - # Find the parameter value - value_start = param_start + name_end + 1 - value_text = tool_text[value_start:] - if value_text.startswith("\n"): - value_text = value_text[1:] - - # Find where this parameter ends - param_end_idx = value_text.find(self.parameter_end_token) - if param_end_idx != -1: - # Complete parameter found - param_value = value_text[:param_end_idx] - if param_value.endswith("\n"): - param_value = param_value[:-1] - - # Build complete JSON fragment for this parameter - if self.param_count == 0: - json_fragment = ( - '"' - + self.current_param_name - + '": "' - + json.dumps(param_value, ensure_ascii=False)[1:-1] - + '"' - ) - else: - json_fragment = ( - ', "' - + self.current_param_name - + '": "' - + json.dumps(param_value, ensure_ascii=False)[1:-1] - + '"' - ) - - self.param_count += 1 - - return DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_index, - function=DeltaFunctionCall( - arguments=json_fragment - ), - ) - ] - ) - - # Continue parameter value - if self.in_param: - if self.parameter_end_token in delta_text: - # End of parameter - end_idx = delta_text.find(self.parameter_end_token) - value_chunk = delta_text[:end_idx] - - # Skip past > if at start - if not self.current_param_value and ">" in value_chunk: - gt_idx = value_chunk.find(">") - value_chunk = value_chunk[gt_idx + 1 :] - - if not self.current_param_value and value_chunk.startswith("\n"): - value_chunk = value_chunk[1:] - - # Calculate incremental JSON - full_value = self.current_param_value + value_chunk - prev_escaped = ( - json.dumps(self.current_param_value, ensure_ascii=False)[1:-1] - if self.current_param_value - else "" - ) - full_escaped = json.dumps(full_value, ensure_ascii=False)[1:-1] - delta_escaped = full_escaped[len(prev_escaped) :] - - self.in_param = False - self.current_param_value = "" - - return DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_index, - function=DeltaFunctionCall( - arguments=delta_escaped + '"' - ), - ) - ] - ) - else: - # Continue accumulating value - value_chunk = delta_text - - # Handle first chunk after param name - if not self.current_param_value and ">" in value_chunk: - gt_idx = value_chunk.find(">") - value_chunk = value_chunk[gt_idx + 1 :] - - if not self.current_param_value and value_chunk.startswith("\n"): - value_chunk = value_chunk[1:] - - if value_chunk: - # Stream the escaped delta - prev_escaped = ( - json.dumps(self.current_param_value, ensure_ascii=False)[ - 1:-1 - ] - if self.current_param_value - else "" - ) - self.current_param_value += value_chunk - full_escaped = json.dumps( - self.current_param_value, ensure_ascii=False - )[1:-1] - delta_escaped = full_escaped[len(prev_escaped) :] - - if delta_escaped: - return DeltaMessage( - tool_calls=[ - DeltaToolCall( - index=self.current_tool_index, - function=DeltaFunctionCall( - arguments=delta_escaped - ), - ) - ] - ) - - return None