diff --git a/pyproject.toml b/pyproject.toml index 031f8d1a0a2..249832ff2e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -129,8 +129,8 @@ extend-exclude = ["tests/models/fixtures/*", "tests/prompts/*", "tests/tokenizer "docs/governance/process.md", "docs/assets/contributing/vllm_bench_serve_timeline.html", "tests/v1/engine/test_fast_incdec_prefix_err.py", ".git/*", "csrc/cpu/sgl-kernels/*", "rust/src/chat/src/renderer/deepseek_v32/fixtures/*", - "rust/src/tool-parser/src/gemma4.rs", "rust/src/text/src/output/decoded.rs", - "rust/src/tokenizer/src/incremental.rs", "rust/src/reasoning-parser/src/tests.rs"] + "rust/src/parser/src/tool/gemma4.rs", "rust/src/text/src/output/decoded.rs", + "rust/src/tokenizer/src/incremental.rs", "rust/src/parser/src/reasoning/tests.rs"] ignore-hidden = false [tool.typos.default] diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 743633b447e..70c325c152d 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -5733,10 +5733,9 @@ dependencies = [ "uuid", "vllm-engine-core-client", "vllm-llm", - "vllm-reasoning-parser", + "vllm-parser", "vllm-text", "vllm-tokenizer", - "vllm-tool-parser", "xgrammar-structural-tag", "zeromq", ] @@ -5872,11 +5871,22 @@ dependencies = [ ] [[package]] -name = "vllm-reasoning-parser" +name = "vllm-parser" version = "0.1.0" dependencies = [ + "criterion", + "easy-ext", + "expect-test", + "futures", + "openai-protocol", + "serde", + "serde_json", "thiserror 2.0.18", + "thiserror-ext", + "tool-parser", "vllm-tokenizer", + "winnow", + "xgrammar-structural-tag", ] [[package]] @@ -5978,24 +5988,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "vllm-tool-parser" -version = "0.1.0" -dependencies = [ - "criterion", - "easy-ext", - "expect-test", - "futures", - "openai-protocol", - "serde", - "serde_json", - "thiserror 2.0.18", - "thiserror-ext", - "tool-parser", - "winnow", - "xgrammar-structural-tag", -] - [[package]] name = "vllm-tool-parser-py" version = "0.1.0" @@ -6004,7 +5996,7 @@ dependencies = [ "pythonize", "serde_json", "thiserror-ext", - "vllm-tool-parser", + "vllm-parser", ] [[package]] diff --git a/rust/Cargo.toml b/rust/Cargo.toml index e31bf07bbd4..dc3895c372d 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -7,12 +7,11 @@ members = [ "src/managed-engine", "src/metrics", "src/mock-engine", - "src/reasoning-parser", + "src/parser", + "src/parser/python", "src/server", "src/text", "src/tokenizer", - "src/tool-parser", - "src/tool-parser/python", ] resolver = "3" @@ -117,11 +116,10 @@ vllm-engine-core-client = { path = "src/engine-core-client" } vllm-llm = { path = "src/llm" } vllm-managed-engine = { path = "src/managed-engine" } vllm-metrics = { path = "src/metrics" } -vllm-reasoning-parser = { path = "src/reasoning-parser" } +vllm-parser = { path = "src/parser" } vllm-server = { path = "src/server" } vllm-text = { path = "src/text" } vllm-tokenizer = { path = "src/tokenizer" } -vllm-tool-parser = { path = "src/tool-parser" } winnow = "1.0.2" xgrammar-structural-tag = "0.1.0" zeromq = { version = "0.6.0", default-features = false, features = [ diff --git a/rust/src/chat/Cargo.toml b/rust/src/chat/Cargo.toml index 85368ad98b9..f5860c18597 100644 --- a/rust/src/chat/Cargo.toml +++ b/rust/src/chat/Cargo.toml @@ -30,10 +30,9 @@ trait-set.workspace = true uuid.workspace = true vllm-engine-core-client.workspace = true vllm-llm.workspace = true -vllm-reasoning-parser.workspace = true +vllm-parser.workspace = true vllm-text.workspace = true vllm-tokenizer.workspace = true -vllm-tool-parser.workspace = true xgrammar-structural-tag.workspace = true [dev-dependencies] diff --git a/rust/src/chat/src/output/default/mod.rs b/rust/src/chat/src/output/default/mod.rs index dbc9cc05a51..8c9a4362d1c 100644 --- a/rust/src/chat/src/output/default/mod.rs +++ b/rust/src/chat/src/output/default/mod.rs @@ -1,44 +1,34 @@ //! Default output processing pipeline. -mod reasoning; mod structural_tag; -mod tool; +mod unified; use std::sync::Once; -use futures::{Stream, StreamExt as _}; +use futures::StreamExt as _; use tracing::info; -use trait_set::trait_set; +use vllm_parser::unified::{CombinedParser, UnifiedParser}; use vllm_text::tokenizer::DynTokenizer; -use self::reasoning::reasoning_event_stream; use self::structural_tag::apply_structural_tag_constraint; -use self::tool::tool_event_stream; +use self::unified::unified_event_stream; use super::structured::structured_chat_event_stream; use crate::error::Result; -use crate::output::{ - AssistantEvent, ChatOutputProcessor, ContentEvent, DynChatEventStream, - DynDecodedTextEventStream, -}; +use crate::output::{ChatOutputProcessor, DynChatEventStream, DynDecodedTextEventStream}; use crate::parser::ParserSelection; use crate::parser::reasoning::{ReasoningParser, ReasoningParserFactory}; use crate::parser::tool::{ToolParser, ToolParserFactory}; use crate::request::ChatRequest; use crate::{Error, Result as ChatResult}; -trait_set! { - trait ContentEventStream = Stream> + Send + 'static; -} - /// Default request-scoped output processor used by Hugging Face style chat /// backends. /// /// This implementation assumes the backend already emitted decoded text deltas, -/// then optionally layers reasoning parsing and tool-call parsing before +/// then optionally layers unified reasoning and tool-call parsing before /// assembling final structured chat events. pub struct DefaultChatOutputProcessor { - reasoning_parser: Option>, - tool_parser: Option>, + parser: Box, parallel_tool_calls: bool, } @@ -66,16 +56,17 @@ impl DefaultChatOutputProcessor { } else { None }; - let reasoning_parser = Self::resolve_optional_reasoning_parser( - request, - model_id, - tokenizer, - reasoning_parser, - )?; + let reasoning_parser = + Self::resolve_optional_reasoning_parser(model_id, tokenizer, reasoning_parser)?; + let parser: Box = + Box::new(CombinedParser::new(reasoning_parser, tool_parser)); + + if parser.preserve_special_tokens() { + request.decode_options.skip_special_tokens = false; + } Ok(Self { - reasoning_parser, - tool_parser, + parser, parallel_tool_calls: request.parallel_tool_calls, }) } @@ -87,8 +78,7 @@ impl DefaultChatOutputProcessor { /// content is treated as opaque text. pub fn plain_text_only() -> Self { Self { - reasoning_parser: None, - tool_parser: None, + parser: Box::new(CombinedParser::plain_text_only()), parallel_tool_calls: true, } } @@ -112,10 +102,6 @@ impl DefaultChatOutputProcessor { let parser = factory.create(parser_name, &request.tools)?; - if parser.preserve_special_tokens() { - request.decode_options.skip_special_tokens = false; - } - apply_structural_tag_constraint(request, parser.as_ref())?; TOOL_PARSER_LOG_ONCE.call_once(|| info!(parser_name, "using tool parser")); @@ -123,7 +109,6 @@ impl DefaultChatOutputProcessor { } fn resolve_optional_reasoning_parser( - request: &mut ChatRequest, model_id: &str, tokenizer: DynTokenizer, selection: &ParserSelection, @@ -142,10 +127,6 @@ impl DefaultChatOutputProcessor { let parser = factory.create(parser_name, tokenizer)?; - if parser.preserve_special_tokens() { - request.decode_options.skip_special_tokens = false; - } - REASONING_PARSER_LOG_ONCE.call_once(|| info!(parser_name, "using reasoning parser")); Ok(Some(parser)) } @@ -156,16 +137,14 @@ static REASONING_PARSER_LOG_ONCE: Once = Once::new(); impl ChatOutputProcessor for DefaultChatOutputProcessor { /// Transforms a raw generate-output token stream into structured chat - /// events through three sequential stages once text decoding has + /// events through two sequential stages once text decoding has /// already happened: /// - /// 1. [`reasoning_event_stream`] — reasoning/content separation - /// 2. [`tool_event_stream`] — tool-call parsing - /// 3. [`structured_chat_event_stream`] — final block assembly + /// 1. [`unified_event_stream`] — reasoning and tool-call parsing + /// 2. [`structured_chat_event_stream`] — final block assembly fn process(self: Box, decoded: DynDecodedTextEventStream) -> Result { - let reasoning = reasoning_event_stream(decoded, self.reasoning_parser); - let tool = tool_event_stream(reasoning, self.tool_parser); - let structured = structured_chat_event_stream(tool, self.parallel_tool_calls); + let parsed = unified_event_stream(decoded, self.parser); + let structured = structured_chat_event_stream(parsed, self.parallel_tool_calls); Ok(structured.boxed()) } diff --git a/rust/src/chat/src/output/default/reasoning.rs b/rust/src/chat/src/output/default/reasoning.rs deleted file mode 100644 index faa9d7894bb..00000000000 --- a/rust/src/chat/src/output/default/reasoning.rs +++ /dev/null @@ -1,509 +0,0 @@ -//! Adapts decoded text updates into reasoning-aware assistant deltas. -//! -//! This stage sits between low-level token decoding and final block assembly. -//! It is the only place in the new pipeline that understands reasoning -//! separation: `decoded.rs` still only produces plain text deltas, while later -//! stages consume the semantic `Text` / `Reasoning` split emitted here. - -use asynk_strim_attr::{TryYielder, try_stream}; -use futures::{StreamExt as _, pin_mut}; -use thiserror_ext::AsReport; -use tracing::warn; -use vllm_text::output::DecodedTextEvent; - -use super::ContentEvent; -use crate::Result; -use crate::error::Error; -use crate::event::AssistantBlockKind; -use crate::output::DecodedTextEventStream; -use crate::parser::reasoning::{ReasoningDelta, ReasoningParser}; - -/// Per-stream reasoning parsing state. -struct ReasoningState { - /// Reasoning parser for the current model family. - parser: Box, - /// Whether reasoning parsing has already failed for this stream. - parser_failed: bool, -} - -impl ReasoningState { - /// Create one fresh reasoning-adaptation state for a new streamed response. - fn new(parser: Box) -> Self { - Self { - parser, - parser_failed: false, - } - } - - /// Convert one decoded text delta into zero or more semantic assistant - /// deltas. - fn process_delta(&mut self, delta: String) -> Vec { - // If the parser has already failed, skip parsing and return plain text deltas. - if self.parser_failed { - return vec![ContentEvent::TextDelta { - kind: AssistantBlockKind::Text, - delta, - }]; - } - - let mut events = Vec::new(); - - match self.parser.push(&delta) { - Ok(result) => { - push_reasoning_delta(&mut events, result); - } - Err(error) => { - if !self.parser_failed { - warn!( - error = %error.as_report(), - "reasoning parser failed; falling back to plain text deltas" - ); - self.parser_failed = true; - } - push_text_delta(&mut events, AssistantBlockKind::Text, delta); - } - } - - events - } - - /// Initialize parser state once prompt token IDs are available. - fn initialize(&mut self, prompt_token_ids: &[u32]) { - if self.parser_failed { - return; - } - - match self.parser.initialize(prompt_token_ids) { - Ok(()) => {} - Err(error) => { - warn!( - error = %error.as_report(), - "failed to initialize reasoning parser; falling back to plain text deltas" - ); - self.parser_failed = true; - } - } - } - - /// Flush any parser-held partial delimiter state at end of stream. - fn finish(&mut self) -> Vec { - if self.parser_failed { - return Vec::new(); - } - - match self.parser.finish() { - Ok(result) => { - let mut events = Vec::new(); - push_reasoning_delta(&mut events, result); - events - } - Err(error) => { - warn!(error = %error.as_report(), "failed to flush reasoning parser state"); - Vec::new() - } - } - } -} - -/// Push one semantic text delta if it is non-empty. -fn push_text_delta(events: &mut Vec, kind: AssistantBlockKind, delta: String) { - if delta.is_empty() { - return; - } - events.push(ContentEvent::TextDelta { kind, delta }); -} - -/// Convert one parsed reasoning delta into zero or more content events. -fn push_reasoning_delta(events: &mut Vec, delta: ReasoningDelta) { - if let Some(reasoning) = delta.reasoning { - push_text_delta(events, AssistantBlockKind::Reasoning, reasoning); - } - if let Some(content) = delta.content { - push_text_delta(events, AssistantBlockKind::Text, content); - } -} - -/// Wrap one decoded-text stream into the internal reasoning event stream. -#[try_stream] -pub(crate) async fn reasoning_event_stream( - decoded_stream: impl DecodedTextEventStream, - reasoning_parser: Option>, - mut y: TryYielder, -) -> Result<()> { - pin_mut!(decoded_stream); - - // Without a parser, pass through as plain text deltas. - let Some(reasoning_parser) = reasoning_parser else { - while let Some(event) = decoded_stream.next().await.transpose()? { - for next in ContentEvent::from_decoded_plain_text(event) { - y.yield_ok(next).await; - } - } - return Ok(()); - }; - - let mut state = ReasoningState::new(reasoning_parser); - - while let Some(event) = decoded_stream.next().await.transpose()? { - match event { - DecodedTextEvent::Start { - prompt_token_ids, - prompt_logprobs, - } => { - state.initialize(&prompt_token_ids); - y.yield_ok(ContentEvent::Start { - prompt_token_ids, - prompt_logprobs, - }) - .await; - } - DecodedTextEvent::TextDelta { - delta, - token_ids, - logprobs, - finished, - } => { - for next in state.process_delta(delta) { - y.yield_ok(next).await; - } - if logprobs.is_some() || !token_ids.is_empty() { - y.yield_ok(ContentEvent::LogprobsDelta { - logprobs, - token_ids, - }) - .await; - } - if let Some(finished) = finished { - for next in state.finish() { - y.yield_ok(next).await; - } - y.yield_ok(ContentEvent::Done { - usage: finished.usage, - finish_reason: finished.finish_reason, - kv_transfer_params: finished.kv_transfer_params, - }) - .await; - } - } - } - } - Ok(()) -} - -#[cfg(test)] -mod tests { - - use std::sync::Arc; - - use futures::{StreamExt as _, stream}; - use vllm_llm::FinishReason; - use vllm_text::output::{ - DecodedLogprobs, DecodedPositionLogprobs, DecodedTextEvent, DecodedTokenLogprob, - }; - use vllm_tokenizer::{DynTokenizer, Tokenizer}; - - use super::super::ContentEvent; - use super::reasoning_event_stream; - use crate::event::AssistantBlockKind; - use crate::parser::reasoning::{ - ReasoningDelta, ReasoningError, ReasoningParser, ReasoningParserFactory, names, - }; - - struct FakeTokenizer; - - impl Tokenizer for FakeTokenizer { - fn encode( - &self, - text: &str, - _add_special_tokens: bool, - ) -> vllm_tokenizer::Result> { - Ok(text.chars().map(u32::from).collect()) - } - - fn decode( - &self, - token_ids: &[u32], - _skip_special_tokens: bool, - ) -> vllm_tokenizer::Result { - Ok(token_ids - .iter() - .map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}')) - .collect()) - } - - fn token_to_id(&self, token: &str) -> Option { - match token { - "" => Some(1), - "" => Some(2), - _ => None, - } - } - } - - struct FailingReasoningParser { - fail_next: bool, - } - - impl ReasoningParser for FailingReasoningParser { - fn create(_tokenizer: DynTokenizer) -> Result, ReasoningError> - where - Self: Sized + 'static, - { - Ok(Box::new(Self { fail_next: true })) - } - - fn push(&mut self, _text: &str) -> Result { - if self.fail_next { - self.fail_next = false; - return Err(ReasoningError::MissingToken { - token: "".to_string(), - }); - } - Ok(ReasoningDelta::default()) - } - } - - fn test_reasoning_parser(factory: &mut ReasoningParserFactory) -> Box { - factory.register_parser::("failing"); - - factory.create("failing", Arc::new(FakeTokenizer)).unwrap() - } - - #[tokio::test] - async fn reasoning_parser_failure_falls_back_to_plain_text() { - let mut factory = ReasoningParserFactory::new(); - let events = stream::iter(vec![ - Ok(DecodedTextEvent::Start { - prompt_token_ids: vec![1, 2, 3].into(), - prompt_logprobs: None, - }), - Ok(DecodedTextEvent::TextDelta { - delta: "abc".to_string(), - token_ids: vec![], - logprobs: None, - finished: None, - }), - Ok(DecodedTextEvent::TextDelta { - delta: "def".to_string(), - token_ids: vec![], - logprobs: None, - finished: Some(vllm_text::Finished { - usage: vllm_llm::TokenUsage { - prompt_token_count: 3, - output_token_count: 0, - cached_token_count: 0, - }, - finish_reason: FinishReason::stop_eos(), - kv_transfer_params: None, - }), - }), - ]); - - let collected = reasoning_event_stream(events, Some(test_reasoning_parser(&mut factory))) - .collect::>() - .await; - - let events = collected - .into_iter() - .collect::>>() - .expect("reasoning stream should not fail"); - - assert_eq!( - events, - vec![ - ContentEvent::Start { - prompt_token_ids: vec![1, 2, 3].into(), - prompt_logprobs: None, - }, - ContentEvent::TextDelta { - kind: AssistantBlockKind::Text, - delta: "abc".to_string(), - }, - ContentEvent::TextDelta { - kind: AssistantBlockKind::Text, - delta: "def".to_string(), - }, - ContentEvent::Done { - usage: vllm_llm::TokenUsage { - prompt_token_count: 3, - output_token_count: 0, - cached_token_count: 0, - }, - finish_reason: FinishReason::stop_eos(), - kv_transfer_params: None, - }, - ] - ); - } - - #[tokio::test] - async fn reasoning_stream_preserves_logprobs_delta() { - let events = stream::iter(vec![ - Ok(DecodedTextEvent::Start { - prompt_token_ids: vec![1].into(), - prompt_logprobs: None, - }), - Ok(DecodedTextEvent::TextDelta { - delta: "abc".to_string(), - token_ids: vec![], - logprobs: Some(DecodedLogprobs { - positions: vec![DecodedPositionLogprobs { - entries: vec![DecodedTokenLogprob { - token_id: 0, - token: "a".to_string(), - logprob: -0.1, - rank: 1, - }], - }], - }), - finished: None, - }), - ]); - - let collected = reasoning_event_stream(events, None) - .collect::>() - .await - .into_iter() - .collect::>>() - .unwrap(); - - assert_eq!( - collected, - vec![ - ContentEvent::Start { - prompt_token_ids: vec![1].into(), - prompt_logprobs: None, - }, - ContentEvent::TextDelta { - kind: AssistantBlockKind::Text, - delta: "abc".to_string(), - }, - ContentEvent::LogprobsDelta { - logprobs: Some(DecodedLogprobs { - positions: vec![DecodedPositionLogprobs { - entries: vec![DecodedTokenLogprob { - token_id: 0, - token: "a".to_string(), - logprob: -0.1, - rank: 1, - }], - }], - }), - token_ids: vec![], - }, - ] - ); - } - - #[tokio::test] - async fn qwen3_parser_uses_prompt_end_marker_to_switch_to_content() { - let tokenizer = Arc::new(FakeTokenizer); - let events = stream::iter(vec![ - Ok(DecodedTextEvent::Start { - prompt_token_ids: vec![2].into(), - prompt_logprobs: None, - }), - Ok(DecodedTextEvent::TextDelta { - delta: "thought ".to_string(), - token_ids: vec![], - logprobs: None, - finished: None, - }), - Ok(DecodedTextEvent::TextDelta { - delta: "doneOK".to_string(), - token_ids: vec![], - logprobs: None, - finished: None, - }), - ]); - - let factory = ReasoningParserFactory::new(); - let collected = reasoning_event_stream( - events, - Some(factory.create(names::QWEN3, tokenizer).unwrap()), - ) - .collect::>() - .await; - - let events = collected - .into_iter() - .collect::>>() - .expect("reasoning stream should not fail"); - - assert_eq!( - events, - vec![ - ContentEvent::Start { - prompt_token_ids: vec![2].into(), - prompt_logprobs: None, - }, - ContentEvent::TextDelta { - kind: AssistantBlockKind::Text, - delta: "thought ".to_string(), - }, - ContentEvent::TextDelta { - kind: AssistantBlockKind::Text, - delta: "doneOK".to_string(), - }, - ] - ); - } - - #[tokio::test] - async fn qwen3_parser_tolerates_prompt_prefill_reasoning() { - let tokenizer = Arc::new(FakeTokenizer); - let events = stream::iter(vec![ - Ok(DecodedTextEvent::Start { - prompt_token_ids: vec![1].into(), - prompt_logprobs: None, - }), - Ok(DecodedTextEvent::TextDelta { - delta: "thought ".to_string(), - token_ids: vec![], - logprobs: None, - finished: None, - }), - Ok(DecodedTextEvent::TextDelta { - delta: "doneOK".to_string(), - token_ids: vec![], - logprobs: None, - finished: None, - }), - ]); - - let factory = ReasoningParserFactory::new(); - let collected = reasoning_event_stream( - events, - Some(factory.create(names::QWEN3, tokenizer).unwrap()), - ) - .collect::>() - .await; - - let events = collected - .into_iter() - .collect::>>() - .expect("reasoning stream should not fail"); - - assert_eq!( - events, - vec![ - ContentEvent::Start { - prompt_token_ids: vec![1].into(), - prompt_logprobs: None, - }, - ContentEvent::TextDelta { - kind: AssistantBlockKind::Reasoning, - delta: "thought ".to_string(), - }, - ContentEvent::TextDelta { - kind: AssistantBlockKind::Reasoning, - delta: "done".to_string(), - }, - ContentEvent::TextDelta { - kind: AssistantBlockKind::Text, - delta: "OK".to_string(), - }, - ] - ); - } -} diff --git a/rust/src/chat/src/output/default/structural_tag.rs b/rust/src/chat/src/output/default/structural_tag.rs index eb1fa3d1436..bdc5d8e14c2 100644 --- a/rust/src/chat/src/output/default/structural_tag.rs +++ b/rust/src/chat/src/output/default/structural_tag.rs @@ -77,7 +77,7 @@ fn structural_tag_tool_choice(request: &ChatRequest) -> Option, - /// Whether tool parsing has already failed for this stream. - parser_failed: bool, - /// The parser-local index of the currently open tool call, if any. - // NOTE: We only allow single open tool call at a time right now, since that's what all - // supported parsers currently emit. Change this to a `BTreeMap` if we need to support multiple - // interleaved calls in the future. - open_call_index: Option, -} - -impl ToolState { - /// Create one fresh tool-parsing state for a new streamed response. - fn new(parser: Box) -> Self { - Self { - parser, - parser_failed: false, - open_call_index: None, - } - } - - /// Convert one semantic assistant text delta into zero or more tool-aware - /// internal events. - fn process_text_delta( - &mut self, - kind: AssistantBlockKind, - delta: String, - ) -> Result> { - let mut events = Vec::new(); - - // Only normal assistant text is eligible for tool parsing. Reasoning - // blocks and plain-text fallback should pass through unchanged. - if kind != AssistantBlockKind::Text || self.parser_failed { - self.open_call_index = None; - events.push(AssistantEvent::TextDelta { kind, delta }); - return Ok(events); - } - - let mut output = ToolParserOutput::default(); - let parse_result = self.parser.parse_into(&delta, &mut output); - - match parse_result { - Ok(()) => self.process_parser_output(kind, output, &mut events)?, - Err(error) => { - warn!( - error = %error.as_report(), - "tool parser failed; falling back to plain text deltas" - ); - // Permanently mark this parser as failed. - // TODO: we may consider recovering from parsing errors in the future. - self.parser_failed = true; - - // On parsing failure, we still apply the partial parser output if any, but we close - // any open tool calls and emit the remaining buffered text as a plain-text delta to - // preserve as much of the output as possible. - self.process_parser_output(kind, output, &mut events)?; - self.open_call_index = None; - push_text_delta(&mut events, kind, self.parser.reset()); - } - } - - Ok(events) - } - - /// Apply one parsed tool output to the current stream state. - fn process_parser_output( - &mut self, - kind: AssistantBlockKind, - output: ToolParserOutput, - events: &mut Vec, - ) -> Result<()> { - // When we are not currently streaming a tool call, preserve plain - // text first and then surface any new tool call items. - if self.open_call_index.is_none() { - push_text_delta(events, kind, output.normal_text); - self.process_tool_items(output.calls, events)?; - } else { - // Once a tool call is open, prioritize tool deltas first. If the - // parser emits normal text again, close the tool call and resume - // plain text output. - self.process_tool_items(output.calls, events)?; - if !output.normal_text.is_empty() { - self.open_call_index = None; - push_text_delta(events, kind, output.normal_text); - } - } - Ok(()) - } - - /// Apply one batch of parsed tool-call deltas emitted by the parser. - fn process_tool_items( - &mut self, - items: Vec, - events: &mut Vec, - ) -> Result<()> { - for item in items { - if let Some(name) = item.name { - let is_new_tool = match self.open_call_index { - Some(open_call_index) => open_call_index != item.tool_index, - None => true, - }; - if is_new_tool { - let id = self - .parser - .tool_call_id(item.tool_index) - .map(str::to_string) - .unwrap_or_else(generate_tool_call_id); - self.open_call_index = Some(item.tool_index); - events.push(AssistantEvent::ToolCallStart { id, name }); - } - } - - if item.arguments.is_empty() { - // No arguments delta to apply. - continue; - } - let Some(open_call_index) = self.open_call_index else { - return Err(Error::ToolCallStreamInvariant { - message: format!( - "received arguments for tool index {} before any tool-call start", - item.tool_index - ), - }); - }; - if open_call_index != item.tool_index { - return Err(Error::ToolCallStreamInvariant { - message: format!( - "received arguments for tool index {} while tool index {} is open", - item.tool_index, open_call_index - ), - }); - } - - events.push(AssistantEvent::ToolCallArgumentsDelta { - delta: item.arguments, - }); - } - Ok(()) - } - - /// Flush parser state at end-of-stream and close any remaining open calls. - fn finish(&mut self) -> Result> { - let mut events = Vec::new(); - - if self.parser_failed { - return Ok(events); - } - - match self.parser.finish() { - Ok(output) => { - self.process_parser_output(AssistantBlockKind::Text, output, &mut events)? - } - Err(error) => { - warn!( - error = %error.as_report(), - "tool parser finish failed; closing open tool calls with buffered state" - ); - self.parser_failed = true; - } - } - - Ok(events) - } -} - -/// Push one plain-text delta if it is non-empty. -fn push_text_delta(events: &mut Vec, kind: AssistantBlockKind, delta: String) { - if delta.is_empty() { - return; - } - events.push(AssistantEvent::TextDelta { kind, delta }); -} - -/// Wrap one semantic assistant stream into the internal tool-aware assistant -/// stream. -#[try_stream] -pub(crate) async fn tool_event_stream( - stream: impl ContentEventStream, - parser: Option>, - mut y: TryYielder, -) -> Result<()> { - // Without a parser, pass through the input stream unchanged. - let Some(parser) = parser else { - pin_mut!(stream); - while let Some(event) = stream.next().await.transpose()? { - y.yield_ok(event.into()).await; - } - return Ok(()); - }; - - pin_mut!(stream); - let mut state = ToolState::new(parser); - - while let Some(event) = stream.next().await.transpose()? { - match event { - ContentEvent::Start { - prompt_token_ids, - prompt_logprobs, - } => { - y.yield_ok(AssistantEvent::Start { - prompt_token_ids, - prompt_logprobs, - }) - .await; - } - ContentEvent::TextDelta { kind, delta } => { - for next in state.process_text_delta(kind, delta)? { - y.yield_ok(next).await; - } - } - ContentEvent::LogprobsDelta { - logprobs, - token_ids, - } => { - y.yield_ok(AssistantEvent::LogprobsDelta { - logprobs, - token_ids, - }) - .await; - } - ContentEvent::Done { - usage, - finish_reason, - kv_transfer_params, - } => { - for next in state.finish()? { - y.yield_ok(next).await; - } - - y.yield_ok(AssistantEvent::Done { - usage, - finish_reason, - kv_transfer_params, - }) - .await; - } - } - } - Ok(()) -} - -#[cfg(test)] -mod tests { - - use futures::{StreamExt as _, stream}; - use vllm_llm::FinishReason; - use vllm_text::{DecodedLogprobs, DecodedPositionLogprobs, DecodedTokenLogprob}; - use vllm_tool_parser::Result; - - use super::super::{AssistantEvent, ContentEvent}; - use super::tool_event_stream; - use crate::error::Error; - use crate::event::{AssistantBlockKind, AssistantMessageExt as _}; - use crate::output::structured::structured_chat_event_stream; - use crate::parser::tool::{ - DeepSeekV4ToolParser, ToolParser, ToolParserError, ToolParserOutput, - }; - use crate::request::ChatTool; - use crate::stream::{ChatEventStream, CollectedAssistantMessage}; - - struct FailingParser { - fail_next: bool, - buffered: String, - } - - struct ScriptedParser { - push_outputs: Vec, - finish_output: ToolParserOutput, - } - - struct PartialThenFailParser { - buffered: String, - } - - struct IdScriptedParser { - output: ToolParserOutput, - tool_call_id: Option, - } - - impl ToolParser for FailingParser { - fn create(_tools: &[ChatTool]) -> vllm_tool_parser::Result> - where - Self: Sized + 'static, - { - Ok(Box::new(Self { - fail_next: false, - buffered: String::new(), - })) - } - - fn parse_into(&mut self, chunk: &str, _output: &mut ToolParserOutput) -> Result<()> { - self.buffered.push_str(chunk); - if self.fail_next { - self.fail_next = false; - return Err(ToolParserError::ParsingFailed { - message: "boom".to_string(), - }); - } - - self.buffered.clear(); - Ok(()) - } - - fn finish(&mut self) -> Result { - Ok(ToolParserOutput::default()) - } - - fn reset(&mut self) -> String { - std::mem::take(&mut self.buffered) - } - } - - impl ToolParser for ScriptedParser { - fn create(_tools: &[ChatTool]) -> vllm_tool_parser::Result> - where - Self: Sized + 'static, - { - Ok(Box::new(Self { - push_outputs: Vec::new(), - finish_output: ToolParserOutput::default(), - })) - } - - fn parse_into(&mut self, _chunk: &str, output: &mut ToolParserOutput) -> Result<()> { - let mut next = self.push_outputs.pop().unwrap_or_default(); - output.normal_text.push_str(&next.normal_text); - output.calls.append(&mut next.calls); - Ok(()) - } - - fn finish(&mut self) -> Result { - Ok(std::mem::take(&mut self.finish_output)) - } - - fn reset(&mut self) -> String { - String::new() - } - } - - impl ToolParser for IdScriptedParser { - fn create(_tools: &[ChatTool]) -> vllm_tool_parser::Result> - where - Self: Sized + 'static, - { - Ok(Box::new(Self { - output: ToolParserOutput::default(), - tool_call_id: None, - })) - } - - fn tool_call_id(&self, tool_index: usize) -> Option<&str> { - (tool_index == 0).then_some(self.tool_call_id.as_deref()).flatten() - } - - fn parse_into(&mut self, _chunk: &str, output: &mut ToolParserOutput) -> Result<()> { - output.append(std::mem::take(&mut self.output)); - Ok(()) - } - - fn finish(&mut self) -> Result { - Ok(ToolParserOutput::default()) - } - - fn reset(&mut self) -> String { - String::new() - } - } - - impl ToolParser for PartialThenFailParser { - fn create(_tools: &[ChatTool]) -> vllm_tool_parser::Result> - where - Self: Sized + 'static, - { - Ok(Box::new(Self { - buffered: String::new(), - })) - } - - fn parse_into(&mut self, _chunk: &str, output: &mut ToolParserOutput) -> Result<()> { - output.calls.extend([ - crate::parser::tool::ToolCallDelta { - tool_index: 0, - name: Some("get_weather".to_string()), - arguments: String::new(), - }, - crate::parser::tool::ToolCallDelta { - tool_index: 0, - name: None, - arguments: r#"{"location":"SF"}"#.to_string(), - }, - ]); - self.buffered.push_str(" trailing text"); - Err(ToolParserError::ParsingFailed { - message: "boom".to_string(), - }) - } - - fn finish(&mut self) -> Result { - Ok(ToolParserOutput::default()) - } - - fn reset(&mut self) -> String { - std::mem::take(&mut self.buffered) - } - } - - fn deepseek_v4_test_tools() -> Vec { - vec![ - ChatTool { - name: "get_weather".to_string(), - description: None, - parameters: serde_json::json!({ - "type": "object", - "properties": { - "location": { "type": "string" } - } - }), - strict: None, - }, - ChatTool { - name: "add".to_string(), - description: None, - parameters: serde_json::json!({ - "type": "object", - "properties": { - "x": { "type": "integer" }, - "y": { "type": "integer" } - } - }), - strict: None, - }, - ] - } - - async fn collect_deepseek_v4_message(chunks: Vec) -> CollectedAssistantMessage { - let events = chunks - .into_iter() - .map(|delta| { - Ok(ContentEvent::TextDelta { - kind: AssistantBlockKind::Text, - delta, - }) - }) - .chain(std::iter::once(Ok(ContentEvent::Done { - usage: vllm_llm::TokenUsage { - prompt_token_count: 1, - output_token_count: 1, - cached_token_count: 0, - }, - finish_reason: FinishReason::stop_eos(), - kv_transfer_params: None, - }))); - let parser = DeepSeekV4ToolParser::create(&deepseek_v4_test_tools()).unwrap(); - let assistant_events = tool_event_stream(stream::iter(events), Some(parser)); - let chat_events = structured_chat_event_stream(assistant_events, true); - - ChatEventStream::new("req_deepseek_v4".to_string(), Box::pin(chat_events)) - .collect_message() - .await - .unwrap() - } - - fn message_tool_projection( - message: &CollectedAssistantMessage, - ) -> (String, Vec<(String, serde_json::Value)>) { - ( - message.message.text(), - message - .message - .tool_calls() - .map(|call| { - ( - call.name.clone(), - serde_json::from_str(&call.arguments).unwrap(), - ) - }) - .collect(), - ) - } - - #[tokio::test] - async fn tool_parser_error_preserves_partial_output_and_flushes_buffer() { - let events = stream::iter(vec![ - Ok(ContentEvent::TextDelta { - kind: AssistantBlockKind::Text, - delta: "ignored".to_string(), - }), - Ok(ContentEvent::Done { - usage: vllm_llm::TokenUsage { - prompt_token_count: 1, - output_token_count: 1, - cached_token_count: 0, - }, - finish_reason: FinishReason::stop_eos(), - kv_transfer_params: None, - }), - ]); - - let events = tool_event_stream( - events, - Some(Box::new(PartialThenFailParser { - buffered: String::new(), - })), - ) - .collect::>() - .await - .into_iter() - .collect::>>() - .unwrap(); - - assert!(matches!( - &events[0], - AssistantEvent::ToolCallStart { name, .. } if name == "get_weather" - )); - assert!(matches!( - &events[1], - AssistantEvent::ToolCallArgumentsDelta { delta } if delta == r#"{"location":"SF"}"# - )); - assert_eq!( - events[2], - AssistantEvent::TextDelta { - kind: AssistantBlockKind::Text, - delta: " trailing text".to_string(), - } - ); - assert!(matches!(events[3], AssistantEvent::Done { .. })); - } - - #[tokio::test] - async fn tool_stream_preserves_parser_provided_tool_call_id() { - let events = stream::iter(vec![Ok(ContentEvent::TextDelta { - kind: AssistantBlockKind::Text, - delta: "ignored".to_string(), - })]); - let parser = IdScriptedParser { - output: ToolParserOutput { - normal_text: String::new(), - calls: vec![crate::parser::tool::ToolCallDelta { - tool_index: 0, - name: Some("get_weather".to_string()), - arguments: "{}".to_string(), - }], - }, - tool_call_id: Some("functions.get_weather:0".to_string()), - }; - - let events = tool_event_stream(events, Some(Box::new(parser))) - .collect::>() - .await - .into_iter() - .collect::>>() - .unwrap(); - - assert!(matches!( - &events[0], - AssistantEvent::ToolCallStart { id, name } - if id == "functions.get_weather:0" && name == "get_weather" - )); - } - - #[tokio::test] - async fn tool_stream_generates_tool_call_id_when_parser_omits_one() { - let events = stream::iter(vec![Ok(ContentEvent::TextDelta { - kind: AssistantBlockKind::Text, - delta: "ignored".to_string(), - })]); - let parser = IdScriptedParser { - output: ToolParserOutput { - normal_text: String::new(), - calls: vec![crate::parser::tool::ToolCallDelta { - tool_index: 0, - name: Some("get_weather".to_string()), - arguments: "{}".to_string(), - }], - }, - tool_call_id: None, - }; - - let events = tool_event_stream(events, Some(Box::new(parser))) - .collect::>() - .await - .into_iter() - .collect::>>() - .unwrap(); - - assert!(matches!( - &events[0], - AssistantEvent::ToolCallStart { id, name } - if id.starts_with("call_") && name == "get_weather" - )); - } - - #[tokio::test] - async fn real_buffered_parser_error_matches_streaming_and_non_streaming() { - let prefix = "I will check both.\n"; - let first_tool_call = concat!( - "<｜DSML｜tool_calls>\n", - "<｜DSML｜invoke name=\"get_weather\">\n", - "<｜DSML｜parameter name=\"location\" string=\"true\">Tokyo\n", - "", - ); - let malformed_second_tool_call = concat!( - "\n<｜DSML｜invoke name=\"add\">\n", - "not a parameter\n", - "\n", - "", - ); - let streaming_chunks = vec![ - prefix.to_string(), - first_tool_call.to_string(), - malformed_second_tool_call.to_string(), - ]; - let full_output = streaming_chunks.concat(); - - let streaming = collect_deepseek_v4_message(streaming_chunks).await; - let non_streaming = collect_deepseek_v4_message(vec![full_output]).await; - - let expected = ( - format!("{prefix}{malformed_second_tool_call}"), - vec![( - "get_weather".to_string(), - serde_json::json!({ "location": "Tokyo" }), - )], - ); - assert_eq!(message_tool_projection(&streaming), expected); - assert_eq!(message_tool_projection(&non_streaming), expected); - } - - #[tokio::test] - async fn tool_parser_failure_falls_back_to_plain_text() { - let events = stream::iter(vec![ - Ok(ContentEvent::Start { - prompt_token_ids: vec![1, 2, 3].into(), - prompt_logprobs: None, - }), - Ok(ContentEvent::TextDelta { - kind: AssistantBlockKind::Text, - delta: "abc".to_string(), - }), - Ok(ContentEvent::TextDelta { - kind: AssistantBlockKind::Text, - delta: "def".to_string(), - }), - Ok(ContentEvent::Done { - usage: vllm_llm::TokenUsage { - prompt_token_count: 3, - output_token_count: 0, - cached_token_count: 0, - }, - finish_reason: FinishReason::stop_eos(), - kv_transfer_params: None, - }), - ]); - - let collected = tool_event_stream( - events, - Some(Box::new(FailingParser { - fail_next: true, - buffered: String::new(), - })), - ) - .collect::>() - .await; - - let events = collected - .into_iter() - .collect::>>() - .expect("tool stream should not fail"); - - assert_eq!( - events, - vec![ - AssistantEvent::Start { - prompt_token_ids: vec![1, 2, 3].into(), - prompt_logprobs: None, - }, - AssistantEvent::TextDelta { - kind: AssistantBlockKind::Text, - delta: "abc".to_string(), - }, - AssistantEvent::TextDelta { - kind: AssistantBlockKind::Text, - delta: "def".to_string(), - }, - AssistantEvent::Done { - usage: vllm_llm::TokenUsage { - prompt_token_count: 3, - output_token_count: 0, - cached_token_count: 0, - }, - finish_reason: FinishReason::stop_eos(), - kv_transfer_params: None, - }, - ] - ); - - let message = ChatEventStream::new( - "req_fallback".to_string(), - Box::pin(structured_chat_event_stream( - stream::iter(events.into_iter().map(Ok)), - true, - )), - ) - .collect_message() - .await - .expect("collect_message should succeed"); - assert_eq!(message.message.text(), "abcdef"); - assert!(message.message.tool_calls().next().is_none()); - } - - #[tokio::test] - async fn tool_stream_preserves_logprobs_delta() { - let events = stream::iter(vec![ - Ok(ContentEvent::Start { - prompt_token_ids: vec![1].into(), - prompt_logprobs: None, - }), - Ok(ContentEvent::LogprobsDelta { - logprobs: Some(DecodedLogprobs { - positions: vec![DecodedPositionLogprobs { - entries: vec![DecodedTokenLogprob { - token_id: 0, - token: "a".to_string(), - logprob: -0.2, - rank: 1, - }], - }], - }), - token_ids: vec![], - }), - Ok(ContentEvent::Done { - usage: vllm_llm::TokenUsage { - prompt_token_count: 1, - output_token_count: 0, - cached_token_count: 0, - }, - finish_reason: FinishReason::stop_eos(), - kv_transfer_params: None, - }), - ]); - let events = tool_event_stream( - events, - Some(Box::new(FailingParser { - fail_next: false, - buffered: String::new(), - })), - ) - .collect::>() - .await - .into_iter() - .collect::>>() - .unwrap(); - - assert_eq!( - events, - vec![ - AssistantEvent::Start { - prompt_token_ids: vec![1].into(), - prompt_logprobs: None, - }, - AssistantEvent::LogprobsDelta { - logprobs: Some(DecodedLogprobs { - positions: vec![DecodedPositionLogprobs { - entries: vec![DecodedTokenLogprob { - token_id: 0, - token: "a".to_string(), - logprob: -0.2, - rank: 1, - }], - }], - }), - token_ids: vec![], - }, - AssistantEvent::Done { - usage: vllm_llm::TokenUsage { - prompt_token_count: 1, - output_token_count: 0, - cached_token_count: 0, - }, - finish_reason: FinishReason::stop_eos(), - kv_transfer_params: None, - }, - ] - ); - } - - #[tokio::test] - async fn tool_stream_rejects_interleaved_tool_indices() { - let events = stream::iter(vec![ - Ok(ContentEvent::TextDelta { - kind: AssistantBlockKind::Text, - delta: "ignored".to_string(), - }), - Ok(ContentEvent::Done { - usage: vllm_llm::TokenUsage { - prompt_token_count: 1, - output_token_count: 1, - cached_token_count: 0, - }, - finish_reason: FinishReason::stop_eos(), - kv_transfer_params: None, - }), - ]); - - let parser = ScriptedParser { - push_outputs: vec![ToolParserOutput { - normal_text: String::new(), - calls: vec![ - crate::parser::tool::ToolCallDelta { - tool_index: 0, - name: Some("first".to_string()), - arguments: String::new(), - }, - crate::parser::tool::ToolCallDelta { - tool_index: 1, - name: None, - arguments: "{}".to_string(), - }, - ], - }], - finish_output: ToolParserOutput::default(), - }; - - let err = tool_event_stream(events, Some(Box::new(parser))) - .collect::>() - .await - .into_iter() - .find_map(|output| output.err()) - .expect("expected invariant error"); - - assert!(matches!(err, Error::ToolCallStreamInvariant { .. })); - } - - #[tokio::test] - async fn tool_stream_resets_open_tool_when_normal_text_interrupts_it() { - let events = stream::iter(vec![ - Ok(ContentEvent::TextDelta { - kind: AssistantBlockKind::Text, - delta: "start".to_string(), - }), - Ok(ContentEvent::TextDelta { - kind: AssistantBlockKind::Text, - delta: "text".to_string(), - }), - Ok(ContentEvent::TextDelta { - kind: AssistantBlockKind::Text, - delta: "args".to_string(), - }), - ]); - - let parser = ScriptedParser { - push_outputs: vec![ - ToolParserOutput { - normal_text: String::new(), - calls: vec![crate::parser::tool::ToolCallDelta { - tool_index: 0, - name: None, - arguments: "}".to_string(), - }], - }, - ToolParserOutput { - normal_text: "plain text".to_string(), - calls: Vec::new(), - }, - ToolParserOutput { - normal_text: String::new(), - calls: vec![crate::parser::tool::ToolCallDelta { - tool_index: 0, - name: Some("first".to_string()), - arguments: "{".to_string(), - }], - }, - ], - finish_output: ToolParserOutput::default(), - }; - - let err = tool_event_stream(events, Some(Box::new(parser))) - .collect::>() - .await - .into_iter() - .find_map(|output| output.err()) - .expect("expected invariant error"); - - assert!(matches!( - err, - Error::ToolCallStreamInvariant { message } - if message == "received arguments for tool index 0 before any tool-call start" - )); - } - - #[tokio::test] - async fn tool_stream_emits_start_and_args_for_terminal_text() { - let events = stream::iter(vec![ - Ok(ContentEvent::Start { - prompt_token_ids: vec![1].into(), - prompt_logprobs: None, - }), - Ok(ContentEvent::TextDelta { - kind: AssistantBlockKind::Text, - delta: "ignored".to_string(), - }), - Ok(ContentEvent::Done { - usage: vllm_llm::TokenUsage { - prompt_token_count: 1, - output_token_count: 1, - cached_token_count: 0, - }, - finish_reason: FinishReason::stop_eos(), - kv_transfer_params: None, - }), - ]); - - let parser = ScriptedParser { - push_outputs: vec![ToolParserOutput { - normal_text: String::new(), - calls: vec![ - crate::parser::tool::ToolCallDelta { - tool_index: 0, - name: Some("first".to_string()), - arguments: r#"{"a":1}"#.to_string(), - }, - crate::parser::tool::ToolCallDelta { - tool_index: 1, - name: Some("second".to_string()), - arguments: r#"{"b":2}"#.to_string(), - }, - ], - }], - finish_output: ToolParserOutput::default(), - }; - - let events = tool_event_stream(events, Some(Box::new(parser))) - .collect::>() - .await - .into_iter() - .collect::>>() - .unwrap(); - - assert!(matches!(events[1], AssistantEvent::ToolCallStart { .. })); - assert!(matches!( - events[2], - AssistantEvent::ToolCallArgumentsDelta { .. } - )); - assert!(matches!(events[3], AssistantEvent::ToolCallStart { .. })); - assert!(matches!( - events[4], - AssistantEvent::ToolCallArgumentsDelta { .. } - )); - let collected = ChatEventStream::new( - "req_final_only".to_string(), - Box::pin(structured_chat_event_stream( - stream::iter(events.into_iter().map(Ok)), - true, - )), - ) - .collect_message() - .await - .unwrap(); - let tool_calls = collected.message.tool_calls().collect::>(); - assert_eq!(tool_calls.len(), 2); - assert_eq!(tool_calls[0].name, "first"); - assert_eq!(tool_calls[1].name, "second"); - } -} diff --git a/rust/src/chat/src/output/default/unified.rs b/rust/src/chat/src/output/default/unified.rs new file mode 100644 index 00000000000..78320e796e9 --- /dev/null +++ b/rust/src/chat/src/output/default/unified.rs @@ -0,0 +1,582 @@ +//! Adapts decoded text updates into parsed assistant deltas. +//! +//! This stage sits between low-level token decoding and final block assembly. +//! It drives one unified parser that may emit normal text, reasoning text, or +//! tool-call deltas, then normalizes those parser events into internal +//! assistant events. + +use asynk_strim_attr::{TryYielder, try_stream}; +use futures::{StreamExt as _, pin_mut}; +use thiserror_ext::AsReport; +use tracing::warn; +use vllm_parser::unified::{UnifiedParser, UnifiedParserEvent, UnifiedParserOutput}; +use vllm_text::output::DecodedTextEvent; + +use crate::Result; +use crate::error::Error; +use crate::event::AssistantBlockKind; +use crate::output::{AssistantEvent, DecodedTextEventStream, generate_tool_call_id}; + +/// Per-stream unified parsing state. +struct UnifiedParserState { + /// Parser for the current request stream. + parser: Box, + /// Whether unified parsing has already failed for this stream. + parser_failed: bool, + /// The parser-local index of the currently open tool call, if any. + /// + /// Supported parsers currently emit at most one active tool call at a time. + /// Change this to an indexed map if a model needs interleaved calls later. + open_call_index: Option, +} + +impl UnifiedParserState { + /// Create one fresh unified parsing state for a new streamed response. + fn new(parser: Box) -> Self { + Self { + parser, + parser_failed: false, + open_call_index: None, + } + } + + /// Initialize parser state once prompt token IDs are available. + fn initialize(&mut self, prompt_token_ids: &[u32]) { + if self.parser_failed { + return; + } + + match self.parser.initialize(prompt_token_ids) { + Ok(()) => {} + Err(error) => { + warn!( + error = %error.as_report(), + "failed to initialize unified parser; falling back to plain text deltas" + ); + self.parser_failed = true; + self.open_call_index = None; + } + } + } + + /// Convert one decoded text delta into zero or more parsed assistant events. + fn process_delta(&mut self, delta: String) -> Result> { + if self.parser_failed { + self.open_call_index = None; + return Ok(text_event(AssistantBlockKind::Text, delta).into_iter().collect()); + } + + let mut output = UnifiedParserOutput::default(); + match self.parser.parse_into(&delta, &mut output) { + Ok(()) => { + let mut events = Vec::new(); + self.process_parser_output(output, &mut events)?; + Ok(events) + } + Err(error) => { + warn!( + error = %error.as_report(), + "unified parser failed; falling back to plain text deltas" + ); + self.parser_failed = true; + + let mut events = Vec::new(); + self.process_parser_output(output, &mut events)?; + self.open_call_index = None; + + let recovered = self.parser.reset(); + if recovered.is_empty() && events.is_empty() { + push_text_delta(&mut events, AssistantBlockKind::Text, delta); + } else { + push_text_delta(&mut events, AssistantBlockKind::Text, recovered); + } + Ok(events) + } + } + } + + /// Flush parser state at end-of-stream and close any remaining open calls. + fn finish(&mut self) -> Result> { + let mut events = Vec::new(); + + if self.parser_failed { + return Ok(events); + } + + match self.parser.finish() { + Ok(output) => self.process_parser_output(output, &mut events)?, + Err(error) => { + warn!( + error = %error.as_report(), + "unified parser finish failed; closing open parser state" + ); + self.parser_failed = true; + self.open_call_index = None; + // TODO: should we reset and emit the buffered text? + } + } + + Ok(events) + } + + /// Apply one parsed unified output to the current stream state. + fn process_parser_output( + &mut self, + output: UnifiedParserOutput, + events: &mut Vec, + ) -> Result<()> { + for event in output.events { + match event { + UnifiedParserEvent::Text(delta) => { + self.open_call_index = None; + push_text_delta(events, AssistantBlockKind::Text, delta); + } + UnifiedParserEvent::Reasoning(delta) => { + self.open_call_index = None; + push_text_delta(events, AssistantBlockKind::Reasoning, delta); + } + UnifiedParserEvent::ToolCall(item) => { + self.process_tool_item(item, events)?; + } + } + } + + Ok(()) + } + + /// Apply one parsed tool-call delta emitted by the parser. + fn process_tool_item( + &mut self, + item: vllm_parser::tool::ToolCallDelta, + events: &mut Vec, + ) -> Result<()> { + if let Some(name) = item.name { + let is_new_tool = match self.open_call_index { + Some(open_call_index) => open_call_index != item.tool_index, + None => true, + }; + if is_new_tool { + let id = self + .parser + .tool_call_id(item.tool_index) + .map(str::to_string) + .unwrap_or_else(generate_tool_call_id); + self.open_call_index = Some(item.tool_index); + events.push(AssistantEvent::ToolCallStart { id, name }); + } + } + + if item.arguments.is_empty() { + return Ok(()); + } + let Some(open_call_index) = self.open_call_index else { + return Err(Error::ToolCallStreamInvariant { + message: format!( + "received arguments for tool index {} before any tool-call start", + item.tool_index + ), + }); + }; + if open_call_index != item.tool_index { + return Err(Error::ToolCallStreamInvariant { + message: format!( + "received arguments for tool index {} while tool index {} is open", + item.tool_index, open_call_index + ), + }); + } + + events.push(AssistantEvent::ToolCallArgumentsDelta { + delta: item.arguments, + }); + Ok(()) + } +} + +/// Build one plain text event if `delta` is non-empty. +fn text_event(kind: AssistantBlockKind, delta: String) -> Option { + if delta.is_empty() { + return None; + } + Some(AssistantEvent::TextDelta { kind, delta }) +} + +/// Push one plain text delta if it is non-empty. +fn push_text_delta(events: &mut Vec, kind: AssistantBlockKind, delta: String) { + if let Some(event) = text_event(kind, delta) { + events.push(event); + } +} + +/// Wrap one decoded-text stream into the internal unified assistant stream. +#[try_stream] +pub(crate) async fn unified_event_stream( + decoded_stream: impl DecodedTextEventStream, + parser: Box, + mut y: TryYielder, +) -> Result<()> { + pin_mut!(decoded_stream); + + let mut state = UnifiedParserState::new(parser); + + while let Some(event) = decoded_stream.next().await.transpose()? { + match event { + DecodedTextEvent::Start { + prompt_token_ids, + prompt_logprobs, + } => { + state.initialize(&prompt_token_ids); + y.yield_ok(AssistantEvent::Start { + prompt_token_ids, + prompt_logprobs, + }) + .await; + } + DecodedTextEvent::TextDelta { + delta, + token_ids, + logprobs, + finished, + } => { + for next in state.process_delta(delta)? { + y.yield_ok(next).await; + } + if logprobs.is_some() || !token_ids.is_empty() { + y.yield_ok(AssistantEvent::LogprobsDelta { + logprobs, + token_ids, + }) + .await; + } + if let Some(finished) = finished { + for next in state.finish()? { + y.yield_ok(next).await; + } + y.yield_ok(AssistantEvent::Done { + usage: finished.usage, + finish_reason: finished.finish_reason, + kv_transfer_params: finished.kv_transfer_params, + }) + .await; + } + } + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::collections::VecDeque; + + use futures::{StreamExt as _, stream}; + use vllm_parser::reasoning::ReasoningError; + use vllm_parser::tool::ToolCallDelta; + use vllm_parser::unified::{UnifiedParserError, UnifiedParserOutput}; + + use super::unified_event_stream; + use crate::event::AssistantBlockKind; + use crate::output::AssistantEvent; + + enum ScriptedStep { + Output(UnifiedParserOutput), + Error { + committed: UnifiedParserOutput, + reset_text: String, + }, + } + + struct ScriptedParser { + steps: VecDeque, + reset_text: String, + tool_call_id: Option, + finish_error_reset_text: Option, + } + + impl ScriptedParser { + fn new(steps: impl IntoIterator) -> Self { + Self { + steps: steps.into_iter().collect(), + reset_text: String::new(), + tool_call_id: Some("call_test".to_string()), + finish_error_reset_text: None, + } + } + + fn with_finish_error(mut self, reset_text: &str) -> Self { + self.finish_error_reset_text = Some(reset_text.to_string()); + self + } + } + + impl vllm_parser::unified::UnifiedParser for ScriptedParser { + fn create( + _tools: &[vllm_parser::tool::Tool], + _tokenizer: vllm_tokenizer::DynTokenizer, + ) -> vllm_parser::unified::Result> + where + Self: Sized + 'static, + { + unreachable!("ScriptedParser is constructed directly in tests") + } + + fn parse_into( + &mut self, + _delta: &str, + output: &mut UnifiedParserOutput, + ) -> vllm_parser::unified::Result<()> { + match self.steps.pop_front().expect("unexpected parser call") { + ScriptedStep::Output(next) => { + output.append(next); + Ok(()) + } + ScriptedStep::Error { + committed, + reset_text, + } => { + output.append(committed); + self.reset_text = reset_text; + Err(UnifiedParserError::Reasoning( + ReasoningError::MissingToken { + token: "".to_string(), + }, + )) + } + } + } + + fn tool_call_id(&self, _tool_index: usize) -> Option<&str> { + self.tool_call_id.as_deref() + } + + fn finish(&mut self) -> vllm_parser::unified::Result { + if let Some(reset_text) = self.finish_error_reset_text.take() { + self.reset_text = reset_text; + return Err(UnifiedParserError::Reasoning( + ReasoningError::MissingToken { + token: "".to_string(), + }, + )); + } + Ok(UnifiedParserOutput::default()) + } + + fn reset(&mut self) -> String { + std::mem::take(&mut self.reset_text) + } + } + + fn decoded_delta(delta: &str) -> vllm_text::output::DecodedTextEvent { + vllm_text::output::DecodedTextEvent::TextDelta { + delta: delta.to_string(), + token_ids: Vec::new(), + logprobs: None, + finished: None, + } + } + + fn finished_delta(delta: &str) -> vllm_text::output::DecodedTextEvent { + vllm_text::output::DecodedTextEvent::TextDelta { + delta: delta.to_string(), + token_ids: Vec::new(), + logprobs: None, + finished: Some(vllm_text::output::Finished { + usage: vllm_llm::TokenUsage::default(), + finish_reason: crate::FinishReason::Stop(None), + kv_transfer_params: None, + }), + } + } + + async fn collect( + parser: ScriptedParser, + events: Vec, + ) -> Vec { + let stream = stream::iter(events.into_iter().map(Ok)); + unified_event_stream(stream, Box::new(parser)) + .collect::>() + .await + .into_iter() + .collect::>>() + .unwrap() + } + + fn text(delta: &str) -> UnifiedParserOutput { + let mut output = UnifiedParserOutput::default(); + output.push_text(delta.to_string()); + output + } + + fn reasoning(delta: &str) -> UnifiedParserOutput { + let mut output = UnifiedParserOutput::default(); + output.push_reasoning(delta.to_string()); + output + } + + fn tool_call(name: &str, arguments: &str) -> UnifiedParserOutput { + UnifiedParserOutput { + events: vec![vllm_parser::unified::UnifiedParserEvent::ToolCall( + ToolCallDelta { + tool_index: 0, + name: Some(name.to_string()), + arguments: arguments.to_string(), + }, + )], + } + } + + fn combined(first: UnifiedParserOutput, second: UnifiedParserOutput) -> UnifiedParserOutput { + let mut output = first; + output.append(second); + output + } + + #[tokio::test] + async fn unified_stream_emits_reasoning_only_deltas() { + let events = collect( + ScriptedParser::new([ScriptedStep::Output(reasoning("thinking"))]), + vec![decoded_delta("raw")], + ) + .await; + + assert_eq!( + events, + vec![AssistantEvent::TextDelta { + kind: AssistantBlockKind::Reasoning, + delta: "thinking".to_string(), + }] + ); + } + + #[tokio::test] + async fn unified_stream_emits_tool_only_deltas() { + let events = collect( + ScriptedParser::new([ScriptedStep::Output(tool_call( + "get_weather", + r#"{"location":"Paris"}"#, + ))]), + vec![decoded_delta("raw")], + ) + .await; + + assert_eq!( + events, + vec![ + AssistantEvent::ToolCallStart { + id: "call_test".to_string(), + name: "get_weather".to_string(), + }, + AssistantEvent::ToolCallArgumentsDelta { + delta: r#"{"location":"Paris"}"#.to_string(), + }, + ] + ); + } + + #[tokio::test] + async fn unified_stream_emits_reasoning_followed_by_tool_call() { + let events = collect( + ScriptedParser::new([ScriptedStep::Output(combined( + reasoning("thinking"), + tool_call("get_weather", r#"{"location":"Paris"}"#), + ))]), + vec![decoded_delta("raw")], + ) + .await; + + assert_eq!( + events, + vec![ + AssistantEvent::TextDelta { + kind: AssistantBlockKind::Reasoning, + delta: "thinking".to_string(), + }, + AssistantEvent::ToolCallStart { + id: "call_test".to_string(), + name: "get_weather".to_string(), + }, + AssistantEvent::ToolCallArgumentsDelta { + delta: r#"{"location":"Paris"}"#.to_string(), + }, + ] + ); + } + + #[tokio::test] + async fn unified_stream_emits_visible_text_followed_by_tool_call() { + let events = collect( + ScriptedParser::new([ScriptedStep::Output(combined( + text("visible "), + tool_call("get_weather", r#"{"location":"Paris"}"#), + ))]), + vec![decoded_delta("raw")], + ) + .await; + + assert_eq!( + events, + vec![ + AssistantEvent::TextDelta { + kind: AssistantBlockKind::Text, + delta: "visible ".to_string(), + }, + AssistantEvent::ToolCallStart { + id: "call_test".to_string(), + name: "get_weather".to_string(), + }, + AssistantEvent::ToolCallArgumentsDelta { + delta: r#"{"location":"Paris"}"#.to_string(), + }, + ] + ); + } + + #[tokio::test] + async fn unified_stream_fallback_keeps_committed_output_and_disables_later_parsing() { + let events = collect( + ScriptedParser::new([ScriptedStep::Error { + committed: text("committed"), + reset_text: "buffered".to_string(), + }]), + vec![decoded_delta("bad"), decoded_delta("later")], + ) + .await; + + assert_eq!( + events, + vec![ + AssistantEvent::TextDelta { + kind: AssistantBlockKind::Text, + delta: "committed".to_string(), + }, + AssistantEvent::TextDelta { + kind: AssistantBlockKind::Text, + delta: "buffered".to_string(), + }, + AssistantEvent::TextDelta { + kind: AssistantBlockKind::Text, + delta: "later".to_string(), + }, + ] + ); + } + + #[tokio::test] + async fn unified_stream_finish_error_closes_parser_without_reset_text() { + let events = collect( + ScriptedParser::new([ScriptedStep::Output(UnifiedParserOutput::default())]) + .with_finish_error("buffered"), + vec![finished_delta("")], + ) + .await; + + assert_eq!( + events, + vec![AssistantEvent::Done { + usage: vllm_llm::TokenUsage::default(), + finish_reason: crate::FinishReason::Stop(None), + kv_transfer_params: None, + }] + ); + } +} diff --git a/rust/src/chat/src/output/mod.rs b/rust/src/chat/src/output/mod.rs index d7b73c4e5e2..836b199eb9b 100644 --- a/rust/src/chat/src/output/mod.rs +++ b/rust/src/chat/src/output/mod.rs @@ -2,7 +2,6 @@ use std::pin::Pin; use std::sync::Arc; use futures::Stream; -use subenum::subenum; use trait_set::trait_set; use uuid::Uuid; use vllm_llm::TokenUsage; @@ -22,23 +21,19 @@ pub(crate) use harmony::validate_harmony_parser_overrides; /// Internal assistant event before final assembly. /// -/// - [`ContentEvent`]: subenum after reasoning parsing, carries only text content. -/// - [`AssistantEvent`]: full event after tool parsing, adds tool-call variants. -#[subenum(ContentEvent)] +/// Unified parsing produces these events, and structured assembly consumes +/// them to build public chat events. #[derive(Debug, Clone, PartialEq)] pub(crate) enum AssistantEvent { - #[subenum(ContentEvent)] Start { prompt_token_ids: Arc<[u32]>, prompt_logprobs: Option, }, - #[subenum(ContentEvent)] TextDelta { kind: AssistantBlockKind, delta: String, }, /// Per-decoded-update sample metadata: logprobs and/or output token IDs. - #[subenum(ContentEvent)] LogprobsDelta { logprobs: Option, token_ids: Vec, @@ -48,7 +43,6 @@ pub(crate) enum AssistantEvent { /// A delta for the arguments of the currently open tool call. Must follow a /// `ToolCallStart`. ToolCallArgumentsDelta { delta: String }, - #[subenum(ContentEvent)] Done { usage: TokenUsage, finish_reason: FinishReason, @@ -57,50 +51,6 @@ pub(crate) enum AssistantEvent { }, } -impl ContentEvent { - /// Convert a [`DecodedTextEvent`] into one or more [`ContentEvent`] values - /// by treating all text as plain (non-reasoning) content. - fn from_decoded_plain_text(event: DecodedTextEvent) -> Vec { - match event { - DecodedTextEvent::Start { - prompt_token_ids, - prompt_logprobs, - } => vec![Self::Start { - prompt_token_ids, - prompt_logprobs, - }], - DecodedTextEvent::TextDelta { - delta, - token_ids, - logprobs, - finished, - } => { - let mut events = Vec::new(); - if !delta.is_empty() { - events.push(Self::TextDelta { - kind: AssistantBlockKind::Text, - delta, - }); - } - if logprobs.is_some() || !token_ids.is_empty() { - events.push(Self::LogprobsDelta { - logprobs, - token_ids, - }); - } - if let Some(finished) = finished { - events.push(Self::Done { - usage: finished.usage, - finish_reason: finished.finish_reason, - kv_transfer_params: finished.kv_transfer_params, - }); - } - events - } - } - } -} - /// Boxed stream of decoded text events coming from [`vllm_text`]. pub type DynDecodedTextEventStream = Pin> + Send>>; /// Boxed stream of structured chat events exposed by [`crate::ChatLlm`]. diff --git a/rust/src/chat/src/parser/reasoning/mod.rs b/rust/src/chat/src/parser/reasoning/mod.rs index 7de8a9d5fa1..a414a8e3ba5 100644 --- a/rust/src/chat/src/parser/reasoning/mod.rs +++ b/rust/src/chat/src/parser/reasoning/mod.rs @@ -2,7 +2,7 @@ use std::sync::LazyLock; -pub use vllm_reasoning_parser::{ +pub use vllm_parser::reasoning::{ CohereCmdReasoningParser, DeepSeekR1ReasoningParser, DeepSeekV3ReasoningParser, DeepSeekV4ReasoningParser, Gemma4ReasoningParser, Glm45ReasoningParser, KimiK2ReasoningParser, KimiReasoningParser, MiniMaxM2ReasoningParser, MiniMaxM3ReasoningParser, @@ -34,7 +34,7 @@ pub mod names { /// Constructor signature for one registered reasoning parser implementation. type ReasoningParserCreator = - fn(DynTokenizer) -> vllm_reasoning_parser::Result>; + fn(DynTokenizer) -> vllm_parser::reasoning::Result>; /// Registry and model matcher for reasoning parsers. pub type ReasoningParserFactory = ParserFactory; diff --git a/rust/src/chat/src/parser/tool/mod.rs b/rust/src/chat/src/parser/tool/mod.rs index 7561aa071ac..9884d1aca2a 100644 --- a/rust/src/chat/src/parser/tool/mod.rs +++ b/rust/src/chat/src/parser/tool/mod.rs @@ -2,13 +2,12 @@ use std::sync::LazyLock; -pub use vllm_tool_parser::{ +pub use vllm_parser::tool::{ DeepSeekV3ToolParser, DeepSeekV4ToolParser, DeepSeekV31ToolParser, DeepSeekV32ToolParser, Gemma4ToolParser, Glm45MoeToolParser, Glm47MoeToolParser, Granite4ToolParser, HermesToolParser, HyV3ToolParser, Internlm2ToolParser, KimiK2ToolParser, Llama3JsonToolParser, MinimaxM2ToolParser, MinimaxM3ToolParser, MistralToolParser, Phi4MiniJsonToolParser, - Qwen3CoderToolParser, Qwen3XmlToolParser, ToolCallDelta, ToolParser, ToolParserError, - ToolParserOutput, + Qwen3CoderToolParser, Qwen3XmlToolParser, ToolParser, ToolParserError, }; use crate::parser::ParserFactory; @@ -41,7 +40,7 @@ pub mod names { } /// Constructor signature for one registered tool parser implementation. -type ToolParserCreator = fn(&[ChatTool]) -> vllm_tool_parser::Result>; +type ToolParserCreator = fn(&[ChatTool]) -> vllm_parser::tool::Result>; /// Registry and model matcher for tool parsers. pub type ToolParserFactory = ParserFactory; diff --git a/rust/src/chat/src/parser/tool/tests.rs b/rust/src/chat/src/parser/tool/tests.rs index c40500adc74..a630f9a951a 100644 --- a/rust/src/chat/src/parser/tool/tests.rs +++ b/rust/src/chat/src/parser/tool/tests.rs @@ -1,6 +1,6 @@ -use vllm_tool_parser::Result; +use vllm_parser::tool::{Result, ToolParserOutput}; -use super::{ToolParser, ToolParserFactory, ToolParserOutput, names}; +use super::{ToolParser, ToolParserFactory, names}; use crate::Error; use crate::request::ChatTool; diff --git a/rust/src/chat/src/request.rs b/rust/src/chat/src/request.rs index 51b2efebd41..72de3d87663 100644 --- a/rust/src/chat/src/request.rs +++ b/rust/src/chat/src/request.rs @@ -4,9 +4,9 @@ use llm_multimodal::ImageDetail; use serde::{Deserialize, Serialize}; use serde_json::Value; use vllm_engine_core_client::protocol::lora::LoraRequest; +pub use vllm_parser::tool::Tool as ChatTool; pub use vllm_text::SamplingParams; use vllm_text::TextDecodeOptions; -pub use vllm_tool_parser::Tool as ChatTool; use crate::AssistantMessageExt; use crate::error::{Error, Result}; diff --git a/rust/src/chat/tests/chat.rs b/rust/src/chat/tests/chat.rs index 611dbe23973..5c4a2c29b7d 100644 --- a/rust/src/chat/tests/chat.rs +++ b/rust/src/chat/tests/chat.rs @@ -1180,7 +1180,7 @@ async fn chat_stream_parses_tool_calls_automatically() { ), request_output( "chat-tool", - bytes_to_token_ids( + bytes_with_special_stop_token( b"\"arguments\":{\"city\":\"Paris\"}}\n", ), Some(EngineCoreFinishReason::Stop), diff --git a/rust/src/tool-parser/Cargo.toml b/rust/src/parser/Cargo.toml similarity index 96% rename from rust/src/tool-parser/Cargo.toml rename to rust/src/parser/Cargo.toml index c4363906aa0..67c74bb5601 100644 --- a/rust/src/tool-parser/Cargo.toml +++ b/rust/src/parser/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "vllm-tool-parser" +name = "vllm-parser" version.workspace = true edition.workspace = true license.workspace = true @@ -13,6 +13,7 @@ serde.workspace = true serde_json.workspace = true thiserror.workspace = true thiserror-ext.workspace = true +vllm-tokenizer.workspace = true winnow.workspace = true xgrammar-structural-tag.workspace = true diff --git a/rust/src/tool-parser/benches/deepseek_v3.rs b/rust/src/parser/benches/deepseek_v3.rs similarity index 96% rename from rust/src/tool-parser/benches/deepseek_v3.rs rename to rust/src/parser/benches/deepseek_v3.rs index 75d2e417ace..4d1ea337a76 100644 --- a/rust/src/tool-parser/benches/deepseek_v3.rs +++ b/rust/src/parser/benches/deepseek_v3.rs @@ -2,8 +2,8 @@ use std::time::Duration; use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main}; use tool_parser::parsers::DeepSeekParser as ExternalDeepSeekParser; -use vllm_tool_parser::test_utils::{split_by_chars, test_tools}; -use vllm_tool_parser::{DeepSeekV3ToolParser, Tool, ToolParser}; +use vllm_parser::tool::test_utils::{split_by_chars, test_tools}; +use vllm_parser::tool::{DeepSeekV3ToolParser, Tool, ToolParser}; mod utils; use utils::{feed_external_parser, feed_parser, openai_tools}; diff --git a/rust/src/tool-parser/benches/deepseek_v31.rs b/rust/src/parser/benches/deepseek_v31.rs similarity index 96% rename from rust/src/tool-parser/benches/deepseek_v31.rs rename to rust/src/parser/benches/deepseek_v31.rs index bb6d029baff..a6f17c9f017 100644 --- a/rust/src/tool-parser/benches/deepseek_v31.rs +++ b/rust/src/parser/benches/deepseek_v31.rs @@ -2,8 +2,8 @@ use std::time::Duration; use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main}; use tool_parser::parsers::DeepSeek31Parser as ExternalDeepSeek31Parser; -use vllm_tool_parser::test_utils::{split_by_chars, test_tools}; -use vllm_tool_parser::{DeepSeekV31ToolParser, Tool, ToolParser}; +use vllm_parser::tool::test_utils::{split_by_chars, test_tools}; +use vllm_parser::tool::{DeepSeekV31ToolParser, Tool, ToolParser}; mod utils; use utils::{feed_external_parser, feed_parser, openai_tools}; diff --git a/rust/src/tool-parser/benches/deepseek_v32.rs b/rust/src/parser/benches/deepseek_v32.rs similarity index 96% rename from rust/src/tool-parser/benches/deepseek_v32.rs rename to rust/src/parser/benches/deepseek_v32.rs index c7a8346120d..1e770d9b136 100644 --- a/rust/src/tool-parser/benches/deepseek_v32.rs +++ b/rust/src/parser/benches/deepseek_v32.rs @@ -1,8 +1,8 @@ use std::time::Duration; use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main}; -use vllm_tool_parser::test_utils::{split_by_chars, test_tools}; -use vllm_tool_parser::{DeepSeekV32ToolParser, Tool, ToolParser}; +use vllm_parser::tool::test_utils::{split_by_chars, test_tools}; +use vllm_parser::tool::{DeepSeekV32ToolParser, Tool, ToolParser}; mod utils; use utils::feed_parser; diff --git a/rust/src/tool-parser/benches/gemma4.rs b/rust/src/parser/benches/gemma4.rs similarity index 97% rename from rust/src/tool-parser/benches/gemma4.rs rename to rust/src/parser/benches/gemma4.rs index c4e8f966c2b..761f8d4e235 100644 --- a/rust/src/tool-parser/benches/gemma4.rs +++ b/rust/src/parser/benches/gemma4.rs @@ -1,8 +1,8 @@ use std::time::Duration; use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main}; -use vllm_tool_parser::test_utils::{split_by_chars, test_tools}; -use vllm_tool_parser::{Gemma4ToolParser, Tool, ToolParser}; +use vllm_parser::tool::test_utils::{split_by_chars, test_tools}; +use vllm_parser::tool::{Gemma4ToolParser, Tool, ToolParser}; mod utils; use utils::feed_parser; diff --git a/rust/src/tool-parser/benches/glm45_moe.rs b/rust/src/parser/benches/glm45_moe.rs similarity index 97% rename from rust/src/tool-parser/benches/glm45_moe.rs rename to rust/src/parser/benches/glm45_moe.rs index 8486885eceb..a55a9e83ac0 100644 --- a/rust/src/tool-parser/benches/glm45_moe.rs +++ b/rust/src/parser/benches/glm45_moe.rs @@ -2,8 +2,8 @@ use std::time::Duration; use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main}; use tool_parser::parsers::Glm4MoeParser as ExternalGlm4MoeParser; -use vllm_tool_parser::test_utils::{split_by_chars, test_tools}; -use vllm_tool_parser::{Glm45MoeToolParser, Glm47MoeToolParser, Tool, ToolParser}; +use vllm_parser::tool::test_utils::{split_by_chars, test_tools}; +use vllm_parser::tool::{Glm45MoeToolParser, Glm47MoeToolParser, Tool, ToolParser}; mod utils; use utils::{feed_external_parser, feed_parser, openai_tools}; diff --git a/rust/src/tool-parser/benches/kimi_k2.rs b/rust/src/parser/benches/kimi_k2.rs similarity index 97% rename from rust/src/tool-parser/benches/kimi_k2.rs rename to rust/src/parser/benches/kimi_k2.rs index 5a80f660673..ab4c98399aa 100644 --- a/rust/src/tool-parser/benches/kimi_k2.rs +++ b/rust/src/parser/benches/kimi_k2.rs @@ -2,8 +2,8 @@ use std::time::Duration; use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main}; use tool_parser::parsers::KimiK2Parser as ExternalKimiK2Parser; -use vllm_tool_parser::test_utils::{split_by_chars, test_tools}; -use vllm_tool_parser::{KimiK2ToolParser, Tool, ToolParser}; +use vllm_parser::tool::test_utils::{split_by_chars, test_tools}; +use vllm_parser::tool::{KimiK2ToolParser, Tool, ToolParser}; mod utils; use utils::{feed_external_parser, feed_parser, openai_tools}; diff --git a/rust/src/tool-parser/benches/llama3_json.rs b/rust/src/parser/benches/llama3_json.rs similarity index 96% rename from rust/src/tool-parser/benches/llama3_json.rs rename to rust/src/parser/benches/llama3_json.rs index 03b5b54ee78..1126daf6f7d 100644 --- a/rust/src/tool-parser/benches/llama3_json.rs +++ b/rust/src/parser/benches/llama3_json.rs @@ -2,8 +2,8 @@ use std::time::Duration; use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main}; use tool_parser::parsers::LlamaParser as ExternalLlamaParser; -use vllm_tool_parser::test_utils::{split_by_chars, test_tools}; -use vllm_tool_parser::{Llama3JsonToolParser, Tool, ToolParser}; +use vllm_parser::tool::test_utils::{split_by_chars, test_tools}; +use vllm_parser::tool::{Llama3JsonToolParser, Tool, ToolParser}; mod utils; use utils::{feed_external_parser, feed_parser, openai_tools}; diff --git a/rust/src/tool-parser/benches/minimax_m2.rs b/rust/src/parser/benches/minimax_m2.rs similarity index 97% rename from rust/src/tool-parser/benches/minimax_m2.rs rename to rust/src/parser/benches/minimax_m2.rs index 4ad20400934..734d7437fbc 100644 --- a/rust/src/tool-parser/benches/minimax_m2.rs +++ b/rust/src/parser/benches/minimax_m2.rs @@ -2,8 +2,8 @@ use std::time::Duration; use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main}; use tool_parser::parsers::MinimaxM2Parser as ExternalMinimaxM2Parser; -use vllm_tool_parser::test_utils::{split_by_chars, test_tools}; -use vllm_tool_parser::{MinimaxM2ToolParser, Tool, ToolParser}; +use vllm_parser::tool::test_utils::{split_by_chars, test_tools}; +use vllm_parser::tool::{MinimaxM2ToolParser, Tool, ToolParser}; mod utils; use utils::{feed_external_parser, feed_parser, openai_tools}; diff --git a/rust/src/tool-parser/benches/qwen3_coder.rs b/rust/src/parser/benches/qwen3_coder.rs similarity index 97% rename from rust/src/tool-parser/benches/qwen3_coder.rs rename to rust/src/parser/benches/qwen3_coder.rs index b4f26ac5cdb..9d70937728f 100644 --- a/rust/src/tool-parser/benches/qwen3_coder.rs +++ b/rust/src/parser/benches/qwen3_coder.rs @@ -2,8 +2,8 @@ use std::time::Duration; use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main}; use tool_parser::parsers::QwenCoderParser as ExternalQwenCoderParser; -use vllm_tool_parser::test_utils::{split_by_chars, test_tools}; -use vllm_tool_parser::{Qwen3CoderToolParser, Tool, ToolParser}; +use vllm_parser::tool::test_utils::{split_by_chars, test_tools}; +use vllm_parser::tool::{Qwen3CoderToolParser, Tool, ToolParser}; mod utils; use utils::{feed_external_parser, feed_parser, openai_tools}; diff --git a/rust/src/tool-parser/benches/qwen3_xml.rs b/rust/src/parser/benches/qwen3_xml.rs similarity index 96% rename from rust/src/tool-parser/benches/qwen3_xml.rs rename to rust/src/parser/benches/qwen3_xml.rs index f2e37551dda..59ea0de47dd 100644 --- a/rust/src/tool-parser/benches/qwen3_xml.rs +++ b/rust/src/parser/benches/qwen3_xml.rs @@ -2,8 +2,8 @@ use std::time::Duration; use criterion::{BatchSize, Criterion, Throughput, black_box, criterion_group, criterion_main}; use tool_parser::parsers::QwenParser as ExternalQwenParser; -use vllm_tool_parser::test_utils::{split_by_chars, test_tools}; -use vllm_tool_parser::{Qwen3XmlToolParser, Tool, ToolParser}; +use vllm_parser::tool::test_utils::{split_by_chars, test_tools}; +use vllm_parser::tool::{Qwen3XmlToolParser, Tool, ToolParser}; mod utils; use utils::{feed_external_parser, feed_parser, openai_tools}; diff --git a/rust/src/tool-parser/benches/utils/mod.rs b/rust/src/parser/benches/utils/mod.rs similarity index 94% rename from rust/src/tool-parser/benches/utils/mod.rs rename to rust/src/parser/benches/utils/mod.rs index a0ad768f115..1acd1e51c0f 100644 --- a/rust/src/tool-parser/benches/utils/mod.rs +++ b/rust/src/parser/benches/utils/mod.rs @@ -3,8 +3,8 @@ use futures::FutureExt as _; use openai_protocol::common::{Function as OpenAiFunction, Tool as OpenAiTool}; use tool_parser::traits::ToolParser as ExternalToolParser; -use vllm_tool_parser::test_utils::collect_stream; -use vllm_tool_parser::{Tool, ToolParser}; +use vllm_parser::tool::test_utils::collect_stream; +use vllm_parser::tool::{Tool, ToolParser}; pub(super) fn openai_tools(tools: &[Tool]) -> Vec { tools diff --git a/rust/src/tool-parser/python/Cargo.toml b/rust/src/parser/python/Cargo.toml similarity index 91% rename from rust/src/tool-parser/python/Cargo.toml rename to rust/src/parser/python/Cargo.toml index c029ad90135..aadae5638f9 100644 --- a/rust/src/tool-parser/python/Cargo.toml +++ b/rust/src/parser/python/Cargo.toml @@ -13,7 +13,7 @@ pyo3.workspace = true pythonize = { workspace = true, features = ["serde_json"] } serde_json.workspace = true thiserror-ext.workspace = true -vllm-tool-parser.workspace = true +vllm-parser.workspace = true [lints] workspace = true diff --git a/rust/src/tool-parser/python/src/lib.rs b/rust/src/parser/python/src/lib.rs similarity index 98% rename from rust/src/tool-parser/python/src/lib.rs rename to rust/src/parser/python/src/lib.rs index e5ae0fa7b69..e988ff3442b 100644 --- a/rust/src/tool-parser/python/src/lib.rs +++ b/rust/src/parser/python/src/lib.rs @@ -1,4 +1,4 @@ -//! Thin PyO3 bindings for `vllm_tool_parser`. +//! Thin PyO3 bindings for `vllm_parser::tool`. //! //! This crate exposes the Rust tool parser trait and data shapes to Python //! while keeping parser state, grammar, and schema-aware argument conversion in @@ -11,7 +11,7 @@ use pyo3::types::{PyAny, PyModule}; use pythonize::{depythonize, pythonize}; use serde_json::Value; use thiserror_ext::AsReport as _; -use vllm_tool_parser::{Tool, ToolCallDelta, ToolParser, ToolParserOutput}; +use vllm_parser::tool::{Tool, ToolCallDelta, ToolParser, ToolParserOutput}; macro_rules! tool_parser_factory { ($($parser:ident),+ $(,)?) => { @@ -22,7 +22,7 @@ macro_rules! tool_parser_factory { match name { $( stringify!($parser) => { - ::create(tools) + ::create(tools) } )+ _ => { diff --git a/rust/src/parser/src/lib.rs b/rust/src/parser/src/lib.rs new file mode 100644 index 00000000000..5ba2cf60edd --- /dev/null +++ b/rust/src/parser/src/lib.rs @@ -0,0 +1,5 @@ +//! Streaming parsers for chat completions. + +pub mod reasoning; +pub mod tool; +pub mod unified; diff --git a/rust/src/reasoning-parser/src/cohere_cmd.rs b/rust/src/parser/src/reasoning/cohere_cmd.rs similarity index 100% rename from rust/src/reasoning-parser/src/cohere_cmd.rs rename to rust/src/parser/src/reasoning/cohere_cmd.rs diff --git a/rust/src/reasoning-parser/src/deepseek_r1.rs b/rust/src/parser/src/reasoning/deepseek_r1.rs similarity index 100% rename from rust/src/reasoning-parser/src/deepseek_r1.rs rename to rust/src/parser/src/reasoning/deepseek_r1.rs diff --git a/rust/src/reasoning-parser/src/delimited.rs b/rust/src/parser/src/reasoning/delimited.rs similarity index 100% rename from rust/src/reasoning-parser/src/delimited.rs rename to rust/src/parser/src/reasoning/delimited.rs diff --git a/rust/src/reasoning-parser/src/gemma4.rs b/rust/src/parser/src/reasoning/gemma4.rs similarity index 99% rename from rust/src/reasoning-parser/src/gemma4.rs rename to rust/src/parser/src/reasoning/gemma4.rs index 86824f2ad40..ac5a6a17165 100644 --- a/rust/src/reasoning-parser/src/gemma4.rs +++ b/rust/src/parser/src/reasoning/gemma4.rs @@ -119,7 +119,7 @@ mod tests { use vllm_tokenizer::Tokenizer; use super::Gemma4ReasoningParser; - use crate::ReasoningParser; + use crate::reasoning::ReasoningParser; struct FakeTokenizer; diff --git a/rust/src/reasoning-parser/src/kimi.rs b/rust/src/parser/src/reasoning/kimi.rs similarity index 100% rename from rust/src/reasoning-parser/src/kimi.rs rename to rust/src/parser/src/reasoning/kimi.rs diff --git a/rust/src/reasoning-parser/src/minimax_m3.rs b/rust/src/parser/src/reasoning/minimax_m3.rs similarity index 100% rename from rust/src/reasoning-parser/src/minimax_m3.rs rename to rust/src/parser/src/reasoning/minimax_m3.rs diff --git a/rust/src/reasoning-parser/src/lib.rs b/rust/src/parser/src/reasoning/mod.rs similarity index 100% rename from rust/src/reasoning-parser/src/lib.rs rename to rust/src/parser/src/reasoning/mod.rs diff --git a/rust/src/reasoning-parser/src/qwen3.rs b/rust/src/parser/src/reasoning/qwen3.rs similarity index 100% rename from rust/src/reasoning-parser/src/qwen3.rs rename to rust/src/parser/src/reasoning/qwen3.rs diff --git a/rust/src/reasoning-parser/src/seed_oss.rs b/rust/src/parser/src/reasoning/seed_oss.rs similarity index 98% rename from rust/src/reasoning-parser/src/seed_oss.rs rename to rust/src/parser/src/reasoning/seed_oss.rs index f514b43a89f..eb996f8477c 100644 --- a/rust/src/reasoning-parser/src/seed_oss.rs +++ b/rust/src/parser/src/reasoning/seed_oss.rs @@ -49,7 +49,7 @@ mod tests { use std::sync::Arc; use super::SeedOssReasoningParser; - use crate::{ReasoningParser, tests::FakeTokenizer}; + use crate::reasoning::{ReasoningParser, tests::FakeTokenizer}; #[test] fn without_prompt_markers_expects_start_token() { diff --git a/rust/src/reasoning-parser/src/step3p5.rs b/rust/src/parser/src/reasoning/step3p5.rs similarity index 99% rename from rust/src/reasoning-parser/src/step3p5.rs rename to rust/src/parser/src/reasoning/step3p5.rs index e369531c92c..d66506538a1 100644 --- a/rust/src/reasoning-parser/src/step3p5.rs +++ b/rust/src/parser/src/reasoning/step3p5.rs @@ -127,7 +127,7 @@ mod tests { use std::sync::Arc; use super::Step3p5ReasoningParser; - use crate::{ReasoningParser, tests::FakeTokenizer}; + use crate::reasoning::{ReasoningParser, tests::FakeTokenizer}; #[test] fn picks_up_prompt_start_boundary() { diff --git a/rust/src/reasoning-parser/src/tests.rs b/rust/src/parser/src/reasoning/tests.rs similarity index 100% rename from rust/src/reasoning-parser/src/tests.rs rename to rust/src/parser/src/reasoning/tests.rs diff --git a/rust/src/tool-parser/src/deepseek_dsml/deepseek_v32.rs b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v32.rs similarity index 98% rename from rust/src/tool-parser/src/deepseek_dsml/deepseek_v32.rs rename to rust/src/parser/src/tool/deepseek_dsml/deepseek_v32.rs index bc636c6035a..201b9dcba8b 100644 --- a/rust/src/tool-parser/src/deepseek_dsml/deepseek_v32.rs +++ b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v32.rs @@ -1,5 +1,5 @@ use super::{DeepSeekDsmlToolParser, DsmlTokens}; -use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput}; +use crate::tool::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput}; /// Tool parser for DeepSeek V3.2 models. /// @@ -67,8 +67,8 @@ mod tests { use thiserror_ext::AsReport; use super::DeepSeekV32ToolParser; - use crate::test_utils::{collect_stream, split_by_chars, test_tools}; - use crate::{ToolParser, ToolParserTestExt as _}; + use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools}; + use crate::tool::{ToolParser, ToolParserTestExt as _}; fn build_tool_call(function_name: &str, params: &[(&str, &str)]) -> String { let params = params diff --git a/rust/src/tool-parser/src/deepseek_dsml/deepseek_v4.rs b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v4.rs similarity index 95% rename from rust/src/tool-parser/src/deepseek_dsml/deepseek_v4.rs rename to rust/src/parser/src/tool/deepseek_dsml/deepseek_v4.rs index 9047b24ced7..344dfff5542 100644 --- a/rust/src/tool-parser/src/deepseek_dsml/deepseek_v4.rs +++ b/rust/src/parser/src/tool/deepseek_dsml/deepseek_v4.rs @@ -1,5 +1,5 @@ use super::{DeepSeekDsmlToolParser, DsmlTokens}; -use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput}; +use crate::tool::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput}; /// Tool parser for DeepSeek V4 models. /// @@ -69,8 +69,8 @@ mod tests { use serde_json::{Value, json}; use super::DeepSeekV4ToolParser; - use crate::test_utils::{collect_stream, test_tools}; - use crate::{StructuralTagModel, ToolParser, ToolParserTestExt as _}; + use crate::tool::test_utils::{collect_stream, test_tools}; + use crate::tool::{StructuralTagModel, ToolParser, ToolParserTestExt as _}; fn build_tool_call(function_name: &str, params: &[(&str, &str)]) -> String { let params = params diff --git a/rust/src/tool-parser/src/deepseek_dsml/mod.rs b/rust/src/parser/src/tool/deepseek_dsml/mod.rs similarity index 99% rename from rust/src/tool-parser/src/deepseek_dsml/mod.rs rename to rust/src/parser/src/tool/deepseek_dsml/mod.rs index 1a2031dd3d7..add70c8d5a9 100644 --- a/rust/src/tool-parser/src/deepseek_dsml/mod.rs +++ b/rust/src/parser/src/tool/deepseek_dsml/mod.rs @@ -7,7 +7,7 @@ use winnow::token::{literal, rest, take_until}; use super::parameters::ToolSchemas; use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker}; use super::{Result, ToolCallDelta, ToolParserOutput}; -use crate::Tool; +use crate::tool::Tool; mod deepseek_v32; mod deepseek_v4; diff --git a/rust/src/tool-parser/src/deepseek_json/deepseek_v3.rs b/rust/src/parser/src/tool/deepseek_json/deepseek_v3.rs similarity index 96% rename from rust/src/tool-parser/src/deepseek_json/deepseek_v3.rs rename to rust/src/parser/src/tool/deepseek_json/deepseek_v3.rs index 6d6062432ab..5b5147450c1 100644 --- a/rust/src/tool-parser/src/deepseek_json/deepseek_v3.rs +++ b/rust/src/parser/src/tool/deepseek_json/deepseek_v3.rs @@ -1,5 +1,5 @@ use super::{DeepSeekJsonFormat, DeepSeekJsonToolParser}; -use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput}; +use crate::tool::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput}; /// Tool parser for DeepSeek V3 JSON-fenced tool calls. /// @@ -55,12 +55,12 @@ mod tests { use thiserror_ext::AsReport; use super::DeepSeekV3ToolParser; - use crate::deepseek_json::{ + use crate::tool::deepseek_json::{ TOOL_CALL_SEPARATOR, TOOL_CALL_START, TOOL_CALLS_END, TOOL_CALLS_START, V3_ARGUMENT_END, V3_JSON_START, }; - use crate::test_utils::{collect_stream, split_by_chars, test_tools}; - use crate::{ToolParser, ToolParserOutput, ToolParserTestExt as _}; + use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools}; + use crate::tool::{ToolParser, ToolParserOutput, ToolParserTestExt as _}; fn v3_tool_call(function_name: &str, arguments: &str) -> String { format!( diff --git a/rust/src/tool-parser/src/deepseek_json/deepseek_v31.rs b/rust/src/parser/src/tool/deepseek_json/deepseek_v31.rs similarity index 96% rename from rust/src/tool-parser/src/deepseek_json/deepseek_v31.rs rename to rust/src/parser/src/tool/deepseek_json/deepseek_v31.rs index 33b362439a8..bf89fb4e841 100644 --- a/rust/src/tool-parser/src/deepseek_json/deepseek_v31.rs +++ b/rust/src/parser/src/tool/deepseek_json/deepseek_v31.rs @@ -1,5 +1,5 @@ use super::{DeepSeekJsonFormat, DeepSeekJsonToolParser}; -use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput}; +use crate::tool::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput}; /// Tool parser for DeepSeek V3.1 raw JSON tool calls. /// @@ -51,11 +51,11 @@ mod tests { use thiserror_ext::AsReport; use super::DeepSeekV31ToolParser; - use crate::deepseek_json::{ + use crate::tool::deepseek_json::{ TOOL_CALL_END, TOOL_CALL_SEPARATOR, TOOL_CALL_START, TOOL_CALLS_END, TOOL_CALLS_START, }; - use crate::test_utils::{collect_stream, split_by_chars, test_tools}; - use crate::{ToolParser, ToolParserOutput, ToolParserTestExt as _}; + use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools}; + use crate::tool::{ToolParser, ToolParserOutput, ToolParserTestExt as _}; fn v31_tool_call(function_name: &str, arguments: &str) -> String { format!("{TOOL_CALL_START}{function_name}{TOOL_CALL_SEPARATOR}{arguments}{TOOL_CALL_END}") diff --git a/rust/src/tool-parser/src/deepseek_json/mod.rs b/rust/src/parser/src/tool/deepseek_json/mod.rs similarity index 100% rename from rust/src/tool-parser/src/deepseek_json/mod.rs rename to rust/src/parser/src/tool/deepseek_json/mod.rs diff --git a/rust/src/tool-parser/src/error.rs b/rust/src/parser/src/tool/error.rs similarity index 87% rename from rust/src/tool-parser/src/error.rs rename to rust/src/parser/src/tool/error.rs index 0ac4a02c658..6a64c257d8c 100644 --- a/rust/src/tool-parser/src/error.rs +++ b/rust/src/parser/src/tool/error.rs @@ -6,7 +6,7 @@ pub type Result = std::result::Result; /// Errors produced while creating or running tool parsers. #[derive(Debug, Error, Macro)] -#[thiserror_ext(macro(path = "crate::error"))] +#[thiserror_ext(macro(path = "crate::tool::error"))] pub enum ToolParserError { #[error("tool parser parsing failed: {message}")] ParsingFailed { message: String }, diff --git a/rust/src/tool-parser/src/gemma4.rs b/rust/src/parser/src/tool/gemma4.rs similarity index 99% rename from rust/src/tool-parser/src/gemma4.rs rename to rust/src/parser/src/tool/gemma4.rs index 2fad84574c0..09d79fd8bcf 100644 --- a/rust/src/tool-parser/src/gemma4.rs +++ b/rust/src/parser/src/tool/gemma4.rs @@ -8,7 +8,7 @@ use winnow::token::{literal, take_till, take_until}; use super::utils::{incomplete, parse_buffered_event, partial_prefix_len, safe_text_len}; use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput}; -use crate::Tool; +use crate::tool::Tool; const TOOL_CALL_START: &str = "<|tool_call>"; const TOOL_CALL_END: &str = ""; @@ -428,7 +428,7 @@ mod tests { Gemma4ToolParser, ToolCallDelta, ToolParser, ToolParserOutput, gemma4_array_content, parse_gemma4_args, }; - use crate::{Tool, ToolParserTestExt as _}; + use crate::tool::{Tool, ToolParserTestExt as _}; fn parse_gemma4_array(array: &str) -> super::Result> { let mut input = array; diff --git a/rust/src/tool-parser/src/glm_xml/glm45_moe.rs b/rust/src/parser/src/tool/glm_xml/glm45_moe.rs similarity index 94% rename from rust/src/tool-parser/src/glm_xml/glm45_moe.rs rename to rust/src/parser/src/tool/glm_xml/glm45_moe.rs index 2a2d2e03813..a8d1ea0f19e 100644 --- a/rust/src/tool-parser/src/glm_xml/glm45_moe.rs +++ b/rust/src/parser/src/tool/glm_xml/glm45_moe.rs @@ -1,5 +1,5 @@ use super::{GlmXmlToolParser, Separator}; -use crate::{Result, Tool, ToolParser, ToolParserOutput}; +use crate::tool::{Result, Tool, ToolParser, ToolParserOutput}; /// Tool parser for GLM-4.5/4.6 MoE XML-style tool calls. /// diff --git a/rust/src/tool-parser/src/glm_xml/glm47_moe.rs b/rust/src/parser/src/tool/glm_xml/glm47_moe.rs similarity index 95% rename from rust/src/tool-parser/src/glm_xml/glm47_moe.rs rename to rust/src/parser/src/tool/glm_xml/glm47_moe.rs index 74afd6c250b..0e8135fdc52 100644 --- a/rust/src/tool-parser/src/glm_xml/glm47_moe.rs +++ b/rust/src/parser/src/tool/glm_xml/glm47_moe.rs @@ -1,5 +1,5 @@ use super::{GlmXmlToolParser, Separator}; -use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput}; +use crate::tool::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput}; /// Tool parser for GLM-4.7 MoE XML-style tool calls. /// @@ -44,8 +44,8 @@ mod tests { use serde_json::{Value, json}; use super::Glm47MoeToolParser; - use crate::ToolParserTestExt as _; - use crate::test_utils::{collect_stream, split_by_chars, test_tools}; + use crate::tool::ToolParserTestExt as _; + use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools}; fn glm47_tool_call(function_name: &str, params: &[(&str, &str)]) -> String { let params = params diff --git a/rust/src/tool-parser/src/glm_xml/mod.rs b/rust/src/parser/src/tool/glm_xml/mod.rs similarity index 98% rename from rust/src/tool-parser/src/glm_xml/mod.rs rename to rust/src/parser/src/tool/glm_xml/mod.rs index ceeb9a75173..7b4cacbb99e 100644 --- a/rust/src/tool-parser/src/glm_xml/mod.rs +++ b/rust/src/parser/src/tool/glm_xml/mod.rs @@ -7,7 +7,7 @@ use winnow::token::{literal, rest, take_until, take_while}; use super::parameters::ToolSchemas; use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker}; use super::{Result, ToolCallDelta, ToolParserOutput}; -use crate::Tool; +use crate::tool::Tool; mod glm45_moe; mod glm47_moe; @@ -264,8 +264,8 @@ mod tests { use thiserror_ext::AsReport; use super::Glm45MoeToolParser; - use crate::test_utils::{collect_stream, split_by_chars, test_tools}; - use crate::{ToolParser, ToolParserTestExt as _}; + use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools}; + use crate::tool::{ToolParser, ToolParserTestExt as _}; fn glm45_tool_call(function_name: &str, params: &[(&str, &str)]) -> String { let params = params diff --git a/rust/src/tool-parser/src/hy_v3.rs b/rust/src/parser/src/tool/hy_v3.rs similarity index 99% rename from rust/src/tool-parser/src/hy_v3.rs rename to rust/src/parser/src/tool/hy_v3.rs index c0cf9446348..94b0c3a9308 100644 --- a/rust/src/tool-parser/src/hy_v3.rs +++ b/rust/src/parser/src/tool/hy_v3.rs @@ -7,7 +7,7 @@ use winnow::token::{literal, rest, take_until}; use super::parameters::ToolSchemas; use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker}; use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput}; -use crate::{StructuralTagModel, Tool}; +use crate::tool::{StructuralTagModel, Tool}; const TOOL_CALLS_START: &str = ""; const TOOL_CALLS_END: &str = ""; @@ -249,8 +249,8 @@ mod tests { use thiserror_ext::AsReport; use super::{HyV3ToolParser, ToolParser}; - use crate::test_utils::{collect_stream, split_by_chars, test_tools}; - use crate::{ToolParserOutput, ToolParserTestExt as _}; + use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools}; + use crate::tool::{ToolParserOutput, ToolParserTestExt as _}; fn build_tool_call(function_name: &str, params: &[(&str, &str)]) -> String { let params = params diff --git a/rust/src/tool-parser/src/json/granite4.rs b/rust/src/parser/src/tool/json/granite4.rs similarity index 98% rename from rust/src/tool-parser/src/json/granite4.rs rename to rust/src/parser/src/tool/json/granite4.rs index a70c0645400..112bd5660e5 100644 --- a/rust/src/tool-parser/src/json/granite4.rs +++ b/rust/src/parser/src/tool/json/granite4.rs @@ -8,10 +8,10 @@ use super::{ JsonToolCallConfig, JsonToolCallEvent, JsonToolCallWhitespace, JsonToolInput, tool_call_header_event, }; -use crate::utils::{ +use crate::tool::utils::{ JsonObjectScanState, json_str, parse_buffered_event, safe_text_len, take_json_object, }; -use crate::{Result, Tool, ToolCallDelta, ToolParser, ToolParserOutput}; +use crate::tool::{Result, Tool, ToolCallDelta, ToolParser, ToolParserOutput}; const TOOL_CALL_START: &str = ""; const TOOL_CALL_END: &str = ""; @@ -279,8 +279,8 @@ mod tests { use thiserror_ext::AsReport; use super::Granite4ToolParser; - use crate::test_utils::{collect_stream, split_by_chars, test_tools}; - use crate::{ToolParser, ToolParserOutput, ToolParserTestExt as _}; + use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools}; + use crate::tool::{ToolParser, ToolParserOutput, ToolParserTestExt as _}; #[test] fn granite4_parse_complete_without_tool_call_keeps_text() { diff --git a/rust/src/tool-parser/src/json/hermes.rs b/rust/src/parser/src/tool/json/hermes.rs similarity index 96% rename from rust/src/tool-parser/src/json/hermes.rs rename to rust/src/parser/src/tool/json/hermes.rs index 04635185176..227c0fec16a 100644 --- a/rust/src/tool-parser/src/json/hermes.rs +++ b/rust/src/parser/src/tool/json/hermes.rs @@ -1,5 +1,5 @@ use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace}; -use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput}; +use crate::tool::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput}; const HERMES_CONFIG: JsonToolCallConfig = JsonToolCallConfig { parser_name: "Hermes", @@ -68,8 +68,8 @@ mod tests { use thiserror_ext::AsReport; use super::HermesToolParser; - use crate::test_utils::{collect_stream, split_by_chars, test_tools}; - use crate::{ToolParser, ToolParserOutput, ToolParserTestExt as _}; + use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools}; + use crate::tool::{ToolParser, ToolParserOutput, ToolParserTestExt as _}; fn build_tool_call(function_name: &str, arguments: &str) -> String { format!(r#"{{"name":"{function_name}","arguments":{arguments}}}"#) diff --git a/rust/src/tool-parser/src/json/internlm2.rs b/rust/src/parser/src/tool/json/internlm2.rs similarity index 98% rename from rust/src/tool-parser/src/json/internlm2.rs rename to rust/src/parser/src/tool/json/internlm2.rs index 8284a4d0e1d..da957fd0614 100644 --- a/rust/src/tool-parser/src/json/internlm2.rs +++ b/rust/src/parser/src/tool/json/internlm2.rs @@ -1,5 +1,5 @@ use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace}; -use crate::{Result, Tool, ToolParser, ToolParserOutput}; +use crate::tool::{Result, Tool, ToolParser, ToolParserOutput}; const INTERNLM2_CONFIG: JsonToolCallConfig = JsonToolCallConfig { parser_name: "InternLM2", @@ -123,8 +123,8 @@ mod tests { use thiserror_ext::AsReport; use super::Internlm2ToolParser; - use crate::test_utils::{collect_stream, split_by_chars, test_tools}; - use crate::{ToolParser, ToolParserOutput, ToolParserTestExt as _}; + use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools}; + use crate::tool::{ToolParser, ToolParserOutput, ToolParserTestExt as _}; const ACTION_START: &str = "<|action_start|><|plugin|>"; const ACTION_END: &str = "<|action_end|>"; diff --git a/rust/src/tool-parser/src/json/llama.rs b/rust/src/parser/src/tool/json/llama.rs similarity index 98% rename from rust/src/tool-parser/src/json/llama.rs rename to rust/src/parser/src/tool/json/llama.rs index d9456487d1f..7bfcb8ac1c9 100644 --- a/rust/src/tool-parser/src/json/llama.rs +++ b/rust/src/parser/src/tool/json/llama.rs @@ -8,8 +8,8 @@ use super::{ JsonToolCallConfig, JsonToolCallEvent, JsonToolCallWhitespace, JsonToolInput, argument_delta_event, tool_call_header_event, }; -use crate::utils::{JsonObjectScanState, parse_buffered_event}; -use crate::{Result, StructuralTagModel, Tool, ToolCallDelta, ToolParser, ToolParserOutput}; +use crate::tool::utils::{JsonObjectScanState, parse_buffered_event}; +use crate::tool::{Result, StructuralTagModel, Tool, ToolCallDelta, ToolParser, ToolParserOutput}; #[derive(Debug, Clone, PartialEq, Eq)] enum LlamaJsonMode { @@ -256,8 +256,8 @@ mod tests { use thiserror_ext::AsReport; use super::Llama3JsonToolParser; - use crate::test_utils::{collect_stream, split_by_chars, test_tools}; - use crate::{ToolParser, ToolParserOutput, ToolParserTestExt as _}; + use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools}; + use crate::tool::{ToolParser, ToolParserOutput, ToolParserTestExt as _}; fn build_tool_call(function_name: &str, parameters: &str) -> String { format!(r#"{{"name":"{function_name}","parameters":{parameters}}}"#) diff --git a/rust/src/tool-parser/src/json/mistral.rs b/rust/src/parser/src/tool/json/mistral.rs similarity index 97% rename from rust/src/tool-parser/src/json/mistral.rs rename to rust/src/parser/src/tool/json/mistral.rs index 9ca40fcaf97..c8d1f51ff71 100644 --- a/rust/src/tool-parser/src/json/mistral.rs +++ b/rust/src/parser/src/tool/json/mistral.rs @@ -1,5 +1,5 @@ use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace}; -use crate::{Result, Tool, ToolParser, ToolParserOutput}; +use crate::tool::{Result, Tool, ToolParser, ToolParserOutput}; const MISTRAL_CONFIG: JsonToolCallConfig = JsonToolCallConfig { parser_name: "Mistral", @@ -61,8 +61,8 @@ mod tests { use thiserror_ext::AsReport; use super::MistralToolParser; - use crate::test_utils::{collect_stream, split_by_chars, test_tools}; - use crate::{ToolParser, ToolParserOutput, ToolParserTestExt as _}; + use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools}; + use crate::tool::{ToolParser, ToolParserOutput, ToolParserTestExt as _}; fn build_tool_call(function_name: &str, arguments: &str) -> String { format!(r#"{{"name":"{function_name}","arguments":{arguments}}}"#) diff --git a/rust/src/tool-parser/src/json/mod.rs b/rust/src/parser/src/tool/json/mod.rs similarity index 99% rename from rust/src/tool-parser/src/json/mod.rs rename to rust/src/parser/src/tool/json/mod.rs index 748f7e49e4d..d7d42c0cecf 100644 --- a/rust/src/tool-parser/src/json/mod.rs +++ b/rust/src/parser/src/tool/json/mod.rs @@ -374,7 +374,7 @@ mod tests { use expect_test::expect; use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace}; - use crate::ToolParserOutput; + use crate::tool::ToolParserOutput; const DELIMITED_CONFIG: JsonToolCallConfig = JsonToolCallConfig { parser_name: "Delimited JSON", diff --git a/rust/src/tool-parser/src/json/phi4mini.rs b/rust/src/parser/src/tool/json/phi4mini.rs similarity index 98% rename from rust/src/tool-parser/src/json/phi4mini.rs rename to rust/src/parser/src/tool/json/phi4mini.rs index 463354b13c9..6e83d4374bf 100644 --- a/rust/src/tool-parser/src/json/phi4mini.rs +++ b/rust/src/parser/src/tool/json/phi4mini.rs @@ -1,5 +1,5 @@ use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace}; -use crate::{Result, Tool, ToolParser, ToolParserOutput}; +use crate::tool::{Result, Tool, ToolParser, ToolParserOutput}; const PHI4MINI_CONFIG: JsonToolCallConfig = JsonToolCallConfig { parser_name: "Phi4Mini", @@ -69,8 +69,8 @@ mod tests { use thiserror_ext::AsReport; use super::Phi4MiniJsonToolParser; - use crate::test_utils::{collect_stream, split_by_chars, test_tools}; - use crate::{ToolParser, ToolParserTestExt as _}; + use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools}; + use crate::tool::{ToolParser, ToolParserTestExt as _}; /// Build one phi-4-mini tool-call object: `{"name":..,"":}`. fn build_call(function_name: &str, args_key: &str, arguments: &str) -> String { diff --git a/rust/src/tool-parser/src/json/qwen.rs b/rust/src/parser/src/tool/json/qwen.rs similarity index 97% rename from rust/src/tool-parser/src/json/qwen.rs rename to rust/src/parser/src/tool/json/qwen.rs index dd943dfffc7..2339cf69fa0 100644 --- a/rust/src/tool-parser/src/json/qwen.rs +++ b/rust/src/parser/src/tool/json/qwen.rs @@ -1,5 +1,5 @@ use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace}; -use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput}; +use crate::tool::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput}; const QWEN_XML_CONFIG: JsonToolCallConfig = JsonToolCallConfig { parser_name: "Qwen XML", @@ -70,8 +70,8 @@ mod tests { use thiserror_ext::AsReport; use super::Qwen3XmlToolParser; - use crate::test_utils::{collect_stream, split_by_chars, test_tools}; - use crate::{ToolParser, ToolParserOutput, ToolParserTestExt as _}; + use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools}; + use crate::tool::{ToolParser, ToolParserOutput, ToolParserTestExt as _}; fn build_tool_call(function_name: &str, arguments: &str) -> String { format!( diff --git a/rust/src/tool-parser/src/kimi_k2.rs b/rust/src/parser/src/tool/kimi_k2.rs similarity index 99% rename from rust/src/tool-parser/src/kimi_k2.rs rename to rust/src/parser/src/tool/kimi_k2.rs index f83611ac79d..14a185011eb 100644 --- a/rust/src/tool-parser/src/kimi_k2.rs +++ b/rust/src/parser/src/tool/kimi_k2.rs @@ -8,7 +8,7 @@ use winnow::token::{literal, rest, take_until, take_while}; use super::utils::{JsonObjectScanState, parse_buffered_event, safe_text_len, take_json_object}; use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput}; -use crate::{StructuralTagModel, Tool}; +use crate::tool::{StructuralTagModel, Tool}; const TOOL_CALLS_START: &str = "<|tool_calls_section_begin|>"; const TOOL_CALLS_END: &str = "<|tool_calls_section_end|>"; @@ -339,8 +339,8 @@ mod tests { KimiK2ToolParser, TOOL_CALL_ARGUMENT_START, TOOL_CALL_END, TOOL_CALL_START, TOOL_CALLS_END, TOOL_CALLS_START, ToolParser, tool_header, }; - use crate::test_utils::{collect_stream, split_by_chars, test_tools}; - use crate::{ToolParserOutput, ToolParserTestExt as _}; + use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools}; + use crate::tool::{ToolParserOutput, ToolParserTestExt as _}; fn build_tool_call(function_name: &str, index: usize, arguments: &str) -> String { format!( diff --git a/rust/src/tool-parser/src/minimax_m2.rs b/rust/src/parser/src/tool/minimax_m2.rs similarity index 99% rename from rust/src/tool-parser/src/minimax_m2.rs rename to rust/src/parser/src/tool/minimax_m2.rs index 16e2b85525f..27519176b2c 100644 --- a/rust/src/tool-parser/src/minimax_m2.rs +++ b/rust/src/parser/src/tool/minimax_m2.rs @@ -7,7 +7,7 @@ use winnow::token::{literal, rest, take_until}; use super::parameters::ToolSchemas; use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker}; use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput}; -use crate::{StructuralTagModel, Tool}; +use crate::tool::{StructuralTagModel, Tool}; const TOOL_CALL_START: &str = ""; const TOOL_CALL_END: &str = ""; @@ -272,8 +272,8 @@ mod tests { use thiserror_ext::AsReport; use super::{MinimaxM2ToolParser, TOOL_CALL_END, TOOL_CALL_START, ToolParser}; - use crate::ToolParserTestExt as _; - use crate::test_utils::{collect_stream, split_by_chars, test_tools}; + use crate::tool::ToolParserTestExt as _; + use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools}; fn build_tool_block(invokes: &[(&str, Vec<(&str, &str)>)]) -> String { let invokes = invokes diff --git a/rust/src/tool-parser/src/minimax_m3.rs b/rust/src/parser/src/tool/minimax_m3.rs similarity index 99% rename from rust/src/tool-parser/src/minimax_m3.rs rename to rust/src/parser/src/tool/minimax_m3.rs index ad40a7f18b7..f6800790723 100644 --- a/rust/src/tool-parser/src/minimax_m3.rs +++ b/rust/src/parser/src/tool/minimax_m3.rs @@ -8,7 +8,7 @@ use winnow::token::{literal, rest, take_until}; use super::parameters::{ParamElement, ParamInput, ToolSchemas}; use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker}; use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput}; -use crate::Tool; +use crate::tool::Tool; const NAMESPACE: &str = "]<]minimax[>["; const TOOL_CALL_START: &str = "]<]minimax[>["; @@ -388,8 +388,8 @@ mod tests { ELEMENT_END_START, ELEMENT_START, INVOKE_END, INVOKE_START, MinimaxM3ToolParser, TOOL_CALL_END, TOOL_CALL_START, ToolParser, }; - use crate::test_utils::{collect_stream, split_by_chars, test_tools}; - use crate::{Tool, ToolParserTestExt as _}; + use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools}; + use crate::tool::{Tool, ToolParserTestExt as _}; fn element(name: &str, body: &str) -> String { format!("{ELEMENT_START}{name}>{body}{ELEMENT_END_START}{name}>") diff --git a/rust/src/tool-parser/src/lib.rs b/rust/src/parser/src/tool/mod.rs similarity index 98% rename from rust/src/tool-parser/src/lib.rs rename to rust/src/parser/src/tool/mod.rs index 6f785ee1d18..8c067f169ed 100644 --- a/rust/src/tool-parser/src/lib.rs +++ b/rust/src/parser/src/tool/mod.rs @@ -1,9 +1,9 @@ //! Streaming tool parsers for chat completions. #[macro_use] -mod error; +pub(crate) mod error; mod deepseek_dsml; -mod deepseek_json; +pub(crate) mod deepseek_json; mod gemma4; mod glm_xml; mod hy_v3; @@ -15,7 +15,7 @@ mod parameters; mod qwen_coder; #[cfg(any(test, feature = "test-util"))] pub mod test_utils; -mod utils; +pub(crate) mod utils; use std::collections::{BTreeMap, btree_map}; diff --git a/rust/src/tool-parser/src/parameters.rs b/rust/src/parser/src/tool/parameters.rs similarity index 99% rename from rust/src/tool-parser/src/parameters.rs rename to rust/src/parser/src/tool/parameters.rs index f857c147cb6..f5661456e3e 100644 --- a/rust/src/tool-parser/src/parameters.rs +++ b/rust/src/parser/src/tool/parameters.rs @@ -2,7 +2,7 @@ use std::collections::BTreeMap; use serde_json::{Map, Number, Value}; -use crate::Tool; +use crate::tool::Tool; /// Normalized parameter schemas for all tools in one request. #[derive(Debug, Clone, Default, PartialEq, Eq)] @@ -416,7 +416,7 @@ mod tests { use serde_json::{Value, json}; use super::{ParamElement, ParamInput, ToolSchema, ToolSchemas}; - use crate::Tool; + use crate::tool::Tool; fn test_tool(name: &str, parameters: serde_json::Value) -> Tool { Tool { diff --git a/rust/src/tool-parser/src/qwen_coder.rs b/rust/src/parser/src/tool/qwen_coder.rs similarity index 99% rename from rust/src/tool-parser/src/qwen_coder.rs rename to rust/src/parser/src/tool/qwen_coder.rs index 5e78d7ae520..c3c792d3d66 100644 --- a/rust/src/tool-parser/src/qwen_coder.rs +++ b/rust/src/parser/src/tool/qwen_coder.rs @@ -7,7 +7,7 @@ use winnow::token::{literal, take_until}; use super::parameters::ToolSchemas; use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker}; use super::{Result, StructuralTagModel, ToolCallDelta, ToolParser, ToolParserOutput}; -use crate::Tool; +use crate::tool::Tool; const TOOL_CALL_START: &str = ""; const TOOL_CALL_END: &str = ""; @@ -241,8 +241,8 @@ mod tests { use thiserror_ext::AsReport; use super::{Qwen3CoderToolParser, StructuralTagModel, ToolParser}; - use crate::test_utils::{collect_stream, split_by_chars, test_tools}; - use crate::{ToolParserOutput, ToolParserTestExt as _}; + use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools}; + use crate::tool::{ToolParserOutput, ToolParserTestExt as _}; fn build_tool_call(function_name: &str, params: &[(&str, &str)]) -> String { let params = params diff --git a/rust/src/tool-parser/src/test_utils.rs b/rust/src/parser/src/tool/test_utils.rs similarity index 98% rename from rust/src/tool-parser/src/test_utils.rs rename to rust/src/parser/src/tool/test_utils.rs index 70178756e4c..b16ef144a33 100644 --- a/rust/src/tool-parser/src/test_utils.rs +++ b/rust/src/parser/src/tool/test_utils.rs @@ -1,7 +1,7 @@ use serde_json::json; use super::{ToolParser, ToolParserOutput}; -use crate::{Tool, ToolParserTestExt as _}; +use crate::tool::{Tool, ToolParserTestExt as _}; /// Build a reusable set of function tools for parser unit tests. pub fn test_tools() -> Vec { diff --git a/rust/src/tool-parser/src/tests.rs b/rust/src/parser/src/tool/tests.rs similarity index 98% rename from rust/src/tool-parser/src/tests.rs rename to rust/src/parser/src/tool/tests.rs index fb9c8e62bf3..db7d0721c87 100644 --- a/rust/src/tool-parser/src/tests.rs +++ b/rust/src/parser/src/tool/tests.rs @@ -1,5 +1,5 @@ use super::{Result, Tool, ToolCallDelta, ToolParser, ToolParserOutput}; -use crate::ToolParserTestExt as _; +use crate::tool::ToolParserTestExt as _; struct DefaultParser; diff --git a/rust/src/tool-parser/src/utils.rs b/rust/src/parser/src/tool/utils.rs similarity index 100% rename from rust/src/tool-parser/src/utils.rs rename to rust/src/parser/src/tool/utils.rs diff --git a/rust/src/parser/src/unified/combined.rs b/rust/src/parser/src/unified/combined.rs new file mode 100644 index 00000000000..549753edbae --- /dev/null +++ b/rust/src/parser/src/unified/combined.rs @@ -0,0 +1,346 @@ +//! Adapter that combines reasoning and tool parsers. + +use vllm_tokenizer::DynTokenizer; + +use crate::reasoning::ReasoningParser; +use crate::tool::{StructuralTagModel, Tool, ToolParser, ToolParserOutput}; + +use super::{Result, UnifiedParser, UnifiedParserError, UnifiedParserOutput}; + +/// Unified parser that composes existing reasoning and tool parsers. +pub struct CombinedParser { + reasoning: Option>, + tool: Option>, +} + +impl CombinedParser { + /// Create a combined parser from optional reasoning and tool parsers. + pub fn new( + reasoning: Option>, + tool: Option>, + ) -> Self { + Self { reasoning, tool } + } + + /// Create a text-only combined parser. + pub fn plain_text_only() -> Self { + Self { + reasoning: None, + tool: None, + } + } + + fn parse_tool(&mut self, content: &str, output: &mut UnifiedParserOutput) -> Result<()> { + let Some(tool) = self.tool.as_mut() else { + output.push_text(content.to_string()); + return Ok(()); + }; + + // Preserve any tool output that was already produced before the error. + let mut tool_output = ToolParserOutput::default(); + let result = tool.parse_into(content, &mut tool_output); + output.append_tool_output(tool_output); + result?; + + Ok(()) + } + + fn flush_tool(&mut self) -> Result { + let Some(tool) = self.tool.as_mut() else { + return Ok(UnifiedParserOutput::default()); + }; + + let output = tool.finish()?; + let mut unified = UnifiedParserOutput::default(); + unified.append_tool_output(output); + Ok(unified) + } +} + +impl UnifiedParser for CombinedParser { + fn create(_tools: &[Tool], _tokenizer: DynTokenizer) -> Result> + where + Self: Sized + 'static, + { + Err(UnifiedParserError::CombinedParserConstructor) + } + + fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> { + if let Some(reasoning) = self.reasoning.as_mut() { + reasoning.initialize(prompt_token_ids)?; + } + Ok(()) + } + + fn preserve_special_tokens(&self) -> bool { + self.reasoning.as_ref().is_some_and(|parser| parser.preserve_special_tokens()) + || self.tool.as_ref().is_some_and(|parser| parser.preserve_special_tokens()) + } + + fn structural_tag_model(&self) -> Option { + self.tool.as_ref().and_then(|parser| parser.structural_tag_model()) + } + + fn tool_call_id(&self, tool_index: usize) -> Option<&str> { + self.tool.as_ref().and_then(|parser| parser.tool_call_id(tool_index)) + } + + fn parse_into(&mut self, delta: &str, output: &mut UnifiedParserOutput) -> Result<()> { + let Some(reasoning) = self.reasoning.as_mut() else { + return self.parse_tool(delta, output); + }; + + let reasoning_delta = reasoning.push(delta)?; + if let Some(reasoning) = reasoning_delta.reasoning { + output.push_reasoning(reasoning); + } + if let Some(content) = reasoning_delta.content { + self.parse_tool(&content, output)?; + } + Ok(()) + } + + fn finish(&mut self) -> Result { + let mut output = UnifiedParserOutput::default(); + if let Some(reasoning) = self.reasoning.as_mut() { + let reasoning_delta = reasoning.finish()?; + if let Some(reasoning) = reasoning_delta.reasoning { + output.push_reasoning(reasoning); + } + if let Some(content) = reasoning_delta.content { + self.parse_tool(&content, &mut output)?; + } + } + output.append(self.flush_tool()?); + Ok(output) + } + + fn reset(&mut self) -> String { + self.tool.as_mut().map_or_else(String::new, |parser| parser.reset()) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use vllm_tokenizer::Tokenizer; + + use super::CombinedParser; + use crate::reasoning::{Qwen3ReasoningParser, ReasoningDelta, ReasoningParser}; + use crate::tool::{Qwen3XmlToolParser, Tool, ToolParser}; + use crate::unified::{UnifiedParser, UnifiedParserEvent, UnifiedParserOutput}; + + struct FakeTokenizer; + + impl Tokenizer for FakeTokenizer { + fn encode( + &self, + text: &str, + _add_special_tokens: bool, + ) -> vllm_tokenizer::Result> { + Ok(text.chars().map(u32::from).collect()) + } + + fn decode( + &self, + token_ids: &[u32], + _skip_special_tokens: bool, + ) -> vllm_tokenizer::Result { + Ok(token_ids + .iter() + .map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}')) + .collect()) + } + + fn token_to_id(&self, token: &str) -> Option { + match token { + "" => Some(1), + "" => Some(2), + _ => None, + } + } + } + + fn test_tools() -> Vec { + vec![Tool { + name: "get_weather".to_string(), + description: None, + parameters: serde_json::json!({ + "type": "object", + "properties": { + "location": { "type": "string" } + }, + }), + strict: None, + }] + } + + fn collect(parser: &mut dyn UnifiedParser, chunks: &[&str]) -> UnifiedParserOutput { + let mut output = UnifiedParserOutput::default(); + for chunk in chunks { + parser.parse_into(chunk, &mut output).unwrap(); + } + output.append(parser.finish().unwrap()); + output + } + + struct PreserveReasoningParser; + + impl ReasoningParser for PreserveReasoningParser { + fn create( + _tokenizer: vllm_tokenizer::DynTokenizer, + ) -> crate::reasoning::Result> + where + Self: Sized + 'static, + { + Ok(Box::new(Self)) + } + + fn preserve_special_tokens(&self) -> bool { + true + } + + fn push(&mut self, delta: &str) -> crate::reasoning::Result { + Ok(ReasoningDelta { + reasoning: None, + content: Some(delta.to_string()), + }) + } + } + + struct PreserveToolParser; + + impl ToolParser for PreserveToolParser { + fn create(_tools: &[Tool]) -> crate::tool::Result> + where + Self: Sized + 'static, + { + Ok(Box::new(Self)) + } + + fn preserve_special_tokens(&self) -> bool { + true + } + + fn parse_into( + &mut self, + chunk: &str, + output: &mut crate::tool::ToolParserOutput, + ) -> crate::tool::Result<()> { + output.normal_text.push_str(chunk); + Ok(()) + } + + fn finish(&mut self) -> crate::tool::Result { + Ok(crate::tool::ToolParserOutput::default()) + } + + fn reset(&mut self) -> String { + String::new() + } + } + + struct PartialThenErrorToolParser; + + impl ToolParser for PartialThenErrorToolParser { + fn create(_tools: &[Tool]) -> crate::tool::Result> + where + Self: Sized + 'static, + { + Ok(Box::new(Self)) + } + + fn parse_into( + &mut self, + _chunk: &str, + output: &mut crate::tool::ToolParserOutput, + ) -> crate::tool::Result<()> { + output.normal_text.push_str("committed"); + Err(crate::tool::ToolParserError::ParsingFailed { + message: "synthetic failure".to_string(), + }) + } + + fn finish(&mut self) -> crate::tool::Result { + Ok(crate::tool::ToolParserOutput::default()) + } + + fn reset(&mut self) -> String { + String::new() + } + } + + #[test] + fn combined_parser_emits_reasoning_and_text() { + let tokenizer = Arc::new(FakeTokenizer); + let reasoning = Qwen3ReasoningParser::create(tokenizer).unwrap(); + let mut parser = CombinedParser::new(Some(reasoning), None); + + let output = collect(&mut parser, &["workanswer"]); + + assert_eq!( + output.events, + vec![ + UnifiedParserEvent::Reasoning("work".to_string()), + UnifiedParserEvent::Text("answer".to_string()), + ] + ); + } + + #[test] + fn combined_parser_emits_tool_calls_from_visible_content() { + let tool = Qwen3XmlToolParser::create(&test_tools()).unwrap(); + let mut parser = CombinedParser::new(None, Some(tool)); + assert!(matches!( + parser.structural_tag_model(), + Some(crate::tool::StructuralTagModel::Qwen3) + )); + + let output = collect( + &mut parser, + &[r#" +{"name":"get_weather","arguments":{"location":"Paris"}} +"#], + ); + + assert_eq!( + output.events, + vec![ + UnifiedParserEvent::ToolCall(crate::tool::ToolCallDelta { + tool_index: 0, + name: Some("get_weather".to_string()), + arguments: String::new(), + }), + UnifiedParserEvent::ToolCall(crate::tool::ToolCallDelta { + tool_index: 0, + name: None, + arguments: r#"{"location":"Paris"}"#.to_string(), + }), + ] + ); + } + + #[test] + fn combined_parser_preserves_tool_output_on_parse_error() { + let mut parser = CombinedParser::new(None, Some(Box::new(PartialThenErrorToolParser))); + let mut output = UnifiedParserOutput::default(); + + let error = parser.parse_into("bad", &mut output).unwrap_err(); + + assert!(matches!(error, crate::unified::UnifiedParserError::Tool(_))); + assert_eq!( + output.events, + vec![UnifiedParserEvent::Text("committed".to_string())] + ); + } + + #[test] + fn combined_parser_preserves_special_tokens_when_either_inner_parser_needs_it() { + let mut parser = CombinedParser::new(Some(Box::new(PreserveReasoningParser)), None); + assert!(parser.preserve_special_tokens()); + + parser = CombinedParser::new(None, Some(Box::new(PreserveToolParser))); + assert!(parser.preserve_special_tokens()); + } +} diff --git a/rust/src/parser/src/unified/mod.rs b/rust/src/parser/src/unified/mod.rs new file mode 100644 index 00000000000..a24ad6952cd --- /dev/null +++ b/rust/src/parser/src/unified/mod.rs @@ -0,0 +1,114 @@ +//! Unified parser interface for reasoning and tool-call deltas. + +mod combined; + +use thiserror::Error; +use vllm_tokenizer::DynTokenizer; + +pub use combined::CombinedParser; + +use crate::reasoning::ReasoningError; +use crate::tool::{StructuralTagModel, Tool, ToolCallDelta, ToolParserError, ToolParserOutput}; + +/// Result alias for unified parser operations. +pub type Result = std::result::Result; + +/// One parsed event emitted by a unified parser. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum UnifiedParserEvent { + /// Normal assistant-visible text. + Text(String), + /// Reasoning text hidden from the normal content stream. + Reasoning(String), + /// A tool-call update extracted from visible assistant text. + ToolCall(ToolCallDelta), +} + +/// Result of advancing unified parsing with one assistant-text input. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct UnifiedParserOutput { + /// Ordered parser events committed by this input. + pub events: Vec, +} + +impl UnifiedParserOutput { + /// Append one visible text event if `delta` is non-empty. + pub fn push_text(&mut self, delta: String) { + if delta.is_empty() { + return; + } + self.events.push(UnifiedParserEvent::Text(delta)); + } + + /// Append one reasoning text event if `delta` is non-empty. + pub fn push_reasoning(&mut self, delta: String) { + if delta.is_empty() { + return; + } + self.events.push(UnifiedParserEvent::Reasoning(delta)); + } + + /// Append parsed tool parser output as unified events. + pub fn append_tool_output(&mut self, output: ToolParserOutput) { + // TODO: make ToolParserOutput carry ordered events and remove this text-first flattening. + self.push_text(output.normal_text); + self.events.extend(output.calls.into_iter().map(UnifiedParserEvent::ToolCall)); + } + + /// Append another parser output onto this one. + pub fn append(&mut self, mut other: Self) { + self.events.append(&mut other.events); + } +} + +/// Incremental parser that extracts reasoning and tool-call events from assistant output. +pub trait UnifiedParser: Send { + /// Construct a boxed parser instance for one request stream. + fn create(tools: &[Tool], tokenizer: DynTokenizer) -> Result> + where + Self: Sized + 'static; + + /// Initialize parser state from prompt token IDs before output deltas arrive. + fn initialize(&mut self, _prompt_token_ids: &[u32]) -> Result<()> { + Ok(()) + } + + /// Return whether decoded output must preserve tokenizer special tokens. + fn preserve_special_tokens(&self) -> bool { + false + } + + /// Return the xgrammar structural-tag model used for strict tool calling. + fn structural_tag_model(&self) -> Option { + None + } + + /// Return the parser-provided ID for a tool call by index, if the model emitted one. + fn tool_call_id(&self, _tool_index: usize) -> Option<&str> { + None + } + + /// Feed one decoded text delta into the parser, appending committed output into `output`. + fn parse_into(&mut self, delta: &str, output: &mut UnifiedParserOutput) -> Result<()>; + + /// Flush any buffered parser state at end of stream. + fn finish(&mut self) -> Result { + Ok(UnifiedParserOutput::default()) + } + + /// Clear parser state and return currently uncommitted buffered text. + fn reset(&mut self) -> String { + String::new() + } +} + +/// Errors produced while creating or running unified parsers. +#[derive(Debug, Error)] +pub enum UnifiedParserError { + #[error("combined parser is constructed from split parser instances")] + CombinedParserConstructor, + #[error(transparent)] + Reasoning(#[from] ReasoningError), + #[error(transparent)] + Tool(#[from] ToolParserError), +} diff --git a/rust/src/reasoning-parser/Cargo.toml b/rust/src/reasoning-parser/Cargo.toml deleted file mode 100644 index d6500a7b0c1..00000000000 --- a/rust/src/reasoning-parser/Cargo.toml +++ /dev/null @@ -1,12 +0,0 @@ -[package] -name = "vllm-reasoning-parser" -version.workspace = true -edition.workspace = true -license.workspace = true - -[dependencies] -thiserror.workspace = true -vllm-tokenizer.workspace = true - -[lints] -workspace = true diff --git a/tools/build_rust.py b/tools/build_rust.py index e5c5d0bb2e4..b5951bfe576 100644 --- a/tools/build_rust.py +++ b/tools/build_rust.py @@ -27,7 +27,7 @@ def rust_extensions(*, optional: bool = False) -> list[RustExtension]: ), RustExtension( target="vllm._rust_tool_parser", - path="rust/src/tool-parser/python/Cargo.toml", + path="rust/src/parser/python/Cargo.toml", features=["pyo3/abi3-py38"], binding=Binding.PyO3, optional=optional,