diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 16329d1fb3a..5a05d9e0b5e 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -2591,6 +2591,7 @@ version = "2.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "328251e58ad8e415be6198888fc207502727dc77945806421ab34f35bf012e7d" dependencies = [ + "indexmap 2.13.0", "memo-map", "serde", "serde_json", @@ -5608,6 +5609,7 @@ dependencies = [ "minijinja", "minijinja-contrib", "openai-harmony", + "paste", "reqwest", "rmp-serde", "serde", @@ -5810,6 +5812,7 @@ dependencies = [ "serde", "serde_json", "serde_with", + "serial_test", "tempfile", "thiserror 2.0.18", "thiserror-ext", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 89e582dbc56..a43aa53862c 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -45,13 +45,14 @@ http-body = "1.0.1" itertools = "0.14.0" libc = "0.2.177" llm-multimodal = { git = "https://github.com/vllm-project/llm-multimodal", rev = "5b558989844d1c7af3e43d0f604069ffd9c06320" } -minijinja = { version = "2.0", features = ["unstable_machinery", "json", "builtins", "loader", "loop_controls"] } +minijinja = { version = "2.0", features = ["unstable_machinery", "json", "builtins", "loader", "loop_controls", "preserve_order"] } minijinja-contrib = { version = "2.0", features = ["pycompat"] } native-tls-vendored = { package = "native-tls", version = "0.2.18", features = ["vendored"] } ndarray = { version = "0.16.1", features = ["serde"] } openai-harmony = "0.0.8" openai-protocol = "1.6.0" parking_lot = "0.12.5" +paste = "1.0.15" prometheus-client = "0.24.0" prometheus-client-derive-encode = "0.5.0" prost = "0.14.3" @@ -65,11 +66,11 @@ rustc-hash = "1.1.0" serde = { version = "1.0.228", features = ["derive"] } serde-json-fmt = "0.1.0" serde_default = "0.2.0" -serde_json = "1.0.145" +serde_json = { version = "1.0.145", features = ["arbitrary_precision", "preserve_order"] } serde_repr = "0.1.20" serde_tuple = "1.1.3" serde_with = "3.18.0" -serial_test = "3.2.0" +serial_test = { version = "3.2.0", features = ["file_locks"] } socket2 = "0.6.3" subenum = "1.1.3" task-local = "0.1.1" diff --git a/rust/src/chat/Cargo.toml b/rust/src/chat/Cargo.toml index 4bb339d43ba..1548c6f5926 100644 --- a/rust/src/chat/Cargo.toml +++ b/rust/src/chat/Cargo.toml @@ -39,8 +39,9 @@ anyhow.workspace = true bytes.workspace = true clap.workspace = true expect-test.workspace = true +paste.workspace = true rmp-serde.workspace = true -serial_test = { workspace = true, features = ["file_locks"] } +serial_test.workspace = true tempfile.workspace = true tokio.workspace = true tracing-subscriber.workspace = true diff --git a/rust/src/chat/src/event.rs b/rust/src/chat/src/event.rs index 08603b43572..9eb8d35042b 100644 --- a/rust/src/chat/src/event.rs +++ b/rust/src/chat/src/event.rs @@ -53,6 +53,25 @@ impl AssistantContentBlock { _ => None, } } + + /// Return a copy of this block with leading and trailing whitespace trimmed from all text + /// fields and tool call arguments, or `None` if the resulting text would be empty. + pub fn trim(mut self) -> Option { + match &mut self { + Self::Text { text } | Self::Reasoning { text } => { + let trimmed_text = text.trim(); + if trimmed_text.is_empty() { + return None; + } else { + *text = trimmed_text.to_string(); + } + } + Self::ToolCall(call) => { + call.arguments = call.arguments.trim().to_string(); + } + } + Some(self) + } } #[easy_ext::ext(AssistantMessageExt)] @@ -119,6 +138,13 @@ impl AssistantMessage { pub(crate) fn push_block(&mut self, block: AssistantContentBlock) { self.content.push(block); } + + /// Return a copy of this message with leading and trailing whitespace trimmed from all text + /// fields and tool call arguments, and with any blocks that are empty after trimming removed. + pub fn trim(mut self) -> Self { + self.content = self.content.into_iter().filter_map(|block| block.trim()).collect(); + self + } } /// Streamed chat event emitted by [`crate::ChatEventStream`]. diff --git a/rust/src/chat/src/renderer/hf/tojson.rs b/rust/src/chat/src/renderer/hf/tojson.rs index 1c5c20f4765..f04c954a79f 100644 --- a/rust/src/chat/src/renderer/hf/tojson.rs +++ b/rust/src/chat/src/renderer/hf/tojson.rs @@ -1,7 +1,7 @@ use minijinja::value::{Kwargs, ViaDeserialize}; use minijinja::{Error as MinijinjaError, ErrorKind, Value}; use serde::Deserialize; -use serde_json::{self, Value as JsonValue}; +use serde_json::Value as JsonValue; use serde_json_fmt::{JsonFormat, JsonSyntaxError}; use thiserror_ext::AsReport; @@ -13,7 +13,7 @@ use thiserror_ext::AsReport; /// - extra kwargs such as `ensure_ascii`, `separators`, and `sort_keys` /// - Python-style `indent` handling pub(super) fn hf_tojson_filter( - value: Value, + ViaDeserialize(value): ViaDeserialize, kwargs: Kwargs, ) -> std::result::Result { let ensure_ascii = kwargs.get::>("ensure_ascii")?.unwrap_or(false); @@ -30,18 +30,11 @@ pub(super) fn hf_tojson_filter( kwargs.assert_all_used()?; - let json_value: serde_json::Value = serde_json::to_value(&value).map_err(|e| { - MinijinjaError::new( - ErrorKind::InvalidOperation, - format!("Failed to convert to JSON value: {e}"), - ) - })?; - let json_str = { let value_to_serialize = if sort_keys { - &sort_json_keys(&json_value) + &sort_json_keys(&value) } else { - &json_value + &value }; build_json_format(indent, separators.0, separators.1, ensure_ascii)? @@ -214,6 +207,14 @@ mod tests { assert_eq!(rendered, "{\"x\":[1,2]}"); } + #[test] + fn tojson_preserves_arbitrary_precision_number_spelling() { + let payload = serde_json::from_str(r#"{"x":2,"y":1.00}"#).unwrap(); + let rendered = render("{{ payload|tojson }}", payload); + + assert_eq!(rendered, "{\"x\": 2, \"y\": 1.00}"); + } + #[test] fn tojson_supports_negative_indent_as_newline_only() { let rendered = render("{{ payload|tojson(indent=-1) }}", json!([1, 2])); diff --git a/rust/src/chat/tests/roundtrip.rs b/rust/src/chat/tests/roundtrip.rs new file mode 100644 index 00000000000..74491cd0243 --- /dev/null +++ b/rust/src/chat/tests/roundtrip.rs @@ -0,0 +1,541 @@ +//! Text-level roundtrip tests for the real chat-template and output-processor pairing. +//! +//! The invariant under test is that a structured assistant message rendered as history can be +//! parsed from the generated assistant completion and then rendered back to the exact same +//! assistant-completion text. + +use std::pin::Pin; +use std::sync::Arc; + +use anyhow::{Context as _, Result, bail, ensure}; +use futures::{Stream, StreamExt as _, stream}; +use serde_json_fmt::JsonFormat as JsonFmt; +use serial_test::file_serial; +use vllm_chat::{ + AssistantContentBlock, AssistantMessage, AssistantMessageExt as _, AssistantToolCall, + ChatEvent, ChatMessage, ChatRequest, ChatRole, ChatTool, ChatToolChoice, FinishReason, + GenerationPromptMode, LoadModelBackendsOptions, NewChatOutputProcessorOptions, ParserSelection, + RendererSelection, load_model_backends, +}; +use vllm_text::{DecodedTextEvent, Finished, Prompt}; + +/// One model/parser configuration used to run the fixed roundtrip fixtures. +struct RoundtripCase { + /// Hugging Face model id resolved through the production backend loader. + model_id: &'static str, + /// Final assistant-history suffix rendered by the chat template but not + /// generated by the model body consumed by the output processor. + // TODO: we should adopt `ContinueFinalAssistant` mode to naturally handle this. + assistant_stop_suffix: &'static str, + /// Tool parser selection used by the output processor. + tool_call_parser: ParserSelection, + /// Reasoning parser selection used by the output processor. + reasoning_parser: ParserSelection, + /// JSON formatting expected after this model's template has materialized + /// tool-call arguments. + json_fmt: JsonFmt, +} + +impl RoundtripCase { + /// Qwen3 XML tool-call format with `qwen3` reasoning tags. + fn qwen3() -> Self { + Self { + model_id: "Qwen/Qwen3-0.6B", + assistant_stop_suffix: "<|im_end|>\n", + tool_call_parser: ParserSelection::Auto, + reasoning_parser: ParserSelection::Auto, + json_fmt: spaced_json_fmt(), + } + } + + /// Qwen3.5 coder-style JSON tool-call format with `qwen3` reasoning tags. + fn qwen35() -> Self { + Self { + model_id: "Qwen/Qwen3.5-4B", + assistant_stop_suffix: "<|im_end|>\n", + tool_call_parser: ParserSelection::Auto, + reasoning_parser: ParserSelection::Auto, + json_fmt: compact_json_fmt(), + } + } + + /// MiniMax M2.5 XML invoke format with `` reasoning tags. + fn minimax_m25() -> Self { + Self { + model_id: "MiniMaxAI/MiniMax-M2.5", + assistant_stop_suffix: "[e~[\n", + tool_call_parser: ParserSelection::Auto, + reasoning_parser: ParserSelection::Auto, + json_fmt: compact_json_fmt(), + } + } + + /// DeepSeek V4 DSML tool-call format. + fn deepseek_v4() -> Self { + Self { + model_id: "deepseek-ai/DeepSeek-V4-Flash", + assistant_stop_suffix: "<｜end▁of▁sentence｜>", + tool_call_parser: ParserSelection::Auto, + reasoning_parser: ParserSelection::Auto, + json_fmt: compact_json_fmt(), + } + } + + /// GLM-4.7 XML-like argument format with `` reasoning tags. + fn glm47() -> Self { + Self { + model_id: "zai-org/GLM-4.7-Flash", + assistant_stop_suffix: "", + tool_call_parser: ParserSelection::Auto, + reasoning_parser: ParserSelection::Auto, + json_fmt: compact_json_fmt(), + } + } + + /// Kimi K2.5 tool-call format with `` reasoning tags. + #[allow(dead_code)] + fn kimi_k25() -> Self { + Self { + model_id: "moonshotai/Kimi-K2.5", + assistant_stop_suffix: "<|im_end|>", + tool_call_parser: ParserSelection::Auto, + reasoning_parser: ParserSelection::Auto, + json_fmt: spaced_json_fmt(), + } + } +} + +macro_rules! roundtrip_tests { + ($($case:ident => [$($fixture:ident),* $(,)?]),+ $(,)?) => { + paste::paste! { + $( + $( + #[tokio::test] + #[file_serial([])] + async fn []() -> Result<()> { + [](RoundtripCase::$case()).await + } + )* + )+ + } + }; +} + +roundtrip_tests! { + qwen3 => [reasoning_and_content, tool_call_mix], + qwen35 => [reasoning_and_content, tool_call_mix], + minimax_m25 => [reasoning_and_content, tool_call_mix], + deepseek_v4 => [reasoning_and_content, tool_call_mix], + glm47 => [reasoning_and_content, tool_call_mix], + + // Note: Kimi K2.5 strips the reasoning content in history. + // TODO: we don't respect model-generated tool call id now so `tool_call_mix` cannot pass. + // kimi_k25 => [tool_call_mix], +} + +/// Run the fixed reasoning+content fixture for one model/parser case. +async fn run_roundtrip_reasoning_and_content(case: RoundtripCase) -> Result<()> { + let backends = load_roundtrip_backends(&case).await?; + let request = roundtrip_request( + "roundtrip-reasoning-content", + vec![ChatMessage::text(ChatRole::User, "What is 2 + 2?")], + Vec::new(), + ); + let expected_reasoning = "Need compute 2 + 2 directly."; + let expected_text = "The answer is 4."; + + let result = run_roundtrip( + &case, + &backends, + &request, + AssistantMessage { + content: vec![ + AssistantContentBlock::Reasoning { + text: expected_reasoning.to_string(), + }, + AssistantContentBlock::Text { + text: expected_text.to_string(), + }, + ], + }, + ) + .await?; + + assert_eq!( + result.parsed_message.reasoning().as_deref().map(str::trim), + Some(expected_reasoning) + ); + assert_eq!(result.parsed_message.text().trim(), expected_text); + assert_eq!(result.parsed_message.tool_calls().count(), 0); + + assert_eq!( + result.rerendered_closed_completion, + result.closed_completion + ); + + Ok(()) +} + +/// Run the fixed reasoning+multiple-tools fixture for one model/parser case. +async fn run_roundtrip_tool_call_mix(case: RoundtripCase) -> Result<()> { + let backends = load_roundtrip_backends(&case).await?; + let request = roundtrip_request( + "roundtrip-reasoning-tools", + vec![ChatMessage::text( + ChatRole::User, + "Check Shanghai weather and add 1.00 plus 2.", + )], + test_tools(), + ); + let expected_reasoning = "Need call the weather and add tools."; + let expected_text = "I will call the tools."; + + let result = run_roundtrip( + &case, + &backends, + &request, + AssistantMessage { + content: vec![ + AssistantContentBlock::Reasoning { + text: expected_reasoning.to_string(), + }, + AssistantContentBlock::Text { + text: expected_text.to_string(), + }, + AssistantContentBlock::ToolCall(AssistantToolCall { + id: "functions.get_weather:0".to_string(), + name: "get_weather".to_string(), + arguments: r#"{"location":"Shanghai"}"#.to_string(), + }), + AssistantContentBlock::ToolCall(AssistantToolCall { + id: "functions.add:1".to_string(), + name: "add".to_string(), + // Intentionally use a non-lexical order of keys and a different number + // formatting style to verify text-level fidelity of the roundtrip. + arguments: r#"{"y":1.00,"x":2}"#.to_string(), + }), + ], + }, + ) + .await?; + + assert_eq!( + result.parsed_message.reasoning().as_deref().map(str::trim), + Some(expected_reasoning) + ); + assert_eq!(result.parsed_message.text().trim(), expected_text); + + let tool_calls = result.parsed_message.tool_calls().collect::>(); + assert_eq!( + tool_calls.len(), + 2, + "parsed message: {:#?}", + result.parsed_message + ); + assert_eq!(tool_calls[0].name, "get_weather"); + assert_eq!( + tool_calls[0].arguments, + expected_arguments(&case, r#"{"location": "Shanghai"}"#)?, + ); + assert_eq!(tool_calls[1].name, "add"); + assert_eq!( + tool_calls[1].arguments, + expected_arguments(&case, r#"{"y": 1.00, "x": 2}"#)?, + ); + + assert_eq!( + result.rerendered_closed_completion, + result.closed_completion + ); + + Ok(()) +} + +/// Compact JSON argument formatting used by JSON-native parsers/renderers. +fn compact_json_fmt() -> JsonFmt { + JsonFmt::new() +} + +/// Python `json.dumps`-style compact formatting with a space after commas and +/// colons. +fn spaced_json_fmt() -> JsonFmt { + JsonFmt::new() + .comma(", ") + .expect("literal comma separator is valid JSON") + .colon(": ") + .expect("literal colon separator is valid JSON") +} + +/// Parse and format expected tool-call arguments from raw JSON text. +/// Pass in a raw JSON string instead of a structured value to ensure the exact precision and +/// formatting of numbers are preserved. +fn expected_arguments(case: &RoundtripCase, raw_json: &str) -> Result { + let value: serde_json::Value = + serde_json::from_str(raw_json).context("invalid expected tool-call arguments")?; + + case.json_fmt + .format_to_string(&value) + .context("failed to format expected tool-call arguments") +} + +/// Load the real model chat/text backend for one roundtrip case. +async fn load_roundtrip_backends(case: &RoundtripCase) -> Result { + load_model_backends( + case.model_id, + LoadModelBackendsOptions { + renderer: RendererSelection::Auto, + ..Default::default() + }, + ) + .await + .with_context(|| format!("failed to load HF model files for {}", case.model_id)) +} + +/// Roundtrip artifacts needed for semantic and exact-text assertions. +struct RoundtripResult { + /// Final assistant message reconstructed by the output processor. + parsed_message: AssistantMessage, + /// Assistant-completion suffix cut from rendering the expected assistant as + /// history. + closed_completion: String, + /// Assistant-completion suffix cut after rendering the parsed assistant + /// back as history. + rerendered_closed_completion: String, +} + +/// Render, parse, and rerender one assistant turn through the production +/// renderer/output-processor boundary. +async fn run_roundtrip( + case: &RoundtripCase, + backends: &vllm_chat::LoadedModelBackends, + request: &ChatRequest, + assistant: AssistantMessage, +) -> Result { + let renderer = backends.chat_backend.chat_renderer(); + let (prompt, closed_completion_text) = + render_closed_completion(renderer.as_ref(), request, &assistant)?; + let completion_body = closed_completion_text + .strip_suffix(case.assistant_stop_suffix) + .with_context(|| { + format!( + "closed assistant completion did not end with {:?}: {:?}", + case.assistant_stop_suffix, closed_completion_text + ) + })?; + + let parsed_message = + parse_completion(case, backends, request, &prompt, completion_body).await?; + let (_, rerendered_closed_completion) = + render_closed_completion(renderer.as_ref(), request, &parsed_message)?; + + Ok(RoundtripResult { + parsed_message, + closed_completion: closed_completion_text, + rerendered_closed_completion, + }) +} + +/// Render `history` as a production prompt and `history + assistant` as closed +/// history, then return the production prompt and assistant-completion suffix. +fn render_closed_completion( + renderer: &dyn vllm_chat::ChatRenderer, + base_request: &ChatRequest, + assistant: &AssistantMessage, +) -> Result<(String, String)> { + let mut prompt_request = base_request.clone(); + prompt_request.chat_options.generation_prompt_mode = GenerationPromptMode::StartNewAssistant; + let prompt = render_text(renderer, &prompt_request).context("failed to render prompt")?; + + let mut full_request = base_request.clone(); + full_request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt; + full_request.messages.push(ChatMessage::from(assistant.clone())); + let full = render_text(renderer, &full_request).context("failed to render full prompt")?; + + ensure!( + full.starts_with(&prompt), + "full prompt must extend production prompt\nprompt: {prompt:?}\nfull: {full:?}" + ); + let completion = full[prompt.len()..].to_string(); + + Ok((prompt, completion)) +} + +/// Render one chat request and require a text prompt. +fn render_text(renderer: &dyn vllm_chat::ChatRenderer, request: &ChatRequest) -> Result { + match renderer.render(request)?.prompt { + Prompt::Text(text) => Ok(text), + other => bail!("roundtrip tests expect text prompts, got {other:?}"), + } +} + +/// Feed one rendered assistant completion body into the real output processor +/// and collect its terminal assistant message. +async fn parse_completion( + case: &RoundtripCase, + backends: &vllm_chat::LoadedModelBackends, + base_request: &ChatRequest, + prompt: &str, + completion_body: &str, +) -> Result { + let tokenizer = backends.text_backend.tokenizer(); + let prompt_token_ids = tokenizer + .encode(prompt, base_request.add_special_tokens) + .context("failed to encode rendered prompt")?; + + let mut request = base_request.clone(); + let processor = backends.chat_backend.new_chat_output_processor( + &mut request, + NewChatOutputProcessorOptions { + tool_call_parser: &case.tool_call_parser, + reasoning_parser: &case.reasoning_parser, + }, + )?; + + let decoded = decoded_completion_stream(prompt_token_ids, completion_body); + let mut events = processor.process(decoded)?; + + while let Some(event) = events.next().await { + if let ChatEvent::Done { message, .. } = event? { + // TODO: currently our parsers are not very strict about preserving or trimming + // whitespace, so we trim here to avoid roundtrip failures due to + // insignificant whitespace differences. However, this may hurt token-level + // fidelity so we should consider improving them. + return Ok(message.trim()); + } + } + + bail!("output processor finished without a Done event") +} + +/// Build a decoded-text stream from an already-rendered completion body. +/// +/// The first event carries real prompt token ids so reasoning parsers can +/// initialize from the same prompt boundary production uses. Completion text is +/// split into small chunks to exercise streaming parser state across marker +/// and JSON boundaries. +fn decoded_completion_stream( + prompt_token_ids: Vec, + completion_body: &str, +) -> Pin> + Send>> { + let prompt_token_count = prompt_token_ids.len(); + let mut events = vec![DecodedTextEvent::Start { + prompt_token_ids: Arc::from(prompt_token_ids.into_boxed_slice()), + prompt_logprobs: None, + }]; + + let chunks = split_by_chars(completion_body, 7); + if chunks.is_empty() { + events.push({ + DecodedTextEvent::TextDelta { + delta: String::new(), + token_ids: Vec::new(), + logprobs: None, + finished: Some(Finished { + prompt_token_count: 0, + output_token_count: 0, + finish_reason: FinishReason::stop_eos(), + kv_transfer_params: None, + }), + } + }); + } else { + let last_index = chunks.len() - 1; + for (index, chunk) in chunks.into_iter().enumerate() { + let finished = (index == last_index).then(|| Finished { + prompt_token_count, + output_token_count: completion_body.chars().count(), + finish_reason: FinishReason::stop_eos(), + kv_transfer_params: None, + }); + events.push(DecodedTextEvent::TextDelta { + delta: chunk, + token_ids: Vec::new(), + logprobs: None, + finished, + }); + } + } + + stream::iter(events).map(Ok).boxed() +} + +/// Split text into chunks containing at most `chunk_chars` Unicode scalar +/// values. +fn split_by_chars(text: &str, chunk_chars: usize) -> Vec { + let mut chunks = Vec::new(); + let mut start = 0; + let mut count = 0; + + for (index, _) in text.char_indices() { + if count == chunk_chars { + chunks.push(text[start..index].to_string()); + start = index; + count = 0; + } + count += 1; + } + + if start < text.len() { + chunks.push(text[start..].to_string()); + } + + chunks +} + +/// Build a chat request fixture with parser-enabling tool-choice semantics. +fn roundtrip_request( + request_id: impl Into, + messages: Vec, + tools: Vec, +) -> ChatRequest { + let mut request = ChatRequest { + request_id: request_id.into(), + messages, + tool_choice: if tools.is_empty() { + ChatToolChoice::None + } else { + ChatToolChoice::Auto + }, + tools, + ..ChatRequest::for_test() + }; + + // Enable thinking for some models so that rendering and parsing the reasoning block is + // exercised in the roundtrip. + for key in ["thinking", "enable_thinking"] { + request.chat_options.template_kwargs.insert(key.to_string(), true.into()); + } + + request +} + +/// Return the function tools used by the multiple-tool-call fixture. +fn test_tools() -> Vec { + vec![ + ChatTool { + name: "get_weather".to_string(), + description: Some("Get weather for a location".to_string()), + parameters: serde_json::json!({ + "type": "object", + "properties": { + "location": { "type": "string" } + }, + "required": ["location"] + }), + strict: None, + }, + ChatTool { + name: "add".to_string(), + description: Some("Add two integers".to_string()), + parameters: serde_json::json!({ + "type": "object", + "properties": { + "y": { "type": "number" }, + "x": { "type": "number" } + }, + "required": ["y", "x"] + }), + strict: None, + }, + ] +} diff --git a/rust/src/text/Cargo.toml b/rust/src/text/Cargo.toml index 04202348182..8be9ee78764 100644 --- a/rust/src/text/Cargo.toml +++ b/rust/src/text/Cargo.toml @@ -26,6 +26,7 @@ vllm-tokenizer.workspace = true [dev-dependencies] expect-test.workspace = true futures.workspace = true +serial_test.workspace = true tempfile.workspace = true tokio.workspace = true vllm-llm = { workspace = true, features = ["test-util"] } diff --git a/rust/src/text/src/backend/hf/model_files.rs b/rust/src/text/src/backend/hf/model_files.rs index f4a66d30dae..7f84c90f39c 100644 --- a/rust/src/text/src/backend/hf/model_files.rs +++ b/rust/src/text/src/backend/hf/model_files.rs @@ -401,7 +401,7 @@ mod tests { } #[tokio::test] - #[ignore = "requires network access to Hugging Face and downloads the real Kimi K2.5 tokenizer"] + #[ignore = "too slow for CI and requires network access to Hugging Face"] async fn tiktoken_real_kimi_k25_tokenizer_files_load_and_handle_special_tokens() { let files = ResolvedModelFiles::new("moonshotai/Kimi-K2.5") .await diff --git a/rust/src/text/src/lower.rs b/rust/src/text/src/lower.rs index 7cbbd53dc6a..54ffe1ff4b8 100644 --- a/rust/src/text/src/lower.rs +++ b/rust/src/text/src/lower.rs @@ -235,6 +235,8 @@ fn merge_unique_token_ids( mod tests { use std::collections::BTreeSet; + use serial_test::file_serial; + use super::*; use crate::backend::hf::HfTextBackend; use crate::backend::{SamplingHints, TextBackend as _}; @@ -386,7 +388,7 @@ mod tests { } #[tokio::test] - #[ignore = "requires network access to Hugging Face"] + #[file_serial(hf_qwen3)] async fn lower_text_request_uses_real_qwen_generation_defaults() { let backend = HfTextBackend::from_model("Qwen/Qwen3-0.6B") .await @@ -410,12 +412,8 @@ mod tests { default_top_k: Some( 20, ), - default_min_p: Some( - 0.1, - ), - default_repetition_penalty: Some( - 1.2, - ), + default_min_p: None, + default_repetition_penalty: None, default_max_tokens: None, max_model_len: Some( 40960, @@ -439,10 +437,10 @@ mod tests { min_tokens: 0, logprobs: None, prompt_logprobs: None, - min_p: 0.1, + min_p: 0.0, frequency_penalty: 0.0, presence_penalty: 0.0, - repetition_penalty: 1.2, + repetition_penalty: 1.0, stop_token_ids: [ 151643, ], @@ -453,6 +451,13 @@ mod tests { 151643, 151645, }, + logit_bias: None, + allowed_token_ids: None, + bad_words_token_ids: None, + structured_outputs: None, + logprob_token_ids: None, + skip_reading_prefix_cache: None, + extra_args: None, } "#]] .assert_debug_eq(¶ms); diff --git a/rust/src/tool-parser/src/minimax_m2.rs b/rust/src/tool-parser/src/minimax_m2.rs index 34f2bc89b11..87e61e9979e 100644 --- a/rust/src/tool-parser/src/minimax_m2.rs +++ b/rust/src/tool-parser/src/minimax_m2.rs @@ -1,5 +1,5 @@ use winnow::ascii::{multispace0 as ws0, multispace1 as ws1}; -use winnow::combinator::{alt, delimited, repeat, seq, terminated}; +use winnow::combinator::{alt, delimited, eof, repeat, seq, terminated}; use winnow::prelude::*; use winnow::stream::Partial; use winnow::token::{literal, rest, take_until}; @@ -183,16 +183,17 @@ fn tool_block_end_event(input: &mut MinimaxM2Input<'_>) -> ModalResult) -> ModalResult { - let (name, raw_params) = seq!( + let (name, body) = seq!( _: ws0, _: literal(INVOKE_START), _: (ws1, literal("name=")), - attr_value, + partial_attr_value, _: literal(">"), - repeat(0.., terminated(parameter, ws0)), + take_until(0.., INVOKE_END), _: literal(INVOKE_END), ) .parse_next(input)?; + let raw_params = parse_invoke_params(body)?; Ok(MinimaxM2Event::Invoke { name: name.trim().to_string(), @@ -200,8 +201,14 @@ fn invoke_event(input: &mut MinimaxM2Input<'_>) -> ModalResult { }) } +/// Parse all parameter blocks inside a complete MiniMax M2 invoke body. +fn parse_invoke_params(invoke_body: &str) -> ModalResult> { + let mut input = invoke_body; + delimited(ws0, repeat(0.., terminated(parameter, ws0)), eof).parse_next(&mut input) +} + /// Parse a MiniMax M2 parameter block. -fn parameter(input: &mut MinimaxM2Input<'_>) -> ModalResult<(String, String)> { +fn parameter(input: &mut &str) -> ModalResult<(String, String)> { let (name, value) = seq!( _: literal(PARAMETER_START), _: (ws1, literal("name=")), @@ -216,7 +223,17 @@ fn parameter(input: &mut MinimaxM2Input<'_>) -> ModalResult<(String, String)> { } /// Parse a quoted or unquoted XML attribute value. -fn attr_value<'i>(input: &mut MinimaxM2Input<'i>) -> ModalResult<&'i str> { +fn attr_value<'i>(input: &mut &'i str) -> ModalResult<&'i str> { + alt(( + delimited(literal("\""), take_until(1.., "\""), literal("\"")), + delimited(literal("'"), take_until(1.., "'"), literal("'")), + take_until(1.., ">"), + )) + .parse_next(input) +} + +/// Parse a quoted or unquoted XML attribute value from partial streaming input. +fn partial_attr_value<'i>(input: &mut MinimaxM2Input<'i>) -> ModalResult<&'i str> { alt(( delimited(literal("\""), take_until(1.., "\""), literal("\"")), delimited(literal("'"), take_until(1.., "'"), literal("'")), @@ -470,6 +487,31 @@ mod tests { assert_eq!(result.calls[1].tool_index, 1); } + #[test] + fn minimax_m2_streaming_handles_template_whitespace_and_split_parameters() { + let text = concat!( + "I will call the tools.\n", + "\n", + "\n", + "Seattle\n", + "\n", + "\n", + "NYC\n", + "\n", + "", + ); + let chunks = split_by_chars(text, 7); + let mut parser = MinimaxM2ToolParser::new(&test_tools()); + let result = collect_stream(&mut parser, &chunks); + + assert_eq!(result.normal_text, "I will call the tools.\n"); + assert_eq!(result.calls.len(), 2); + assert_eq!(result.calls[0].tool_index, 0); + assert_eq!(result.calls[0].name.as_deref(), Some("get_weather")); + assert_eq!(result.calls[1].tool_index, 1); + assert_eq!(result.calls[1].name.as_deref(), Some("get_weather")); + } + #[test] fn minimax_m2_streaming_ignores_text_after_tool_block() { let text = format!( diff --git a/rust/src/tool-parser/src/parameters.rs b/rust/src/tool-parser/src/parameters.rs index cf21bad16cd..66c67eaab91 100644 --- a/rust/src/tool-parser/src/parameters.rs +++ b/rust/src/tool-parser/src/parameters.rs @@ -224,10 +224,11 @@ fn convert_value(param_type: &JsonParamType, value: &str) -> Option { /// Convert one raw string value to a JSON number. fn convert_number(value: &str) -> Option { - if let Ok(parsed) = value.parse::() { - return Some(Value::Number(Number::from(parsed))); - } - Number::from_f64(value.parse::().ok()?).map(Value::Number) + serde_json::from_str::(value) + .or_else(|_| value.parse::().map(Number::from)) + .or_else(|_| value.parse::().ok().and_then(Number::from_f64).ok_or(())) + .ok() + .map(Value::Number) } /// Convert one raw string value to a boolean. @@ -304,7 +305,7 @@ mod tests { } #[test] - fn number_conversion_parses_int_then_float() { + fn number_conversion_preserves_json_number_spelling_with_legacy_fallback() { let params = ToolSchema::from_schema(&json!({ "type": "object", "properties": { @@ -312,17 +313,23 @@ mod tests { } })); - assert_eq!(params.convert("value", "5"), json!(5)); - assert_eq!(params.convert("value", "5.0"), json!(5.0)); - assert_eq!(params.convert("value", "5."), json!(5.0)); - assert_eq!(params.convert("value", "+1"), json!(1)); - assert_eq!(params.convert("value", "+1.0"), json!(1.0)); + assert_eq!(converted_number_text(¶ms, "5"), "5"); + assert_eq!(converted_number_text(¶ms, "5.0"), "5.0"); + assert_eq!(converted_number_text(¶ms, "5.00"), "5.00"); + assert_eq!(converted_number_text(¶ms, "1e0"), "1e+0"); + assert_eq!(converted_number_text(¶ms, "5."), "5.0"); + assert_eq!(converted_number_text(¶ms, "+1"), "1"); + assert_eq!(converted_number_text(¶ms, "+1.0"), "1.0"); assert_eq!( - params.convert("value", "9223372036854775807.5"), - json!(9223372036854775808.0) + converted_number_text(¶ms, "9223372036854775807.5"), + "9223372036854775807.5" ); } + fn converted_number_text(params: &ToolSchema, value: &str) -> String { + serde_json::to_string(¶ms.convert("value", value)).unwrap() + } + #[test] fn converts_upstream_aliases() { let params = ToolSchema::from_schema(&json!({