diff --git a/rust/Cargo.lock b/rust/Cargo.lock index da782e41e68..7b6cb928e19 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -5310,6 +5310,7 @@ dependencies = [ "vllm-llm", "vllm-metrics", "vllm-text", + "vllm-tokenizer", "zeromq", ] diff --git a/rust/src/chat/Cargo.toml b/rust/src/chat/Cargo.toml index 40498bac3fe..95ce5ff2e42 100644 --- a/rust/src/chat/Cargo.toml +++ b/rust/src/chat/Cargo.toml @@ -50,6 +50,7 @@ tokio.workspace = true tracing-subscriber.workspace = true uuid.workspace = true vllm-engine-core-client = { workspace = true, features = ["test-util"] } +vllm-tokenizer = { workspace = true, features = ["test-utils"] } zeromq.workspace = true [lints] diff --git a/rust/src/chat/src/backend/hf.rs b/rust/src/chat/src/backend/hf.rs index 9dff25ea49b..9997bc24736 100644 --- a/rust/src/chat/src/backend/hf.rs +++ b/rust/src/chat/src/backend/hf.rs @@ -154,7 +154,8 @@ mod tests { use thiserror_ext::AsReport as _; use vllm_text::Prompt; use vllm_text::backend::hf::TokenizerSource; - use vllm_text::tokenizer::{DynTokenizer, Tokenizer}; + use vllm_text::tokenizer::DynTokenizer; + use vllm_tokenizer::test_utils::TestTokenizer; use super::HfChatBackend; use crate::backend::{ChatBackend, LoadModelBackendsOptions, NewChatOutputProcessorOptions}; @@ -196,32 +197,8 @@ mod tests { } } - struct TestTokenizer; - - impl Tokenizer for TestTokenizer { - fn encode( - &self, - _text: &str, - _add_special_tokens: bool, - ) -> vllm_text::tokenizer::Result> { - Ok(Vec::new()) - } - - fn decode( - &self, - _token_ids: &[u32], - _skip_special_tokens: bool, - ) -> vllm_text::tokenizer::Result { - Ok(String::new()) - } - - fn token_to_id(&self, _token: &str) -> Option { - None - } - } - fn test_tokenizer() -> DynTokenizer { - Arc::new(TestTokenizer) + Arc::new(TestTokenizer::new()) } fn backend_for_selection( diff --git a/rust/src/chat/src/multimodal.rs b/rust/src/chat/src/multimodal.rs index 024e4b63ea3..8fd44376f99 100644 --- a/rust/src/chat/src/multimodal.rs +++ b/rust/src/chat/src/multimodal.rs @@ -563,7 +563,7 @@ mod tests { use llm_multimodal::TokenId; use vllm_engine_core_client::protocol::tensor::WireArrayData; - use vllm_text::tokenizer::{IncrementalDecoder, Tokenizer, TokenizerError}; + use vllm_tokenizer::test_utils::TestTokenizer; use super::*; @@ -574,60 +574,14 @@ mod tests { const LLAMA4_TILE_X_SEPARATOR_ID: u32 = 200093; const LLAMA4_TILE_Y_SEPARATOR_ID: u32 = 200094; - struct TestTokenizer; - - impl Tokenizer for TestTokenizer { - fn encode( - &self, - text: &str, - _add_special_tokens: bool, - ) -> std::result::Result, TokenizerError> { - Ok(match text { - "<|image|>" => vec![LLAMA4_IMAGE_ID], - text => text.bytes().map(u32::from).collect(), - }) - } - - fn decode( - &self, - _token_ids: &[u32], - _skip_special_tokens: bool, - ) -> std::result::Result { - Ok(String::new()) - } - - fn token_to_id(&self, token: &str) -> Option { - match token { - "<|image_start|>" => Some(LLAMA4_IMAGE_START_ID), - "<|image_end|>" => Some(LLAMA4_IMAGE_END_ID), - "<|image|>" => Some(LLAMA4_IMAGE_ID), - "<|patch|>" => Some(LLAMA4_PATCH_ID), - "<|tile_x_separator|>" => Some(LLAMA4_TILE_X_SEPARATOR_ID), - "<|tile_y_separator|>" => Some(LLAMA4_TILE_Y_SEPARATOR_ID), - _ => None, - } - } - - fn id_to_token(&self, id: u32) -> Option { - match id { - LLAMA4_IMAGE_START_ID => Some("<|image_start|>".to_string()), - LLAMA4_IMAGE_END_ID => Some("<|image_end|>".to_string()), - LLAMA4_IMAGE_ID => Some("<|image|>".to_string()), - LLAMA4_PATCH_ID => Some("<|patch|>".to_string()), - LLAMA4_TILE_X_SEPARATOR_ID => Some("<|tile_x_separator|>".to_string()), - LLAMA4_TILE_Y_SEPARATOR_ID => Some("<|tile_y_separator|>".to_string()), - _ => None, - } - } - - fn create_decode_stream( - &self, - _prompt_token_ids: &[u32], - _skip_special_tokens: bool, - _min_bytes_to_buffer: usize, - ) -> Box { - unreachable!("not used") - } + fn llama4_tokenizer() -> TestTokenizer { + TestTokenizer::new() + .with_regular_token("<|image_start|>", LLAMA4_IMAGE_START_ID) + .with_regular_token("<|image_end|>", LLAMA4_IMAGE_END_ID) + .with_regular_token("<|image|>", LLAMA4_IMAGE_ID) + .with_regular_token("<|patch|>", LLAMA4_PATCH_ID) + .with_regular_token("<|tile_x_separator|>", LLAMA4_TILE_X_SEPARATOR_ID) + .with_regular_token("<|tile_y_separator|>", LLAMA4_TILE_Y_SEPARATOR_ID) } fn test_info(model_type: &str, config: serde_json::Value) -> MultimodalModelInfo { @@ -635,7 +589,7 @@ mod tests { model_id: format!("{model_type}-test"), model_type: Some(model_type.to_string()), config, - tokenizer: TokenizerResolver(Arc::new(TestTokenizer)), + tokenizer: TokenizerResolver(Arc::new(llama4_tokenizer())), }; let spec = context .resolve_model_spec() diff --git a/rust/src/chat/src/output/default/mod.rs b/rust/src/chat/src/output/default/mod.rs index c494df600f4..b42d24dcaa6 100644 --- a/rust/src/chat/src/output/default/mod.rs +++ b/rust/src/chat/src/output/default/mod.rs @@ -189,46 +189,19 @@ impl ChatOutputProcessor for DefaultChatOutputProcessor { mod tests { use std::sync::Arc; - use vllm_tokenizer::Tokenizer; + use vllm_tokenizer::test_utils::TestTokenizer; use super::DefaultChatOutputProcessor; use crate::Error; use crate::parser::ParserSelection; use crate::request::ChatRequest; - struct FakeTokenizer; - - impl Tokenizer for FakeTokenizer { - fn encode( - &self, - text: &str, - _add_special_tokens: bool, - ) -> vllm_tokenizer::Result> { - Ok(text.chars().map(u32::from).collect()) - } - - fn decode( - &self, - token_ids: &[u32], - _skip_special_tokens: bool, - ) -> vllm_tokenizer::Result { - Ok(token_ids - .iter() - .map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}')) - .collect()) - } - - fn token_to_id(&self, token: &str) -> Option { - match token { - "<|channel>" => Some(1), - "" => Some(2), - _ => None, - } - } - } - - fn tokenizer() -> Arc { - Arc::new(FakeTokenizer) + fn tokenizer() -> Arc { + Arc::new( + TestTokenizer::new() + .with_regular_token("<|channel>", 256) + .with_regular_token("", 257), + ) } #[test] diff --git a/rust/src/chat/src/parser/reasoning/tests.rs b/rust/src/chat/src/parser/reasoning/tests.rs index e6255d14a00..b6ae5ba9c38 100644 --- a/rust/src/chat/src/parser/reasoning/tests.rs +++ b/rust/src/chat/src/parser/reasoning/tests.rs @@ -1,32 +1,9 @@ use std::sync::Arc; -use vllm_tokenizer::Tokenizer; +use vllm_tokenizer::test_utils::TestTokenizer; use super::{ReasoningParserFactory, names}; -struct FakeTokenizer; - -impl Tokenizer for FakeTokenizer { - fn encode(&self, text: &str, _add_special_tokens: bool) -> vllm_tokenizer::Result> { - Ok(text.chars().map(u32::from).collect()) - } - - fn decode( - &self, - token_ids: &[u32], - _skip_special_tokens: bool, - ) -> vllm_tokenizer::Result { - Ok(token_ids - .iter() - .map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}')) - .collect()) - } - - fn token_to_id(&self, _token: &str) -> Option { - None - } -} - #[test] fn factory_contains_and_lists_registered_parsers() { let factory = ReasoningParserFactory::new(); @@ -107,7 +84,7 @@ fn factory_resolves_minimax_m3_before_generic_minimax() { #[test] fn factory_rejects_unknown_parser_names() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(TestTokenizer::new()); let factory = ReasoningParserFactory::new(); let error = match factory.create("missing", tokenizer) { Ok(_) => panic!("expected parser lookup to fail"), diff --git a/rust/src/chat/src/parser/unified.rs b/rust/src/chat/src/parser/unified.rs index 6456cfda754..50f2104d619 100644 --- a/rust/src/chat/src/parser/unified.rs +++ b/rust/src/chat/src/parser/unified.rs @@ -75,39 +75,14 @@ impl UnifiedParserFactory { mod tests { use std::sync::Arc; - use vllm_tokenizer::Tokenizer; + use vllm_tokenizer::test_utils::TestTokenizer; use super::{UnifiedParserFactory, names}; - struct FakeTokenizer; - - impl Tokenizer for FakeTokenizer { - fn encode( - &self, - text: &str, - _add_special_tokens: bool, - ) -> vllm_tokenizer::Result> { - Ok(text.chars().map(u32::from).collect()) - } - - fn decode( - &self, - token_ids: &[u32], - _skip_special_tokens: bool, - ) -> vllm_tokenizer::Result { - Ok(token_ids - .iter() - .map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}')) - .collect()) - } - - fn token_to_id(&self, token: &str) -> Option { - match token { - "<|channel>" => Some(1), - "" => Some(2), - _ => None, - } - } + fn tokenizer() -> TestTokenizer { + TestTokenizer::new() + .with_regular_token("<|channel>", 256) + .with_regular_token("", 257) } #[test] @@ -119,6 +94,6 @@ mod tests { factory.resolve_name_for_model("google/gemma-4-27b-it"), Some(names::GEMMA4) ); - factory.create(names::GEMMA4, &[], Arc::new(FakeTokenizer)).unwrap(); + factory.create(names::GEMMA4, &[], Arc::new(tokenizer())).unwrap(); } } diff --git a/rust/src/parser/Cargo.toml b/rust/src/parser/Cargo.toml index 09c3d6b5dc1..3c49362c5d2 100644 --- a/rust/src/parser/Cargo.toml +++ b/rust/src/parser/Cargo.toml @@ -23,6 +23,7 @@ expect-test.workspace = true futures.workspace = true openai-protocol.workspace = true tool-parser.workspace = true +vllm-tokenizer = { workspace = true, features = ["test-utils"] } [[bench]] name = "deepseek_v3" diff --git a/rust/src/parser/src/reasoning/seed_oss.rs b/rust/src/parser/src/reasoning/seed_oss.rs index eb996f8477c..580e95e7957 100644 --- a/rust/src/parser/src/reasoning/seed_oss.rs +++ b/rust/src/parser/src/reasoning/seed_oss.rs @@ -49,11 +49,14 @@ mod tests { use std::sync::Arc; use super::SeedOssReasoningParser; - use crate::reasoning::{ReasoningParser, tests::FakeTokenizer}; + use crate::reasoning::{ + ReasoningParser, + tests::{SEED_THINK_END_ID, SEED_THINK_START_ID, fake_tokenizer}, + }; #[test] fn without_prompt_markers_expects_start_token() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap(); let delta = parser.push("implicit reasoninganswer").unwrap(); @@ -66,10 +69,10 @@ mod tests { #[test] fn picks_up_prompt_start_boundary() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap(); - // Prompt prefills `` (id 10), opening reasoning before the stream. - parser.initialize(&[10]).unwrap(); + // Prompt prefills ``, opening reasoning before the stream. + parser.initialize(&[SEED_THINK_START_ID]).unwrap(); let delta = parser.push("reasonanswer").unwrap(); assert_eq!(delta.reasoning.as_deref(), Some("reason")); @@ -78,10 +81,10 @@ mod tests { #[test] fn respects_prompt_end_boundary() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap(); - // Prompt already closed reasoning with `` (id 11). - parser.initialize(&[11]).unwrap(); + // Prompt already closed reasoning with ``. + parser.initialize(&[SEED_THINK_END_ID]).unwrap(); let delta = parser.push("answer").unwrap(); assert_eq!(delta.reasoning, None); @@ -91,7 +94,7 @@ mod tests { #[test] fn handles_explicit_start_token() { // An explicit start delimiter must not leak into reasoning text. - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap(); let delta = parser.push("reasonanswer").unwrap(); @@ -103,7 +106,7 @@ mod tests { fn streams_explicit_start_token_across_pushes() { // Start token, reasoning body, end token, and content arrive in separate // streaming deltas. - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap(); let mut reasoning = String::new(); @@ -131,9 +134,9 @@ mod tests { #[test] fn handles_partial_delimiters_across_pushes() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap(); - parser.initialize(&[10]).unwrap(); + parser.initialize(&[SEED_THINK_START_ID]).unwrap(); // Closing delimiter `` arrives in two halves. let first = parser.push("reason` (id 1), opening reasoning before the stream. - parser.initialize(&[1]).unwrap(); + // Prompt prefills ``, opening reasoning before the stream. + parser.initialize(&[THINK_START_ID]).unwrap(); let delta = parser.push("This is a reasoning sectionThis is the rest").unwrap(); assert_eq!( @@ -146,7 +149,7 @@ mod tests { #[test] fn handles_unterminated_reasoning() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap(); let pushed = parser.push("reason without end").unwrap(); @@ -159,7 +162,7 @@ mod tests { #[test] fn handles_empty_input() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap(); let pushed = parser.push("").unwrap(); @@ -172,9 +175,9 @@ mod tests { fn complex_newline_pattern_trims_only_single_framing_newline_each_side() { // Only the immediately-adjacent framing `\n` is dropped on each side of // ``; surrounding newlines remain part of reasoning/content. - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap(); - parser.initialize(&[1]).unwrap(); + parser.initialize(&[THINK_START_ID]).unwrap(); let delta = parser .push("\n This is a \n reasoning section\n\n\n\n\nThis is the rest") @@ -188,7 +191,7 @@ mod tests { #[test] fn drops_framing_newlines_in_single_push() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap(); let delta = parser.push("reason\n\nanswer").unwrap(); @@ -198,7 +201,7 @@ mod tests { #[test] fn drops_framing_newlines_across_pushes() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap(); // The trailing `\n` from the first push is held until we know whether @@ -219,7 +222,7 @@ mod tests { #[test] fn replays_held_newline_when_more_reasoning_follows() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap(); let first = parser.push("reason\n").unwrap(); @@ -232,7 +235,7 @@ mod tests { #[test] fn finish_flushes_held_newline_in_unterminated_stream() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap(); let first = parser.push("reason\n").unwrap(); @@ -245,7 +248,7 @@ mod tests { #[test] fn preserves_inner_newlines_in_reasoning() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap(); let delta = parser.push("line1\nline2tail").unwrap(); @@ -257,7 +260,7 @@ mod tests { fn trims_only_one_trailing_reasoning_newline() { // Only the single framing newline immediately before `` is // dropped; earlier newlines in the reasoning body are preserved. - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap(); let delta = parser.push("reason\n\nanswer").unwrap(); @@ -269,7 +272,7 @@ mod tests { fn drops_only_first_content_newline_after_transition() { // The leading-`\n` drop applies only to the first content delta after // ``; later deltas pass through untouched. - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap(); let first = parser.push("reason").unwrap(); @@ -288,7 +291,7 @@ mod tests { #[test] fn passes_through_clean_boundary_without_framing_newlines() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap(); let delta = parser.push("reasontail").unwrap(); @@ -298,7 +301,7 @@ mod tests { #[test] fn handles_empty_reasoning_section() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap(); let delta = parser.push("answer").unwrap(); diff --git a/rust/src/parser/src/reasoning/tests.rs b/rust/src/parser/src/reasoning/tests.rs index 22c026d3581..5c148079e3a 100644 --- a/rust/src/parser/src/reasoning/tests.rs +++ b/rust/src/parser/src/reasoning/tests.rs @@ -1,54 +1,42 @@ use std::sync::Arc; -use vllm_tokenizer::Tokenizer; +use vllm_tokenizer::test_utils::TestTokenizer; use super::{ DeepSeekR1ReasoningParser, DelimitedReasoningParser, MiniMaxM3ReasoningParser, Qwen3ReasoningParser, ReasoningParser, }; -pub(crate) struct FakeTokenizer; +pub(crate) const THINK_START_ID: u32 = 256; +pub(crate) const THINK_END_ID: u32 = 257; +pub(crate) const START_THINKING_ID: u32 = 258; +pub(crate) const END_THINKING_ID: u32 = 259; +pub(crate) const MINIMAX_THINK_START_ID: u32 = 260; +pub(crate) const MINIMAX_THINK_END_ID: u32 = 261; +pub(crate) const SPECIAL_BOUNDARY_ID: u32 = 262; +pub(crate) const MM_THINK_START_ID: u32 = 263; +pub(crate) const MM_THINK_END_ID: u32 = 264; +pub(crate) const SEED_THINK_START_ID: u32 = 265; +pub(crate) const SEED_THINK_END_ID: u32 = 266; -impl Tokenizer for FakeTokenizer { - fn encode(&self, text: &str, _add_special_tokens: bool) -> vllm_tokenizer::Result> { - Ok(text.chars().map(u32::from).collect()) - } - - fn decode( - &self, - token_ids: &[u32], - _skip_special_tokens: bool, - ) -> vllm_tokenizer::Result { - Ok(token_ids - .iter() - .map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}')) - .collect()) - } - - fn token_to_id(&self, token: &str) -> Option { - match token { - "" => Some(1), - "" => Some(2), - "<|START_THINKING|>" => Some(3), - "<|END_THINKING|>" => Some(4), - "◁think▷" => Some(5), - "◁/think▷" => Some(6), - "" => Some(8), - "" => Some(9), - "" => Some(10), - "" => Some(11), - _ => None, - } - } - - fn is_special_id(&self, token_id: u32) -> bool { - token_id == 7 - } +pub(crate) fn fake_tokenizer() -> TestTokenizer { + TestTokenizer::new() + .with_regular_token("", THINK_START_ID) + .with_regular_token("", THINK_END_ID) + .with_regular_token("<|START_THINKING|>", START_THINKING_ID) + .with_regular_token("<|END_THINKING|>", END_THINKING_ID) + .with_regular_token("◁think▷", MINIMAX_THINK_START_ID) + .with_regular_token("◁/think▷", MINIMAX_THINK_END_ID) + .with_special_token("", SPECIAL_BOUNDARY_ID) + .with_regular_token("", MM_THINK_START_ID) + .with_regular_token("", MM_THINK_END_ID) + .with_regular_token("", SEED_THINK_START_ID) + .with_regular_token("", SEED_THINK_END_ID) } #[test] fn delimited_content_only_stream() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = DelimitedReasoningParser::new(tokenizer, "", "", false).unwrap(); @@ -60,7 +48,7 @@ fn delimited_content_only_stream() { #[test] fn delimited_single_chunk_with_reasoning_and_content() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = DelimitedReasoningParser::new(tokenizer, "", "", false).unwrap(); @@ -71,7 +59,7 @@ fn delimited_single_chunk_with_reasoning_and_content() { #[test] fn delimited_partial_tokens_across_chunks() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = DelimitedReasoningParser::new(tokenizer, "", "", false).unwrap(); @@ -83,10 +71,10 @@ fn delimited_partial_tokens_across_chunks() { #[test] fn delimited_finish_flushes_buffer() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = DelimitedReasoningParser::new(tokenizer, "", "", false).unwrap(); - parser.initialize(&[1]); + parser.initialize(&[THINK_START_ID]); let delta = parser.push("unfinishedanswer").unwrap(); @@ -106,9 +94,9 @@ fn qwen3_without_prompt_markers_expects_start_token() { #[test] fn qwen3_prompt_end_marker_starts_in_content() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = Qwen3ReasoningParser::new(tokenizer).unwrap(); - parser.initialize(&[2]).unwrap(); + parser.initialize(&[THINK_END_ID]).unwrap(); let delta = parser.push("answer").unwrap(); assert_eq!(delta.reasoning, None); @@ -117,7 +105,7 @@ fn qwen3_prompt_end_marker_starts_in_content() { #[test] fn qwen3_tolerates_old_and_new_formats() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut old_parser = Qwen3ReasoningParser::new(tokenizer.clone()).unwrap(); let old = old_parser.push("reasonanswer").unwrap(); @@ -125,7 +113,7 @@ fn qwen3_tolerates_old_and_new_formats() { assert_eq!(old.content.as_deref(), Some("answer")); let mut new_parser = Qwen3ReasoningParser::new(tokenizer).unwrap(); - new_parser.initialize(&[1]).unwrap(); + new_parser.initialize(&[THINK_START_ID]).unwrap(); let new = new_parser.push("reasonanswer").unwrap(); assert_eq!(new.reasoning.as_deref(), Some("reason")); assert_eq!(new.content.as_deref(), Some("answer")); @@ -133,10 +121,10 @@ fn qwen3_tolerates_old_and_new_formats() { #[test] fn qwen3_stops_scanning_at_last_special_token() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = Qwen3ReasoningParser::new(tokenizer).unwrap(); - parser.initialize(&[1, 7]).unwrap(); + parser.initialize(&[THINK_START_ID, SPECIAL_BOUNDARY_ID]).unwrap(); let delta = parser.push("answer").unwrap(); assert_eq!(delta.reasoning, None); @@ -145,7 +133,7 @@ fn qwen3_stops_scanning_at_last_special_token() { #[test] fn deepseek_r1_defaults_to_reasoning_without_prompt_boundary() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = DeepSeekR1ReasoningParser::new(tokenizer).unwrap(); let delta = parser.push("reasonanswer").unwrap(); @@ -155,10 +143,10 @@ fn deepseek_r1_defaults_to_reasoning_without_prompt_boundary() { #[test] fn deepseek_r1_stops_scanning_at_last_special_token() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = DeepSeekR1ReasoningParser::new(tokenizer).unwrap(); - parser.initialize(&[2, 7]).unwrap(); + parser.initialize(&[THINK_END_ID, SPECIAL_BOUNDARY_ID]).unwrap(); let delta = parser.push("reasonanswer").unwrap(); assert_eq!(delta.reasoning.as_deref(), Some("reason")); @@ -167,7 +155,7 @@ fn deepseek_r1_stops_scanning_at_last_special_token() { #[test] fn minimax_m3_handles_explicit_think_delimiters() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap(); let delta = parser.push("reasonanswer").unwrap(); @@ -177,7 +165,7 @@ fn minimax_m3_handles_explicit_think_delimiters() { #[test] fn minimax_m3_drops_leading_end_marker() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap(); let delta = parser.push("answer").unwrap(); @@ -187,7 +175,7 @@ fn minimax_m3_drops_leading_end_marker() { #[test] fn minimax_m3_preserves_non_leading_end_marker() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap(); let delta = parser.push("XXXYYY").unwrap(); @@ -197,7 +185,7 @@ fn minimax_m3_preserves_non_leading_end_marker() { #[test] fn minimax_m3_drops_split_leading_end_marker() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap(); assert!(parser.push("answer").unwrap(); assert_eq!(delta.reasoning.as_deref(), Some("reason")); @@ -219,9 +207,9 @@ fn minimax_m3_uses_prompt_prefilled_start_marker() { #[test] fn minimax_m3_uses_prompt_prefilled_end_marker() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(fake_tokenizer()); let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap(); - parser.initialize(&[9]).unwrap(); + parser.initialize(&[MM_THINK_END_ID]).unwrap(); let delta = parser.push("answer").unwrap(); assert_eq!(delta.reasoning, None); diff --git a/rust/src/parser/src/unified/combined.rs b/rust/src/parser/src/unified/combined.rs index 3f1c669013d..f06334c72ee 100644 --- a/rust/src/parser/src/unified/combined.rs +++ b/rust/src/parser/src/unified/combined.rs @@ -124,42 +124,17 @@ impl UnifiedParser for CombinedParser { mod tests { use std::sync::Arc; - use vllm_tokenizer::Tokenizer; + use vllm_tokenizer::test_utils::TestTokenizer; use super::CombinedParser; use crate::reasoning::{Qwen3ReasoningParser, ReasoningDelta, ReasoningParser}; use crate::tool::{Qwen3XmlToolParser, Tool, ToolParser}; use crate::unified::{UnifiedParser, UnifiedParserEvent, UnifiedParserOutput}; - struct FakeTokenizer; - - impl Tokenizer for FakeTokenizer { - fn encode( - &self, - text: &str, - _add_special_tokens: bool, - ) -> vllm_tokenizer::Result> { - Ok(text.chars().map(u32::from).collect()) - } - - fn decode( - &self, - token_ids: &[u32], - _skip_special_tokens: bool, - ) -> vllm_tokenizer::Result { - Ok(token_ids - .iter() - .map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}')) - .collect()) - } - - fn token_to_id(&self, token: &str) -> Option { - match token { - "" => Some(1), - "" => Some(2), - _ => None, - } - } + fn tokenizer() -> TestTokenizer { + TestTokenizer::new() + .with_regular_token("", 256) + .with_regular_token("", 257) } fn test_tools() -> Vec { @@ -273,7 +248,7 @@ mod tests { #[test] fn combined_parser_emits_reasoning_and_text() { - let tokenizer = Arc::new(FakeTokenizer); + let tokenizer = Arc::new(tokenizer()); let reasoning = Qwen3ReasoningParser::create(tokenizer).unwrap(); let mut parser = CombinedParser::new(Some(reasoning), None); diff --git a/rust/src/parser/src/unified/gemma4.rs b/rust/src/parser/src/unified/gemma4.rs index 3276088913d..71541f94c69 100644 --- a/rust/src/parser/src/unified/gemma4.rs +++ b/rust/src/parser/src/unified/gemma4.rs @@ -515,7 +515,7 @@ mod tests { use serde_json::{Value, json}; use thiserror_ext::AsReport; - use vllm_tokenizer::Tokenizer; + use vllm_tokenizer::test_utils::TestTokenizer; use winnow::combinator::{eof, terminated}; use winnow::error::ErrMode; use winnow::prelude::*; @@ -527,66 +527,15 @@ mod tests { use crate::tool::Tool; use crate::unified::{UnifiedParserEvent, parsing_failed}; - struct FakeTokenizer; + const CHANNEL_START_ID: u32 = 256; + const CHANNEL_END_ID: u32 = 257; + const TURN_BOUNDARY_ID: u32 = 258; - impl Tokenizer for FakeTokenizer { - fn encode( - &self, - text: &str, - _add_special_tokens: bool, - ) -> vllm_tokenizer::Result> { - Ok(text.chars().map(u32::from).collect()) - } - - fn decode( - &self, - token_ids: &[u32], - _skip_special_tokens: bool, - ) -> vllm_tokenizer::Result { - Ok(token_ids - .iter() - .map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}')) - .collect()) - } - - fn token_to_id(&self, token: &str) -> Option { - match token { - CHANNEL_START => Some(100), - CHANNEL_END => Some(101), - _ => None, - } - } - - fn is_special_id(&self, token_id: u32) -> bool { - matches!(token_id, 100..=105) - } - } - - struct MissingTokenTokenizer; - - impl Tokenizer for MissingTokenTokenizer { - fn encode( - &self, - text: &str, - _add_special_tokens: bool, - ) -> vllm_tokenizer::Result> { - Ok(text.chars().map(u32::from).collect()) - } - - fn decode( - &self, - token_ids: &[u32], - _skip_special_tokens: bool, - ) -> vllm_tokenizer::Result { - Ok(token_ids - .iter() - .map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}')) - .collect()) - } - - fn token_to_id(&self, _token: &str) -> Option { - None - } + fn tokenizer() -> TestTokenizer { + TestTokenizer::new() + .with_special_token(CHANNEL_START, CHANNEL_START_ID) + .with_special_token(CHANNEL_END, CHANNEL_END_ID) + .with_special_token("", TURN_BOUNDARY_ID) } trait UnifiedParserTestExt { @@ -716,12 +665,12 @@ mod tests { } fn test_parser() -> Gemma4UnifiedParser { - Gemma4UnifiedParser::new(&test_tools(), Arc::new(FakeTokenizer)).unwrap() + Gemma4UnifiedParser::new(&test_tools(), Arc::new(tokenizer())).unwrap() } #[test] fn gemma4_create_requires_channel_start_token() { - let error = match Gemma4UnifiedParser::new(&test_tools(), Arc::new(MissingTokenTokenizer)) { + let error = match Gemma4UnifiedParser::new(&test_tools(), Arc::new(TestTokenizer::new())) { Ok(_) => panic!("expected missing token error"), Err(error) => error, }; @@ -1046,7 +995,7 @@ mod tests { #[test] fn gemma4_initialize_open_channel_prompt_starts_in_reasoning() { let mut parser = test_parser(); - parser.initialize(&[100, 3000, 3001]).unwrap(); + parser.initialize(&[CHANNEL_START_ID, 3000, 3001]).unwrap(); let output = parser.parse_complete("reasonanswer").unwrap(); @@ -1057,7 +1006,7 @@ mod tests { #[test] fn gemma4_initialize_turn_prompt_starts_in_text() { let mut parser = test_parser(); - parser.initialize(&[104, 3000, 3001]).unwrap(); + parser.initialize(&[TURN_BOUNDARY_ID, 3000, 3001]).unwrap(); let output = parser.parse_complete("<|channel>thought\nreasonanswer").unwrap(); @@ -1068,7 +1017,7 @@ mod tests { #[test] fn gemma4_initialize_special_token_caps_boundary_scan() { let mut parser = test_parser(); - parser.initialize(&[100, 3000, 104, 3001]).unwrap(); + parser.initialize(&[CHANNEL_START_ID, 3000, TURN_BOUNDARY_ID, 3001]).unwrap(); let output = parser.parse_complete("answer").unwrap(); @@ -1079,7 +1028,7 @@ mod tests { #[test] fn gemma4_initialize_closed_channel_prompt_starts_in_text() { let mut parser = test_parser(); - parser.initialize(&[100, 3000, 3001, 101]).unwrap(); + parser.initialize(&[CHANNEL_START_ID, 3000, 3001, CHANNEL_END_ID]).unwrap(); let output = parser.parse_complete("answer").unwrap(); diff --git a/rust/src/server/Cargo.toml b/rust/src/server/Cargo.toml index 59c7f8d1744..3d71838545f 100644 --- a/rust/src/server/Cargo.toml +++ b/rust/src/server/Cargo.toml @@ -63,6 +63,7 @@ serial_test.workspace = true tempfile.workspace = true tower.workspace = true vllm-engine-core-client = { workspace = true, features = ["test-util"] } +vllm-tokenizer = { workspace = true, features = ["test-utils"] } zeromq.workspace = true [lints] diff --git a/rust/src/server/src/grpc/tests.rs b/rust/src/server/src/grpc/tests.rs index 311b819e0c8..852a030930e 100644 --- a/rust/src/server/src/grpc/tests.rs +++ b/rust/src/server/src/grpc/tests.rs @@ -24,8 +24,9 @@ use vllm_engine_core_client::protocol::{ use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task}; use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, EngineId}; use vllm_llm::Llm; -use vllm_text::tokenizer::{DynTokenizer, Tokenizer}; +use vllm_text::tokenizer::DynTokenizer; use vllm_text::{Prompt, TextBackend}; +use vllm_tokenizer::test_utils::TestTokenizer; use zeromq::prelude::{SocketRecv, SocketSend}; use zeromq::{DealerSocket, PushSocket, ZmqMessage}; @@ -155,37 +156,9 @@ fn test_llm(client: EngineCoreClient) -> Llm { #[derive(Clone, Debug)] struct FakeTextBackend; -#[derive(Debug)] -struct FakeTokenizer; - -impl Tokenizer for FakeTokenizer { - fn encode( - &self, - text: &str, - _add_special_tokens: bool, - ) -> vllm_text::tokenizer::Result> { - Ok(text.bytes().map(u32::from).collect()) - } - - fn decode( - &self, - token_ids: &[u32], - _skip_special_tokens: bool, - ) -> vllm_text::tokenizer::Result { - Ok( - String::from_utf8_lossy(&token_ids.iter().map(|id| *id as u8).collect::>()) - .into_owned(), - ) - } - - fn token_to_id(&self, token: &str) -> Option { - token.bytes().next().map(u32::from) - } -} - impl TextBackend for FakeTextBackend { fn tokenizer(&self) -> DynTokenizer { - Arc::new(FakeTokenizer) + Arc::new(TestTokenizer::new()) } fn model_id(&self) -> &str { diff --git a/rust/src/server/src/routes/http_client_tests.rs b/rust/src/server/src/routes/http_client_tests.rs index b23c95fa6d9..b3c6977f5ec 100644 --- a/rust/src/server/src/routes/http_client_tests.rs +++ b/rust/src/server/src/routes/http_client_tests.rs @@ -24,8 +24,9 @@ use vllm_engine_core_client::protocol::{ use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task}; use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, EngineId}; use vllm_llm::Llm; -use vllm_text::tokenizer::{DynTokenizer, Tokenizer}; +use vllm_text::tokenizer::DynTokenizer; use vllm_text::{Prompt, TextBackend}; +use vllm_tokenizer::test_utils::TestTokenizer; use zeromq::prelude::{SocketRecv, SocketSend}; use zeromq::{DealerSocket, PushSocket, ZmqMessage}; @@ -151,37 +152,9 @@ fn test_llm(client: EngineCoreClient) -> Llm { #[derive(Clone, Debug)] struct FakeChatBackend; -#[derive(Debug)] -struct FakeChatTokenizer; - -impl Tokenizer for FakeChatTokenizer { - fn encode( - &self, - text: &str, - _add_special_tokens: bool, - ) -> vllm_text::tokenizer::Result> { - Ok(text.bytes().map(u32::from).collect()) - } - - fn decode( - &self, - token_ids: &[u32], - _skip_special_tokens: bool, - ) -> vllm_text::tokenizer::Result { - Ok( - String::from_utf8_lossy(&token_ids.iter().map(|id| *id as u8).collect::>()) - .into_owned(), - ) - } - - fn token_to_id(&self, token: &str) -> Option { - token.bytes().next().map(u32::from) - } -} - impl TextBackend for FakeChatBackend { fn tokenizer(&self) -> DynTokenizer { - Arc::new(FakeChatTokenizer) + Arc::new(TestTokenizer::new()) } fn model_id(&self) -> &str { diff --git a/rust/src/server/src/routes/openai/completions/convert.rs b/rust/src/server/src/routes/openai/completions/convert.rs index 0541346438e..46d552385a4 100644 --- a/rust/src/server/src/routes/openai/completions/convert.rs +++ b/rust/src/server/src/routes/openai/completions/convert.rs @@ -194,7 +194,7 @@ mod tests { use axum::http::HeaderMap; use serde_json::json; use vllm_text::Prompt; - use vllm_text::tokenizer::Tokenizer; + use vllm_tokenizer::test_utils::TestTokenizer; use super::prepare_completion_request; use crate::lora::LoraModelResolution; @@ -212,32 +212,8 @@ mod tests { } } - #[derive(Debug)] - struct TestTokenizer; - - impl Tokenizer for TestTokenizer { - fn encode( - &self, - text: &str, - _add_special_tokens: bool, - ) -> vllm_text::tokenizer::Result> { - Ok(text.bytes().map(u32::from).collect()) - } - - fn decode( - &self, - token_ids: &[u32], - _skip_special_tokens: bool, - ) -> vllm_text::tokenizer::Result { - Ok( - String::from_utf8_lossy(&token_ids.iter().map(|id| *id as u8).collect::>()) - .into_owned(), - ) - } - - fn token_to_id(&self, _token: &str) -> Option { - None - } + fn test_tokenizer() -> TestTokenizer { + TestTokenizer::new() } fn base_request_json() -> serde_json::Value { @@ -297,7 +273,7 @@ mod tests { request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]), ResolvedRequestContext::default(), - &TestTokenizer, + &test_tokenizer(), ) .expect("prepare"); @@ -340,7 +316,7 @@ mod tests { request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]), ResolvedRequestContext::default(), - &TestTokenizer, + &test_tokenizer(), ) .expect("prepare") .text_request @@ -374,7 +350,7 @@ mod tests { request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]), ResolvedRequestContext::default(), - &TestTokenizer, + &test_tokenizer(), ) .expect("prepare"); @@ -399,7 +375,7 @@ mod tests { request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]), ResolvedRequestContext::default(), - &TestTokenizer, + &test_tokenizer(), ) .expect("prepare"); @@ -422,7 +398,7 @@ mod tests { request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]), ResolvedRequestContext::default(), - &TestTokenizer, + &test_tokenizer(), ) .expect("prepare"); @@ -446,7 +422,7 @@ mod tests { request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]), ResolvedRequestContext::default(), - &TestTokenizer, + &test_tokenizer(), ) .expect("prepare"); @@ -471,7 +447,7 @@ mod tests { request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]), ResolvedRequestContext::default(), - &TestTokenizer, + &test_tokenizer(), ) .expect("prepare"); @@ -498,7 +474,7 @@ mod tests { request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]), ResolvedRequestContext::default(), - &TestTokenizer, + &test_tokenizer(), ) .expect("prepare"); @@ -523,7 +499,7 @@ mod tests { request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]), ResolvedRequestContext::default(), - &TestTokenizer, + &test_tokenizer(), ) .expect("prepare"); @@ -549,7 +525,7 @@ mod tests { request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]), ResolvedRequestContext::default(), - &TestTokenizer, + &test_tokenizer(), ) .expect("prepare"); assert_eq!(prepared.text_request.sampling_params.logprobs, Some(1)); @@ -574,7 +550,7 @@ mod tests { request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]), request_context(&headers, None), - &TestTokenizer, + &test_tokenizer(), ) .expect("prepare"); assert_eq!(prepared.text_request.data_parallel_rank, Some(3)); @@ -593,7 +569,7 @@ mod tests { request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]), ResolvedRequestContext::default(), - &TestTokenizer, + &test_tokenizer(), ) .expect("prepare"); assert_eq!(prepared.text_request.data_parallel_rank, None); diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index d7e99e8fa91..c79bae98833 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -42,6 +42,7 @@ use vllm_llm::Llm; use vllm_metrics::METRICS; use vllm_text::tokenizer::{DynTokenizer, Tokenizer}; use vllm_text::{Prompt, TextBackend}; +use vllm_tokenizer::test_utils::TestTokenizer; use zeromq::prelude::{SocketRecv, SocketSend}; use zeromq::{DealerSocket, PushSocket, ZmqMessage}; @@ -417,82 +418,19 @@ struct FakeChatBackend { } /// Synthetic BOS id used when `add_special_tokens` is true in tests. -const FAKE_BOS_TOKEN_ID: u32 = 1; +const FAKE_BOS_TOKEN_ID: u32 = 256; -#[derive(Debug)] -struct FakeChatTokenizer; - -impl Tokenizer for FakeChatTokenizer { - fn encode( - &self, - text: &str, - add_special_tokens: bool, - ) -> vllm_text::tokenizer::Result> { - let mut token_ids = Vec::new(); - if add_special_tokens { - token_ids.push(FAKE_BOS_TOKEN_ID); - } - let mut rest = text; - while !rest.is_empty() { - if let Some(stripped) = rest.strip_prefix("") { - token_ids.push(999); - rest = stripped; - continue; - } - if let Some(stripped) = rest.strip_prefix("<|image_pad|>") { - token_ids.push(151655); - rest = stripped; - continue; - } - - let ch = rest.chars().next().expect("rest is not empty"); - let mut buf = [0; 4]; - token_ids.extend(ch.encode_utf8(&mut buf).bytes().map(u32::from)); - rest = &rest[ch.len_utf8()..]; - } - Ok(token_ids) - } - - fn decode( - &self, - token_ids: &[u32], - _skip_special_tokens: bool, - ) -> vllm_text::tokenizer::Result { - Ok( - String::from_utf8_lossy(&token_ids.iter().map(|id| *id as u8).collect::>()) - .into_owned(), - ) - } - - fn token_to_id(&self, token: &str) -> Option { - match token { - "" => Some(999), - "<|image_pad|>" => Some(151655), - "" => Some(0xF001), - "" => Some(0xF002), - "<|START_THINKING|>" => Some(0xF003), - "<|END_THINKING|>" => Some(0xF004), - "◁think▷" => Some(0xF005), - "◁/think▷" => Some(0xF006), - _ => None, - } - } - - fn id_to_token(&self, id: u32) -> Option { - match id { - FAKE_BOS_TOKEN_ID => Some("".to_string()), - 999 => Some("".to_string()), - 151655 => Some("<|image_pad|>".to_string()), - 0xF001 => Some("".to_string()), - 0xF002 => Some("".to_string()), - 0xF003 => Some("<|START_THINKING|>".to_string()), - 0xF004 => Some("<|END_THINKING|>".to_string()), - 0xF005 => Some("◁think▷".to_string()), - 0xF006 => Some("◁/think▷".to_string()), - id if id < 128 => char::from_u32(id).map(|ch| ch.to_string()), - _ => None, - } - } +fn fake_chat_tokenizer() -> TestTokenizer { + TestTokenizer::new() + .with_bos_token("", FAKE_BOS_TOKEN_ID) + .with_regular_token("", 999) + .with_regular_token("<|image_pad|>", 151655) + .with_regular_token("", 0xF001) + .with_regular_token("", 0xF002) + .with_regular_token("<|START_THINKING|>", 0xF003) + .with_regular_token("<|END_THINKING|>", 0xF004) + .with_regular_token("◁think▷", 0xF005) + .with_regular_token("◁/think▷", 0xF006) } impl FakeChatBackend { @@ -530,7 +468,7 @@ impl fmt::Debug for FakeChatBackend { impl TextBackend for FakeChatBackend { fn tokenizer(&self) -> DynTokenizer { - Arc::new(FakeChatTokenizer) + Arc::new(fake_chat_tokenizer()) } fn model_id(&self) -> &str { @@ -630,7 +568,7 @@ fn qwen_multimodal_model_info() -> vllm_chat::multimodal::MultimodalModelInfo { Some("qwen2_vl".to_string()), Some(&config_path), None, - Arc::new(FakeChatTokenizer), + Arc::new(fake_chat_tokenizer()), ) .expect("load multimodal info") .expect("qwen multimodal info is registered"); @@ -650,7 +588,7 @@ impl Tokenizer for FailingDecodeTokenizer { text: &str, add_special_tokens: bool, ) -> vllm_text::tokenizer::Result> { - FakeChatTokenizer.encode(text, add_special_tokens) + fake_chat_tokenizer().encode(text, add_special_tokens) } fn decode( @@ -664,11 +602,11 @@ impl Tokenizer for FailingDecodeTokenizer { )); } - FakeChatTokenizer.decode(token_ids, skip_special_tokens) + fake_chat_tokenizer().decode(token_ids, skip_special_tokens) } fn token_to_id(&self, token: &str) -> Option { - FakeChatTokenizer.token_to_id(token) + fake_chat_tokenizer().token_to_id(token) } } diff --git a/rust/src/text/Cargo.toml b/rust/src/text/Cargo.toml index 7ed02c07fca..7bda7f976e1 100644 --- a/rust/src/text/Cargo.toml +++ b/rust/src/text/Cargo.toml @@ -31,6 +31,7 @@ serial_test.workspace = true tempfile.workspace = true tokio.workspace = true vllm-llm = { workspace = true, features = ["test-util"] } +vllm-tokenizer = { workspace = true, features = ["test-utils"] } [lints] workspace = true diff --git a/rust/src/text/src/lower.rs b/rust/src/text/src/lower.rs index 6cd18195bb9..06adc8c7821 100644 --- a/rust/src/text/src/lower.rs +++ b/rust/src/text/src/lower.rs @@ -275,6 +275,7 @@ mod tests { use std::collections::{BTreeSet, HashMap}; use serial_test::file_serial; + use vllm_tokenizer::test_utils::TestTokenizer; use super::*; use crate::backend::hf::HfTextBackend; @@ -282,34 +283,8 @@ mod tests { use crate::error::{LogprobsError, TokenIdsError}; use crate::request::{Prompt, TextRequest}; - /// Stub tokenizer that returns empty token IDs — sufficient for tests that - /// don't exercise bad-words tokenization. - struct StubTokenizer; - - impl Tokenizer for StubTokenizer { - fn encode( - &self, - _text: &str, - _add_special_tokens: bool, - ) -> vllm_tokenizer::Result> { - Ok(vec![]) - } - - fn decode( - &self, - _token_ids: &[u32], - _skip_special_tokens: bool, - ) -> vllm_tokenizer::Result { - Ok(String::new()) - } - - fn token_to_id(&self, _token: &str) -> Option { - None - } - } - - fn stub_tokenizer() -> StubTokenizer { - StubTokenizer + fn stub_tokenizer() -> TestTokenizer { + TestTokenizer::new() } struct FixedTokenizer { diff --git a/rust/src/text/src/output/logprobs.rs b/rust/src/text/src/output/logprobs.rs index 7024c52b779..069a8cba272 100644 --- a/rust/src/text/src/output/logprobs.rs +++ b/rust/src/text/src/output/logprobs.rs @@ -129,40 +129,13 @@ fn decode_position_logprobs( #[cfg(test)] mod tests { use vllm_llm::{Logprobs, PositionLogprobs, TokenLogprob}; + use vllm_tokenizer::test_utils::TestTokenizer; use super::*; - #[derive(Debug)] - struct ByteTokenizer; - - impl vllm_tokenizer::Tokenizer for ByteTokenizer { - fn encode( - &self, - _text: &str, - _add_special_tokens: bool, - ) -> vllm_tokenizer::Result> { - unreachable!() - } - - fn decode( - &self, - token_ids: &[u32], - _skip_special_tokens: bool, - ) -> vllm_tokenizer::Result { - Ok(String::from_utf8_lossy( - &token_ids.iter().map(|token_id| *token_id as u8).collect::>(), - ) - .into_owned()) - } - - fn token_to_id(&self, _token: &str) -> Option { - unreachable!() - } - } - #[test] fn decode_logprobs_decodes_every_candidate_token() { - let tokenizer = ByteTokenizer; + let tokenizer = TestTokenizer::new(); let logprobs = Logprobs { positions: vec![PositionLogprobs { entries: vec![ @@ -205,7 +178,7 @@ mod tests { #[test] fn decode_prompt_logprobs_separates_first_prompt_token() { - let tokenizer = ByteTokenizer; + let tokenizer = TestTokenizer::new(); let logprobs = Logprobs { positions: vec![PositionLogprobs { entries: vec![TokenLogprob {