karylab_agents/vllm
1
0
Fork 0
forked from Karylab-cklius/vllm
Code Pull Requests 2 Actions 1 Packages Activity

update callsite

Browse Source 
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
Bugen Zhao
2026-06-30 14:53:50 +08:00
parent 82f53366f5
commit 7ff1a4e47a
21 changed files with 182 additions and 595 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rust/Cargo.lock
Generated
+1
View File
@@ -5310,6 +5310,7 @@ dependencies = [
"vllm-llm",
"vllm-metrics",
"vllm-text",
"vllm-tokenizer",
"zeromq",
]
rust/src/chat/Cargo.toml
+1
View File
@@ -50,6 +50,7 @@ tokio.workspace = true
tracing-subscriber.workspace = true
uuid.workspace = true
vllm-engine-core-client = { workspace = true, features = ["test-util"] }
vllm-tokenizer = { workspace = true, features = ["test-utils"] }
zeromq.workspace = true
[lints]
rust/src/chat/src/backend/hf.rs
+3 -26
View File
@@ -154,7 +154,8 @@ mod tests {
use thiserror_ext::AsReport as _;
use vllm_text::Prompt;
use vllm_text::backend::hf::TokenizerSource;
use vllm_text::tokenizer::{DynTokenizer, Tokenizer};
use vllm_text::tokenizer::DynTokenizer;
use vllm_tokenizer::test_utils::TestTokenizer;
use super::HfChatBackend;
use crate::backend::{ChatBackend, LoadModelBackendsOptions, NewChatOutputProcessorOptions};
@@ -196,32 +197,8 @@ mod tests {
}
}
struct TestTokenizer;
impl Tokenizer for TestTokenizer {
fn encode(
&self,
_text: &str,
_add_special_tokens: bool,
) -> vllm_text::tokenizer::Result<Vec<u32>> {
Ok(Vec::new())
}
fn decode(
&self,
_token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_text::tokenizer::Result<String> {
Ok(String::new())
}
fn token_to_id(&self, _token: &str) -> Option<u32> {
None
}
}
fn test_tokenizer() -> DynTokenizer {
Arc::new(TestTokenizer)
Arc::new(TestTokenizer::new())
}
fn backend_for_selection(
rust/src/chat/src/multimodal.rs
+10 -56
View File
@@ -563,7 +563,7 @@ mod tests {
use llm_multimodal::TokenId;
use vllm_engine_core_client::protocol::tensor::WireArrayData;
use vllm_text::tokenizer::{IncrementalDecoder, Tokenizer, TokenizerError};
use vllm_tokenizer::test_utils::TestTokenizer;
use super::*;
@@ -574,60 +574,14 @@ mod tests {
const LLAMA4_TILE_X_SEPARATOR_ID: u32 = 200093;
const LLAMA4_TILE_Y_SEPARATOR_ID: u32 = 200094;
struct TestTokenizer;
impl Tokenizer for TestTokenizer {
fn encode(
&self,
text: &str,
_add_special_tokens: bool,
) -> std::result::Result<Vec<u32>, TokenizerError> {
Ok(match text {
"<|image|>" => vec![LLAMA4_IMAGE_ID],
text => text.bytes().map(u32::from).collect(),
})
}
fn decode(
&self,
_token_ids: &[u32],
_skip_special_tokens: bool,
) -> std::result::Result<String, TokenizerError> {
Ok(String::new())
}
fn token_to_id(&self, token: &str) -> Option<u32> {
match token {
"<|image_start|>" => Some(LLAMA4_IMAGE_START_ID),
"<|image_end|>" => Some(LLAMA4_IMAGE_END_ID),
"<|image|>" => Some(LLAMA4_IMAGE_ID),
"<|patch|>" => Some(LLAMA4_PATCH_ID),
"<|tile_x_separator|>" => Some(LLAMA4_TILE_X_SEPARATOR_ID),
"<|tile_y_separator|>" => Some(LLAMA4_TILE_Y_SEPARATOR_ID),
_ => None,
}
}
fn id_to_token(&self, id: u32) -> Option<String> {
match id {
LLAMA4_IMAGE_START_ID => Some("<|image_start|>".to_string()),
LLAMA4_IMAGE_END_ID => Some("<|image_end|>".to_string()),
LLAMA4_IMAGE_ID => Some("<|image|>".to_string()),
LLAMA4_PATCH_ID => Some("<|patch|>".to_string()),
LLAMA4_TILE_X_SEPARATOR_ID => Some("<|tile_x_separator|>".to_string()),
LLAMA4_TILE_Y_SEPARATOR_ID => Some("<|tile_y_separator|>".to_string()),
_ => None,
}
}
fn create_decode_stream(
&self,
_prompt_token_ids: &[u32],
_skip_special_tokens: bool,
_min_bytes_to_buffer: usize,
) -> Box<dyn IncrementalDecoder + '_> {
unreachable!("not used")
}
fn llama4_tokenizer() -> TestTokenizer {
TestTokenizer::new()
.with_regular_token("<|image_start|>", LLAMA4_IMAGE_START_ID)
.with_regular_token("<|image_end|>", LLAMA4_IMAGE_END_ID)
.with_regular_token("<|image|>", LLAMA4_IMAGE_ID)
.with_regular_token("<|patch|>", LLAMA4_PATCH_ID)
.with_regular_token("<|tile_x_separator|>", LLAMA4_TILE_X_SEPARATOR_ID)
.with_regular_token("<|tile_y_separator|>", LLAMA4_TILE_Y_SEPARATOR_ID)
}
fn test_info(model_type: &str, config: serde_json::Value) -> MultimodalModelInfo {
@@ -635,7 +589,7 @@ mod tests {
model_id: format!("{model_type}-test"),
model_type: Some(model_type.to_string()),
config,
tokenizer: TokenizerResolver(Arc::new(TestTokenizer)),
tokenizer: TokenizerResolver(Arc::new(llama4_tokenizer())),
};
let spec = context
.resolve_model_spec()
rust/src/chat/src/output/default/mod.rs
+7 -34
View File
@@ -189,46 +189,19 @@ impl ChatOutputProcessor for DefaultChatOutputProcessor {
mod tests {
use std::sync::Arc;
use vllm_tokenizer::Tokenizer;
use vllm_tokenizer::test_utils::TestTokenizer;
use super::DefaultChatOutputProcessor;
use crate::Error;
use crate::parser::ParserSelection;
use crate::request::ChatRequest;
struct FakeTokenizer;
impl Tokenizer for FakeTokenizer {
fn encode(
&self,
text: &str,
_add_special_tokens: bool,
) -> vllm_tokenizer::Result<Vec<u32>> {
Ok(text.chars().map(u32::from).collect())
}
fn decode(
&self,
token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_tokenizer::Result<String> {
Ok(token_ids
.iter()
.map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
.collect())
}
fn token_to_id(&self, token: &str) -> Option<u32> {
match token {
"<|channel>" => Some(1),
"<channel|>" => Some(2),
_ => None,
}
}
}
fn tokenizer() -> Arc<FakeTokenizer> {
Arc::new(FakeTokenizer)
fn tokenizer() -> Arc<TestTokenizer> {
Arc::new(
TestTokenizer::new()
.with_regular_token("<|channel>", 256)
.with_regular_token("<channel|>", 257),
)
}
#[test]
rust/src/chat/src/parser/reasoning/tests.rs
+2 -25
View File
@@ -1,32 +1,9 @@
use std::sync::Arc;
use vllm_tokenizer::Tokenizer;
use vllm_tokenizer::test_utils::TestTokenizer;
use super::{ReasoningParserFactory, names};
struct FakeTokenizer;
impl Tokenizer for FakeTokenizer {
fn encode(&self, text: &str, _add_special_tokens: bool) -> vllm_tokenizer::Result<Vec<u32>> {
Ok(text.chars().map(u32::from).collect())
}
fn decode(
&self,
token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_tokenizer::Result<String> {
Ok(token_ids
.iter()
.map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
.collect())
}
fn token_to_id(&self, _token: &str) -> Option<u32> {
None
}
}
#[test]
fn factory_contains_and_lists_registered_parsers() {
let factory = ReasoningParserFactory::new();
@@ -107,7 +84,7 @@ fn factory_resolves_minimax_m3_before_generic_minimax() {
#[test]
fn factory_rejects_unknown_parser_names() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(TestTokenizer::new());
let factory = ReasoningParserFactory::new();
let error = match factory.create("missing", tokenizer) {
Ok(_) => panic!("expected parser lookup to fail"),
rust/src/chat/src/parser/unified.rs
+6 -31
View File
@@ -75,39 +75,14 @@ impl UnifiedParserFactory {
mod tests {
use std::sync::Arc;
use vllm_tokenizer::Tokenizer;
use vllm_tokenizer::test_utils::TestTokenizer;
use super::{UnifiedParserFactory, names};
struct FakeTokenizer;
impl Tokenizer for FakeTokenizer {
fn encode(
&self,
text: &str,
_add_special_tokens: bool,
) -> vllm_tokenizer::Result<Vec<u32>> {
Ok(text.chars().map(u32::from).collect())
}
fn decode(
&self,
token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_tokenizer::Result<String> {
Ok(token_ids
.iter()
.map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
.collect())
}
fn token_to_id(&self, token: &str) -> Option<u32> {
match token {
"<|channel>" => Some(1),
"<channel|>" => Some(2),
_ => None,
}
}
fn tokenizer() -> TestTokenizer {
TestTokenizer::new()
.with_regular_token("<|channel>", 256)
.with_regular_token("<channel|>", 257)
}
#[test]
@@ -119,6 +94,6 @@ mod tests {
factory.resolve_name_for_model("google/gemma-4-27b-it"),
Some(names::GEMMA4)
);
factory.create(names::GEMMA4, &[], Arc::new(FakeTokenizer)).unwrap();
factory.create(names::GEMMA4, &[], Arc::new(tokenizer())).unwrap();
}
}
rust/src/parser/Cargo.toml
+1
View File
@@ -23,6 +23,7 @@ expect-test.workspace = true
futures.workspace = true
openai-protocol.workspace = true
tool-parser.workspace = true
vllm-tokenizer = { workspace = true, features = ["test-utils"] }
[[bench]]
name = "deepseek_v3"
rust/src/parser/src/reasoning/seed_oss.rs
+15 -12
View File
@@ -49,11 +49,14 @@ mod tests {
use std::sync::Arc;
use super::SeedOssReasoningParser;
use crate::reasoning::{ReasoningParser, tests::FakeTokenizer};
use crate::reasoning::{
ReasoningParser,
tests::{SEED_THINK_END_ID, SEED_THINK_START_ID, fake_tokenizer},
};
#[test]
fn without_prompt_markers_expects_start_token() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap();
let delta = parser.push("implicit reasoning</seed:think>answer").unwrap();
@@ -66,10 +69,10 @@ mod tests {
#[test]
fn picks_up_prompt_start_boundary() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap();
// Prompt prefills `<seed:think>` (id 10), opening reasoning before the stream.
parser.initialize(&[10]).unwrap();
// Prompt prefills `<seed:think>`, opening reasoning before the stream.
parser.initialize(&[SEED_THINK_START_ID]).unwrap();
let delta = parser.push("reason</seed:think>answer").unwrap();
assert_eq!(delta.reasoning.as_deref(), Some("reason"));
@@ -78,10 +81,10 @@ mod tests {
#[test]
fn respects_prompt_end_boundary() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap();
// Prompt already closed reasoning with `</seed:think>` (id 11).
parser.initialize(&[11]).unwrap();
// Prompt already closed reasoning with `</seed:think>`.
parser.initialize(&[SEED_THINK_END_ID]).unwrap();
let delta = parser.push("answer").unwrap();
assert_eq!(delta.reasoning, None);
@@ -91,7 +94,7 @@ mod tests {
#[test]
fn handles_explicit_start_token() {
// An explicit start delimiter must not leak into reasoning text.
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap();
let delta = parser.push("<seed:think>reason</seed:think>answer").unwrap();
@@ -103,7 +106,7 @@ mod tests {
fn streams_explicit_start_token_across_pushes() {
// Start token, reasoning body, end token, and content arrive in separate
// streaming deltas.
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap();
let mut reasoning = String::new();
@@ -131,9 +134,9 @@ mod tests {
#[test]
fn handles_partial_delimiters_across_pushes() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap();
parser.initialize(&[10]).unwrap();
parser.initialize(&[SEED_THINK_START_ID]).unwrap();
// Closing delimiter `</seed:think>` arrives in two halves.
let first = parser.push("reason</seed:").unwrap();
rust/src/parser/src/reasoning/step3p5.rs
+20 -17
View File
@@ -127,14 +127,17 @@ mod tests {
use std::sync::Arc;
use super::Step3p5ReasoningParser;
use crate::reasoning::{ReasoningParser, tests::FakeTokenizer};
use crate::reasoning::{
ReasoningParser,
tests::{THINK_START_ID, fake_tokenizer},
};
#[test]
fn picks_up_prompt_start_boundary() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
// Prompt prefills `<think>` (id 1), opening reasoning before the stream.
parser.initialize(&[1]).unwrap();
// Prompt prefills `<think>`, opening reasoning before the stream.
parser.initialize(&[THINK_START_ID]).unwrap();
let delta = parser.push("This is a reasoning section</think>This is the rest").unwrap();
assert_eq!(
@@ -146,7 +149,7 @@ mod tests {
#[test]
fn handles_unterminated_reasoning() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
let pushed = parser.push("<think>reason without end").unwrap();
@@ -159,7 +162,7 @@ mod tests {
#[test]
fn handles_empty_input() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
let pushed = parser.push("").unwrap();
@@ -172,9 +175,9 @@ mod tests {
fn complex_newline_pattern_trims_only_single_framing_newline_each_side() {
// Only the immediately-adjacent framing `\n` is dropped on each side of
// `</think>`; surrounding newlines remain part of reasoning/content.
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
parser.initialize(&[1]).unwrap();
parser.initialize(&[THINK_START_ID]).unwrap();
let delta = parser
.push("\n This is a \n reasoning section\n\n\n</think>\n\nThis is the rest")
@@ -188,7 +191,7 @@ mod tests {
#[test]
fn drops_framing_newlines_in_single_push() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
let delta = parser.push("<think>reason\n</think>\nanswer").unwrap();
@@ -198,7 +201,7 @@ mod tests {
#[test]
fn drops_framing_newlines_across_pushes() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
// The trailing `\n` from the first push is held until we know whether
@@ -219,7 +222,7 @@ mod tests {
#[test]
fn replays_held_newline_when_more_reasoning_follows() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
let first = parser.push("<think>reason\n").unwrap();
@@ -232,7 +235,7 @@ mod tests {
#[test]
fn finish_flushes_held_newline_in_unterminated_stream() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
let first = parser.push("<think>reason\n").unwrap();
@@ -245,7 +248,7 @@ mod tests {
#[test]
fn preserves_inner_newlines_in_reasoning() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
let delta = parser.push("<think>line1\nline2</think>tail").unwrap();
@@ -257,7 +260,7 @@ mod tests {
fn trims_only_one_trailing_reasoning_newline() {
// Only the single framing newline immediately before `</think>` is
// dropped; earlier newlines in the reasoning body are preserved.
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
let delta = parser.push("<think>reason\n\n</think>answer").unwrap();
@@ -269,7 +272,7 @@ mod tests {
fn drops_only_first_content_newline_after_transition() {
// The leading-`\n` drop applies only to the first content delta after
// `</think>`; later deltas pass through untouched.
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
let first = parser.push("<think>reason</think>").unwrap();
@@ -288,7 +291,7 @@ mod tests {
#[test]
fn passes_through_clean_boundary_without_framing_newlines() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
let delta = parser.push("<think>reason</think>tail").unwrap();
@@ -298,7 +301,7 @@ mod tests {
#[test]
fn handles_empty_reasoning_section() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
let delta = parser.push("<think></think>answer").unwrap();
rust/src/parser/src/reasoning/tests.rs
+48 -60
View File
@@ -1,54 +1,42 @@
use std::sync::Arc;
use vllm_tokenizer::Tokenizer;
use vllm_tokenizer::test_utils::TestTokenizer;
use super::{
DeepSeekR1ReasoningParser, DelimitedReasoningParser, MiniMaxM3ReasoningParser,
Qwen3ReasoningParser, ReasoningParser,
};
pub(crate) struct FakeTokenizer;
pub(crate) const THINK_START_ID: u32 = 256;
pub(crate) const THINK_END_ID: u32 = 257;
pub(crate) const START_THINKING_ID: u32 = 258;
pub(crate) const END_THINKING_ID: u32 = 259;
pub(crate) const MINIMAX_THINK_START_ID: u32 = 260;
pub(crate) const MINIMAX_THINK_END_ID: u32 = 261;
pub(crate) const SPECIAL_BOUNDARY_ID: u32 = 262;
pub(crate) const MM_THINK_START_ID: u32 = 263;
pub(crate) const MM_THINK_END_ID: u32 = 264;
pub(crate) const SEED_THINK_START_ID: u32 = 265;
pub(crate) const SEED_THINK_END_ID: u32 = 266;
impl Tokenizer for FakeTokenizer {
fn encode(&self, text: &str, _add_special_tokens: bool) -> vllm_tokenizer::Result<Vec<u32>> {
Ok(text.chars().map(u32::from).collect())
}
fn decode(
&self,
token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_tokenizer::Result<String> {
Ok(token_ids
.iter()
.map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
.collect())
}
fn token_to_id(&self, token: &str) -> Option<u32> {
match token {
"<think>" => Some(1),
"</think>" => Some(2),
"<|START_THINKING|>" => Some(3),
"<|END_THINKING|>" => Some(4),
"◁think▷" => Some(5),
"◁/think▷" => Some(6),
"<mm:think>" => Some(8),
"</mm:think>" => Some(9),
"<seed:think>" => Some(10),
"</seed:think>" => Some(11),
_ => None,
}
}
fn is_special_id(&self, token_id: u32) -> bool {
token_id == 7
}
pub(crate) fn fake_tokenizer() -> TestTokenizer {
TestTokenizer::new()
.with_regular_token("<think>", THINK_START_ID)
.with_regular_token("</think>", THINK_END_ID)
.with_regular_token("<|START_THINKING|>", START_THINKING_ID)
.with_regular_token("<|END_THINKING|>", END_THINKING_ID)
.with_regular_token("◁think▷", MINIMAX_THINK_START_ID)
.with_regular_token("◁/think▷", MINIMAX_THINK_END_ID)
.with_special_token("<special-boundary>", SPECIAL_BOUNDARY_ID)
.with_regular_token("<mm:think>", MM_THINK_START_ID)
.with_regular_token("</mm:think>", MM_THINK_END_ID)
.with_regular_token("<seed:think>", SEED_THINK_START_ID)
.with_regular_token("</seed:think>", SEED_THINK_END_ID)
}
#[test]
fn delimited_content_only_stream() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser =
DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false).unwrap();
@@ -60,7 +48,7 @@ fn delimited_content_only_stream() {
#[test]
fn delimited_single_chunk_with_reasoning_and_content() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser =
DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false).unwrap();
@@ -71,7 +59,7 @@ fn delimited_single_chunk_with_reasoning_and_content() {
#[test]
fn delimited_partial_tokens_across_chunks() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser =
DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false).unwrap();
@@ -83,10 +71,10 @@ fn delimited_partial_tokens_across_chunks() {
#[test]
fn delimited_finish_flushes_buffer() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser =
DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false).unwrap();
parser.initialize(&[1]);
parser.initialize(&[THINK_START_ID]);
let delta = parser.push("unfinished</thi");
assert_eq!(delta.reasoning.as_deref(), Some("unfinished"));
@@ -96,7 +84,7 @@ fn delimited_finish_flushes_buffer() {
#[test]
fn qwen3_without_prompt_markers_expects_start_token() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = Qwen3ReasoningParser::new(tokenizer).unwrap();
let delta = parser.push("reason</think>answer").unwrap();
@@ -106,9 +94,9 @@ fn qwen3_without_prompt_markers_expects_start_token() {
#[test]
fn qwen3_prompt_end_marker_starts_in_content() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = Qwen3ReasoningParser::new(tokenizer).unwrap();
parser.initialize(&[2]).unwrap();
parser.initialize(&[THINK_END_ID]).unwrap();
let delta = parser.push("answer").unwrap();
assert_eq!(delta.reasoning, None);
@@ -117,7 +105,7 @@ fn qwen3_prompt_end_marker_starts_in_content() {
#[test]
fn qwen3_tolerates_old_and_new_formats() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut old_parser = Qwen3ReasoningParser::new(tokenizer.clone()).unwrap();
let old = old_parser.push("<think>reason</think>answer").unwrap();
@@ -125,7 +113,7 @@ fn qwen3_tolerates_old_and_new_formats() {
assert_eq!(old.content.as_deref(), Some("answer"));
let mut new_parser = Qwen3ReasoningParser::new(tokenizer).unwrap();
new_parser.initialize(&[1]).unwrap();
new_parser.initialize(&[THINK_START_ID]).unwrap();
let new = new_parser.push("reason</think>answer").unwrap();
assert_eq!(new.reasoning.as_deref(), Some("reason"));
assert_eq!(new.content.as_deref(), Some("answer"));
@@ -133,10 +121,10 @@ fn qwen3_tolerates_old_and_new_formats() {
#[test]
fn qwen3_stops_scanning_at_last_special_token() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = Qwen3ReasoningParser::new(tokenizer).unwrap();
parser.initialize(&[1, 7]).unwrap();
parser.initialize(&[THINK_START_ID, SPECIAL_BOUNDARY_ID]).unwrap();
let delta = parser.push("answer").unwrap();
assert_eq!(delta.reasoning, None);
@@ -145,7 +133,7 @@ fn qwen3_stops_scanning_at_last_special_token() {
#[test]
fn deepseek_r1_defaults_to_reasoning_without_prompt_boundary() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = DeepSeekR1ReasoningParser::new(tokenizer).unwrap();
let delta = parser.push("reason</think>answer").unwrap();
@@ -155,10 +143,10 @@ fn deepseek_r1_defaults_to_reasoning_without_prompt_boundary() {
#[test]
fn deepseek_r1_stops_scanning_at_last_special_token() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = DeepSeekR1ReasoningParser::new(tokenizer).unwrap();
parser.initialize(&[2, 7]).unwrap();
parser.initialize(&[THINK_END_ID, SPECIAL_BOUNDARY_ID]).unwrap();
let delta = parser.push("reason</think>answer").unwrap();
assert_eq!(delta.reasoning.as_deref(), Some("reason"));
@@ -167,7 +155,7 @@ fn deepseek_r1_stops_scanning_at_last_special_token() {
#[test]
fn minimax_m3_handles_explicit_think_delimiters() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
let delta = parser.push("<mm:think>reason</mm:think>answer").unwrap();
@@ -177,7 +165,7 @@ fn minimax_m3_handles_explicit_think_delimiters() {
#[test]
fn minimax_m3_drops_leading_end_marker() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
let delta = parser.push("</mm:think>answer").unwrap();
@@ -187,7 +175,7 @@ fn minimax_m3_drops_leading_end_marker() {
#[test]
fn minimax_m3_preserves_non_leading_end_marker() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
let delta = parser.push("XXX</mm:think>YYY").unwrap();
@@ -197,7 +185,7 @@ fn minimax_m3_preserves_non_leading_end_marker() {
#[test]
fn minimax_m3_drops_split_leading_end_marker() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
assert!(parser.push("</mm").unwrap().is_empty());
@@ -208,9 +196,9 @@ fn minimax_m3_drops_split_leading_end_marker() {
#[test]
fn minimax_m3_uses_prompt_prefilled_start_marker() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
parser.initialize(&[8]).unwrap();
parser.initialize(&[MM_THINK_START_ID]).unwrap();
let delta = parser.push("reason</mm:think>answer").unwrap();
assert_eq!(delta.reasoning.as_deref(), Some("reason"));
@@ -219,9 +207,9 @@ fn minimax_m3_uses_prompt_prefilled_start_marker() {
#[test]
fn minimax_m3_uses_prompt_prefilled_end_marker() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(fake_tokenizer());
let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
parser.initialize(&[9]).unwrap();
parser.initialize(&[MM_THINK_END_ID]).unwrap();
let delta = parser.push("answer").unwrap();
assert_eq!(delta.reasoning, None);
rust/src/parser/src/unified/combined.rs
+6 -31
View File
@@ -124,42 +124,17 @@ impl UnifiedParser for CombinedParser {
mod tests {
use std::sync::Arc;
use vllm_tokenizer::Tokenizer;
use vllm_tokenizer::test_utils::TestTokenizer;
use super::CombinedParser;
use crate::reasoning::{Qwen3ReasoningParser, ReasoningDelta, ReasoningParser};
use crate::tool::{Qwen3XmlToolParser, Tool, ToolParser};
use crate::unified::{UnifiedParser, UnifiedParserEvent, UnifiedParserOutput};
struct FakeTokenizer;
impl Tokenizer for FakeTokenizer {
fn encode(
&self,
text: &str,
_add_special_tokens: bool,
) -> vllm_tokenizer::Result<Vec<u32>> {
Ok(text.chars().map(u32::from).collect())
}
fn decode(
&self,
token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_tokenizer::Result<String> {
Ok(token_ids
.iter()
.map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
.collect())
}
fn token_to_id(&self, token: &str) -> Option<u32> {
match token {
"<think>" => Some(1),
"</think>" => Some(2),
_ => None,
}
}
fn tokenizer() -> TestTokenizer {
TestTokenizer::new()
.with_regular_token("<think>", 256)
.with_regular_token("</think>", 257)
}
fn test_tools() -> Vec<Tool> {
@@ -273,7 +248,7 @@ mod tests {
#[test]
fn combined_parser_emits_reasoning_and_text() {
let tokenizer = Arc::new(FakeTokenizer);
let tokenizer = Arc::new(tokenizer());
let reasoning = Qwen3ReasoningParser::create(tokenizer).unwrap();
let mut parser = CombinedParser::new(Some(reasoning), None);
rust/src/parser/src/unified/gemma4.rs
+15 -66
View File
@@ -515,7 +515,7 @@ mod tests {
use serde_json::{Value, json};
use thiserror_ext::AsReport;
use vllm_tokenizer::Tokenizer;
use vllm_tokenizer::test_utils::TestTokenizer;
use winnow::combinator::{eof, terminated};
use winnow::error::ErrMode;
use winnow::prelude::*;
@@ -527,66 +527,15 @@ mod tests {
use crate::tool::Tool;
use crate::unified::{UnifiedParserEvent, parsing_failed};
struct FakeTokenizer;
const CHANNEL_START_ID: u32 = 256;
const CHANNEL_END_ID: u32 = 257;
const TURN_BOUNDARY_ID: u32 = 258;
impl Tokenizer for FakeTokenizer {
fn encode(
&self,
text: &str,
_add_special_tokens: bool,
) -> vllm_tokenizer::Result<Vec<u32>> {
Ok(text.chars().map(u32::from).collect())
}
fn decode(
&self,
token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_tokenizer::Result<String> {
Ok(token_ids
.iter()
.map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
.collect())
}
fn token_to_id(&self, token: &str) -> Option<u32> {
match token {
CHANNEL_START => Some(100),
CHANNEL_END => Some(101),
_ => None,
}
}
fn is_special_id(&self, token_id: u32) -> bool {
matches!(token_id, 100..=105)
}
}
struct MissingTokenTokenizer;
impl Tokenizer for MissingTokenTokenizer {
fn encode(
&self,
text: &str,
_add_special_tokens: bool,
) -> vllm_tokenizer::Result<Vec<u32>> {
Ok(text.chars().map(u32::from).collect())
}
fn decode(
&self,
token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_tokenizer::Result<String> {
Ok(token_ids
.iter()
.map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
.collect())
}
fn token_to_id(&self, _token: &str) -> Option<u32> {
None
}
fn tokenizer() -> TestTokenizer {
TestTokenizer::new()
.with_special_token(CHANNEL_START, CHANNEL_START_ID)
.with_special_token(CHANNEL_END, CHANNEL_END_ID)
.with_special_token("<turn-boundary>", TURN_BOUNDARY_ID)
}
trait UnifiedParserTestExt {
@@ -716,12 +665,12 @@ mod tests {
}
fn test_parser() -> Gemma4UnifiedParser {
Gemma4UnifiedParser::new(&test_tools(), Arc::new(FakeTokenizer)).unwrap()
Gemma4UnifiedParser::new(&test_tools(), Arc::new(tokenizer())).unwrap()
}
#[test]
fn gemma4_create_requires_channel_start_token() {
let error = match Gemma4UnifiedParser::new(&test_tools(), Arc::new(MissingTokenTokenizer)) {
let error = match Gemma4UnifiedParser::new(&test_tools(), Arc::new(TestTokenizer::new())) {
Ok(_) => panic!("expected missing token error"),
Err(error) => error,
};
@@ -1046,7 +995,7 @@ mod tests {
#[test]
fn gemma4_initialize_open_channel_prompt_starts_in_reasoning() {
let mut parser = test_parser();
parser.initialize(&[100, 3000, 3001]).unwrap();
parser.initialize(&[CHANNEL_START_ID, 3000, 3001]).unwrap();
let output = parser.parse_complete("reason<channel|>answer").unwrap();
@@ -1057,7 +1006,7 @@ mod tests {
#[test]
fn gemma4_initialize_turn_prompt_starts_in_text() {
let mut parser = test_parser();
parser.initialize(&[104, 3000, 3001]).unwrap();
parser.initialize(&[TURN_BOUNDARY_ID, 3000, 3001]).unwrap();
let output = parser.parse_complete("<|channel>thought\nreason<channel|>answer").unwrap();
@@ -1068,7 +1017,7 @@ mod tests {
#[test]
fn gemma4_initialize_special_token_caps_boundary_scan() {
let mut parser = test_parser();
parser.initialize(&[100, 3000, 104, 3001]).unwrap();
parser.initialize(&[CHANNEL_START_ID, 3000, TURN_BOUNDARY_ID, 3001]).unwrap();
let output = parser.parse_complete("answer").unwrap();
@@ -1079,7 +1028,7 @@ mod tests {
#[test]
fn gemma4_initialize_closed_channel_prompt_starts_in_text() {
let mut parser = test_parser();
parser.initialize(&[100, 3000, 3001, 101]).unwrap();
parser.initialize(&[CHANNEL_START_ID, 3000, 3001, CHANNEL_END_ID]).unwrap();
let output = parser.parse_complete("answer").unwrap();
rust/src/server/Cargo.toml
+1
View File
@@ -63,6 +63,7 @@ serial_test.workspace = true
tempfile.workspace = true
tower.workspace = true
vllm-engine-core-client = { workspace = true, features = ["test-util"] }
vllm-tokenizer = { workspace = true, features = ["test-utils"] }
zeromq.workspace = true
[lints]
rust/src/server/src/grpc/tests.rs
+3 -30
View File
@@ -24,8 +24,9 @@ use vllm_engine_core_client::protocol::{
use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, EngineId};
use vllm_llm::Llm;
use vllm_text::tokenizer::{DynTokenizer, Tokenizer};
use vllm_text::tokenizer::DynTokenizer;
use vllm_text::{Prompt, TextBackend};
use vllm_tokenizer::test_utils::TestTokenizer;
use zeromq::prelude::{SocketRecv, SocketSend};
use zeromq::{DealerSocket, PushSocket, ZmqMessage};
@@ -155,37 +156,9 @@ fn test_llm(client: EngineCoreClient) -> Llm {
#[derive(Clone, Debug)]
struct FakeTextBackend;
#[derive(Debug)]
struct FakeTokenizer;
impl Tokenizer for FakeTokenizer {
fn encode(
&self,
text: &str,
_add_special_tokens: bool,
) -> vllm_text::tokenizer::Result<Vec<u32>> {
Ok(text.bytes().map(u32::from).collect())
}
fn decode(
&self,
token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_text::tokenizer::Result<String> {
Ok(
String::from_utf8_lossy(&token_ids.iter().map(|id| *id as u8).collect::<Vec<_>>())
.into_owned(),
)
}
fn token_to_id(&self, token: &str) -> Option<u32> {
token.bytes().next().map(u32::from)
}
}
impl TextBackend for FakeTextBackend {
fn tokenizer(&self) -> DynTokenizer {
Arc::new(FakeTokenizer)
Arc::new(TestTokenizer::new())
}
fn model_id(&self) -> &str {
rust/src/server/src/routes/http_client_tests.rs
+3 -30
View File
@@ -24,8 +24,9 @@ use vllm_engine_core_client::protocol::{
use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};
use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, EngineId};
use vllm_llm::Llm;
use vllm_text::tokenizer::{DynTokenizer, Tokenizer};
use vllm_text::tokenizer::DynTokenizer;
use vllm_text::{Prompt, TextBackend};
use vllm_tokenizer::test_utils::TestTokenizer;
use zeromq::prelude::{SocketRecv, SocketSend};
use zeromq::{DealerSocket, PushSocket, ZmqMessage};
@@ -151,37 +152,9 @@ fn test_llm(client: EngineCoreClient) -> Llm {
#[derive(Clone, Debug)]
struct FakeChatBackend;
#[derive(Debug)]
struct FakeChatTokenizer;
impl Tokenizer for FakeChatTokenizer {
fn encode(
&self,
text: &str,
_add_special_tokens: bool,
) -> vllm_text::tokenizer::Result<Vec<u32>> {
Ok(text.bytes().map(u32::from).collect())
}
fn decode(
&self,
token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_text::tokenizer::Result<String> {
Ok(
String::from_utf8_lossy(&token_ids.iter().map(|id| *id as u8).collect::<Vec<_>>())
.into_owned(),
)
}
fn token_to_id(&self, token: &str) -> Option<u32> {
token.bytes().next().map(u32::from)
}
}
impl TextBackend for FakeChatBackend {
fn tokenizer(&self) -> DynTokenizer {
Arc::new(FakeChatTokenizer)
Arc::new(TestTokenizer::new())
}
fn model_id(&self) -> &str {
rust/src/server/src/routes/openai/completions/convert.rs
+15 -39
View File
@@ -194,7 +194,7 @@ mod tests {
use axum::http::HeaderMap;
use serde_json::json;
use vllm_text::Prompt;
use vllm_text::tokenizer::Tokenizer;
use vllm_tokenizer::test_utils::TestTokenizer;
use super::prepare_completion_request;
use crate::lora::LoraModelResolution;
@@ -212,32 +212,8 @@ mod tests {
}
}
#[derive(Debug)]
struct TestTokenizer;
impl Tokenizer for TestTokenizer {
fn encode(
&self,
text: &str,
_add_special_tokens: bool,
) -> vllm_text::tokenizer::Result<Vec<u32>> {
Ok(text.bytes().map(u32::from).collect())
}
fn decode(
&self,
token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_text::tokenizer::Result<String> {
Ok(
String::from_utf8_lossy(&token_ids.iter().map(|id| *id as u8).collect::<Vec<_>>())
.into_owned(),
)
}
fn token_to_id(&self, _token: &str) -> Option<u32> {
None
}
fn test_tokenizer() -> TestTokenizer {
TestTokenizer::new()
}
fn base_request_json() -> serde_json::Value {
@@ -297,7 +273,7 @@ mod tests {
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
&TestTokenizer,
&test_tokenizer(),
)
.expect("prepare");
@@ -340,7 +316,7 @@ mod tests {
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
&TestTokenizer,
&test_tokenizer(),
)
.expect("prepare")
.text_request
@@ -374,7 +350,7 @@ mod tests {
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
&TestTokenizer,
&test_tokenizer(),
)
.expect("prepare");
@@ -399,7 +375,7 @@ mod tests {
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
&TestTokenizer,
&test_tokenizer(),
)
.expect("prepare");
@@ -422,7 +398,7 @@ mod tests {
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
&TestTokenizer,
&test_tokenizer(),
)
.expect("prepare");
@@ -446,7 +422,7 @@ mod tests {
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
&TestTokenizer,
&test_tokenizer(),
)
.expect("prepare");
@@ -471,7 +447,7 @@ mod tests {
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
&TestTokenizer,
&test_tokenizer(),
)
.expect("prepare");
@@ -498,7 +474,7 @@ mod tests {
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
&TestTokenizer,
&test_tokenizer(),
)
.expect("prepare");
@@ -523,7 +499,7 @@ mod tests {
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
&TestTokenizer,
&test_tokenizer(),
)
.expect("prepare");
@@ -549,7 +525,7 @@ mod tests {
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
&TestTokenizer,
&test_tokenizer(),
)
.expect("prepare");
assert_eq!(prepared.text_request.sampling_params.logprobs, Some(1));
@@ -574,7 +550,7 @@ mod tests {
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
request_context(&headers, None),
&TestTokenizer,
&test_tokenizer(),
)
.expect("prepare");
assert_eq!(prepared.text_request.data_parallel_rank, Some(3));
@@ -593,7 +569,7 @@ mod tests {
request,
&served(&["Qwen/Qwen1.5-0.5B-Chat"]),
ResolvedRequestContext::default(),
&TestTokenizer,
&test_tokenizer(),
)
.expect("prepare");
assert_eq!(prepared.text_request.data_parallel_rank, None);
rust/src/server/src/routes/tests.rs
+18 -80
View File
@@ -42,6 +42,7 @@ use vllm_llm::Llm;
use vllm_metrics::METRICS;
use vllm_text::tokenizer::{DynTokenizer, Tokenizer};
use vllm_text::{Prompt, TextBackend};
use vllm_tokenizer::test_utils::TestTokenizer;
use zeromq::prelude::{SocketRecv, SocketSend};
use zeromq::{DealerSocket, PushSocket, ZmqMessage};
@@ -417,82 +418,19 @@ struct FakeChatBackend {
}
/// Synthetic BOS id used when `add_special_tokens` is true in tests.
const FAKE_BOS_TOKEN_ID: u32 = 1;
const FAKE_BOS_TOKEN_ID: u32 = 256;
#[derive(Debug)]
struct FakeChatTokenizer;
impl Tokenizer for FakeChatTokenizer {
fn encode(
&self,
text: &str,
add_special_tokens: bool,
) -> vllm_text::tokenizer::Result<Vec<u32>> {
let mut token_ids = Vec::new();
if add_special_tokens {
token_ids.push(FAKE_BOS_TOKEN_ID);
}
let mut rest = text;
while !rest.is_empty() {
if let Some(stripped) = rest.strip_prefix("<image>") {
token_ids.push(999);
rest = stripped;
continue;
}
if let Some(stripped) = rest.strip_prefix("<|image_pad|>") {
token_ids.push(151655);
rest = stripped;
continue;
}
let ch = rest.chars().next().expect("rest is not empty");
let mut buf = [0; 4];
token_ids.extend(ch.encode_utf8(&mut buf).bytes().map(u32::from));
rest = &rest[ch.len_utf8()..];
}
Ok(token_ids)
}
fn decode(
&self,
token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_text::tokenizer::Result<String> {
Ok(
String::from_utf8_lossy(&token_ids.iter().map(|id| *id as u8).collect::<Vec<_>>())
.into_owned(),
)
}
fn token_to_id(&self, token: &str) -> Option<u32> {
match token {
"<image>" => Some(999),
"<|image_pad|>" => Some(151655),
"<think>" => Some(0xF001),
"</think>" => Some(0xF002),
"<|START_THINKING|>" => Some(0xF003),
"<|END_THINKING|>" => Some(0xF004),
"◁think▷" => Some(0xF005),
"◁/think▷" => Some(0xF006),
_ => None,
}
}
fn id_to_token(&self, id: u32) -> Option<String> {
match id {
FAKE_BOS_TOKEN_ID => Some("<bos>".to_string()),
999 => Some("<image>".to_string()),
151655 => Some("<|image_pad|>".to_string()),
0xF001 => Some("<think>".to_string()),
0xF002 => Some("</think>".to_string()),
0xF003 => Some("<|START_THINKING|>".to_string()),
0xF004 => Some("<|END_THINKING|>".to_string()),
0xF005 => Some("◁think▷".to_string()),
0xF006 => Some("◁/think▷".to_string()),
id if id < 128 => char::from_u32(id).map(|ch| ch.to_string()),
_ => None,
}
}
fn fake_chat_tokenizer() -> TestTokenizer {
TestTokenizer::new()
.with_bos_token("<bos>", FAKE_BOS_TOKEN_ID)
.with_regular_token("<image>", 999)
.with_regular_token("<|image_pad|>", 151655)
.with_regular_token("<think>", 0xF001)
.with_regular_token("</think>", 0xF002)
.with_regular_token("<|START_THINKING|>", 0xF003)
.with_regular_token("<|END_THINKING|>", 0xF004)
.with_regular_token("◁think▷", 0xF005)
.with_regular_token("◁/think▷", 0xF006)
}
impl FakeChatBackend {
@@ -530,7 +468,7 @@ impl fmt::Debug for FakeChatBackend {
impl TextBackend for FakeChatBackend {
fn tokenizer(&self) -> DynTokenizer {
Arc::new(FakeChatTokenizer)
Arc::new(fake_chat_tokenizer())
}
fn model_id(&self) -> &str {
@@ -630,7 +568,7 @@ fn qwen_multimodal_model_info() -> vllm_chat::multimodal::MultimodalModelInfo {
Some("qwen2_vl".to_string()),
Some(&config_path),
None,
Arc::new(FakeChatTokenizer),
Arc::new(fake_chat_tokenizer()),
)
.expect("load multimodal info")
.expect("qwen multimodal info is registered");
@@ -650,7 +588,7 @@ impl Tokenizer for FailingDecodeTokenizer {
text: &str,
add_special_tokens: bool,
) -> vllm_text::tokenizer::Result<Vec<u32>> {
FakeChatTokenizer.encode(text, add_special_tokens)
fake_chat_tokenizer().encode(text, add_special_tokens)
}
fn decode(
@@ -664,11 +602,11 @@ impl Tokenizer for FailingDecodeTokenizer {
));
}
FakeChatTokenizer.decode(token_ids, skip_special_tokens)
fake_chat_tokenizer().decode(token_ids, skip_special_tokens)
}
fn token_to_id(&self, token: &str) -> Option<u32> {
FakeChatTokenizer.token_to_id(token)
fake_chat_tokenizer().token_to_id(token)
}
}
rust/src/text/Cargo.toml
+1
View File
@@ -31,6 +31,7 @@ serial_test.workspace = true
tempfile.workspace = true
tokio.workspace = true
vllm-llm = { workspace = true, features = ["test-util"] }
vllm-tokenizer = { workspace = true, features = ["test-utils"] }
[lints]
workspace = true
rust/src/text/src/lower.rs
+3 -28
View File
@@ -275,6 +275,7 @@ mod tests {
use std::collections::{BTreeSet, HashMap};
use serial_test::file_serial;
use vllm_tokenizer::test_utils::TestTokenizer;
use super::*;
use crate::backend::hf::HfTextBackend;
@@ -282,34 +283,8 @@ mod tests {
use crate::error::{LogprobsError, TokenIdsError};
use crate::request::{Prompt, TextRequest};
/// Stub tokenizer that returns empty token IDs — sufficient for tests that
/// don't exercise bad-words tokenization.
struct StubTokenizer;
impl Tokenizer for StubTokenizer {
fn encode(
&self,
_text: &str,
_add_special_tokens: bool,
) -> vllm_tokenizer::Result<Vec<u32>> {
Ok(vec![])
}
fn decode(
&self,
_token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_tokenizer::Result<String> {
Ok(String::new())
}
fn token_to_id(&self, _token: &str) -> Option<u32> {
None
}
}
fn stub_tokenizer() -> StubTokenizer {
StubTokenizer
fn stub_tokenizer() -> TestTokenizer {
TestTokenizer::new()
}
struct FixedTokenizer {
rust/src/text/src/output/logprobs.rs
+3 -30
View File
@@ -129,40 +129,13 @@ fn decode_position_logprobs<T: Tokenizer + ?Sized>(
#[cfg(test)]
mod tests {
use vllm_llm::{Logprobs, PositionLogprobs, TokenLogprob};
use vllm_tokenizer::test_utils::TestTokenizer;
use super::*;
#[derive(Debug)]
struct ByteTokenizer;
impl vllm_tokenizer::Tokenizer for ByteTokenizer {
fn encode(
&self,
_text: &str,
_add_special_tokens: bool,
) -> vllm_tokenizer::Result<Vec<u32>> {
unreachable!()
}
fn decode(
&self,
token_ids: &[u32],
_skip_special_tokens: bool,
) -> vllm_tokenizer::Result<String> {
Ok(String::from_utf8_lossy(
&token_ids.iter().map(|token_id| *token_id as u8).collect::<Vec<_>>(),
)
.into_owned())
}
fn token_to_id(&self, _token: &str) -> Option<u32> {
unreachable!()
}
}
#[test]
fn decode_logprobs_decodes_every_candidate_token() {
let tokenizer = ByteTokenizer;
let tokenizer = TestTokenizer::new();
let logprobs = Logprobs {
positions: vec![PositionLogprobs {
entries: vec![
@@ -205,7 +178,7 @@ mod tests {
#[test]
fn decode_prompt_logprobs_separates_first_prompt_token() {
let tokenizer = ByteTokenizer;
let tokenizer = TestTokenizer::new();
let logprobs = Logprobs {
positions: vec![PositionLogprobs {
entries: vec![TokenLogprob {