[Rust Frontend] Tolerate out-of-vocab tokens from prompt in detokenizer

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
fix clippy
2026-06-30 15:37:56 +08:00 · 2026-06-30 15:20:47 +08:00 · 2026-06-30 15:14:45 +08:00 · 2026-06-30 15:10:18 +08:00 · 2026-06-30 15:08:15 +08:00 · 2026-06-30 14:53:50 +08:00
28 changed files with 754 additions and 832 deletions
@@ -5310,6 +5310,7 @@ dependencies = [
 "vllm-llm",
 "vllm-metrics",
 "vllm-text",
+ "vllm-tokenizer",
 "zeromq",
 ]

@@ -50,6 +50,7 @@ tokio.workspace = true
 tracing-subscriber.workspace = true
 uuid.workspace = true
 vllm-engine-core-client = { workspace = true, features = ["test-util"] }
+vllm-tokenizer = { workspace = true, features = ["test-utils"] }
 zeromq.workspace = true

 [lints]
@@ -154,7 +154,8 @@ mod tests {
    use thiserror_ext::AsReport as _;
    use vllm_text::Prompt;
    use vllm_text::backend::hf::TokenizerSource;
-    use vllm_text::tokenizer::{DynTokenizer, Tokenizer};
+    use vllm_text::tokenizer::DynTokenizer;
+    use vllm_tokenizer::test_utils::TestTokenizer;

    use super::HfChatBackend;
    use crate::backend::{ChatBackend, LoadModelBackendsOptions, NewChatOutputProcessorOptions};
@@ -196,32 +197,8 @@ mod tests {
        }
    }

-    struct TestTokenizer;
-
-    impl Tokenizer for TestTokenizer {
-        fn encode(
-            &self,
-            _text: &str,
-            _add_special_tokens: bool,
-        ) -> vllm_text::tokenizer::Result<Vec<u32>> {
-            Ok(Vec::new())
-        }
-
-        fn decode(
-            &self,
-            _token_ids: &[u32],
-            _skip_special_tokens: bool,
-        ) -> vllm_text::tokenizer::Result<String> {
-            Ok(String::new())
-        }
-
-        fn token_to_id(&self, _token: &str) -> Option<u32> {
-            None
-        }
-    }
-
    fn test_tokenizer() -> DynTokenizer {
-        Arc::new(TestTokenizer)
+        Arc::new(TestTokenizer::new())
    }

    fn backend_for_selection(
@@ -563,7 +563,7 @@ mod tests {

    use llm_multimodal::TokenId;
    use vllm_engine_core_client::protocol::tensor::WireArrayData;
-    use vllm_text::tokenizer::{IncrementalDecoder, Tokenizer, TokenizerError};
+    use vllm_tokenizer::test_utils::TestTokenizer;

    use super::*;

@@ -574,60 +574,14 @@ mod tests {
    const LLAMA4_TILE_X_SEPARATOR_ID: u32 = 200093;
    const LLAMA4_TILE_Y_SEPARATOR_ID: u32 = 200094;

-    struct TestTokenizer;
-
-    impl Tokenizer for TestTokenizer {
-        fn encode(
-            &self,
-            text: &str,
-            _add_special_tokens: bool,
-        ) -> std::result::Result<Vec<u32>, TokenizerError> {
-            Ok(match text {
-                "<|image|>" => vec![LLAMA4_IMAGE_ID],
-                text => text.bytes().map(u32::from).collect(),
-            })
-        }
-
-        fn decode(
-            &self,
-            _token_ids: &[u32],
-            _skip_special_tokens: bool,
-        ) -> std::result::Result<String, TokenizerError> {
-            Ok(String::new())
-        }
-
-        fn token_to_id(&self, token: &str) -> Option<u32> {
-            match token {
-                "<|image_start|>" => Some(LLAMA4_IMAGE_START_ID),
-                "<|image_end|>" => Some(LLAMA4_IMAGE_END_ID),
-                "<|image|>" => Some(LLAMA4_IMAGE_ID),
-                "<|patch|>" => Some(LLAMA4_PATCH_ID),
-                "<|tile_x_separator|>" => Some(LLAMA4_TILE_X_SEPARATOR_ID),
-                "<|tile_y_separator|>" => Some(LLAMA4_TILE_Y_SEPARATOR_ID),
-                _ => None,
-            }
-        }
-
-        fn id_to_token(&self, id: u32) -> Option<String> {
-            match id {
-                LLAMA4_IMAGE_START_ID => Some("<|image_start|>".to_string()),
-                LLAMA4_IMAGE_END_ID => Some("<|image_end|>".to_string()),
-                LLAMA4_IMAGE_ID => Some("<|image|>".to_string()),
-                LLAMA4_PATCH_ID => Some("<|patch|>".to_string()),
-                LLAMA4_TILE_X_SEPARATOR_ID => Some("<|tile_x_separator|>".to_string()),
-                LLAMA4_TILE_Y_SEPARATOR_ID => Some("<|tile_y_separator|>".to_string()),
-                _ => None,
-            }
-        }
-
-        fn create_decode_stream(
-            &self,
-            _prompt_token_ids: &[u32],
-            _skip_special_tokens: bool,
-            _min_bytes_to_buffer: usize,
-        ) -> Box<dyn IncrementalDecoder + '_> {
-            unreachable!("not used")
-        }
+    fn llama4_tokenizer() -> TestTokenizer {
+        TestTokenizer::new()
+            .with_regular_token("<|image_start|>", LLAMA4_IMAGE_START_ID)
+            .with_regular_token("<|image_end|>", LLAMA4_IMAGE_END_ID)
+            .with_regular_token("<|image|>", LLAMA4_IMAGE_ID)
+            .with_regular_token("<|patch|>", LLAMA4_PATCH_ID)
+            .with_regular_token("<|tile_x_separator|>", LLAMA4_TILE_X_SEPARATOR_ID)
+            .with_regular_token("<|tile_y_separator|>", LLAMA4_TILE_Y_SEPARATOR_ID)
    }

    fn test_info(model_type: &str, config: serde_json::Value) -> MultimodalModelInfo {
@@ -635,7 +589,7 @@ mod tests {
            model_id: format!("{model_type}-test"),
            model_type: Some(model_type.to_string()),
            config,
-            tokenizer: TokenizerResolver(Arc::new(TestTokenizer)),
+            tokenizer: TokenizerResolver(Arc::new(llama4_tokenizer())),
        };
        let spec = context
            .resolve_model_spec()
@@ -189,46 +189,19 @@ impl ChatOutputProcessor for DefaultChatOutputProcessor {
 mod tests {
    use std::sync::Arc;

-    use vllm_tokenizer::Tokenizer;
+    use vllm_tokenizer::test_utils::TestTokenizer;

    use super::DefaultChatOutputProcessor;
    use crate::Error;
    use crate::parser::ParserSelection;
    use crate::request::ChatRequest;

-    struct FakeTokenizer;
-
-    impl Tokenizer for FakeTokenizer {
-        fn encode(
-            &self,
-            text: &str,
-            _add_special_tokens: bool,
-        ) -> vllm_tokenizer::Result<Vec<u32>> {
-            Ok(text.chars().map(u32::from).collect())
-        }
-
-        fn decode(
-            &self,
-            token_ids: &[u32],
-            _skip_special_tokens: bool,
-        ) -> vllm_tokenizer::Result<String> {
-            Ok(token_ids
-                .iter()
-                .map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
-                .collect())
-        }
-
-        fn token_to_id(&self, token: &str) -> Option<u32> {
-            match token {
-                "<|channel>" => Some(1),
-                "<channel|>" => Some(2),
-                _ => None,
-            }
-        }
-    }
-
-    fn tokenizer() -> Arc<FakeTokenizer> {
-        Arc::new(FakeTokenizer)
+    fn tokenizer() -> Arc<TestTokenizer> {
+        Arc::new(
+            TestTokenizer::new()
+                .with_regular_token("<|channel>", 256)
+                .with_regular_token("<channel|>", 257),
+        )
    }

    #[test]
@@ -1,32 +1,9 @@
 use std::sync::Arc;

-use vllm_tokenizer::Tokenizer;
+use vllm_tokenizer::test_utils::TestTokenizer;

 use super::{ReasoningParserFactory, names};

-struct FakeTokenizer;
-
-impl Tokenizer for FakeTokenizer {
-    fn encode(&self, text: &str, _add_special_tokens: bool) -> vllm_tokenizer::Result<Vec<u32>> {
-        Ok(text.chars().map(u32::from).collect())
-    }
-
-    fn decode(
-        &self,
-        token_ids: &[u32],
-        _skip_special_tokens: bool,
-    ) -> vllm_tokenizer::Result<String> {
-        Ok(token_ids
-            .iter()
-            .map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
-            .collect())
-    }
-
-    fn token_to_id(&self, _token: &str) -> Option<u32> {
-        None
-    }
-}
-
 #[test]
 fn factory_contains_and_lists_registered_parsers() {
    let factory = ReasoningParserFactory::new();
@@ -107,7 +84,7 @@ fn factory_resolves_minimax_m3_before_generic_minimax() {

 #[test]
 fn factory_rejects_unknown_parser_names() {
-    let tokenizer = Arc::new(FakeTokenizer);
+    let tokenizer = Arc::new(TestTokenizer::new());
    let factory = ReasoningParserFactory::new();
    let error = match factory.create("missing", tokenizer) {
        Ok(_) => panic!("expected parser lookup to fail"),
@@ -75,39 +75,14 @@ impl UnifiedParserFactory {
 mod tests {
    use std::sync::Arc;

-    use vllm_tokenizer::Tokenizer;
+    use vllm_tokenizer::test_utils::TestTokenizer;

    use super::{UnifiedParserFactory, names};

-    struct FakeTokenizer;
-
-    impl Tokenizer for FakeTokenizer {
-        fn encode(
-            &self,
-            text: &str,
-            _add_special_tokens: bool,
-        ) -> vllm_tokenizer::Result<Vec<u32>> {
-            Ok(text.chars().map(u32::from).collect())
-        }
-
-        fn decode(
-            &self,
-            token_ids: &[u32],
-            _skip_special_tokens: bool,
-        ) -> vllm_tokenizer::Result<String> {
-            Ok(token_ids
-                .iter()
-                .map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
-                .collect())
-        }
-
-        fn token_to_id(&self, token: &str) -> Option<u32> {
-            match token {
-                "<|channel>" => Some(1),
-                "<channel|>" => Some(2),
-                _ => None,
-            }
-        }
+    fn tokenizer() -> TestTokenizer {
+        TestTokenizer::new()
+            .with_regular_token("<|channel>", 256)
+            .with_regular_token("<channel|>", 257)
    }

    #[test]
@@ -119,6 +94,6 @@ mod tests {
            factory.resolve_name_for_model("google/gemma-4-27b-it"),
            Some(names::GEMMA4)
        );
-        factory.create(names::GEMMA4, &[], Arc::new(FakeTokenizer)).unwrap();
+        factory.create(names::GEMMA4, &[], Arc::new(tokenizer())).unwrap();
    }
 }
@@ -21,15 +21,17 @@ use vllm_engine_core_client::protocol::{
 use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};
 use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig};
 use vllm_llm::Llm;
-use vllm_text::tokenizer::{DynTokenizer, Tokenizer};
+use vllm_text::tokenizer::DynTokenizer;
 use vllm_text::{
    DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs, DecodedTokenLogprob, Prompt,
    TextBackend,
 };
+use vllm_tokenizer::test_utils::TestTokenizer;
 use zeromq::prelude::{SocketRecv, SocketSend};
 use zeromq::{DealerSocket, PushSocket, ZmqMessage};

 const SPECIAL_STOP_TOKEN_ID: u32 = 256;
+const UNKNOWN_DECODE_TOKEN_ID: u32 = 10_000;

 fn request_output(
    request_id: &str,
@@ -158,45 +160,18 @@ async fn connect_chat_llm_with_ipc(
 struct FakeChatBackend {
    has_template: bool,
    model_id: String,
+    tokenizer: DynTokenizer,
 }

-#[derive(Debug)]
-struct FakeChatTokenizer;
-
-impl Tokenizer for FakeChatTokenizer {
-    fn encode(&self, text: &str, _add_special_tokens: bool) -> vllm_tokenizer::Result<Vec<u32>> {
-        Ok(text.bytes().map(u32::from).collect())
-    }
-
-    fn decode(
-        &self,
-        token_ids: &[u32],
-        skip_special_tokens: bool,
-    ) -> vllm_tokenizer::Result<String> {
-        let bytes = token_ids
-            .iter()
-            .filter_map(|id| {
-                if skip_special_tokens && *id == SPECIAL_STOP_TOKEN_ID {
-                    None
-                } else {
-                    Some(*id as u8)
-                }
-            })
-            .collect::<Vec<_>>();
-        Ok(String::from_utf8_lossy(&bytes).into_owned())
-    }
-
-    fn token_to_id(&self, token: &str) -> Option<u32> {
-        match token {
-            "<think>" => Some(0xF001),
-            "</think>" => Some(0xF002),
-            "<|START_THINKING|>" => Some(0xF003),
-            "<|END_THINKING|>" => Some(0xF004),
-            "◁think▷" => Some(0xF005),
-            "◁/think▷" => Some(0xF006),
-            _ => None,
-        }
-    }
+fn fake_chat_tokenizer() -> TestTokenizer {
+    TestTokenizer::new()
+        .with_special_token("<stop>", SPECIAL_STOP_TOKEN_ID)
+        .with_regular_token("<think>", 0xF001)
+        .with_regular_token("</think>", 0xF002)
+        .with_regular_token("<|START_THINKING|>", 0xF003)
+        .with_regular_token("<|END_THINKING|>", 0xF004)
+        .with_regular_token("◁think▷", 0xF005)
+        .with_regular_token("◁/think▷", 0xF006)
 }

 impl fmt::Debug for FakeChatBackend {
@@ -210,6 +185,7 @@ impl FakeChatBackend {
        Self {
            has_template: true,
            model_id: "test-model".to_string(),
+            tokenizer: Arc::new(fake_chat_tokenizer()),
        }
    }

@@ -217,6 +193,7 @@ impl FakeChatBackend {
        Self {
            has_template: false,
            model_id: "test-model".to_string(),
+            tokenizer: Arc::new(fake_chat_tokenizer()),
        }
    }

@@ -224,13 +201,19 @@ impl FakeChatBackend {
        Self {
            has_template: true,
            model_id: model_id.into(),
+            tokenizer: Arc::new(fake_chat_tokenizer()),
        }
    }
+
+    fn with_tokenizer(mut self, tokenizer: DynTokenizer) -> Self {
+        self.tokenizer = tokenizer;
+        self
+    }
 }

 impl TextBackend for FakeChatBackend {
    fn tokenizer(&self) -> DynTokenizer {
-        Arc::new(FakeChatTokenizer)
+        Arc::clone(&self.tokenizer)
    }

    fn model_id(&self) -> &str {
@@ -282,65 +265,6 @@ impl ChatRenderer for FakeChatBackend {
    }
 }

-#[derive(Clone, Debug)]
-struct FailingDecodeBackend {
-    inner: FakeChatBackend,
-}
-
-#[derive(Debug)]
-struct FailingDecodeTokenizer;
-
-impl Tokenizer for FailingDecodeTokenizer {
-    fn encode(&self, text: &str, add_special_tokens: bool) -> vllm_tokenizer::Result<Vec<u32>> {
-        FakeChatTokenizer.encode(text, add_special_tokens)
-    }
-
-    fn decode(
-        &self,
-        token_ids: &[u32],
-        skip_special_tokens: bool,
-    ) -> vllm_tokenizer::Result<String> {
-        if token_ids.contains(&(b'i' as u32)) {
-            return Err(vllm_tokenizer::TokenizerError("decode failed".to_string()));
-        }
-        FakeChatTokenizer.decode(token_ids, skip_special_tokens)
-    }
-
-    fn token_to_id(&self, token: &str) -> Option<u32> {
-        FakeChatTokenizer.token_to_id(token)
-    }
-}
-
-impl TextBackend for FailingDecodeBackend {
-    fn tokenizer(&self) -> DynTokenizer {
-        Arc::new(FailingDecodeTokenizer)
-    }
-
-    fn model_id(&self) -> &str {
-        self.inner.model_id()
-    }
-}
-
-impl ChatBackend for FailingDecodeBackend {
-    fn chat_renderer(&self) -> DynChatRenderer {
-        Arc::new(self.clone())
-    }
-
-    fn new_chat_output_processor(
-        &self,
-        _request: &mut ChatRequest,
-        _options: NewChatOutputProcessorOptions<'_>,
-    ) -> vllm_chat::Result<DynChatOutputProcessor> {
-        Ok(Box::new(DefaultChatOutputProcessor::plain_text_only()))
-    }
-}
-
-impl ChatRenderer for FailingDecodeBackend {
-    fn render(&self, request: &ChatRequest) -> vllm_chat::Result<RenderedPrompt> {
-        self.inner.render(request)
-    }
-}
-
 /// Skip `LogprobsDelta` events that carry only token_ids (no logprobs),
 /// returning the next semantically interesting event.
 async fn next_semantic<S>(stream: &mut S) -> Option<Result<ChatEvent, vllm_chat::Error>>
@@ -738,7 +662,12 @@ async fn chat_stream_reports_decode_failure_as_error_event() {
                send_outputs(
                    push,
                    EngineCoreOutputs {
-                        outputs: vec![request_output("chat-4", vec![b'i' as u32], None, None)],
+                        outputs: vec![request_output(
+                            "chat-4",
+                            vec![UNKNOWN_DECODE_TOKEN_ID],
+                            None,
+                            None,
+                        )],
                        ..Default::default()
                    },
                )
@@ -747,9 +676,8 @@ async fn chat_stream_reports_decode_failure_as_error_event() {
        },
    );

-    let backend: Arc<dyn ChatTextBackend> = Arc::new(FailingDecodeBackend {
-        inner: FakeChatBackend::new(),
-    });
+    let backend: Arc<dyn ChatTextBackend> =
+        Arc::new(FakeChatBackend::new().with_tokenizer(Arc::new(TestTokenizer::new())));
    let chat = connect_chat_llm_with_ipc(
        EngineCoreClientConfig::new_single(handshake_address),
        &ipc,
@@ -769,7 +697,10 @@ async fn chat_stream_reports_decode_failure_as_error_event() {

    match timeout(Duration::from_secs(2), stream.next()).await.unwrap() {
        Some(Err(vllm_chat::Error::Text(vllm_text::Error::Tokenizer(message)))) => {
-            assert_eq!(message, "decode failed");
+            assert_eq!(
+                message,
+                format!("test tokenizer cannot decode unknown token id {UNKNOWN_DECODE_TOKEN_ID}")
+            );
        }
        other => panic!("unexpected event after close: {other:?}"),
    }
@@ -23,6 +23,7 @@ expect-test.workspace = true
 futures.workspace = true
 openai-protocol.workspace = true
 tool-parser.workspace = true
+vllm-tokenizer = { workspace = true, features = ["test-utils"] }

 [[bench]]
 name = "deepseek_v3"
@@ -27,6 +27,10 @@ impl Tokenizer for BenchTokenizer {
    fn token_to_id(&self, _token: &str) -> Option<u32> {
        Some(u32::MAX)
    }
+
+    fn id_to_token(&self, _id: u32) -> Option<String> {
+        Some("\u{FFFD}".to_string())
+    }
 }

 /// Bench-only adapter that exposes a unified parser through the tool-parser
@@ -49,11 +49,14 @@ mod tests {
    use std::sync::Arc;

    use super::SeedOssReasoningParser;
-    use crate::reasoning::{ReasoningParser, tests::FakeTokenizer};
+    use crate::reasoning::{
+        ReasoningParser,
+        tests::{SEED_THINK_END_ID, SEED_THINK_START_ID, fake_tokenizer},
+    };

    #[test]
    fn without_prompt_markers_expects_start_token() {
-        let tokenizer = Arc::new(FakeTokenizer);
+        let tokenizer = Arc::new(fake_tokenizer());
        let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap();

        let delta = parser.push("implicit reasoning</seed:think>answer").unwrap();
@@ -66,10 +69,10 @@ mod tests {

    #[test]
    fn picks_up_prompt_start_boundary() {
-        let tokenizer = Arc::new(FakeTokenizer);
+        let tokenizer = Arc::new(fake_tokenizer());
        let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap();
-        // Prompt prefills `<seed:think>` (id 10), opening reasoning before the stream.
-        parser.initialize(&[10]).unwrap();
+        // Prompt prefills `<seed:think>`, opening reasoning before the stream.
+        parser.initialize(&[SEED_THINK_START_ID]).unwrap();

        let delta = parser.push("reason</seed:think>answer").unwrap();
        assert_eq!(delta.reasoning.as_deref(), Some("reason"));
@@ -78,10 +81,10 @@ mod tests {

    #[test]
    fn respects_prompt_end_boundary() {
-        let tokenizer = Arc::new(FakeTokenizer);
+        let tokenizer = Arc::new(fake_tokenizer());
        let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap();
-        // Prompt already closed reasoning with `</seed:think>` (id 11).
-        parser.initialize(&[11]).unwrap();
+        // Prompt already closed reasoning with `</seed:think>`.
+        parser.initialize(&[SEED_THINK_END_ID]).unwrap();

        let delta = parser.push("answer").unwrap();
        assert_eq!(delta.reasoning, None);
@@ -91,7 +94,7 @@ mod tests {
    #[test]
    fn handles_explicit_start_token() {
        // An explicit start delimiter must not leak into reasoning text.
-        let tokenizer = Arc::new(FakeTokenizer);
+        let tokenizer = Arc::new(fake_tokenizer());
        let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap();

        let delta = parser.push("<seed:think>reason</seed:think>answer").unwrap();
@@ -103,7 +106,7 @@ mod tests {
    fn streams_explicit_start_token_across_pushes() {
        // Start token, reasoning body, end token, and content arrive in separate
        // streaming deltas.
-        let tokenizer = Arc::new(FakeTokenizer);
+        let tokenizer = Arc::new(fake_tokenizer());
        let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap();

        let mut reasoning = String::new();
@@ -131,9 +134,9 @@ mod tests {

    #[test]
    fn handles_partial_delimiters_across_pushes() {
-        let tokenizer = Arc::new(FakeTokenizer);
+        let tokenizer = Arc::new(fake_tokenizer());
        let mut parser = SeedOssReasoningParser::new(tokenizer).unwrap();
-        parser.initialize(&[10]).unwrap();
+        parser.initialize(&[SEED_THINK_START_ID]).unwrap();

        // Closing delimiter `</seed:think>` arrives in two halves.
        let first = parser.push("reason</seed:").unwrap();
@@ -127,14 +127,17 @@ mod tests {
    use std::sync::Arc;

    use super::Step3p5ReasoningParser;
-    use crate::reasoning::{ReasoningParser, tests::FakeTokenizer};
+    use crate::reasoning::{
+        ReasoningParser,
+        tests::{THINK_START_ID, fake_tokenizer},
+    };

    #[test]
    fn picks_up_prompt_start_boundary() {
-        let tokenizer = Arc::new(FakeTokenizer);
+        let tokenizer = Arc::new(fake_tokenizer());
        let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
-        // Prompt prefills `<think>` (id 1), opening reasoning before the stream.
-        parser.initialize(&[1]).unwrap();
+        // Prompt prefills `<think>`, opening reasoning before the stream.
+        parser.initialize(&[THINK_START_ID]).unwrap();

        let delta = parser.push("This is a reasoning section</think>This is the rest").unwrap();
        assert_eq!(
@@ -146,7 +149,7 @@ mod tests {

    #[test]
    fn handles_unterminated_reasoning() {
-        let tokenizer = Arc::new(FakeTokenizer);
+        let tokenizer = Arc::new(fake_tokenizer());
        let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();

        let pushed = parser.push("<think>reason without end").unwrap();
@@ -159,7 +162,7 @@ mod tests {

    #[test]
    fn handles_empty_input() {
-        let tokenizer = Arc::new(FakeTokenizer);
+        let tokenizer = Arc::new(fake_tokenizer());
        let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();

        let pushed = parser.push("").unwrap();
@@ -172,9 +175,9 @@ mod tests {
    fn complex_newline_pattern_trims_only_single_framing_newline_each_side() {
        // Only the immediately-adjacent framing `\n` is dropped on each side of
        // `</think>`; surrounding newlines remain part of reasoning/content.
-        let tokenizer = Arc::new(FakeTokenizer);
+        let tokenizer = Arc::new(fake_tokenizer());
        let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
-        parser.initialize(&[1]).unwrap();
+        parser.initialize(&[THINK_START_ID]).unwrap();

        let delta = parser
            .push("\n This is a \n reasoning section\n\n\n</think>\n\nThis is the rest")
@@ -188,7 +191,7 @@ mod tests {

    #[test]
    fn drops_framing_newlines_in_single_push() {
-        let tokenizer = Arc::new(FakeTokenizer);
+        let tokenizer = Arc::new(fake_tokenizer());
        let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();

        let delta = parser.push("<think>reason\n</think>\nanswer").unwrap();
@@ -198,7 +201,7 @@ mod tests {

    #[test]
    fn drops_framing_newlines_across_pushes() {
-        let tokenizer = Arc::new(FakeTokenizer);
+        let tokenizer = Arc::new(fake_tokenizer());
        let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();

        // The trailing `\n` from the first push is held until we know whether
@@ -219,7 +222,7 @@ mod tests {

    #[test]
    fn replays_held_newline_when_more_reasoning_follows() {
-        let tokenizer = Arc::new(FakeTokenizer);
+        let tokenizer = Arc::new(fake_tokenizer());
        let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();

        let first = parser.push("<think>reason\n").unwrap();
@@ -232,7 +235,7 @@ mod tests {

    #[test]
    fn finish_flushes_held_newline_in_unterminated_stream() {
-        let tokenizer = Arc::new(FakeTokenizer);
+        let tokenizer = Arc::new(fake_tokenizer());
        let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();

        let first = parser.push("<think>reason\n").unwrap();
@@ -245,7 +248,7 @@ mod tests {

    #[test]
    fn preserves_inner_newlines_in_reasoning() {
-        let tokenizer = Arc::new(FakeTokenizer);
+        let tokenizer = Arc::new(fake_tokenizer());
        let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();

        let delta = parser.push("<think>line1\nline2</think>tail").unwrap();
@@ -257,7 +260,7 @@ mod tests {
    fn trims_only_one_trailing_reasoning_newline() {
        // Only the single framing newline immediately before `</think>` is
        // dropped; earlier newlines in the reasoning body are preserved.
-        let tokenizer = Arc::new(FakeTokenizer);
+        let tokenizer = Arc::new(fake_tokenizer());
        let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();

        let delta = parser.push("<think>reason\n\n</think>answer").unwrap();
@@ -269,7 +272,7 @@ mod tests {
    fn drops_only_first_content_newline_after_transition() {
        // The leading-`\n` drop applies only to the first content delta after
        // `</think>`; later deltas pass through untouched.
-        let tokenizer = Arc::new(FakeTokenizer);
+        let tokenizer = Arc::new(fake_tokenizer());
        let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();

        let first = parser.push("<think>reason</think>").unwrap();
@@ -288,7 +291,7 @@ mod tests {

    #[test]
    fn passes_through_clean_boundary_without_framing_newlines() {
-        let tokenizer = Arc::new(FakeTokenizer);
+        let tokenizer = Arc::new(fake_tokenizer());
        let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();

        let delta = parser.push("<think>reason</think>tail").unwrap();
@@ -298,7 +301,7 @@ mod tests {

    #[test]
    fn handles_empty_reasoning_section() {
-        let tokenizer = Arc::new(FakeTokenizer);
+        let tokenizer = Arc::new(fake_tokenizer());
        let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();

        let delta = parser.push("<think></think>answer").unwrap();
@@ -1,54 +1,42 @@
 use std::sync::Arc;

-use vllm_tokenizer::Tokenizer;
+use vllm_tokenizer::test_utils::TestTokenizer;

 use super::{
    DeepSeekR1ReasoningParser, DelimitedReasoningParser, MiniMaxM3ReasoningParser,
    Qwen3ReasoningParser, ReasoningParser,
 };

-pub(crate) struct FakeTokenizer;
+pub(crate) const THINK_START_ID: u32 = 256;
+pub(crate) const THINK_END_ID: u32 = 257;
+pub(crate) const START_THINKING_ID: u32 = 258;
+pub(crate) const END_THINKING_ID: u32 = 259;
+pub(crate) const MINIMAX_THINK_START_ID: u32 = 260;
+pub(crate) const MINIMAX_THINK_END_ID: u32 = 261;
+pub(crate) const SPECIAL_BOUNDARY_ID: u32 = 262;
+pub(crate) const MM_THINK_START_ID: u32 = 263;
+pub(crate) const MM_THINK_END_ID: u32 = 264;
+pub(crate) const SEED_THINK_START_ID: u32 = 265;
+pub(crate) const SEED_THINK_END_ID: u32 = 266;

-impl Tokenizer for FakeTokenizer {
-    fn encode(&self, text: &str, _add_special_tokens: bool) -> vllm_tokenizer::Result<Vec<u32>> {
-        Ok(text.chars().map(u32::from).collect())
-    }
-
-    fn decode(
-        &self,
-        token_ids: &[u32],
-        _skip_special_tokens: bool,
-    ) -> vllm_tokenizer::Result<String> {
-        Ok(token_ids
-            .iter()
-            .map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
-            .collect())
-    }
-
-    fn token_to_id(&self, token: &str) -> Option<u32> {
-        match token {
-            "<think>" => Some(1),
-            "</think>" => Some(2),
-            "<|START_THINKING|>" => Some(3),
-            "<|END_THINKING|>" => Some(4),
-            "◁think▷" => Some(5),
-            "◁/think▷" => Some(6),
-            "<mm:think>" => Some(8),
-            "</mm:think>" => Some(9),
-            "<seed:think>" => Some(10),
-            "</seed:think>" => Some(11),
-            _ => None,
-        }
-    }
-
-    fn is_special_id(&self, token_id: u32) -> bool {
-        token_id == 7
-    }
+pub(crate) fn fake_tokenizer() -> TestTokenizer {
+    TestTokenizer::new()
+        .with_regular_token("<think>", THINK_START_ID)
+        .with_regular_token("</think>", THINK_END_ID)
+        .with_regular_token("<|START_THINKING|>", START_THINKING_ID)
+        .with_regular_token("<|END_THINKING|>", END_THINKING_ID)
+        .with_regular_token("◁think▷", MINIMAX_THINK_START_ID)
+        .with_regular_token("◁/think▷", MINIMAX_THINK_END_ID)
+        .with_special_token("<special-boundary>", SPECIAL_BOUNDARY_ID)
+        .with_regular_token("<mm:think>", MM_THINK_START_ID)
+        .with_regular_token("</mm:think>", MM_THINK_END_ID)
+        .with_regular_token("<seed:think>", SEED_THINK_START_ID)
+        .with_regular_token("</seed:think>", SEED_THINK_END_ID)
 }

 #[test]
 fn delimited_content_only_stream() {
-    let tokenizer = Arc::new(FakeTokenizer);
+    let tokenizer = Arc::new(fake_tokenizer());
    let mut parser =
        DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false).unwrap();

@@ -60,7 +48,7 @@ fn delimited_content_only_stream() {

 #[test]
 fn delimited_single_chunk_with_reasoning_and_content() {
-    let tokenizer = Arc::new(FakeTokenizer);
+    let tokenizer = Arc::new(fake_tokenizer());
    let mut parser =
        DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false).unwrap();

@@ -71,7 +59,7 @@ fn delimited_single_chunk_with_reasoning_and_content() {

 #[test]
 fn delimited_partial_tokens_across_chunks() {
-    let tokenizer = Arc::new(FakeTokenizer);
+    let tokenizer = Arc::new(fake_tokenizer());
    let mut parser =
        DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false).unwrap();

@@ -83,10 +71,10 @@ fn delimited_partial_tokens_across_chunks() {

 #[test]
 fn delimited_finish_flushes_buffer() {
-    let tokenizer = Arc::new(FakeTokenizer);
+    let tokenizer = Arc::new(fake_tokenizer());
    let mut parser =
        DelimitedReasoningParser::new(tokenizer, "<think>", "</think>", false).unwrap();
-    parser.initialize(&[1]);
+    parser.initialize(&[THINK_START_ID]);

    let delta = parser.push("unfinished</thi");
    assert_eq!(delta.reasoning.as_deref(), Some("unfinished"));
@@ -96,7 +84,7 @@ fn delimited_finish_flushes_buffer() {

 #[test]
 fn qwen3_without_prompt_markers_expects_start_token() {
-    let tokenizer = Arc::new(FakeTokenizer);
+    let tokenizer = Arc::new(fake_tokenizer());
    let mut parser = Qwen3ReasoningParser::new(tokenizer).unwrap();

    let delta = parser.push("reason</think>answer").unwrap();
@@ -106,9 +94,9 @@ fn qwen3_without_prompt_markers_expects_start_token() {

 #[test]
 fn qwen3_prompt_end_marker_starts_in_content() {
-    let tokenizer = Arc::new(FakeTokenizer);
+    let tokenizer = Arc::new(fake_tokenizer());
    let mut parser = Qwen3ReasoningParser::new(tokenizer).unwrap();
-    parser.initialize(&[2]).unwrap();
+    parser.initialize(&[THINK_END_ID]).unwrap();

    let delta = parser.push("answer").unwrap();
    assert_eq!(delta.reasoning, None);
@@ -117,7 +105,7 @@ fn qwen3_prompt_end_marker_starts_in_content() {

 #[test]
 fn qwen3_tolerates_old_and_new_formats() {
-    let tokenizer = Arc::new(FakeTokenizer);
+    let tokenizer = Arc::new(fake_tokenizer());

    let mut old_parser = Qwen3ReasoningParser::new(tokenizer.clone()).unwrap();
    let old = old_parser.push("<think>reason</think>answer").unwrap();
@@ -125,7 +113,7 @@ fn qwen3_tolerates_old_and_new_formats() {
    assert_eq!(old.content.as_deref(), Some("answer"));

    let mut new_parser = Qwen3ReasoningParser::new(tokenizer).unwrap();
-    new_parser.initialize(&[1]).unwrap();
+    new_parser.initialize(&[THINK_START_ID]).unwrap();
    let new = new_parser.push("reason</think>answer").unwrap();
    assert_eq!(new.reasoning.as_deref(), Some("reason"));
    assert_eq!(new.content.as_deref(), Some("answer"));
@@ -133,10 +121,10 @@ fn qwen3_tolerates_old_and_new_formats() {

 #[test]
 fn qwen3_stops_scanning_at_last_special_token() {
-    let tokenizer = Arc::new(FakeTokenizer);
+    let tokenizer = Arc::new(fake_tokenizer());
    let mut parser = Qwen3ReasoningParser::new(tokenizer).unwrap();

-    parser.initialize(&[1, 7]).unwrap();
+    parser.initialize(&[THINK_START_ID, SPECIAL_BOUNDARY_ID]).unwrap();

    let delta = parser.push("answer").unwrap();
    assert_eq!(delta.reasoning, None);
@@ -145,7 +133,7 @@ fn qwen3_stops_scanning_at_last_special_token() {

 #[test]
 fn deepseek_r1_defaults_to_reasoning_without_prompt_boundary() {
-    let tokenizer = Arc::new(FakeTokenizer);
+    let tokenizer = Arc::new(fake_tokenizer());
    let mut parser = DeepSeekR1ReasoningParser::new(tokenizer).unwrap();

    let delta = parser.push("reason</think>answer").unwrap();
@@ -155,10 +143,10 @@ fn deepseek_r1_defaults_to_reasoning_without_prompt_boundary() {

 #[test]
 fn deepseek_r1_stops_scanning_at_last_special_token() {
-    let tokenizer = Arc::new(FakeTokenizer);
+    let tokenizer = Arc::new(fake_tokenizer());
    let mut parser = DeepSeekR1ReasoningParser::new(tokenizer).unwrap();

-    parser.initialize(&[2, 7]).unwrap();
+    parser.initialize(&[THINK_END_ID, SPECIAL_BOUNDARY_ID]).unwrap();

    let delta = parser.push("reason</think>answer").unwrap();
    assert_eq!(delta.reasoning.as_deref(), Some("reason"));
@@ -167,7 +155,7 @@ fn deepseek_r1_stops_scanning_at_last_special_token() {

 #[test]
 fn minimax_m3_handles_explicit_think_delimiters() {
-    let tokenizer = Arc::new(FakeTokenizer);
+    let tokenizer = Arc::new(fake_tokenizer());
    let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();

    let delta = parser.push("<mm:think>reason</mm:think>answer").unwrap();
@@ -177,7 +165,7 @@ fn minimax_m3_handles_explicit_think_delimiters() {

 #[test]
 fn minimax_m3_drops_leading_end_marker() {
-    let tokenizer = Arc::new(FakeTokenizer);
+    let tokenizer = Arc::new(fake_tokenizer());
    let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();

    let delta = parser.push("</mm:think>answer").unwrap();
@@ -187,7 +175,7 @@ fn minimax_m3_drops_leading_end_marker() {

 #[test]
 fn minimax_m3_preserves_non_leading_end_marker() {
-    let tokenizer = Arc::new(FakeTokenizer);
+    let tokenizer = Arc::new(fake_tokenizer());
    let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();

    let delta = parser.push("XXX</mm:think>YYY").unwrap();
@@ -197,7 +185,7 @@ fn minimax_m3_preserves_non_leading_end_marker() {

 #[test]
 fn minimax_m3_drops_split_leading_end_marker() {
-    let tokenizer = Arc::new(FakeTokenizer);
+    let tokenizer = Arc::new(fake_tokenizer());
    let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();

    assert!(parser.push("</mm").unwrap().is_empty());
@@ -208,9 +196,9 @@ fn minimax_m3_drops_split_leading_end_marker() {

 #[test]
 fn minimax_m3_uses_prompt_prefilled_start_marker() {
-    let tokenizer = Arc::new(FakeTokenizer);
+    let tokenizer = Arc::new(fake_tokenizer());
    let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
-    parser.initialize(&[8]).unwrap();
+    parser.initialize(&[MM_THINK_START_ID]).unwrap();

    let delta = parser.push("reason</mm:think>answer").unwrap();
    assert_eq!(delta.reasoning.as_deref(), Some("reason"));
@@ -219,9 +207,9 @@ fn minimax_m3_uses_prompt_prefilled_start_marker() {

 #[test]
 fn minimax_m3_uses_prompt_prefilled_end_marker() {
-    let tokenizer = Arc::new(FakeTokenizer);
+    let tokenizer = Arc::new(fake_tokenizer());
    let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
-    parser.initialize(&[9]).unwrap();
+    parser.initialize(&[MM_THINK_END_ID]).unwrap();

    let delta = parser.push("answer").unwrap();
    assert_eq!(delta.reasoning, None);
@@ -124,42 +124,17 @@ impl UnifiedParser for CombinedParser {
 mod tests {
    use std::sync::Arc;

-    use vllm_tokenizer::Tokenizer;
+    use vllm_tokenizer::test_utils::TestTokenizer;

    use super::CombinedParser;
    use crate::reasoning::{Qwen3ReasoningParser, ReasoningDelta, ReasoningParser};
    use crate::tool::{Qwen3XmlToolParser, Tool, ToolParser};
    use crate::unified::{UnifiedParser, UnifiedParserEvent, UnifiedParserOutput};

-    struct FakeTokenizer;
-
-    impl Tokenizer for FakeTokenizer {
-        fn encode(
-            &self,
-            text: &str,
-            _add_special_tokens: bool,
-        ) -> vllm_tokenizer::Result<Vec<u32>> {
-            Ok(text.chars().map(u32::from).collect())
-        }
-
-        fn decode(
-            &self,
-            token_ids: &[u32],
-            _skip_special_tokens: bool,
-        ) -> vllm_tokenizer::Result<String> {
-            Ok(token_ids
-                .iter()
-                .map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
-                .collect())
-        }
-
-        fn token_to_id(&self, token: &str) -> Option<u32> {
-            match token {
-                "<think>" => Some(1),
-                "</think>" => Some(2),
-                _ => None,
-            }
-        }
+    fn tokenizer() -> TestTokenizer {
+        TestTokenizer::new()
+            .with_regular_token("<think>", 256)
+            .with_regular_token("</think>", 257)
    }

    fn test_tools() -> Vec<Tool> {
@@ -273,7 +248,7 @@ mod tests {

    #[test]
    fn combined_parser_emits_reasoning_and_text() {
-        let tokenizer = Arc::new(FakeTokenizer);
+        let tokenizer = Arc::new(tokenizer());
        let reasoning = Qwen3ReasoningParser::create(tokenizer).unwrap();
        let mut parser = CombinedParser::new(Some(reasoning), None);

@@ -515,7 +515,7 @@ mod tests {

    use serde_json::{Value, json};
    use thiserror_ext::AsReport;
-    use vllm_tokenizer::Tokenizer;
+    use vllm_tokenizer::test_utils::TestTokenizer;
    use winnow::combinator::{eof, terminated};
    use winnow::error::ErrMode;
    use winnow::prelude::*;
@@ -527,66 +527,15 @@ mod tests {
    use crate::tool::Tool;
    use crate::unified::{UnifiedParserEvent, parsing_failed};

-    struct FakeTokenizer;
+    const CHANNEL_START_ID: u32 = 256;
+    const CHANNEL_END_ID: u32 = 257;
+    const TURN_BOUNDARY_ID: u32 = 258;

-    impl Tokenizer for FakeTokenizer {
-        fn encode(
-            &self,
-            text: &str,
-            _add_special_tokens: bool,
-        ) -> vllm_tokenizer::Result<Vec<u32>> {
-            Ok(text.chars().map(u32::from).collect())
-        }
-
-        fn decode(
-            &self,
-            token_ids: &[u32],
-            _skip_special_tokens: bool,
-        ) -> vllm_tokenizer::Result<String> {
-            Ok(token_ids
-                .iter()
-                .map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
-                .collect())
-        }
-
-        fn token_to_id(&self, token: &str) -> Option<u32> {
-            match token {
-                CHANNEL_START => Some(100),
-                CHANNEL_END => Some(101),
-                _ => None,
-            }
-        }
-
-        fn is_special_id(&self, token_id: u32) -> bool {
-            matches!(token_id, 100..=105)
-        }
-    }
-
-    struct MissingTokenTokenizer;
-
-    impl Tokenizer for MissingTokenTokenizer {
-        fn encode(
-            &self,
-            text: &str,
-            _add_special_tokens: bool,
-        ) -> vllm_tokenizer::Result<Vec<u32>> {
-            Ok(text.chars().map(u32::from).collect())
-        }
-
-        fn decode(
-            &self,
-            token_ids: &[u32],
-            _skip_special_tokens: bool,
-        ) -> vllm_tokenizer::Result<String> {
-            Ok(token_ids
-                .iter()
-                .map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
-                .collect())
-        }
-
-        fn token_to_id(&self, _token: &str) -> Option<u32> {
-            None
-        }
+    fn tokenizer() -> TestTokenizer {
+        TestTokenizer::new()
+            .with_special_token(CHANNEL_START, CHANNEL_START_ID)
+            .with_special_token(CHANNEL_END, CHANNEL_END_ID)
+            .with_special_token("<turn-boundary>", TURN_BOUNDARY_ID)
    }

    trait UnifiedParserTestExt {
@@ -716,12 +665,12 @@ mod tests {
    }

    fn test_parser() -> Gemma4UnifiedParser {
-        Gemma4UnifiedParser::new(&test_tools(), Arc::new(FakeTokenizer)).unwrap()
+        Gemma4UnifiedParser::new(&test_tools(), Arc::new(tokenizer())).unwrap()
    }

    #[test]
    fn gemma4_create_requires_channel_start_token() {
-        let error = match Gemma4UnifiedParser::new(&test_tools(), Arc::new(MissingTokenTokenizer)) {
+        let error = match Gemma4UnifiedParser::new(&test_tools(), Arc::new(TestTokenizer::new())) {
            Ok(_) => panic!("expected missing token error"),
            Err(error) => error,
        };
@@ -1046,7 +995,7 @@ mod tests {
    #[test]
    fn gemma4_initialize_open_channel_prompt_starts_in_reasoning() {
        let mut parser = test_parser();
-        parser.initialize(&[100, 3000, 3001]).unwrap();
+        parser.initialize(&[CHANNEL_START_ID, 3000, 3001]).unwrap();

        let output = parser.parse_complete("reason<channel|>answer").unwrap();

@@ -1057,7 +1006,7 @@ mod tests {
    #[test]
    fn gemma4_initialize_turn_prompt_starts_in_text() {
        let mut parser = test_parser();
-        parser.initialize(&[104, 3000, 3001]).unwrap();
+        parser.initialize(&[TURN_BOUNDARY_ID, 3000, 3001]).unwrap();

        let output = parser.parse_complete("<|channel>thought\nreason<channel|>answer").unwrap();

@@ -1068,7 +1017,7 @@ mod tests {
    #[test]
    fn gemma4_initialize_special_token_caps_boundary_scan() {
        let mut parser = test_parser();
-        parser.initialize(&[100, 3000, 104, 3001]).unwrap();
+        parser.initialize(&[CHANNEL_START_ID, 3000, TURN_BOUNDARY_ID, 3001]).unwrap();

        let output = parser.parse_complete("answer").unwrap();

@@ -1079,7 +1028,7 @@ mod tests {
    #[test]
    fn gemma4_initialize_closed_channel_prompt_starts_in_text() {
        let mut parser = test_parser();
-        parser.initialize(&[100, 3000, 3001, 101]).unwrap();
+        parser.initialize(&[CHANNEL_START_ID, 3000, 3001, CHANNEL_END_ID]).unwrap();

        let output = parser.parse_complete("answer").unwrap();

@@ -63,6 +63,7 @@ serial_test.workspace = true
 tempfile.workspace = true
 tower.workspace = true
 vllm-engine-core-client = { workspace = true, features = ["test-util"] }
+vllm-tokenizer = { workspace = true, features = ["test-utils"] }
 zeromq.workspace = true

 [lints]
@@ -24,8 +24,9 @@ use vllm_engine_core_client::protocol::{
 use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};
 use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, EngineId};
 use vllm_llm::Llm;
-use vllm_text::tokenizer::{DynTokenizer, Tokenizer};
+use vllm_text::tokenizer::DynTokenizer;
 use vllm_text::{Prompt, TextBackend};
+use vllm_tokenizer::test_utils::TestTokenizer;
 use zeromq::prelude::{SocketRecv, SocketSend};
 use zeromq::{DealerSocket, PushSocket, ZmqMessage};

@@ -155,37 +156,9 @@ fn test_llm(client: EngineCoreClient) -> Llm {
 #[derive(Clone, Debug)]
 struct FakeTextBackend;

-#[derive(Debug)]
-struct FakeTokenizer;
-
-impl Tokenizer for FakeTokenizer {
-    fn encode(
-        &self,
-        text: &str,
-        _add_special_tokens: bool,
-    ) -> vllm_text::tokenizer::Result<Vec<u32>> {
-        Ok(text.bytes().map(u32::from).collect())
-    }
-
-    fn decode(
-        &self,
-        token_ids: &[u32],
-        _skip_special_tokens: bool,
-    ) -> vllm_text::tokenizer::Result<String> {
-        Ok(
-            String::from_utf8_lossy(&token_ids.iter().map(|id| *id as u8).collect::<Vec<_>>())
-                .into_owned(),
-        )
-    }
-
-    fn token_to_id(&self, token: &str) -> Option<u32> {
-        token.bytes().next().map(u32::from)
-    }
-}
-
 impl TextBackend for FakeTextBackend {
    fn tokenizer(&self) -> DynTokenizer {
-        Arc::new(FakeTokenizer)
+        Arc::new(TestTokenizer::new())
    }

    fn model_id(&self) -> &str {
@@ -24,8 +24,9 @@ use vllm_engine_core_client::protocol::{
 use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};
 use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, EngineId};
 use vllm_llm::Llm;
-use vllm_text::tokenizer::{DynTokenizer, Tokenizer};
+use vllm_text::tokenizer::DynTokenizer;
 use vllm_text::{Prompt, TextBackend};
+use vllm_tokenizer::test_utils::TestTokenizer;
 use zeromq::prelude::{SocketRecv, SocketSend};
 use zeromq::{DealerSocket, PushSocket, ZmqMessage};

@@ -151,37 +152,9 @@ fn test_llm(client: EngineCoreClient) -> Llm {
 #[derive(Clone, Debug)]
 struct FakeChatBackend;

-#[derive(Debug)]
-struct FakeChatTokenizer;
-
-impl Tokenizer for FakeChatTokenizer {
-    fn encode(
-        &self,
-        text: &str,
-        _add_special_tokens: bool,
-    ) -> vllm_text::tokenizer::Result<Vec<u32>> {
-        Ok(text.bytes().map(u32::from).collect())
-    }
-
-    fn decode(
-        &self,
-        token_ids: &[u32],
-        _skip_special_tokens: bool,
-    ) -> vllm_text::tokenizer::Result<String> {
-        Ok(
-            String::from_utf8_lossy(&token_ids.iter().map(|id| *id as u8).collect::<Vec<_>>())
-                .into_owned(),
-        )
-    }
-
-    fn token_to_id(&self, token: &str) -> Option<u32> {
-        token.bytes().next().map(u32::from)
-    }
-}
-
 impl TextBackend for FakeChatBackend {
    fn tokenizer(&self) -> DynTokenizer {
-        Arc::new(FakeChatTokenizer)
+        Arc::new(TestTokenizer::new())
    }

    fn model_id(&self) -> &str {
@@ -194,7 +194,7 @@ mod tests {
    use axum::http::HeaderMap;
    use serde_json::json;
    use vllm_text::Prompt;
-    use vllm_text::tokenizer::Tokenizer;
+    use vllm_tokenizer::test_utils::TestTokenizer;

    use super::prepare_completion_request;
    use crate::lora::LoraModelResolution;
@@ -212,32 +212,8 @@ mod tests {
        }
    }

-    #[derive(Debug)]
-    struct TestTokenizer;
-
-    impl Tokenizer for TestTokenizer {
-        fn encode(
-            &self,
-            text: &str,
-            _add_special_tokens: bool,
-        ) -> vllm_text::tokenizer::Result<Vec<u32>> {
-            Ok(text.bytes().map(u32::from).collect())
-        }
-
-        fn decode(
-            &self,
-            token_ids: &[u32],
-            _skip_special_tokens: bool,
-        ) -> vllm_text::tokenizer::Result<String> {
-            Ok(
-                String::from_utf8_lossy(&token_ids.iter().map(|id| *id as u8).collect::<Vec<_>>())
-                    .into_owned(),
-            )
-        }
-
-        fn token_to_id(&self, _token: &str) -> Option<u32> {
-            None
-        }
+    fn test_tokenizer() -> TestTokenizer {
+        TestTokenizer::new()
    }

    fn base_request_json() -> serde_json::Value {
@@ -297,7 +273,7 @@ mod tests {
            request,
            &served(&["Qwen/Qwen1.5-0.5B-Chat"]),
            ResolvedRequestContext::default(),
-            &TestTokenizer,
+            &test_tokenizer(),
        )
        .expect("prepare");

@@ -340,7 +316,7 @@ mod tests {
                request,
                &served(&["Qwen/Qwen1.5-0.5B-Chat"]),
                ResolvedRequestContext::default(),
-                &TestTokenizer,
+                &test_tokenizer(),
            )
            .expect("prepare")
            .text_request
@@ -374,7 +350,7 @@ mod tests {
            request,
            &served(&["Qwen/Qwen1.5-0.5B-Chat"]),
            ResolvedRequestContext::default(),
-            &TestTokenizer,
+            &test_tokenizer(),
        )
        .expect("prepare");

@@ -399,7 +375,7 @@ mod tests {
            request,
            &served(&["Qwen/Qwen1.5-0.5B-Chat"]),
            ResolvedRequestContext::default(),
-            &TestTokenizer,
+            &test_tokenizer(),
        )
        .expect("prepare");

@@ -422,7 +398,7 @@ mod tests {
            request,
            &served(&["Qwen/Qwen1.5-0.5B-Chat"]),
            ResolvedRequestContext::default(),
-            &TestTokenizer,
+            &test_tokenizer(),
        )
        .expect("prepare");

@@ -446,7 +422,7 @@ mod tests {
            request,
            &served(&["Qwen/Qwen1.5-0.5B-Chat"]),
            ResolvedRequestContext::default(),
-            &TestTokenizer,
+            &test_tokenizer(),
        )
        .expect("prepare");

@@ -471,7 +447,7 @@ mod tests {
            request,
            &served(&["Qwen/Qwen1.5-0.5B-Chat"]),
            ResolvedRequestContext::default(),
-            &TestTokenizer,
+            &test_tokenizer(),
        )
        .expect("prepare");

@@ -498,7 +474,7 @@ mod tests {
            request,
            &served(&["Qwen/Qwen1.5-0.5B-Chat"]),
            ResolvedRequestContext::default(),
-            &TestTokenizer,
+            &test_tokenizer(),
        )
        .expect("prepare");

@@ -523,7 +499,7 @@ mod tests {
            request,
            &served(&["Qwen/Qwen1.5-0.5B-Chat"]),
            ResolvedRequestContext::default(),
-            &TestTokenizer,
+            &test_tokenizer(),
        )
        .expect("prepare");

@@ -549,7 +525,7 @@ mod tests {
            request,
            &served(&["Qwen/Qwen1.5-0.5B-Chat"]),
            ResolvedRequestContext::default(),
-            &TestTokenizer,
+            &test_tokenizer(),
        )
        .expect("prepare");
        assert_eq!(prepared.text_request.sampling_params.logprobs, Some(1));
@@ -574,7 +550,7 @@ mod tests {
            request,
            &served(&["Qwen/Qwen1.5-0.5B-Chat"]),
            request_context(&headers, None),
-            &TestTokenizer,
+            &test_tokenizer(),
        )
        .expect("prepare");
        assert_eq!(prepared.text_request.data_parallel_rank, Some(3));
@@ -593,7 +569,7 @@ mod tests {
            request,
            &served(&["Qwen/Qwen1.5-0.5B-Chat"]),
            ResolvedRequestContext::default(),
-            &TestTokenizer,
+            &test_tokenizer(),
        )
        .expect("prepare");
        assert_eq!(prepared.text_request.data_parallel_rank, None);
@@ -40,8 +40,9 @@ use vllm_engine_core_client::{
 };
 use vllm_llm::Llm;
 use vllm_metrics::METRICS;
-use vllm_text::tokenizer::{DynTokenizer, Tokenizer};
+use vllm_text::tokenizer::DynTokenizer;
 use vllm_text::{Prompt, TextBackend};
+use vllm_tokenizer::test_utils::TestTokenizer;
 use zeromq::prelude::{SocketRecv, SocketSend};
 use zeromq::{DealerSocket, PushSocket, ZmqMessage};

@@ -417,82 +418,20 @@ struct FakeChatBackend {
 }

 /// Synthetic BOS id used when `add_special_tokens` is true in tests.
-const FAKE_BOS_TOKEN_ID: u32 = 1;
+const FAKE_BOS_TOKEN_ID: u32 = 256;
+const UNKNOWN_DECODE_TOKEN_ID: u32 = 10_000;

-#[derive(Debug)]
-struct FakeChatTokenizer;
-
-impl Tokenizer for FakeChatTokenizer {
-    fn encode(
-        &self,
-        text: &str,
-        add_special_tokens: bool,
-    ) -> vllm_text::tokenizer::Result<Vec<u32>> {
-        let mut token_ids = Vec::new();
-        if add_special_tokens {
-            token_ids.push(FAKE_BOS_TOKEN_ID);
-        }
-        let mut rest = text;
-        while !rest.is_empty() {
-            if let Some(stripped) = rest.strip_prefix("<image>") {
-                token_ids.push(999);
-                rest = stripped;
-                continue;
-            }
-            if let Some(stripped) = rest.strip_prefix("<|image_pad|>") {
-                token_ids.push(151655);
-                rest = stripped;
-                continue;
-            }
-
-            let ch = rest.chars().next().expect("rest is not empty");
-            let mut buf = [0; 4];
-            token_ids.extend(ch.encode_utf8(&mut buf).bytes().map(u32::from));
-            rest = &rest[ch.len_utf8()..];
-        }
-        Ok(token_ids)
-    }
-
-    fn decode(
-        &self,
-        token_ids: &[u32],
-        _skip_special_tokens: bool,
-    ) -> vllm_text::tokenizer::Result<String> {
-        Ok(
-            String::from_utf8_lossy(&token_ids.iter().map(|id| *id as u8).collect::<Vec<_>>())
-                .into_owned(),
-        )
-    }
-
-    fn token_to_id(&self, token: &str) -> Option<u32> {
-        match token {
-            "<image>" => Some(999),
-            "<|image_pad|>" => Some(151655),
-            "<think>" => Some(0xF001),
-            "</think>" => Some(0xF002),
-            "<|START_THINKING|>" => Some(0xF003),
-            "<|END_THINKING|>" => Some(0xF004),
-            "◁think▷" => Some(0xF005),
-            "◁/think▷" => Some(0xF006),
-            _ => None,
-        }
-    }
-
-    fn id_to_token(&self, id: u32) -> Option<String> {
-        match id {
-            FAKE_BOS_TOKEN_ID => Some("<bos>".to_string()),
-            999 => Some("<image>".to_string()),
-            151655 => Some("<|image_pad|>".to_string()),
-            0xF001 => Some("<think>".to_string()),
-            0xF002 => Some("</think>".to_string()),
-            0xF003 => Some("<|START_THINKING|>".to_string()),
-            0xF004 => Some("<|END_THINKING|>".to_string()),
-            0xF005 => Some("◁think▷".to_string()),
-            0xF006 => Some("◁/think▷".to_string()),
-            id if id < 128 => char::from_u32(id).map(|ch| ch.to_string()),
-            _ => None,
-        }
-    }
+fn fake_chat_tokenizer() -> TestTokenizer {
+    TestTokenizer::new()
+        .with_bos_token("<bos>", FAKE_BOS_TOKEN_ID)
+        .with_regular_token("<image>", 999)
+        .with_regular_token("<|image_pad|>", 151655)
+        .with_regular_token("<think>", 0xF001)
+        .with_regular_token("</think>", 0xF002)
+        .with_regular_token("<|START_THINKING|>", 0xF003)
+        .with_regular_token("<|END_THINKING|>", 0xF004)
+        .with_regular_token("◁think▷", 0xF005)
+        .with_regular_token("◁/think▷", 0xF006)
 }

 impl FakeChatBackend {
@@ -530,7 +469,7 @@ impl fmt::Debug for FakeChatBackend {

 impl TextBackend for FakeChatBackend {
    fn tokenizer(&self) -> DynTokenizer {
-        Arc::new(FakeChatTokenizer)
+        Arc::new(fake_chat_tokenizer())
    }

    fn model_id(&self) -> &str {
@@ -630,7 +569,7 @@ fn qwen_multimodal_model_info() -> vllm_chat::multimodal::MultimodalModelInfo {
        Some("qwen2_vl".to_string()),
        Some(&config_path),
        None,
-        Arc::new(FakeChatTokenizer),
+        Arc::new(fake_chat_tokenizer()),
    )
    .expect("load multimodal info")
    .expect("qwen multimodal info is registered");
@@ -638,70 +577,6 @@ fn qwen_multimodal_model_info() -> vllm_chat::multimodal::MultimodalModelInfo {
    info
 }

-#[derive(Clone, Debug)]
-struct FailingDecodeChatBackend;
-
-#[derive(Debug)]
-struct FailingDecodeTokenizer;
-
-impl Tokenizer for FailingDecodeTokenizer {
-    fn encode(
-        &self,
-        text: &str,
-        add_special_tokens: bool,
-    ) -> vllm_text::tokenizer::Result<Vec<u32>> {
-        FakeChatTokenizer.encode(text, add_special_tokens)
-    }
-
-    fn decode(
-        &self,
-        token_ids: &[u32],
-        skip_special_tokens: bool,
-    ) -> vllm_text::tokenizer::Result<String> {
-        if token_ids.contains(&(b'i' as u32)) {
-            return Err(vllm_text::tokenizer::TokenizerError(
-                "forced decode failure for streaming test".to_string(),
-            ));
-        }
-
-        FakeChatTokenizer.decode(token_ids, skip_special_tokens)
-    }
-
-    fn token_to_id(&self, token: &str) -> Option<u32> {
-        FakeChatTokenizer.token_to_id(token)
-    }
-}
-
-impl TextBackend for FailingDecodeChatBackend {
-    fn tokenizer(&self) -> DynTokenizer {
-        Arc::new(FailingDecodeTokenizer)
-    }
-
-    fn model_id(&self) -> &str {
-        "test-model"
-    }
-}
-
-impl ChatBackend for FailingDecodeChatBackend {
-    fn chat_renderer(&self) -> DynChatRenderer {
-        Arc::new(self.clone())
-    }
-
-    fn new_chat_output_processor(
-        &self,
-        _request: &mut ChatRequest,
-        _options: NewChatOutputProcessorOptions<'_>,
-    ) -> vllm_chat::Result<DynChatOutputProcessor> {
-        Ok(Box::new(DefaultChatOutputProcessor::plain_text_only()))
-    }
-}
-
-impl ChatRenderer for FailingDecodeChatBackend {
-    fn render(&self, request: &ChatRequest) -> vllm_chat::Result<vllm_chat::RenderedPrompt> {
-        FakeChatBackend::new().render(request)
-    }
-}
-
 async fn test_models_with_engine_outputs_and_backend_inner(
    engine_id: impl Into<EngineId>,
    output_specs: Vec<(Vec<u32>, Option<EngineCoreFinishReason>)>,
@@ -2755,8 +2630,8 @@ async fn load_endpoint_resets_when_stream_response_is_dropped() {
 #[serial]
 async fn stream_error_is_returned_as_openai_error_sse() {
    let (app, engine_task) = test_app_with_backend_and_stream_output_specs(
-        Arc::new(FailingDecodeChatBackend),
-        default_stream_output_specs(),
+        Arc::new(FakeChatBackend::new()),
+        vec![(vec![UNKNOWN_DECODE_TOKEN_ID], None)],
    )
    .await;
    let response = app
@@ -2789,7 +2664,9 @@ async fn stream_error_is_returned_as_openai_error_sse() {
    assert!(text.contains("\"role\":\"assistant\""), "{text}");
    assert!(text.contains("\"type\":\"server_error\""), "{text}");
    assert!(
-        text.contains("forced decode failure for streaming test"),
+        text.contains(&format!(
+            "test tokenizer cannot decode unknown token id {UNKNOWN_DECODE_TOKEN_ID}"
+        )),
        "{text}"
    );
    assert!(!text.contains("\"usage\":"), "{text}");
@@ -31,6 +31,7 @@ serial_test.workspace = true
 tempfile.workspace = true
 tokio.workspace = true
 vllm-llm = { workspace = true, features = ["test-util"] }
+vllm-tokenizer = { workspace = true, features = ["test-utils"] }

 [lints]
 workspace = true
@@ -275,6 +275,7 @@ mod tests {
    use std::collections::{BTreeSet, HashMap};

    use serial_test::file_serial;
+    use vllm_tokenizer::test_utils::TestTokenizer;

    use super::*;
    use crate::backend::hf::HfTextBackend;
@@ -282,60 +283,8 @@ mod tests {
    use crate::error::{LogprobsError, TokenIdsError};
    use crate::request::{Prompt, TextRequest};

-    /// Stub tokenizer that returns empty token IDs — sufficient for tests that
-    /// don't exercise bad-words tokenization.
-    struct StubTokenizer;
-
-    impl Tokenizer for StubTokenizer {
-        fn encode(
-            &self,
-            _text: &str,
-            _add_special_tokens: bool,
-        ) -> vllm_tokenizer::Result<Vec<u32>> {
-            Ok(vec![])
-        }
-
-        fn decode(
-            &self,
-            _token_ids: &[u32],
-            _skip_special_tokens: bool,
-        ) -> vllm_tokenizer::Result<String> {
-            Ok(String::new())
-        }
-
-        fn token_to_id(&self, _token: &str) -> Option<u32> {
-            None
-        }
-    }
-
-    fn stub_tokenizer() -> StubTokenizer {
-        StubTokenizer
-    }
-
-    struct FixedTokenizer {
-        token_ids: Vec<u32>,
-    }
-
-    impl Tokenizer for FixedTokenizer {
-        fn encode(
-            &self,
-            _text: &str,
-            _add_special_tokens: bool,
-        ) -> vllm_tokenizer::Result<Vec<u32>> {
-            Ok(self.token_ids.clone())
-        }
-
-        fn decode(
-            &self,
-            _token_ids: &[u32],
-            _skip_special_tokens: bool,
-        ) -> vllm_tokenizer::Result<String> {
-            Ok(String::new())
-        }
-
-        fn token_to_id(&self, _token: &str) -> Option<u32> {
-            None
-        }
+    fn stub_tokenizer() -> TestTokenizer {
+        TestTokenizer::new()
    }

    fn sample_request() -> TextRequest {
@@ -952,9 +901,7 @@ mod tests {

    #[test]
    fn lower_sampling_params_rejects_out_of_vocab_bad_words() {
-        let tokenizer = FixedTokenizer {
-            token_ids: vec![1999, 2000],
-        };
+        let tokenizer = TestTokenizer::new().with_regular_token("blocked", 2000);
        let error = lower_sampling_params(
            SamplingParams {
                bad_words: Some(vec!["blocked".to_string()]),
@@ -323,37 +323,11 @@ mod tests {
    use futures::{Stream, stream};
    use vllm_engine_core_client::AbortCause;
    use vllm_llm::GenerateOutput;
-    use vllm_tokenizer::Tokenizer;
+    use vllm_tokenizer::test_utils::TestTokenizer;

    use super::*;
    use crate::output::TextOutputStreamExt as _;

-    /// Backend that treats each token ID as a raw byte, producing lossy UTF-8.
-    struct ByteTokenizer;
-
-    impl Tokenizer for ByteTokenizer {
-        fn encode(
-            &self,
-            _text: &str,
-            _add_special_tokens: bool,
-        ) -> vllm_tokenizer::Result<Vec<u32>> {
-            unreachable!()
-        }
-
-        fn decode(
-            &self,
-            token_ids: &[u32],
-            _skip_special_tokens: bool,
-        ) -> vllm_tokenizer::Result<String> {
-            let bytes = token_ids.iter().map(|id| *id as u8).collect::<Vec<_>>();
-            Ok(String::from_utf8_lossy(&bytes).into_owned())
-        }
-
-        fn token_to_id(&self, _token: &str) -> Option<u32> {
-            unreachable!()
-        }
-    }
-
    /// Helper: run `decoded_text_event_stream` to completion and return the
    /// collected output.
    async fn run_to_completion(
@@ -366,7 +340,7 @@ mod tests {
            token_ids,
            Some(FinishReason::Length),
        ))]);
-        let tokenizer: DynTokenizer = Arc::new(ByteTokenizer);
+        let tokenizer: DynTokenizer = Arc::new(TestTokenizer::new());
        decoded_text_event_stream("test".into(), tokenizer, raw_stream, decode_options, false)
            .collect_output()
            .await
@@ -419,7 +393,7 @@ mod tests {
            ))),
            dropped_cause: Arc::clone(&dropped_cause),
        };
-        let tokenizer: DynTokenizer = Arc::new(ByteTokenizer);
+        let tokenizer: DynTokenizer = Arc::new(TestTokenizer::new());

        let output = decoded_text_event_stream(
            "test".into(),
@@ -129,40 +129,13 @@ fn decode_position_logprobs<T: Tokenizer + ?Sized>(
 #[cfg(test)]
 mod tests {
    use vllm_llm::{Logprobs, PositionLogprobs, TokenLogprob};
+    use vllm_tokenizer::test_utils::TestTokenizer;

    use super::*;

-    #[derive(Debug)]
-    struct ByteTokenizer;
-
-    impl vllm_tokenizer::Tokenizer for ByteTokenizer {
-        fn encode(
-            &self,
-            _text: &str,
-            _add_special_tokens: bool,
-        ) -> vllm_tokenizer::Result<Vec<u32>> {
-            unreachable!()
-        }
-
-        fn decode(
-            &self,
-            token_ids: &[u32],
-            _skip_special_tokens: bool,
-        ) -> vllm_tokenizer::Result<String> {
-            Ok(String::from_utf8_lossy(
-                &token_ids.iter().map(|token_id| *token_id as u8).collect::<Vec<_>>(),
-            )
-            .into_owned())
-        }
-
-        fn token_to_id(&self, _token: &str) -> Option<u32> {
-            unreachable!()
-        }
-    }
-
    #[test]
    fn decode_logprobs_decodes_every_candidate_token() {
-        let tokenizer = ByteTokenizer;
+        let tokenizer = TestTokenizer::new();
        let logprobs = Logprobs {
            positions: vec![PositionLogprobs {
                entries: vec![
@@ -205,7 +178,7 @@ mod tests {

    #[test]
    fn decode_prompt_logprobs_separates_first_prompt_token() {
-        let tokenizer = ByteTokenizer;
+        let tokenizer = TestTokenizer::new();
        let logprobs = Logprobs {
            positions: vec![PositionLogprobs {
                entries: vec![TokenLogprob {
@@ -4,6 +4,9 @@ version.workspace = true
 edition.workspace = true
 license.workspace = true

+[features]
+test-utils = []
+
 [dependencies]
 base64.workspace = true
 fastokens.workspace = true
@@ -32,6 +32,7 @@ pub(crate) struct DecodeStream<'a, T: Tokenizer + ?Sized> {
    ids: Vec<u32>,
    prefix: String,
    prefix_index: usize,
+    prefix_seeded: bool,
    cumulative_output: String,
    output_index: usize,
 }
@@ -50,6 +51,7 @@ impl<'a, T: Tokenizer + ?Sized> DecodeStream<'a, T> {
            ids: prompt_token_ids.to_vec(),
            prefix: String::new(),
            prefix_index: 0,
+            prefix_seeded: prompt_token_ids.is_empty(),
            cumulative_output: String::new(),
            output_index: 0,
        }
@@ -63,29 +65,48 @@ const SAFE_SUFFIX_MIN: usize = 4;
 const SAFE_SUFFIX_MAX: usize = 6;

 impl<T: Tokenizer + ?Sized> DecodeStream<'_, T> {
+    /// Return prompt-context ids that have a tokenizer-local raw token string.
+    ///
+    /// `DecodeStream` uses prompt ids only to seed left context before the
+    /// first generated token. Some prompt ids may come from a wider model
+    /// vocabulary than the local tokenizer can decode, so context seeding drops
+    /// ids that [`Tokenizer::id_to_token`] cannot resolve before calling strict
+    /// [`Tokenizer::decode`]. Generated token ids keep the normal strict decode
+    /// path.
+    fn decodable_context_ids(&self, ids: &[u32]) -> Vec<u32> {
+        ids.iter()
+            .copied()
+            .filter(|&id| self.tokenizer.id_to_token(id).is_some())
+            .collect()
+    }
+
    /// Seed `self.prefix` from the shortest trailing suffix whose decoded text
    /// has no U+FFFD — a clean decode means the suffix starts and ends at
    /// valid UTF-8/token boundaries, so priming from it is equivalent to
-    /// priming from the full prompt.
+    /// priming from the full prompt. Prompt-only ids that the tokenizer cannot
+    /// map back to token text are ignored for decode context; generated ids
+    /// still go through strict decode.
    fn seed_prefix(&mut self) -> Result<()> {
        let prompt_len = self.ids.len();
        if prompt_len > SAFE_SUFFIX_MIN {
            let max_try = SAFE_SUFFIX_MAX.min(prompt_len - 1);
            for suffix_len in SAFE_SUFFIX_MIN..=max_try {
                let start = prompt_len - suffix_len;
-                let decoded =
-                    self.tokenizer.decode(&self.ids[start..], self.skip_special_tokens)?;
+                let candidate = self.decodable_context_ids(&self.ids[start..]);
+                let decoded = self.tokenizer.decode(&candidate, self.skip_special_tokens)?;
                if !decoded.contains('\u{FFFD}') {
                    self.prefix = decoded;
-                    self.ids.drain(..start);
+                    self.ids = candidate;
                    self.prefix_index = self.ids.len();
                    return Ok(());
                }
            }
        }
-        let decoded = self.tokenizer.decode(&self.ids, self.skip_special_tokens)?;
+        let candidate = self.decodable_context_ids(&self.ids);
+        let decoded = self.tokenizer.decode(&candidate, self.skip_special_tokens)?;
        if !decoded.ends_with('\u{FFFD}') {
            self.prefix = decoded;
+            self.ids = candidate;
            self.prefix_index = self.ids.len();
        }
        Ok(())
@@ -94,8 +115,9 @@ impl<T: Tokenizer + ?Sized> DecodeStream<'_, T> {

 impl<T: Tokenizer + ?Sized> IncrementalDecoder for DecodeStream<'_, T> {
    fn push_token(&mut self, token_id: u32) -> Result<usize> {
-        if self.prefix.is_empty() && !self.ids.is_empty() {
+        if !self.prefix_seeded && !self.ids.is_empty() {
            self.seed_prefix()?;
+            self.prefix_seeded = true;
        }

        self.ids.push(token_id);
@@ -131,6 +153,7 @@ impl<T: Tokenizer + ?Sized> IncrementalDecoder for DecodeStream<'_, T> {
            self.ids.clear();
            self.prefix.clear();
            self.prefix_index = 0;
+            self.prefix_seeded = true;
            // Ensure we split at a utf-8 char boundary.
            self.cumulative_output
                .push_str(&string[string.floor_char_boundary(prefix_len)..]);
@@ -152,6 +175,7 @@ impl<T: Tokenizer + ?Sized> IncrementalDecoder for DecodeStream<'_, T> {
 #[cfg(test)]
 mod tests {
    use super::*;
+    use crate::test_utils::TestTokenizer;

    /// Backend that treats each token ID as a raw byte, producing lossy UTF-8.
    #[derive(Debug)]
@@ -170,6 +194,13 @@ mod tests {
        fn token_to_id(&self, _token: &str) -> Option<u32> {
            unreachable!()
        }
+
+        fn id_to_token(&self, id: u32) -> Option<String> {
+            u8::try_from(id).ok().map(|byte| {
+                let bytes = [byte];
+                String::from_utf8_lossy(&bytes).into_owned()
+            })
+        }
    }

    #[test]
@@ -248,6 +279,10 @@ mod tests {
        fn token_to_id(&self, _token: &str) -> Option<u32> {
            unreachable!()
        }
+
+        fn id_to_token(&self, _id: u32) -> Option<String> {
+            unreachable!()
+        }
    }

    #[test]
@@ -273,6 +308,47 @@ mod tests {
        assert_eq!(decoder.output(), "!");
    }

+    #[test]
+    fn seed_prefix_filters_unknown_prompt_ids_from_suffix_context() {
+        let tokenizer = TestTokenizer::new();
+        let prompt = &[
+            b'a' as u32,
+            b'b' as u32,
+            b'c' as u32,
+            10_000,
+            b'H' as u32,
+            b'i' as u32,
+        ];
+        let mut decoder = tokenizer.create_decode_stream(prompt, false, 0);
+
+        assert_eq!(decoder.push_token(b'!' as u32).unwrap(), 1);
+        assert_eq!(decoder.output(), "!");
+    }
+
+    #[test]
+    fn seed_prefix_filters_unknown_prompt_ids_from_full_prompt_fallback() {
+        let tokenizer = TestTokenizer::new();
+        let prompt = &[10_000, b'H' as u32, b'i' as u32];
+        let mut decoder = tokenizer.create_decode_stream(prompt, false, 0);
+
+        assert_eq!(decoder.push_token(b'!' as u32).unwrap(), 1);
+        assert_eq!(decoder.output(), "!");
+    }
+
+    #[test]
+    fn generated_unknown_ids_still_return_decode_error() {
+        let tokenizer = TestTokenizer::new();
+        let prompt = &[10_000, b'H' as u32, b'i' as u32];
+        let mut decoder = tokenizer.create_decode_stream(prompt, false, 0);
+
+        let error = decoder.push_token(10_000).unwrap_err();
+        assert!(
+            error
+                .to_string()
+                .contains("test tokenizer cannot decode unknown token id 10000")
+        );
+    }
+
    #[test]
    fn chunks_concatenate_to_full_text() {
        let backend = Utf8Backend;
@@ -320,6 +396,10 @@ mod tests {
        fn token_to_id(&self, _token: &str) -> Option<u32> {
            unreachable!()
        }
+
+        fn id_to_token(&self, _id: u32) -> Option<String> {
+            unreachable!()
+        }
    }

    /// Without the char-boundary fix, this panics slicing mid-emoji.
@@ -8,6 +8,8 @@ mod error;
 mod hf;
 mod incremental;
 mod tekken;
+#[cfg(any(test, feature = "test-utils"))]
+pub mod test_utils;
 mod tiktoken;

 pub use error::{Result, TokenizerError};
@@ -28,11 +30,7 @@ pub trait Tokenizer: Send + Sync {
    fn token_to_id(&self, token: &str) -> Option<u32>;

    /// Convert one token ID into the tokenizer's raw token string.
-    fn id_to_token(&self, _id: u32) -> Option<String> {
-        // TODO: remove default impl and require this to be implemented by all
-        // tokenizers
-        None
-    }
+    fn id_to_token(&self, id: u32) -> Option<String>;

    /// Return the vocabulary size. Backends that cannot report it fall back to
    /// `usize::MAX`, an effectively unbounded value used only by test stubs.
@@ -0,0 +1,434 @@
+use std::collections::BTreeMap;
+
+use crate::{Result, Tokenizer, TokenizerError};
+
+const FIRST_CONFIGURED_TOKEN_ID: u32 = 256;
+
+/// Whether a configured test token should be treated as special.
+///
+/// Special tokens are skipped by [`Tokenizer::decode`] when
+/// `skip_special_tokens` is set. Regular configured tokens are always emitted.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum TestTokenKind {
+    /// Token is skipped when `skip_special_tokens = true`.
+    Special,
+    /// Token is emitted regardless of `skip_special_tokens`.
+    Regular,
+}
+
+impl TestTokenKind {
+    fn is_special(self) -> bool {
+        matches!(self, Self::Special)
+    }
+}
+
+/// Decode behavior for token ids that are neither configured tokens nor byte ids.
+///
+/// The default is [`UnknownDecode::Error`] so tests notice missing tokenizer
+/// fixtures instead of silently accepting impossible ids. Individual tests can
+/// opt into empty or replacement output when they are explicitly modeling a
+/// lenient detokenization path.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum UnknownDecode {
+    /// Return a tokenizer error on the first unknown id.
+    Error,
+    /// Drop unknown ids from decoded output.
+    Empty,
+    /// Emit U+FFFD for each unknown id.
+    Replacement,
+}
+
+#[derive(Debug, Clone)]
+struct TestToken {
+    text: String,
+    kind: TestTokenKind,
+}
+
+/// Configurable tokenizer for Rust frontend tests.
+///
+/// `TestTokenizer` is intentionally small, but its methods obey the same basic
+/// contract as production tokenizers:
+///
+/// - ordinary text encodes as UTF-8 byte ids;
+/// - configured token ids start at 256, leaving `0..=255` for byte fallback;
+/// - configured token ids and token text are unique;
+/// - configured tokens are matched before ordinary bytes, using longest-prefix matching so
+///   multi-character markers such as `<think>` work naturally;
+/// - `token_to_id` and `id_to_token` are consistent for configured tokens;
+/// - `decode` is strict by default for ids outside the byte range and the configured token table;
+/// - `vocab_size` is an exclusive upper bound covering byte ids and configured token ids unless a
+///   test sets it explicitly.
+///
+/// Prefer this helper over ad-hoc fake tokenizers for tests that rely on
+/// tokenizer semantics. Keep dedicated tiny fakes for error injection or for
+/// tests that deliberately need a degenerate tokenizer.
+#[derive(Debug, Clone)]
+pub struct TestTokenizer {
+    token_to_id: BTreeMap<String, u32>,
+    id_to_token: BTreeMap<u32, TestToken>,
+    unknown_decode: UnknownDecode,
+    vocab_size: Option<usize>,
+    bos_token_id: Option<u32>,
+}
+
+impl Default for TestTokenizer {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl TestTokenizer {
+    /// Create a byte-level test tokenizer with strict unknown-id decode.
+    pub fn new() -> Self {
+        Self {
+            token_to_id: BTreeMap::new(),
+            id_to_token: BTreeMap::new(),
+            unknown_decode: UnknownDecode::Error,
+            vocab_size: None,
+            bos_token_id: None,
+        }
+    }
+
+    /// Add a configured token and return the updated tokenizer.
+    ///
+    /// Configured tokens must use ids outside the byte range and may be marked
+    /// special or regular.
+    pub fn with_token(mut self, token: impl Into<String>, id: u32, kind: TestTokenKind) -> Self {
+        self.insert_token(token, id, kind);
+        self
+    }
+
+    /// Add a special configured token and return the updated tokenizer.
+    pub fn with_special_token(self, token: impl Into<String>, id: u32) -> Self {
+        self.with_token(token, id, TestTokenKind::Special)
+    }
+
+    /// Add a regular configured token and return the updated tokenizer.
+    pub fn with_regular_token(self, token: impl Into<String>, id: u32) -> Self {
+        self.with_token(token, id, TestTokenKind::Regular)
+    }
+
+    /// Add a special BOS token inserted by `encode(..., true)`.
+    ///
+    /// This also registers the token in the normal token/id maps so
+    /// `token_to_id`, `id_to_token`, `decode`, and `is_special_id` stay
+    /// consistent for the inserted id.
+    pub fn with_bos_token(mut self, token: impl Into<String>, id: u32) -> Self {
+        self.insert_token(token, id, TestTokenKind::Special);
+        self.bos_token_id = Some(id);
+        self
+    }
+
+    /// Set decode behavior for unknown non-byte ids.
+    pub fn with_unknown_decode(mut self, behavior: UnknownDecode) -> Self {
+        self.unknown_decode = behavior;
+        self
+    }
+
+    /// Set an explicit vocabulary size.
+    ///
+    /// Use this when a test needs a model-like vocabulary bound that differs
+    /// from the highest configured token id plus one.
+    pub fn with_vocab_size(mut self, vocab_size: usize) -> Self {
+        self.vocab_size = Some(vocab_size);
+        self
+    }
+
+    fn insert_token(&mut self, token: impl Into<String>, id: u32, kind: TestTokenKind) {
+        let token = token.into();
+        assert!(
+            !token.is_empty(),
+            "configured test token text must be non-empty"
+        );
+        assert!(
+            id >= FIRST_CONFIGURED_TOKEN_ID,
+            "configured test token id {id} overlaps byte fallback range 0..=255"
+        );
+        assert!(
+            token.len() > 1,
+            "configured test token text {token:?} overlaps byte fallback token text"
+        );
+        if self.token_to_id.insert(token.clone(), id).is_some() {
+            panic!("configured test token text {token:?} was registered more than once");
+        }
+        if self.id_to_token.insert(id, TestToken { text: token, kind }).is_some() {
+            panic!("configured test token id {id} was registered more than once");
+        }
+    }
+
+    fn byte_to_token(id: u32) -> Option<String> {
+        u8::try_from(id).ok().map(|byte| String::from_utf8_lossy(&[byte]).into_owned())
+    }
+
+    fn flush_bytes(bytes: &mut Vec<u8>, output: &mut String) {
+        if !bytes.is_empty() {
+            output.push_str(&String::from_utf8_lossy(bytes));
+            bytes.clear();
+        }
+    }
+
+    fn configured_token_prefix(&self, text: &str) -> Option<(&str, u32)> {
+        self.token_to_id
+            .iter()
+            .filter_map(|(token, &id)| text.starts_with(token).then_some((token.as_str(), id)))
+            .max_by_key(|(token, _)| token.len())
+    }
+
+    fn inferred_vocab_size(&self) -> usize {
+        let max_configured =
+            self.id_to_token.last_key_value().map(|(&id, _)| id as usize + 1).unwrap_or(0);
+        256.max(max_configured)
+    }
+}
+
+impl Tokenizer for TestTokenizer {
+    fn encode(&self, text: &str, add_special_tokens: bool) -> Result<Vec<u32>> {
+        let mut ids = Vec::new();
+        if add_special_tokens && let Some(bos_token_id) = self.bos_token_id {
+            ids.push(bos_token_id);
+        }
+
+        let mut rest = text;
+        while !rest.is_empty() {
+            if let Some((token, id)) = self.configured_token_prefix(rest) {
+                ids.push(id);
+                rest = &rest[token.len()..];
+                continue;
+            }
+
+            let ch = rest.chars().next().expect("rest is not empty");
+            let mut buf = [0_u8; 4];
+            ids.extend(ch.encode_utf8(&mut buf).bytes().map(u32::from));
+            rest = &rest[ch.len_utf8()..];
+        }
+
+        Ok(ids)
+    }
+
+    fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result<String> {
+        let mut output = String::new();
+        let mut pending_bytes = Vec::new();
+        for &id in token_ids {
+            if let Some(token) = self.id_to_token.get(&id) {
+                Self::flush_bytes(&mut pending_bytes, &mut output);
+                if !(skip_special_tokens && token.kind.is_special()) {
+                    output.push_str(&token.text);
+                }
+            } else if let Ok(byte) = u8::try_from(id) {
+                pending_bytes.push(byte);
+            } else {
+                Self::flush_bytes(&mut pending_bytes, &mut output);
+                match self.unknown_decode {
+                    UnknownDecode::Error => {
+                        return Err(TokenizerError(format!(
+                            "test tokenizer cannot decode unknown token id {id}"
+                        )));
+                    }
+                    UnknownDecode::Empty => {}
+                    UnknownDecode::Replacement => output.push('\u{FFFD}'),
+                }
+            }
+        }
+        Self::flush_bytes(&mut pending_bytes, &mut output);
+        Ok(output)
+    }
+
+    fn token_to_id(&self, token: &str) -> Option<u32> {
+        self.token_to_id.get(token).copied().or_else(|| {
+            let bytes = token.as_bytes();
+            (bytes.len() == 1).then(|| u32::from(bytes[0]))
+        })
+    }
+
+    fn id_to_token(&self, id: u32) -> Option<String> {
+        self.id_to_token
+            .get(&id)
+            .map(|token| token.text.clone())
+            .or_else(|| Self::byte_to_token(id))
+    }
+
+    fn vocab_size(&self) -> usize {
+        self.vocab_size.unwrap_or_else(|| self.inferred_vocab_size())
+    }
+
+    fn is_special_id(&self, token_id: u32) -> bool {
+        self.id_to_token.get(&token_id).is_some_and(|token| token.kind.is_special())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn byte_text_roundtrips_and_reports_byte_ids() {
+        let tokenizer = TestTokenizer::new();
+
+        let ids = tokenizer.encode("hi", false).unwrap();
+        assert_eq!(ids, vec![b'h' as u32, b'i' as u32]);
+        assert_eq!(tokenizer.decode(&ids, false).unwrap(), "hi");
+        assert_eq!(tokenizer.token_to_id("h"), Some(b'h' as u32));
+        assert_eq!(tokenizer.id_to_token(b'h' as u32).as_deref(), Some("h"));
+        assert_eq!(tokenizer.vocab_size(), 256);
+    }
+
+    #[test]
+    fn configured_tokens_use_longest_prefix_matching() {
+        let tokenizer = TestTokenizer::new()
+            .with_regular_token("<image>", 999)
+            .with_regular_token("<image></image>", 1000);
+
+        assert_eq!(
+            tokenizer.encode("a<image></image>b", false).unwrap(),
+            vec![b'a' as u32, 1000, b'b' as u32,]
+        );
+        assert_eq!(
+            tokenizer.decode(&[b'a' as u32, 1000, b'b' as u32], false).unwrap(),
+            "a<image></image>b"
+        );
+        assert_eq!(tokenizer.token_to_id("<image>"), Some(999));
+        assert_eq!(
+            tokenizer.id_to_token(1000).as_deref(),
+            Some("<image></image>")
+        );
+        assert_eq!(tokenizer.vocab_size(), 1001);
+    }
+
+    #[test]
+    fn non_ascii_text_roundtrips_through_buffered_byte_decode() {
+        let tokenizer = TestTokenizer::new();
+        let text = "你好, café, 🚀";
+
+        let ids = tokenizer.encode(text, false).unwrap();
+        assert_eq!(
+            ids,
+            text.as_bytes().iter().copied().map(u32::from).collect::<Vec<_>>()
+        );
+        assert_eq!(tokenizer.decode(&ids, false).unwrap(), text);
+    }
+
+    #[test]
+    fn buffered_byte_decode_flushes_around_configured_tokens() {
+        let tokenizer = TestTokenizer::new()
+            .with_regular_token("<image>", 999)
+            .with_special_token("<skip>", 1000);
+
+        assert_eq!(
+            tokenizer.encode("你<image>好<skip>🚀", false).unwrap(),
+            vec![228, 189, 160, 999, 229, 165, 189, 1000, 240, 159, 154, 128]
+        );
+        assert_eq!(
+            tokenizer
+                .decode(
+                    &[228, 189, 160, 999, 229, 165, 189, 1000, 240, 159, 154, 128],
+                    false
+                )
+                .unwrap(),
+            "你<image>好<skip>🚀"
+        );
+        assert_eq!(
+            tokenizer
+                .decode(
+                    &[228, 189, 160, 999, 229, 165, 189, 1000, 240, 159, 154, 128],
+                    true
+                )
+                .unwrap(),
+            "你<image>好🚀"
+        );
+    }
+
+    #[test]
+    fn invalid_utf8_bytes_decode_lossily_as_a_sequence() {
+        let tokenizer = TestTokenizer::new();
+
+        assert_eq!(tokenizer.decode(&[0xE4, 0xBD], false).unwrap(), "\u{FFFD}");
+        assert_eq!(
+            tokenizer.decode(&[0xFF, b'a' as u32], false).unwrap(),
+            "\u{FFFD}a"
+        );
+    }
+
+    #[test]
+    fn special_tokens_respect_skip_special_tokens() {
+        let tokenizer = TestTokenizer::new()
+            .with_bos_token("<bos>", 256)
+            .with_special_token("<think>", 0xF001)
+            .with_regular_token("</think>", 0xF002);
+
+        assert_eq!(
+            tokenizer.encode("<think>x</think>", true).unwrap(),
+            vec![256, 0xF001, b'x' as u32, 0xF002,]
+        );
+        assert_eq!(
+            tokenizer.decode(&[256, 0xF001, b'x' as u32, 0xF002], false).unwrap(),
+            "<bos><think>x</think>"
+        );
+        assert_eq!(
+            tokenizer.decode(&[256, 0xF001, b'x' as u32, 0xF002], true).unwrap(),
+            "x</think>"
+        );
+        assert!(tokenizer.is_special_id(0xF001));
+        assert!(!tokenizer.is_special_id(0xF002));
+    }
+
+    #[test]
+    #[should_panic(expected = "configured test token id 255 overlaps byte fallback range 0..=255")]
+    fn configured_token_id_must_stay_outside_byte_range() {
+        let _ = TestTokenizer::new().with_regular_token("<token>", 255);
+    }
+
+    #[test]
+    #[should_panic(expected = "configured test token text \"a\" overlaps byte fallback token text")]
+    fn configured_token_text_must_not_shadow_byte_tokens() {
+        let _ = TestTokenizer::new().with_regular_token("a", 256);
+    }
+
+    #[test]
+    #[should_panic(
+        expected = "configured test token text \"<token>\" was registered more than once"
+    )]
+    fn configured_token_text_must_be_unique() {
+        let _ = TestTokenizer::new()
+            .with_regular_token("<token>", 256)
+            .with_regular_token("<token>", 257);
+    }
+
+    #[test]
+    #[should_panic(expected = "configured test token id 256 was registered more than once")]
+    fn configured_token_id_must_be_unique() {
+        let _ = TestTokenizer::new()
+            .with_regular_token("<token-a>", 256)
+            .with_regular_token("<token-b>", 256);
+    }
+
+    #[test]
+    fn unknown_decode_is_strict_by_default_and_configurable() {
+        let strict = TestTokenizer::new();
+        assert!(strict.decode(&[300], false).is_err());
+        assert_eq!(
+            TestTokenizer::new()
+                .with_unknown_decode(UnknownDecode::Empty)
+                .decode(&[b'a' as u32, 300, b'b' as u32], false)
+                .unwrap(),
+            "ab"
+        );
+        assert_eq!(
+            TestTokenizer::new()
+                .with_unknown_decode(UnknownDecode::Replacement)
+                .decode(&[300], false)
+                .unwrap(),
+            "\u{FFFD}"
+        );
+        assert_eq!(strict.id_to_token(300), None);
+    }
+
+    #[test]
+    fn explicit_vocab_size_overrides_inferred_bound() {
+        let tokenizer = TestTokenizer::new()
+            .with_regular_token("<high>", 10_000)
+            .with_vocab_size(20_000);
+
+        assert_eq!(tokenizer.vocab_size(), 20_000);
+        assert_eq!(tokenizer.id_to_token(10_000).as_deref(), Some("<high>"));
+    }
+}
Author	SHA1	Message	Date
Bugen Zhao	61b40475c7	[Rust Frontend] Tolerate out-of-vocab tokens from prompt in detokenizer Signed-off-by: Bugen Zhao <i@bugenzhao.com>	2026-06-30 15:37:56 +08:00
Bugen Zhao	a1f41ee1ca	fix clippy Signed-off-by: Bugen Zhao <i@bugenzhao.com>	2026-06-30 15:20:47 +08:00
Bugen Zhao	a99bc9e6f9	remove default impl on `id_to_token` Signed-off-by: Bugen Zhao <i@bugenzhao.com>	2026-06-30 15:14:45 +08:00
Bugen Zhao	ebc444a533	replace more Signed-off-by: Bugen Zhao <i@bugenzhao.com>	2026-06-30 15:10:18 +08:00
Bugen Zhao	8fd4cfa611	non-ascii roundtrip Signed-off-by: Bugen Zhao <i@bugenzhao.com>	2026-06-30 15:08:15 +08:00
Bugen Zhao	7ff1a4e47a	update callsite Signed-off-by: Bugen Zhao <i@bugenzhao.com>	2026-06-30 14:53:50 +08:00
Bugen Zhao	82f53366f5	stronger invariant Signed-off-by: Bugen Zhao <i@bugenzhao.com>	2026-06-30 14:39:05 +08:00
Bugen Zhao	777fa7cf20	introduce `TestTokenizer` Signed-off-by: Bugen Zhao <i@bugenzhao.com>	2026-06-30 14:23:56 +08:00