stash

Signed-off-by: Bugen Zhao <i@bugenzhao.com>

rust/Cargo.lock

@@ -5099,6 +5099,7 @@ dependencies = [
  "uuid",
  "vllm-engine-core-client",
  "vllm-llm",
+ "vllm-model-files",
  "vllm-parser",
  "vllm-text",
  "vllm-tokenizer",
@@ -5236,6 +5237,20 @@ dependencies = [
  "zeromq",
 ]
 
+[[package]]
+name = "vllm-model-files"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "hf-hub",
+ "serde",
+ "serde_json",
+ "tempfile",
+ "thiserror 2.0.18",
+ "thiserror-ext",
+ "tokio",
+]
+
 [[package]]
 name = "vllm-parser"
 version = "0.1.0"
@@ -5324,7 +5339,6 @@ dependencies = [
  "enum-as-inner",
  "expect-test",
  "futures",
- "hf-hub",
  "itertools 0.14.0",
  "reqwest",
  "serde",
@@ -5339,6 +5353,7 @@ dependencies = [
  "trait-set",
  "vllm-engine-core-client",
  "vllm-llm",
+ "vllm-model-files",
  "vllm-tokenizer",
 ]
 

rust/Cargo.toml

@@ -6,6 +6,7 @@ members = [
     "src/llm",
     "src/managed-engine",
     "src/metrics",
+    "src/model-files",
     "src/mock-engine",
     "src/parser",
     "src/parser/python",
@@ -126,6 +127,7 @@ vllm-chat = { path = "src/chat" }
 vllm-engine-core-client = { path = "src/engine-core-client" }
 vllm-llm = { path = "src/llm" }
 vllm-managed-engine = { path = "src/managed-engine" }
+vllm-model-files = { path = "src/model-files" }
 vllm-metrics = { path = "src/metrics" }
 vllm-parser = { path = "src/parser" }
 vllm-server = { path = "src/server" }

rust/src/chat/Cargo.toml

@@ -32,6 +32,7 @@ trait-set.workspace = true
 uuid.workspace = true
 vllm-engine-core-client.workspace = true
 vllm-llm.workspace = true
+vllm-model-files.workspace = true
 vllm-parser.workspace = true
 vllm-text.workspace = true
 vllm-tokenizer.workspace = true

rust/src/chat/src/error.rs

@@ -1,5 +1,6 @@
 use thiserror::Error;
 use thiserror_ext::Macro;
+use vllm_model_files::Error as ModelFilesError;
 
 type BoxedError = Box<dyn std::error::Error + Send + Sync>;
 
@@ -69,6 +70,8 @@ pub enum Error {
     #[error(transparent)]
     Text(#[from] vllm_text::Error),
     #[error(transparent)]
+    ModelFiles(#[from] ModelFilesError),
+    #[error(transparent)]
     Tokenizer(#[from] vllm_tokenizer::TokenizerError),
 }
 

rust/src/chat/src/renderer/hf/mod.rs

@@ -6,7 +6,7 @@ use thiserror_ext::AsReport as _;
 use tracing::{info, trace, warn};
 use vllm_text::Prompt;
 use vllm_text::backend::hf::{
-    HfSpecialTokens, HfTokenizerConfig, ResolvedModelFiles, load_tokenizer_config,
+    ResolvedModelFiles, SpecialTokens, TokenizerConfig, load_tokenizer_config,
 };
 
 use self::format::{
@@ -42,7 +42,7 @@ pub struct HfChatRenderer {
     default_template: Option<CompiledChatTemplate>,
     default_template_kwargs: HashMap<String, JsonValue>,
     content_format: ContentFormatOption,
-    special_tokens: Option<HfSpecialTokens>,
+    special_tokens: Option<SpecialTokens>,
     multimodal: Option<MultimodalRenderInfo>,
 }
 
@@ -67,7 +67,7 @@ impl HfChatRenderer {
         })
     }
 
-    pub fn with_special_tokens(mut self, special_tokens: Option<HfSpecialTokens>) -> Self {
+    pub fn with_special_tokens(mut self, special_tokens: Option<SpecialTokens>) -> Self {
         self.special_tokens = special_tokens;
         self
     }
@@ -83,7 +83,7 @@ impl HfChatRenderer {
         options: LoadModelBackendsOptions,
         multimodal: Option<MultimodalRenderInfo>,
     ) -> Result<Self> {
-        let HfTokenizerConfig {
+        let TokenizerConfig {
             special_tokens,
             chat_template,
             ..
@@ -451,7 +451,7 @@ mod tests {
     use expect_test::expect;
     use serde_json::Value;
     use vllm_text::Prompt;
-    use vllm_text::backend::hf::{HfSpecialTokens, NamedSpecialToken};
+    use vllm_text::backend::hf::{NamedSpecialToken, SpecialTokens};
 
     use super::{ChatTemplateContentFormatOption, HfChatRenderer, MultimodalRenderInfo};
     use crate::request::{
@@ -675,7 +675,7 @@ mod tests {
     #[test]
     fn chat_template_injects_special_tokens_into_context() {
         let request = sample_request(vec![ChatMessage::text(ChatRole::User, "hello")]);
-        let special_tokens = HfSpecialTokens {
+        let special_tokens = SpecialTokens {
             bos_token: Some(NamedSpecialToken::Text("<bos>".to_string())),
             ..Default::default()
         };

rust/src/chat/src/renderer/hf/template.rs

@@ -2,7 +2,7 @@
 //!
 //! This module is inlined from SMG's tokenizer crate with local adaptations:
 //! - thinking-related detection/state is removed
-//! - special tokens are wired to `vllm_text::backends::hf::HfSpecialTokens`
+//! - special tokens are wired to `vllm_text::backends::hf::SpecialTokens`
 
 use std::collections::HashMap;
 use std::fs;
@@ -11,7 +11,7 @@ use std::path::Path;
 use minijinja::Environment;
 use serde::{Deserialize, Serialize};
 use serde_json::{self};
-use vllm_text::backend::hf::HfSpecialTokens;
+use vllm_text::backend::hf::SpecialTokens;
 
 use super::error::TemplateError;
 use super::format::{
@@ -46,7 +46,7 @@ pub(super) struct TemplateContext<'a> {
     pub(super) tools: Option<&'a [TemplateTool]>,
     pub(super) documents: Option<&'a [serde_json::Value]>,
     #[serde(flatten)]
-    pub(super) special_tokens: Option<&'a HfSpecialTokens>,
+    pub(super) special_tokens: Option<&'a SpecialTokens>,
     #[serde(flatten)]
     pub(super) template_kwargs: Option<&'a HashMap<String, serde_json::Value>>,
 }
@@ -133,7 +133,7 @@ mod tests {
     use std::fs;
 
     use tempfile::TempDir;
-    use vllm_text::backend::hf::{HfSpecialTokens, NamedSpecialToken};
+    use vllm_text::backend::hf::{NamedSpecialToken, SpecialTokens};
 
     use super::*;
 
@@ -170,7 +170,7 @@ mod tests {
             CompiledChatTemplate::new(template.to_string(), ChatTemplateContentFormatOption::Auto)
                 .unwrap();
 
-        let special_tokens = HfSpecialTokens {
+        let special_tokens = SpecialTokens {
             bos_token: Some(NamedSpecialToken::Text("<s>".to_string())),
             eos_token: Some(NamedSpecialToken::Text("</s>".to_string())),
             ..Default::default()
@@ -205,7 +205,7 @@ mod tests {
             CompiledChatTemplate::new(template.to_string(), ChatTemplateContentFormatOption::Auto)
                 .unwrap();
 
-        let special_tokens = HfSpecialTokens {
+        let special_tokens = SpecialTokens {
             bos_token: Some(NamedSpecialToken::Text("<s>".to_string())),
             eos_token: None,
             ..Default::default()

rust/src/model-files/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "vllm-model-files"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+
+[dependencies]
+anyhow.workspace = true
+hf-hub.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+thiserror.workspace = true
+thiserror-ext.workspace = true
+
+[dev-dependencies]
+tempfile.workspace = true
+tokio.workspace = true
+
+[lints]
+workspace = true

rust/src/model-files/src/config.rs

@@ -0,0 +1,21 @@
+use std::path::Path;
+
+use serde::Deserialize;
+
+use crate::error::Result;
+use crate::json::read_json_file;
+
+/// Minimal subset of `tokenizer_config.json` needed by tokenizer selection.
+#[derive(Debug, Default, Deserialize)]
+#[serde(default)]
+pub(crate) struct TokenizerConfig {
+    /// The `tokenizer_class` field from HuggingFace tokenizer configs. Some
+    /// tiktoken-based models (e.g. DeepSeek, Kimi K2) set this to a value
+    /// containing "Tiktoken" which can be used as a hint for backend
+    /// selection.
+    pub tokenizer_class: Option<String>,
+}
+
+pub(crate) fn load_tokenizer_config(path: Option<&Path>) -> Result<TokenizerConfig> {
+    read_json_file(path)
+}

rust/src/model-files/src/error.rs

@@ -0,0 +1,15 @@
+use thiserror::Error as ThisError;
+
+/// Error returned while resolving or reading model files.
+#[derive(Debug, ThisError)]
+#[error("model file error: {0}")]
+pub struct Error(String);
+
+impl Error {
+    pub(crate) fn new(message: impl Into<String>) -> Self {
+        Self(message.into())
+    }
+}
+
+/// Result type used by model-file discovery helpers.
+pub type Result<T> = std::result::Result<T, Error>;

rust/src/model-files/src/json.rs

@@ -0,0 +1,31 @@
+use std::fs;
+use std::path::Path;
+
+use serde::Deserialize;
+use thiserror_ext::AsReport as _;
+
+use crate::error::{Error, Result};
+
+/// Read an optional JSON file into `T`, returning `T::default()` when absent.
+pub fn read_json_file<T>(path: Option<&Path>) -> Result<T>
+where
+    T: for<'de> Deserialize<'de> + Default,
+{
+    let Some(path) = path else {
+        return Ok(T::default());
+    };
+    let content = fs::read_to_string(path).map_err(|error| {
+        Error::new(format!(
+            "failed to read {}: {}",
+            path.display(),
+            error.as_report()
+        ))
+    })?;
+    serde_json::from_str(&content).map_err(|error| {
+        Error::new(format!(
+            "failed to parse {}: {}",
+            path.display(),
+            error.as_report()
+        ))
+    })
+}

rust/src/model-files/src/lib.rs

@@ -0,0 +1,10 @@
+//! Hugging Face model file discovery shared by Rust frontend crates.
+
+mod config;
+mod error;
+mod json;
+mod model_files;
+
+pub use error::{Error, Result};
+pub use json::read_json_file;
+pub use model_files::{ResolvedModelFiles, TokenizerSource};

rust/src/model-files/src/model_files.rs

@@ -4,7 +4,7 @@ use hf_hub::Cache;
 use hf_hub::api::tokio::{Api, ApiBuilder, ApiRepo};
 use thiserror_ext::AsReport as _;
 
-use super::config::{HfTokenizerConfig, load_tokenizer_config};
+use crate::config::{TokenizerConfig, load_tokenizer_config};
 use crate::error::{Error, Result};
 
 const HF_TOKEN_ENV: &str = "HF_TOKEN";
@@ -26,6 +26,28 @@ pub enum TokenizerSource {
 }
 
 impl TokenizerSource {
+    /// Select a tokenizer source from a tokenizer file path.
+    pub fn from_path(path: impl Into<PathBuf>) -> Result<Self> {
+        let path = path.into();
+        let file_name = path.file_name().and_then(|name| name.to_str()).ok_or_else(|| {
+            Error::new(format!(
+                "tokenizer path has no file name: {}",
+                path.display()
+            ))
+        })?;
+
+        match file_name {
+            "tekken.json" => Ok(Self::Tekken(path)),
+            "tokenizer.json" => Ok(Self::HuggingFace(path)),
+            _ if is_tiktoken_file(&path) => Ok(Self::Tiktoken(path)),
+            _ => Err(Error::new(format!(
+                "unsupported tokenizer file '{}'",
+                path.display()
+            ))),
+        }
+    }
+
+    /// Return the local filesystem path for this tokenizer source.
     pub fn path(&self) -> &Path {
         match self {
             Self::HuggingFace(path) | Self::Tiktoken(path) | Self::Tekken(path) => path,
@@ -38,10 +60,15 @@ impl TokenizerSource {
 pub struct ResolvedModelFiles {
     /// The selected tokenizer source for this model.
     pub tokenizer: TokenizerSource,
+    /// Path to `tokenizer_config.json` when present.
     pub tokenizer_config_path: Option<PathBuf>,
+    /// Path to `generation_config.json` when present.
     pub generation_config_path: Option<PathBuf>,
+    /// Path to `preprocessor_config.json` when present.
     pub preprocessor_config_path: Option<PathBuf>,
+    /// Path to a discovered chat template file when present.
     pub chat_template_path: Option<PathBuf>,
+    /// Path to `config.json` when present.
     pub config_path: Option<PathBuf>,
 }
 
@@ -76,10 +103,10 @@ fn resolve_local_model_files(model_dir: &Path) -> Result<ResolvedModelFiles> {
 }
 
 async fn resolve_remote_model_files(model_id: &str) -> Result<ResolvedModelFiles> {
-    let api = build_api().map_err(|error| Error::Tokenizer(error.to_report_string()))?;
+    let api = build_api().map_err(|error| Error::new(format!("{}", error.as_report())))?;
     let repo = api.model(model_id.to_string());
     let info = repo.info().await.map_err(|error| {
-        Error::Tokenizer(format!(
+        Error::new(format!(
             "failed to fetch model '{model_id}': {}",
             error.as_report()
         ))
@@ -138,9 +165,10 @@ fn resolve_cached_model_files(model_id: &str) -> Result<Option<ResolvedModelFile
         None => return Ok(None),
     };
 
-    let model_dir = tokenizer.path().parent().ok_or_else(|| {
-        Error::Tokenizer("resolved tokenizer file has no parent directory".to_string())
-    })?;
+    let model_dir = tokenizer
+        .path()
+        .parent()
+        .ok_or_else(|| Error::new("resolved tokenizer file has no parent directory"))?;
     let generation_config_path = cache_repo.get("generation_config.json");
     let preprocessor_config_path = cache_repo.get("preprocessor_config.json");
     let chat_template_path = discover_chat_template_in_dir(model_dir);
@@ -162,113 +190,88 @@ async fn resolve_remote_tokenizer_source(
     siblings: &std::collections::BTreeSet<&str>,
     tokenizer_class: Option<&str>,
 ) -> Result<TokenizerSource> {
-    if let Some(tekken_path) = download_if_present(repo, model_id, siblings, "tekken.json").await? {
-        return Ok(TokenizerSource::Tekken(tekken_path));
-    }
-
-    let tokenizer_path = if siblings.contains("tokenizer.json") {
+    let tokenizer_path = if siblings.contains("tekken.json") {
+        download_known_file(repo, model_id, "tekken.json").await?
+    } else if siblings.contains("tokenizer.json") {
         download_known_file(repo, model_id, "tokenizer.json").await?
     } else if let Some(tiktoken_name) = find_tiktoken_sibling(siblings) {
         download_known_file(repo, model_id, tiktoken_name).await?
     } else {
-        return Err(Error::Tokenizer(format!(
+        return Err(Error::new(format!(
             "model '{model_id}' does not expose a supported tokenizer file \
              (tokenizer.json, tiktoken.model, or *.tiktoken) on Hugging Face"
         )));
     };
 
-    Ok(resolve_tokenizer_source(
-        tokenizer_path,
-        tokenizer_class,
-        None,
-    ))
+    resolve_tokenizer_source(tokenizer_path, tokenizer_class)
 }
 
 fn resolve_cached_tokenizer_source(
     cache_repo: &hf_hub::CacheRepo,
-    tokenizer_config: &HfTokenizerConfig,
+    tokenizer_config: &TokenizerConfig,
 ) -> Result<Option<TokenizerSource>> {
-    let tekken_path = cache_repo.get("tekken.json");
-
-    if let Some(tekken_path) = tekken_path {
-        return Ok(Some(TokenizerSource::Tekken(tekken_path)));
-    }
-
-    let Some(tokenizer_path) = cache_repo.get("tokenizer.json").or_else(|| {
-        // tiktoken.model is the most common name, try it first.
-        cache_repo.get("tiktoken.model").or_else(|| {
-            // Scan for any *.tiktoken file in the cache snapshot directory.
-            let snapshot_dir = cache_repo.get("config.json")?.parent()?.to_path_buf();
-            discover_tiktoken_in_dir(&snapshot_dir)
+    let Some(tokenizer_path) = cache_repo
+        .get("tekken.json")
+        .or_else(|| cache_repo.get("tokenizer.json"))
+        .or_else(|| {
+            // tiktoken.model is the most common name, try it first.
+            cache_repo.get("tiktoken.model").or_else(|| {
+                // Scan for any *.tiktoken file in the cache snapshot directory.
+                let snapshot_dir = cache_repo.get("config.json")?.parent()?.to_path_buf();
+                discover_tiktoken_in_dir(&snapshot_dir)
+            })
         })
-    }) else {
+    else {
         return Ok(None);
     };
 
     Ok(Some(resolve_tokenizer_source(
         tokenizer_path,
         tokenizer_config.tokenizer_class.as_deref(),
-        None,
-    )))
+    )?))
 }
 
 fn resolve_local_tokenizer_source(
     model_dir: &Path,
-    tokenizer_config: &HfTokenizerConfig,
+    tokenizer_config: &TokenizerConfig,
 ) -> Result<TokenizerSource> {
-    let tekken_path = local_file_if_exists(model_dir, "tekken.json");
-    if let Some(tekken_path) = tekken_path {
-        return Ok(TokenizerSource::Tekken(tekken_path));
-    }
-
-    let tokenizer_path = local_file_if_exists(model_dir, "tokenizer.json")
+    let tokenizer_path = local_file_if_exists(model_dir, "tekken.json")
+        .or_else(|| local_file_if_exists(model_dir, "tokenizer.json"))
         .or_else(|| local_file_if_exists(model_dir, "tiktoken.model"))
         .or_else(|| discover_tiktoken_in_dir(model_dir))
         .ok_or_else(|| {
-            Error::Tokenizer(format!(
+            Error::new(format!(
                 "local model directory '{}' does not contain a supported tokenizer file \
                  (tokenizer.json, tiktoken.model, or *.tiktoken)",
                 model_dir.display()
             ))
         })?;
 
-    Ok(resolve_tokenizer_source(
-        tokenizer_path,
-        tokenizer_config.tokenizer_class.as_deref(),
-        None,
-    ))
+    resolve_tokenizer_source(tokenizer_path, tokenizer_config.tokenizer_class.as_deref())
 }
 
 /// Choose the tokenizer.
 ///
 /// Selection order:
-/// 1. `tekken.json` — Mistral native tokenizer (preferred over HF `tokenizer.json` because the HF
-///    version has a known regex bug for Mistral models).
-/// 2. File extension — `.tiktoken` / `tiktoken.model` files use tiktoken from BPE data.
-/// 3. `tokenizer_class` in `tokenizer_config.json` — classes containing "Tiktoken" (case-
+/// 1. File extension — `.tiktoken` / `tiktoken.model` files use tiktoken from BPE data.
+/// 2. `tokenizer_class` in `tokenizer_config.json` — classes containing "Tiktoken" (case-
 ///    insensitive) trigger tiktoken loading from a sibling BPE file.
-/// 4. Default — `tokenizer.json` in HuggingFace format.
+/// 3. Default — `tokenizer.json` in HuggingFace format.
 fn resolve_tokenizer_source(
     tokenizer_path: PathBuf,
     tokenizer_class: Option<&str>,
-    tekken_path: Option<PathBuf>,
-) -> TokenizerSource {
-    if let Some(tekken_path) = tekken_path {
-        return TokenizerSource::Tekken(tekken_path);
-    }
+) -> Result<TokenizerSource> {
+    let tokenizer = TokenizerSource::from_path(tokenizer_path)?;
 
-    if is_tiktoken_file(&tokenizer_path) {
-        return TokenizerSource::Tiktoken(tokenizer_path);
-    }
-
-    if tokenizer_class.is_some_and(|cls| cls.to_ascii_lowercase().contains("tiktoken"))
-        && let Some(dir) = tokenizer_path.parent()
+    if let TokenizerSource::HuggingFace(path) = &tokenizer
+        && tokenizer_class.is_some_and(|cls| cls.to_ascii_lowercase().contains("tiktoken"))
+        && let Some(dir) = path.parent()
         && let Some(tiktoken_path) = discover_tiktoken_in_dir(dir)
     {
-        return TokenizerSource::Tiktoken(tiktoken_path);
+        return Ok(TokenizerSource::Tiktoken(tiktoken_path));
     }
 
-    TokenizerSource::HuggingFace(tokenizer_path)
+    Ok(tokenizer)
 }
 
 /// Download `filename` only if it exists in `siblings`.
@@ -286,7 +289,7 @@ async fn download_if_present(
 
 async fn download_known_file(repo: &ApiRepo, model_id: &str, filename: &str) -> Result<PathBuf> {
     repo.get(filename).await.map_err(|error| {
-        Error::Tokenizer(format!(
+        Error::new(format!(
             "failed to download '{filename}' for model '{model_id}': {}",
             error.as_report()
         ))
@@ -317,7 +320,7 @@ fn find_tiktoken_sibling<'a>(siblings: &std::collections::BTreeSet<&'a str>) ->
 }
 
 /// Discover a tiktoken model file in a local directory.
-pub(super) fn discover_tiktoken_in_dir(dir: &std::path::Path) -> Option<PathBuf> {
+fn discover_tiktoken_in_dir(dir: &std::path::Path) -> Option<PathBuf> {
     let tiktoken_model = dir.join("tiktoken.model");
     if tiktoken_model.exists() {
         return Some(tiktoken_model);
@@ -337,7 +340,7 @@ pub(super) fn discover_tiktoken_in_dir(dir: &std::path::Path) -> Option<PathBuf>
 }
 
 /// Returns `true` if `path` points to a tiktoken-format file (by name).
-pub(super) fn is_tiktoken_file(path: &std::path::Path) -> bool {
+fn is_tiktoken_file(path: &std::path::Path) -> bool {
     path.file_name()
         .and_then(|n| n.to_str())
         .is_some_and(|name| name == "tiktoken.model" || name.ends_with(".tiktoken"))
@@ -368,7 +371,6 @@ mod tests {
     use std::fs;
 
     use tempfile::tempdir;
-    use vllm_tokenizer::{TiktokenTokenizer, Tokenizer};
 
     use super::{ResolvedModelFiles, TokenizerSource};
 
@@ -399,61 +401,4 @@ mod tests {
             Some(dir.path().join("tokenizer_config.json"))
         );
     }
-
-    #[tokio::test]
-    #[ignore = "too slow for CI and requires network access to Hugging Face"]
-    async fn tiktoken_real_kimi_k25_tokenizer_files_load_and_handle_special_tokens() {
-        let files = ResolvedModelFiles::new("moonshotai/Kimi-K2.5")
-            .await
-            .expect("resolve real Kimi K2.5 model files");
-
-        let tokenizer_path = match &files.tokenizer {
-            TokenizerSource::Tiktoken(path) => path.clone(),
-            other => panic!("expected tiktoken tokenizer source, got {other:?}"),
-        };
-
-        for backend in [
-            TiktokenTokenizer::new_riptoken(&tokenizer_path).expect("load riptoken backend"),
-            TiktokenTokenizer::new_tiktoken_rs(&tokenizer_path).expect("load tiktoken-rs backend"),
-        ] {
-            let think_id = backend.token_to_id("<think>").expect("resolve <think>");
-            let end_think_id = backend.token_to_id("</think>").expect("resolve </think>");
-            let tool_section_id = backend
-                .token_to_id("<|tool_calls_section_begin|>")
-                .expect("resolve tool call section marker");
-            let contraction_heavy_text =
-                "I'm sure it's fine, but I can't say I'd trust that it's what we'd ship.";
-            let contraction_heavy_ids = backend.encode(contraction_heavy_text, false).unwrap();
-
-            assert_eq!(
-                (think_id, end_think_id, tool_section_id),
-                (163606, 163607, 163595)
-            );
-            assert_eq!(backend.decode(&[think_id], true).unwrap(), "<think>");
-            assert_eq!(backend.decode(&[end_think_id], true).unwrap(), "</think>");
-            assert_eq!(
-                backend.decode(&[tool_section_id], true).unwrap(),
-                "<|tool_calls_section_begin|>"
-            );
-
-            // This demonstrates that we're using Kimi's custom BPE pattern.
-            // With CL100K this will be 23 tokens instead.
-            assert_eq!(
-                contraction_heavy_ids,
-                vec![
-                    17172, 3287, 4643, 8201, 11, 996, 374, 8971, 3637, 20020, 8173, 473, 4643,
-                    1573, 56229, 13922, 13,
-                ]
-            );
-            assert_eq!(contraction_heavy_ids.len(), 17);
-            assert_eq!(
-                backend.decode(&contraction_heavy_ids, false).unwrap(),
-                contraction_heavy_text
-            );
-
-            // Special-looking text that is not actually registered should fail gracefully.
-            assert_eq!(backend.token_to_id("◁think▷"), None);
-            assert_eq!(backend.token_to_id("<|definitely_not_registered|>"), None);
-        }
-    }
 }

rust/src/text/Cargo.toml

@@ -10,7 +10,6 @@ asynk-strim-attr.workspace = true
 easy-ext.workspace = true
 enum-as-inner.workspace = true
 futures.workspace = true
-hf-hub.workspace = true
 itertools.workspace = true
 reqwest.workspace = true
 serde.workspace = true
@@ -22,6 +21,7 @@ tracing.workspace = true
 trait-set.workspace = true
 vllm-engine-core-client.workspace = true
 vllm-llm.workspace = true
+vllm-model-files.workspace = true
 vllm-tokenizer.workspace = true
 
 [dev-dependencies]

rust/src/text/src/backend/hf/config.rs

@@ -1,24 +1,18 @@
 use std::collections::BTreeSet;
-use std::fs;
 use std::path::Path;
 
 use serde::{Deserialize, Serialize};
-use thiserror_ext::AsReport as _;
+use vllm_model_files::read_json_file;
 
 use crate::error::{Error, Result};
 
 /// Minimal subset of `tokenizer_config.json` needed by chat/EOS handling.
 #[derive(Debug, Default, Deserialize)]
 #[serde(default)]
-pub struct HfTokenizerConfig {
+pub struct TokenizerConfig {
     #[serde(flatten)]
-    pub special_tokens: HfSpecialTokens,
+    pub special_tokens: SpecialTokens,
     pub chat_template: Option<String>,
-    /// The `tokenizer_class` field from HuggingFace tokenizer configs. Some
-    /// tiktoken-based models (e.g. DeepSeek, Kimi K2) set this to a value
-    /// containing "Tiktoken" which can be used as a hint for backend
-    /// selection.
-    pub tokenizer_class: Option<String>,
 }
 
 /// Hugging Face named special tokens may be serialized as a string or an
@@ -61,14 +55,14 @@ impl NamedSpecialToken {
 #[serde_with::skip_serializing_none]
 #[derive(Debug, Clone, Default, Deserialize, Serialize)]
 #[serde(default)]
-pub struct HfSpecialTokens {
+pub struct SpecialTokens {
     pub bos_token: Option<NamedSpecialToken>,
     pub eos_token: Option<NamedSpecialToken>,
     pub unk_token: Option<NamedSpecialToken>,
     pub pad_token: Option<NamedSpecialToken>,
 }
 
-impl HfSpecialTokens {
+impl SpecialTokens {
     /// Returns true if we don't discover any special tokens in the config.
     pub fn is_empty(&self) -> bool {
         self.bos_token.is_none()
@@ -234,42 +228,19 @@ impl ModelConfig {
     }
 }
 
-/// Load the tokenizer-side EOS metadata if a config file is present.
-pub fn load_tokenizer_config(path: Option<&Path>) -> Result<HfTokenizerConfig> {
-    read_json_file(path)
-}
-
 /// Load the generation-side EOS metadata if a config file is present.
 pub(super) fn load_generation_config(path: Option<&Path>) -> Result<GenerationConfig> {
-    read_json_file(path)
+    Ok(read_json_file(path)?)
+}
+
+/// Load the tokenizer-side EOS metadata if a config file is present.
+pub fn load_tokenizer_config(path: Option<&Path>) -> Result<TokenizerConfig> {
+    Ok(read_json_file(path)?)
 }
 
 /// Load the model-side config (`config.json`) if present.
 pub fn load_model_config(path: Option<&Path>) -> Result<ModelConfig> {
-    read_json_file(path)
-}
-
-fn read_json_file<T>(path: Option<&Path>) -> Result<T>
-where
-    T: for<'de> Deserialize<'de> + Default,
-{
-    let Some(path) = path else {
-        return Ok(T::default());
-    };
-    let content = fs::read_to_string(path).map_err(|error| {
-        Error::Tokenizer(format!(
-            "failed to read {}: {}",
-            path.display(),
-            error.as_report()
-        ))
-    })?;
-    serde_json::from_str(&content).map_err(|error| {
-        Error::Tokenizer(format!(
-            "failed to parse {}: {}",
-            path.display(),
-            error.as_report()
-        ))
-    })
+    Ok(read_json_file(path)?)
 }
 
 #[cfg(test)]

rust/src/text/src/backend/hf/mod.rs

@@ -1,5 +1,4 @@
 mod config;
-mod model_files;
 
 use std::collections::BTreeSet;
 use std::sync::Arc;
@@ -9,12 +8,12 @@ use vllm_tokenizer::{DynTokenizer, HuggingFaceTokenizer, TekkenTokenizer, Tiktok
 
 use self::config::{GenerationConfig, load_generation_config};
 pub use self::config::{
-    HfSpecialTokens, HfTokenizerConfig, ModelConfig, NamedSpecialToken, load_model_config,
+    ModelConfig, NamedSpecialToken, SpecialTokens, TokenizerConfig, load_model_config,
     load_tokenizer_config,
 };
-pub use self::model_files::{ResolvedModelFiles, TokenizerSource};
 use crate::backend::{SamplingHints, TextBackend};
 use crate::error::Result;
+pub use vllm_model_files::{ResolvedModelFiles, TokenizerSource};
 
 fn load_tokenizer(tokenizer: &TokenizerSource) -> Result<DynTokenizer> {
     match tokenizer {
@@ -125,3 +124,67 @@ impl TextBackend for HfTextBackend {
         })
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use vllm_tokenizer::{TiktokenTokenizer, Tokenizer};
+
+    use super::{ResolvedModelFiles, TokenizerSource};
+
+    #[tokio::test]
+    #[ignore = "too slow for CI and requires network access to Hugging Face"]
+    async fn tiktoken_real_kimi_k25_tokenizer_files_load_and_handle_special_tokens() {
+        let files = ResolvedModelFiles::new("moonshotai/Kimi-K2.5")
+            .await
+            .expect("resolve real Kimi K2.5 model files");
+
+        let tokenizer_path = match &files.tokenizer {
+            TokenizerSource::Tiktoken(path) => path.clone(),
+            other => panic!("expected tiktoken tokenizer source, got {other:?}"),
+        };
+
+        for backend in [
+            TiktokenTokenizer::new_riptoken(&tokenizer_path).expect("load riptoken backend"),
+            TiktokenTokenizer::new_tiktoken_rs(&tokenizer_path).expect("load tiktoken-rs backend"),
+        ] {
+            let think_id = backend.token_to_id("<think>").expect("resolve <think>");
+            let end_think_id = backend.token_to_id("</think>").expect("resolve </think>");
+            let tool_section_id = backend
+                .token_to_id("<|tool_calls_section_begin|>")
+                .expect("resolve tool call section marker");
+            let contraction_heavy_text =
+                "I'm sure it's fine, but I can't say I'd trust that it's what we'd ship.";
+            let contraction_heavy_ids = backend.encode(contraction_heavy_text, false).unwrap();
+
+            assert_eq!(
+                (think_id, end_think_id, tool_section_id),
+                (163606, 163607, 163595)
+            );
+            assert_eq!(backend.decode(&[think_id], true).unwrap(), "<think>");
+            assert_eq!(backend.decode(&[end_think_id], true).unwrap(), "</think>");
+            assert_eq!(
+                backend.decode(&[tool_section_id], true).unwrap(),
+                "<|tool_calls_section_begin|>"
+            );
+
+            // This demonstrates that we're using Kimi's custom BPE pattern.
+            // With CL100K this will be 23 tokens instead.
+            assert_eq!(
+                contraction_heavy_ids,
+                vec![
+                    17172, 3287, 4643, 8201, 11, 996, 374, 8971, 3637, 20020, 8173, 473, 4643,
+                    1573, 56229, 13922, 13,
+                ]
+            );
+            assert_eq!(contraction_heavy_ids.len(), 17);
+            assert_eq!(
+                backend.decode(&contraction_heavy_ids, false).unwrap(),
+                contraction_heavy_text
+            );
+
+            // Special-looking text that is not actually registered should fail gracefully.
+            assert_eq!(backend.token_to_id("◁think▷"), None);
+            assert_eq!(backend.token_to_id("<|definitely_not_registered|>"), None);
+        }
+    }
+}

rust/src/text/src/error.rs

@@ -1,6 +1,7 @@
 use thiserror::Error;
 use vllm_engine_core_client::Error as EngineCoreError;
 use vllm_llm::Error as LlmError;
+use vllm_model_files::Error as ModelFilesError;
 
 pub use crate::lower::logprobs::LogprobsError;
 pub use crate::lower::token_ids::TokenIdsError;
@@ -20,6 +21,8 @@ pub enum Error {
     Logprobs(#[from] LogprobsError),
     #[error(transparent)]
     TokenIds(#[from] TokenIdsError),
+    #[error(transparent)]
+    ModelFiles(#[from] ModelFilesError),
     #[error(
         "`min_tokens` must be less than or equal to `max_tokens`, \
          got min_tokens={min_tokens}, max_tokens={max_tokens}"