diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 7b6cb928e19..cc055505dc4 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -489,9 +489,9 @@ checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" [[package]] name = "bytes" -version = "1.11.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" dependencies = [ "serde", ] @@ -2112,6 +2112,16 @@ version = "0.2.183" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + [[package]] name = "libm" version = "0.2.16" @@ -2156,21 +2166,27 @@ checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" [[package]] name = "llm-multimodal" -version = "1.5.0" -source = "git+https://github.com/vllm-project/llm-multimodal?rev=046b669bd1c4faa2a7e05344d8cbf7b2befb37d5#046b669bd1c4faa2a7e05344d8cbf7b2befb37d5" +version = "1.7.1" +source = "git+https://github.com/smg-project/llm-multimodal?rev=7d74582aeaf0e4086a44964382655d22f1af0686#7d74582aeaf0e4086a44964382655d22f1af0686" dependencies = [ + "anyhow", "base64 0.22.1", "blake3", "bytes", "fast_image_resize", + "hf-hub", "image", + "libloading", "ndarray 0.17.2", "once_cell", "reqwest", "serde", "serde_json", + "serde_with", + "tempfile", "thiserror 2.0.18", "tokio", + "tracing", "url", ] @@ -2365,9 +2381,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.1.1" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" dependencies = [ "libc", "wasi", @@ -2532,9 +2548,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.21.3" +version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "once_cell_polyfill" @@ -4452,9 +4468,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.50.0" +version = "1.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ "bytes", "libc", @@ -4469,9 +4485,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.6.1" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 60ba138e8b5..344f10b3f28 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -31,7 +31,7 @@ axum = "0.8.8" base64 = "0.22.1" bytemuck = { version = "1.25.0", features = ["extern_crate_alloc"] } byteorder = "1.5.0" -bytes = "1.11.1" +bytes = "1.12.0" clap = { version = "4.5.38", features = ["derive", "env"] } criterion = "0.5.1" easy-ext = "1.0.3" @@ -53,7 +53,7 @@ hyper-util = { version = "0.1.20", features = [ indexmap = "2.13.0" itertools = "0.14.0" libc = "0.2.177" -llm-multimodal = { git = "https://github.com/vllm-project/llm-multimodal", rev = "046b669bd1c4faa2a7e05344d8cbf7b2befb37d5" } +llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "7d74582aeaf0e4086a44964382655d22f1af0686" } mimalloc = "0.1.52" minijinja = { version = "2.0", features = ["unstable_machinery", "json", "builtins", "loader", "loop_controls", "preserve_order"] } minijinja-contrib = { version = "2.0", features = ["pycompat"] } diff --git a/rust/src/chat/src/backend/hf.rs b/rust/src/chat/src/backend/hf.rs index 9997bc24736..fdfe8620b20 100644 --- a/rust/src/chat/src/backend/hf.rs +++ b/rust/src/chat/src/backend/hf.rs @@ -303,7 +303,7 @@ mod tests { .unwrap() .join("preprocessor_config.json"); write_json(&preprocessor_config_path, r#"{"size":[672,672]}"#); - files.preprocessor_config_path = Some(preprocessor_config_path); + files.preprocessor_config_path = Some(preprocessor_config_path.clone()); let backend = HfChatBackend::from_resolved_model_files( files.clone(), @@ -321,6 +321,9 @@ mod tests { assert!(backend.multimodal_model_info().is_none()); + let invalid_preprocessor_config = r#"{"size":[672,672]"#; + write_json(&preprocessor_config_path, invalid_preprocessor_config); + let error = HfChatBackend::from_resolved_model_files( files, "test-model".to_string(), diff --git a/rust/src/chat/src/multimodal.rs b/rust/src/chat/src/multimodal.rs index 9ec67cdd32e..7d950d581da 100644 --- a/rust/src/chat/src/multimodal.rs +++ b/rust/src/chat/src/multimodal.rs @@ -16,10 +16,11 @@ use std::sync::{Arc, LazyLock}; use itertools::izip; use llm_multimodal::{ - AsyncMultiModalTracker, FieldLayout, ImagePreProcessor, ImageProcessorRegistry, MediaConnector, - MediaConnectorConfig, MediaContentPart, Modality, ModelMetadata, ModelProcessorSpec, - ModelRegistry, PreProcessorConfig, PreprocessedImages, PromptReplacement, TokenResolver, - TrackedMedia, + AsyncMultiModalTracker, FieldLayout, MediaConnector, MediaConnectorConfig, MediaContentPart, + Modality, ModelMetadata, ModelProcessorSpec, ModelRegistry, PreProcessorConfig, + PreprocessedEncoderInputs as PreprocessedImages, PromptReplacement, Tokenizer as TokenResolver, + TrackedMedia, VisionPreProcessor as ImagePreProcessor, + VisionProcessorRegistry as ImageProcessorRegistry, }; use tracing::warn; use vllm_engine_core_client::protocol::dtype::ModelDtype; @@ -555,6 +556,10 @@ impl TokenResolver for TokenizerResolver { fn id_to_token(&self, id: u32) -> Option { self.0.id_to_token(id) } + + fn encode_text(&self, text: &str) -> Option> { + self.0.encode(text, false).ok() + } } #[cfg(test)] diff --git a/rust/src/chat/src/multimodal/tensor.rs b/rust/src/chat/src/multimodal/tensor.rs index b5a5f78f264..95259f1a93f 100644 --- a/rust/src/chat/src/multimodal/tensor.rs +++ b/rust/src/chat/src/multimodal/tensor.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use half::{bf16, f16}; -use llm_multimodal::{ModelSpecificValue, PreprocessedImages}; +use llm_multimodal::{ModelSpecificValue, PreprocessedEncoderInputs as PreprocessedImages}; use vllm_engine_core_client::protocol::dtype::ModelDtype; use vllm_engine_core_client::protocol::multimodal::MmKwargValue as ProtocolKwargValue; use vllm_engine_core_client::protocol::tensor::{ShapeExt as _, WireTensor}; @@ -31,14 +31,14 @@ pub(super) fn collect_tensors( float_dtype: ModelDtype, ) -> Result> { let PreprocessedImages { - pixel_values, + encoder_input, model_specific, .. } = preprocessed; let pixel_values = { - let shape = pixel_values.shape().to_vec(); - let data = pixel_values.into_iter().collect(); + let shape = encoder_input.shape().to_vec(); + let data = encoder_input.into_iter().collect(); KwargValue::from_f32_tensor(data, shape, float_dtype)? }; diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index 4590afef31e..02348151ea7 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -559,7 +559,7 @@ fn qwen_multimodal_model_info() -> vllm_chat::multimodal::MultimodalModelInfo { )); fs::write( &config_path, - r#"{"model_type":"qwen2_vl","vision_token_id":151655}"#, + r#"{"model_type":"qwen2_vl","image_token_id":151655}"#, ) .expect("write qwen test config"); let info = vllm_chat::multimodal::MultimodalModelInfo::from_paths(