diff --git a/rust/Cargo.lock b/rust/Cargo.lock index d155e07edad..2a7b365ab4f 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -182,7 +182,7 @@ dependencies = [ "hex", "hmac", "rand 0.9.2", - "reqwest", + "reqwest 0.12.28", "reqwest-eventsource", "secrecy", "serde", @@ -379,6 +379,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + [[package]] name = "bit-set" version = "0.5.3" @@ -665,6 +671,35 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" +[[package]] +name = "cookie" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747" +dependencies = [ + "percent-encoding", + "time", + "version_check", +] + +[[package]] +name = "cookie_store" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b2c103cf610ec6cae3da84a766285b42fd16aad564758459e6ecf128c75206" +dependencies = [ + "cookie", + "document-features", + "idna", + "indexmap 2.13.0", + "log", + "serde", + "serde_derive", + "serde_json", + "time", + "url", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -888,6 +923,16 @@ dependencies = [ "serde", ] +[[package]] +name = "der" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a69dedd701da44b0536442edf09c81a64b0ab97a7a4a5e3d1971f00027cbc63d" +dependencies = [ + "pem-rfc7468", + "zeroize", +] + [[package]] name = "deranged" version = "0.5.8" @@ -1171,6 +1216,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "extended" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365" + [[package]] name = "fancy-regex" version = "0.13.0" @@ -1589,13 +1640,15 @@ dependencies = [ "indicatif", "libc", "log", + "native-tls", "num_cpus", "rand 0.9.2", - "reqwest", + "reqwest 0.12.28", "serde", "serde_json", "thiserror 2.0.18", "tokio", + "ureq", "windows-sys 0.61.2", ] @@ -2167,22 +2220,27 @@ checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" [[package]] name = "llm-multimodal" version = "1.7.1" -source = "git+https://github.com/smg-project/llm-multimodal?rev=c8a29dcc755139fdc26185f400ea48c6d6d48273#c8a29dcc755139fdc26185f400ea48c6d6d48273" +source = "git+https://github.com/smg-project/llm-multimodal?rev=7df38e53f99aefaebe86934f010aa8084ec99b2f#7df38e53f99aefaebe86934f010aa8084ec99b2f" dependencies = [ "anyhow", "base64 0.22.1", "blake3", "bytes", + "cc", "fast_image_resize", "hf-hub", "image", "libloading", "ndarray 0.17.2", "once_cell", - "reqwest", + "pkg-config", + "rayon", + "reqwest 0.13.4", + "rustfft", "serde", "serde_json", "serde_with", + "symphonia", "tempfile", "thiserror 2.0.18", "tokio", @@ -2740,6 +2798,15 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "pem-rfc7468" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -3245,9 +3312,9 @@ checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" [[package]] name = "rayon" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" dependencies = [ "either", "rayon-core", @@ -3346,6 +3413,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + [[package]] name = "regex-syntax" version = "0.8.10" @@ -3392,7 +3465,43 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams", + "wasm-streams 0.4.2", + "web-sys", +] + +[[package]] +name = "reqwest" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-tls", + "hyper-util", + "js-sys", + "log", + "native-tls", + "percent-encoding", + "pin-project-lite", + "rustls-pki-types", + "sync_wrapper", + "tokio", + "tokio-native-tls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams 0.5.0", "web-sys", ] @@ -3408,7 +3517,7 @@ dependencies = [ "mime", "nom", "pin-project-lite", - "reqwest", + "reqwest 0.12.28", "thiserror 1.0.69", ] @@ -3526,7 +3635,9 @@ version = "0.23.37" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" dependencies = [ + "log", "once_cell", + "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -4052,6 +4163,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "socks" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b" +dependencies = [ + "byteorder", + "libc", + "winapi", +] + [[package]] name = "spm_precompiled" version = "0.1.4" @@ -4127,6 +4249,203 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "symphonia" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1758d6c853020a7244de03cc3e0185eaea3f58715122422dd3cc7452e6d4c16a" +dependencies = [ + "lazy_static", + "symphonia-bundle-flac", + "symphonia-bundle-mp3", + "symphonia-codec-aac", + "symphonia-codec-adpcm", + "symphonia-codec-alac", + "symphonia-codec-pcm", + "symphonia-codec-vorbis", + "symphonia-core", + "symphonia-format-caf", + "symphonia-format-isomp4", + "symphonia-format-mkv", + "symphonia-format-ogg", + "symphonia-format-riff", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-bundle-flac" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee69ad01236a67260b82fd1ff9790dd75ead29f2f46af145e63b7e72273e0e03" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-bundle-mp3" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "350f1f2f2e19ad4dd315db94304d1eb361b29af070681f94e51b8fdaad769546" +dependencies = [ + "lazy_static", + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-aac" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1979c515a76371b186aad2feff5f23e21cbec775bf95de08bf1e3af92a2ad76" +dependencies = [ + "lazy_static", + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-adpcm" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebbdfd76d6cc5a601c6292a44357c5b7c82f2cd7cdc0f171421f5c5cff0ea1f" +dependencies = [ + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-alac" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a149cbfc7fb5c405d123a273227d31de17138419552112bf1aa7b73e65827b8" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-pcm" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50baee168f0e9dcf6ba7fc06e8b57eb62072a4490cc7cf13af77e72baae5d328" +dependencies = [ + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-vorbis" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45b07b4423cd8e0fc472575909a5554b12c2f58e3c190b38c24f042e732fd8de" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-common" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8257891ffa7f05e02b58f4761e2abf7e5278c8744fd59e981559e050f86eef55" +dependencies = [ + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-core" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95ec293b5f288383b72a7bffcade6b2860b642cf66f28b3bd5967349a49938b1" +dependencies = [ + "bitflags", + "bytemuck", + "lazy_static", + "log", + "num-complex", + "smallvec", +] + +[[package]] +name = "symphonia-format-caf" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cde3ca76633d3400ab57195456c09f8a58d775ff5452329f3f212b6efc8622f5" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-format-isomp4" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d179a01305b3505940135a9f0180d6ef4b487912748fe97554756f120fbd05e" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-format-mkv" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb17713e134f5ad316c2690fa3104590ccc85842cdbcf82c3cd1a845cb08aa74" +dependencies = [ + "lazy_static", + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-format-ogg" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05a67e02b1e4fca1a261ba4fe06910a9357489ad8c36aafdd2960e9c6559433" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-format-riff" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17424452a777666d3eaf09a5c651029b15b6a333812fcc5b5474f2a3f0cff3f0" +dependencies = [ + "extended", + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-metadata" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31acf5cd623398a6208e2225d18f4b20f761c55098a796a5247ad516a4a8681" +dependencies = [ + "lazy_static", + "log", + "regex-lite", + "smallvec", + "symphonia-core", +] + [[package]] name = "syn" version = "1.0.109" @@ -4982,6 +5301,42 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0" +dependencies = [ + "base64 0.22.1", + "cookie_store", + "der", + "flate2", + "log", + "native-tls", + "percent-encoding", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "socks", + "ureq-proto", + "utf8-zero", + "webpki-root-certs", + "webpki-roots", +] + +[[package]] +name = "ureq-proto" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c" +dependencies = [ + "base64 0.22.1", + "http", + "httparse", + "log", +] + [[package]] name = "url" version = "2.5.8" @@ -5006,6 +5361,12 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -5097,7 +5458,7 @@ dependencies = [ "ndarray 0.17.2", "oss-harmony", "paste", - "reqwest", + "reqwest 0.13.4", "rmp-serde", "serde", "serde-json-fmt", @@ -5344,7 +5705,7 @@ dependencies = [ "futures", "hf-hub", "itertools 0.14.0", - "reqwest", + "reqwest 0.12.28", "serde", "serde_json", "serde_with", @@ -5368,7 +5729,7 @@ dependencies = [ "criterion", "fastokens", "hf-hub", - "reqwest", + "reqwest 0.12.28", "riptoken", "rustc-hash 1.1.0", "serde", @@ -5531,6 +5892,19 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasmparser" version = "0.244.0" @@ -5563,6 +5937,24 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "webpki-roots" +version = "1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "weezl" version = "0.1.12" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index db9764672f9..7c1793b448d 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -53,7 +53,7 @@ hyper-util = { version = "0.1.20", features = [ indexmap = "2.13.0" itertools = "0.14.0" libc = "0.2.177" -llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "c8a29dcc755139fdc26185f400ea48c6d6d48273" } +llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "7df38e53f99aefaebe86934f010aa8084ec99b2f", default-features = false, features = ["native-tls"] } mimalloc = "0.1.52" minijinja = { version = "2.0", features = ["unstable_machinery", "json", "builtins", "loader", "loop_controls", "preserve_order"] } minijinja-contrib = { version = "2.0", features = ["pycompat"] } @@ -73,6 +73,7 @@ pythonize = "0.28.0" rand = "0.9.2" reasoning-parser = "1.2.2" reqwest = { version = "0.12.8", default-features = false, features = ["native-tls"] } +reqwest-0-13 = { package = "reqwest", version = "0.13.4", default-features = false, features = ["native-tls"] } riptoken = { version = "0.3.0", default-features = false } rmp-serde = "1.3.1" rmpv = { version = "1.3.1", features = ["with-serde"] } diff --git a/rust/src/chat/Cargo.toml b/rust/src/chat/Cargo.toml index 00a6e223b77..7bc66dd5987 100644 --- a/rust/src/chat/Cargo.toml +++ b/rust/src/chat/Cargo.toml @@ -16,7 +16,7 @@ llm-multimodal.workspace = true minijinja.workspace = true minijinja-contrib.workspace = true openai-harmony.workspace = true -reqwest.workspace = true +reqwest-0-13.workspace = true serde.workspace = true serde-json-fmt.workspace = true serde_json.workspace = true diff --git a/rust/src/chat/src/backend/hf.rs b/rust/src/chat/src/backend/hf.rs index 5f3a656ae04..f938e1905cb 100644 --- a/rust/src/chat/src/backend/hf.rs +++ b/rust/src/chat/src/backend/hf.rs @@ -151,6 +151,7 @@ fn resolve_multimodal_render_info( info.map(|info| MultimodalRenderInfo { image_token: info.placeholder_token(Modality::Image).map(str::to_string), video_token: info.placeholder_token(Modality::Video).map(str::to_string), + audio_token: info.placeholder_token(Modality::Audio).map(str::to_string), }) } diff --git a/rust/src/chat/src/multimodal.rs b/rust/src/chat/src/multimodal.rs index 662e6fd0649..9c4f57c0413 100644 --- a/rust/src/chat/src/multimodal.rs +++ b/rust/src/chat/src/multimodal.rs @@ -5,24 +5,24 @@ //! //! This module owns the multimodal path for chat requests: it extracts media //! parts from structured chat messages, fetches and preprocesses them through -//! `llm-multimodal` one modality at a time, expands rendered prompt -//! placeholders after tokenization, and builds the engine-facing -//! `MmFeatures` payload. +//! `llm-multimodal`, expands rendered prompt placeholders after tokenization, +//! and builds the engine-facing `MmFeatures` payload. //! //! Raw media stays above `vllm-text`; this module lowers it into token IDs and //! opaque tensor payloads before the request is handed to text generation. -use std::collections::{HashMap, HashSet}; +use std::collections::HashSet; use std::fs; use std::path::Path; use std::sync::{Arc, LazyLock}; use itertools::izip; use llm_multimodal::{ - AsyncMultiModalTracker, FieldLayout, ImageFrame, MediaConnector, MediaConnectorConfig, - MediaContentPart, Modality, ModelMetadata, ModelProcessorSpec, ModelRegistry, - PreProcessorConfig, PreprocessedEncoderInputs, PromptReplacement, Tokenizer as TokenResolver, - TrackedMedia, VideoClip, VisionPreProcessor, VisionProcessorRegistry, + AsyncMultiModalTracker, AudioClip, AudioPreProcessor, EncoderFieldLayouts, FieldLayout, + ImageFrame, MediaConnector, MediaConnectorConfig, MediaContentPart, Modality, ModelMetadata, + ModelProcessorSpec, ModelRegistry, PreProcessorConfig, PreprocessedEncoderInputs, + PromptReplacement, Tokenizer as TokenResolver, TrackedMedia, VideoClip, VisionPreProcessor, + VisionProcessorRegistry, }; use thiserror_ext::AsReport as _; use tracing::warn; @@ -35,8 +35,10 @@ use crate::error::{Error, Result, bail_multimodal, multimodal}; use crate::renderer::RenderedPrompt; use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest}; +mod audio; mod expand; mod image; +mod item; mod tensor; mod video; @@ -46,9 +48,9 @@ use self::expand::expand_prompt_token_ids; #[derive(Clone)] pub struct MultimodalModelInfo { context: MultimodalModelContext, - spec: ResolvedMultimodalSpec, image: Option, video: Option, + audio: Option, media_connector: Arc, } @@ -88,22 +90,50 @@ impl MultimodalModelContext { LazyLock::new(VisionProcessorRegistry::with_defaults); REGISTRY.find(&self.model_id, self.model_type.as_deref()) } + + /// Resolve an audio preprocessor for one loaded model. + fn resolve_audio_processor( + &self, + model_spec: &'static dyn ModelProcessorSpec, + preprocessor_config: &PreProcessorConfig, + ) -> Option> { + model_spec.audio_processor(&self.config, preprocessor_config).map(Arc::from) + } } -/// Static model-specific tensor-layout behavior shared across modalities. +/// Static model-specific tensor-layout behavior for one modality. #[derive(Clone)] struct ResolvedMultimodalSpec { raw: &'static dyn ModelProcessorSpec, - field_layouts: HashMap, + modality: Modality, + field_layouts: EncoderFieldLayouts, keep_on_cpu_keys: HashSet, } impl ResolvedMultimodalSpec { - fn new(raw: &'static dyn ModelProcessorSpec) -> Self { + fn new(raw: &'static dyn ModelProcessorSpec, modality: Modality) -> Self { Self { raw, - field_layouts: raw.field_layouts(), - keep_on_cpu_keys: raw.keep_on_cpu_keys().into_iter().collect(), + modality, + field_layouts: raw.encoder_field_layouts_for(modality), + keep_on_cpu_keys: raw.keep_on_cpu_keys_for(modality).into_iter().collect(), + } + } + + fn primary_key(&self) -> &'static str { + match self.modality { + Modality::Image => image::IMAGE_PRIMARY_KEY, + Modality::Video => video::VIDEO_PRIMARY_KEY, + Modality::Audio => audio::AUDIO_PRIMARY_KEY, + Modality::ImageEmbeds => unreachable!("image embeds use no encoder preprocessor"), + } + } + + fn field_layout_for(&self, key: &str) -> Option<&FieldLayout> { + if key == self.primary_key() { + Some(&self.field_layouts.encoder_input) + } else { + self.field_layouts.model_specific.get(key) } } @@ -111,14 +141,15 @@ impl ResolvedMultimodalSpec { &self, context: &MultimodalModelContext, preprocessed: &PreprocessedEncoderInputs, - modality: Modality, ) -> Result> { - Ok(self.raw.prompt_replacements_for(&context.metadata(), preprocessed, modality)?) + Ok(self + .raw + .prompt_replacements_for(&context.metadata(), preprocessed, self.modality)?) } } /// Resolved placeholder tokens for one modality. -#[derive(Clone)] +#[derive(Debug, Clone)] struct ResolvedPlaceholder { token: String, /// The token ID emitted for `token` in the rendered prompt. @@ -152,19 +183,30 @@ impl ResolvedPlaceholder { } } -/// Static per-modality vision preprocessor plus its loaded config and -/// resolved placeholder tokens. +/// Static per-modality vision preprocessor plus its loaded config, resolved +/// placeholder tokens, and the model's shared tensor-layout spec. #[derive(Clone)] struct ModalitySupport { + spec: ResolvedMultimodalSpec, placeholder: ResolvedPlaceholder, processor: &'static dyn VisionPreProcessor, config: PreProcessorConfig, } +/// Static audio preprocessor plus the resolved model contract for its output. +#[derive(Clone)] +struct AudioModalitySupport { + spec: ResolvedMultimodalSpec, + placeholder: ResolvedPlaceholder, + processor: Arc, +} + /// Model-repo config file locations consumed by multimodal support. #[derive(Debug, Default, Clone, Copy)] pub struct MultimodalConfigFiles<'a> { pub config: Option<&'a Path>, + /// Shared preprocessor config (`preprocessor_config.json`), consumed by + /// image/audio processors and used as the video fallback. pub preprocessor_config: Option<&'a Path>, /// Video-specific preprocessor config (`video_preprocessor_config.json`). pub video_preprocessor_config: Option<&'a Path>, @@ -212,6 +254,8 @@ struct FetchedMedia { image_uuids: Vec>, videos: Vec>, video_uuids: Vec>, + audios: Vec>, + audio_uuids: Vec>, } /// One modality's preprocessed output, ready for the shared expansion and @@ -253,7 +297,7 @@ impl MultimodalModelInfo { } None => serde_json::Value::Object(Default::default()), }; - let image_preprocessor_config = load_preprocessor_config( + let preprocessor_config = load_preprocessor_config( files.preprocessor_config, "preprocessor_config.json", files.processor_config, @@ -266,7 +310,7 @@ impl MultimodalModelInfo { files.processor_config, "video_processor", )? - .unwrap_or_else(|| image_preprocessor_config.clone()); + .unwrap_or_else(|| preprocessor_config.clone()); let context = MultimodalModelContext { model_id, @@ -275,41 +319,76 @@ impl MultimodalModelInfo { tokenizer: TokenizerResolver(tokenizer), }; - Self::from_loaded( - context, - image_preprocessor_config, - video_preprocessor_config, - ) + Self::from_loaded(context, preprocessor_config, video_preprocessor_config) } /// Resolve multimodal support from an assembled context and parsed /// preprocessor configs. fn from_loaded( context: MultimodalModelContext, - image_preprocessor_config: PreProcessorConfig, + preprocessor_config: PreProcessorConfig, video_preprocessor_config: PreProcessorConfig, ) -> Result> { + let (image, video) = Self::resolve_vision_lanes( + &context, + preprocessor_config.clone(), + video_preprocessor_config, + ); + let audio = Self::resolve_audio_lane(&context, &preprocessor_config)?; + + if image.is_none() && video.is_none() && audio.is_none() { + warn!( + model_id = context.model_id, + model_type = context.model_type, + "no multimodal modality resolved; disabling multimodal support for this model" + ); + return Ok(None); + } + + let media_connector = Arc::new(MediaConnector::new( + reqwest_0_13::Client::new(), + MediaConnectorConfig::default(), + )?); + + Ok(Some(Self { + context, + image, + video, + audio, + media_connector, + })) + } + + /// Resolve the spec-backed vision lanes (image and video). + /// + /// A missing model spec or vision processor disables both lanes but is + /// not an error: unsupported model families are logged and skipped. + fn resolve_vision_lanes( + context: &MultimodalModelContext, + preprocessor_config: PreProcessorConfig, + video_preprocessor_config: PreProcessorConfig, + ) -> (Option, Option) { let Some(raw_spec) = context.resolve_model_spec() else { warn!( model_id = context.model_id, model_type = context.model_type, - "multimodal model spec is not registered; disabling multimodal support for this model" + "multimodal model spec is not registered; disabling image/video support for this model" ); - return Ok(None); + return (None, None); }; let Some(processor) = context.resolve_vision_processor() else { warn!( model_id = context.model_id, model_type = context.model_type, - "vision processor is not registered; disabling multimodal support for this model" + "vision processor is not registered; disabling image/video support for this model" ); - return Ok(None); + return (None, None); }; // Warn and disable the modality if the placeholder resolution fails. let resolve_placeholder = - |modality: Modality| match ResolvedPlaceholder::resolve(raw_spec, &context, modality) { + |modality: Modality| match ResolvedPlaceholder::resolve(raw_spec, context, modality) { Ok(placeholder) => Some(placeholder), Err(error) => { warn!( @@ -323,9 +402,10 @@ impl MultimodalModelInfo { }; let image = resolve_placeholder(Modality::Image).map(|placeholder| ModalitySupport { + spec: ResolvedMultimodalSpec::new(raw_spec, Modality::Image), placeholder, processor, - config: image_preprocessor_config, + config: preprocessor_config.clone(), }); let video = resolve_placeholder(Modality::Video).and_then(|placeholder| { @@ -341,6 +421,7 @@ impl MultimodalModelInfo { None } else { Some(ModalitySupport { + spec: ResolvedMultimodalSpec::new(raw_spec, Modality::Video), placeholder, processor, config: video_preprocessor_config, @@ -348,26 +429,33 @@ impl MultimodalModelInfo { } }); - if image.is_none() && video.is_none() { - warn!( - model_id = context.model_id, - model_type = context.model_type, - "no multimodal modality resolved; disabling multimodal support for this model" - ); + (image, video) + } + + /// Resolve a spec-backed audio lane when the model advertises audio support. + fn resolve_audio_lane( + context: &MultimodalModelContext, + preprocessor_config: &PreProcessorConfig, + ) -> Result> { + let Some(raw_spec) = context.resolve_model_spec() else { + return Ok(None); + }; + if !raw_spec.modality_limits(&context.metadata())?.contains_key(&Modality::Audio) { return Ok(None); } - let media_connector = Arc::new(MediaConnector::new( - reqwest::Client::new(), - MediaConnectorConfig::default(), - )?); - - Ok(Some(Self { - context, - spec: ResolvedMultimodalSpec::new(raw_spec), - image, - video, - media_connector, + let Some(processor) = context.resolve_audio_processor(raw_spec, preprocessor_config) else { + warn!( + model_id = context.model_id, + model_spec = raw_spec.name(), + "model spec does not provide an audio processor; disabling audio support for this model" + ); + return Ok(None); + }; + Ok(Some(AudioModalitySupport { + spec: ResolvedMultimodalSpec::new(raw_spec, Modality::Audio), + placeholder: ResolvedPlaceholder::resolve(raw_spec, context, Modality::Audio)?, + processor, })) } @@ -380,7 +468,8 @@ impl MultimodalModelInfo { match modality { Modality::Image => self.image.as_ref()?.placeholder.token.as_str().into(), Modality::Video => self.video.as_ref()?.placeholder.token.as_str().into(), - _ => None, + Modality::Audio => self.audio.as_ref()?.placeholder.token.as_str().into(), + Modality::ImageEmbeds => None, } } } @@ -449,12 +538,35 @@ fn extract_media_parts(request: &ChatRequest) -> Result> { uuid: uuid.clone(), }) } + ChatContentPart::InputAudio { data, format, uuid } => { + all_parts.push(MediaContentPart::AudioUrl { + url: input_audio_data_url(data, format.as_deref())?, + uuid: uuid.clone(), + }) + } + ChatContentPart::AudioUrl { audio_url, uuid } => { + all_parts.push(MediaContentPart::AudioUrl { + url: audio_url.clone(), + uuid: uuid.clone(), + }) + } } } } Ok(all_parts) } +/// Wrap OpenAI base64 audio in a data URL consumed by `MediaConnector`. +fn input_audio_data_url(data: &str, format: Option<&str>) -> Result { + let mime_type = match format { + Some(format) if format.eq_ignore_ascii_case("wav") => "audio/wav", + Some(format) if format.eq_ignore_ascii_case("mp3") => "audio/mpeg", + Some(format) => bail_multimodal!("unsupported input_audio format `{format}`"), + None => "application/octet-stream", + }; + Ok(format!("data:{mime_type};base64,{data}")) +} + impl MultimodalModelInfo { /// Run media fetch, per-modality preprocessing, prompt expansion, and /// feature build. @@ -468,11 +580,10 @@ impl MultimodalModelInfo { prompt_token_ids: &mut Vec, model_dtype: ModelDtype, ) -> Result { - if media_parts.is_empty() { + let media_parts_len = media_parts.len(); + if media_parts_len == 0 { return Ok(Vec::new()); } - let media_parts_len = media_parts.len(); - // TODO: enforce per-modality item-count limits, aligned with the // engine's `--limit-mm-per-prompt` semantics. let fetched = self.fetch_media(media_parts).await?; @@ -486,6 +597,9 @@ impl MultimodalModelInfo { prepared .push(self.prepare_videos(fetched.videos, fetched.video_uuids, model_dtype).await?); } + if !fetched.audios.is_empty() { + prepared.push(self.prepare_audios(fetched.audios, fetched.audio_uuids).await?); + } let mut ranges = expand_prompt_token_ids(prompt_token_ids, &prepared)?; @@ -524,8 +638,8 @@ impl MultimodalModelInfo { Ok(features) } - /// Fetch all media parts and split them per modality, preserving their - /// request-order UUID metadata. + /// Fetch all connector-backed media parts and split them per modality, + /// preserving their request-order UUID metadata. async fn fetch_media(&self, media_parts: Vec) -> Result { let mut tracker = AsyncMultiModalTracker::new(Arc::clone(&self.media_connector)); for part in media_parts { @@ -562,11 +676,27 @@ impl MultimodalModelInfo { .collect::>>()?; let video_uuids = tracker_output.uuids.remove(&Modality::Video).unwrap_or_default(); + let audios = tracker_output + .data + .remove(&Modality::Audio) + .unwrap_or_default() + .into_iter() + .map(|media| match media { + TrackedMedia::Audio(clip) => Ok(clip), + _ => Err(multimodal!( + "tracker returned non-audio media for the audio modality" + )), + }) + .collect::>>()?; + let audio_uuids = tracker_output.uuids.remove(&Modality::Audio).unwrap_or_default(); + Ok(FetchedMedia { images, image_uuids, videos, video_uuids, + audios, + audio_uuids, }) } } @@ -665,42 +795,6 @@ mod tests { test_info("qwen3_vl", config, qwen3_vl_tokenizer()) } - #[test] - fn from_paths_resolves_image_config_from_processor_config() { - let dir = tempfile::tempdir().unwrap(); - let config_path = dir.path().join("config.json"); - std::fs::write( - &config_path, - serde_json::json!({ - "model_type": "qwen3_vl", - "image_token_id": QWEN3_IMAGE_PAD_ID, - }) - .to_string(), - ) - .unwrap(); - let processor_config_path = dir.path().join("processor_config.json"); - std::fs::write( - &processor_config_path, - r#"{"image_processor":{"size":{"shortest_edge":64}}}"#, - ) - .unwrap(); - - let info = MultimodalModelInfo::from_paths( - "qwen3-vl-test".to_string(), - Some("qwen3_vl".to_string()), - MultimodalConfigFiles { - config: Some(&config_path), - processor_config: Some(&processor_config_path), - ..Default::default() - }, - Arc::new(qwen3_vl_tokenizer()), - ) - .unwrap() - .unwrap(); - - assert_eq!(info.image.unwrap().config.get_shortest_edge(), Some(64)); - } - #[test] fn qwen3_vl_resolves_image_and_video_support() { let info = qwen3_vl_info(); @@ -742,4 +836,21 @@ mod tests { assert_eq!(info.placeholder_token(Modality::Image), Some("<|image|>")); assert_eq!(info.placeholder_token(Modality::Video), None); } + + #[test] + fn input_audio_uses_connector_data_urls() { + assert_eq!( + input_audio_data_url("AAAA", Some("wav")).unwrap(), + "data:audio/wav;base64,AAAA" + ); + assert_eq!( + input_audio_data_url("BBBB", Some("MP3")).unwrap(), + "data:audio/mpeg;base64,BBBB" + ); + assert_eq!( + input_audio_data_url("CCCC", None).unwrap(), + "data:application/octet-stream;base64,CCCC" + ); + assert!(input_audio_data_url("AAAA", Some("flac")).is_err()); + } } diff --git a/rust/src/chat/src/multimodal/audio.rs b/rust/src/chat/src/multimodal/audio.rs new file mode 100644 index 00000000000..27596377545 --- /dev/null +++ b/rust/src/chat/src/multimodal/audio.rs @@ -0,0 +1,213 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +//! Audio-modality preparation through `llm-multimodal`. + +use std::sync::Arc; + +use llm_multimodal::{AudioClip, Modality, PreprocessedEncoderInputs}; +use vllm_engine_core_client::protocol::dtype::ModelDtype; + +use super::{AudioModalitySupport, MultimodalModelInfo, PreparedMedia, item}; +use crate::error::{Error, Result, bail_multimodal, multimodal}; + +/// Forward-kwargs name of the primary audio encoder input. +pub(super) const AUDIO_PRIMARY_KEY: &str = "input_audio_features"; + +impl MultimodalModelInfo { + /// Preprocess fetched audio clips as one batch and build per-item features. + pub(super) async fn prepare_audios( + &self, + clips: Vec>, + uuids: Vec>, + ) -> Result { + let support = self.audio.as_ref().ok_or_else(|| Error::UnsupportedModality { + modality: Modality::Audio.to_string(), + })?; + let preprocessed = self.preprocess_audios(support, &clips).await?; + let replacements = support.spec.prompt_replacements_for(&self.context, &preprocessed)?; + if replacements.len() != clips.len() { + bail_multimodal!( + "number of audio prompt replacements {} does not match number of audio clips {}", + replacements.len(), + clips.len() + ); + } + + let hashes = clips.iter().map(|clip| clip.hash.clone()).collect(); + let items = item::build_batched_items( + &support.spec, + preprocessed, + hashes, + uuids, + ModelDtype::Float32, + )?; + + Ok(PreparedMedia { + modality: Modality::Audio, + placeholder: support.placeholder.clone(), + replacements, + items, + }) + } + + /// Run CPU-heavy audio preprocessing in a blocking task. + async fn preprocess_audios( + &self, + support: &AudioModalitySupport, + clips: &[Arc], + ) -> Result { + let processor = Arc::clone(&support.processor); + let clips = clips.to_vec(); + tokio::task::spawn_blocking(move || Ok(processor.preprocess(&clips)?)) + .await + .map_err(|error| multimodal!("audio preprocessing task failed: {error}"))? + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::sync::Arc; + + use llm_multimodal::{MediaContentPart, ModelSpecificValue, PreProcessorConfig}; + use ndarray::ArrayD; + use vllm_engine_core_client::protocol::multimodal::{MmField, MmKwargValue}; + use vllm_tokenizer::test_utils::TestTokenizer; + + use super::super::{MultimodalModelContext, TokenizerResolver}; + use super::*; + + const AUDIO_PAD_ID: u32 = 151_676; + + fn qwen3_asr_info() -> MultimodalModelInfo { + let context = MultimodalModelContext { + model_id: "Qwen/Qwen3-ASR-1.7B".to_string(), + model_type: Some("qwen3_asr".to_string()), + config: serde_json::json!({"model_type": "qwen3_asr"}), + tokenizer: TokenizerResolver(Arc::new( + TestTokenizer::new().with_regular_token("<|audio_pad|>", AUDIO_PAD_ID), + )), + }; + + MultimodalModelInfo::from_loaded( + context, + PreProcessorConfig::default(), + PreProcessorConfig::default(), + ) + .unwrap() + .expect("Qwen3-ASR multimodal support") + } + + fn wav_i16_mono(sample_rate: u32, samples: &[i16]) -> Vec { + let data_bytes = samples.len() as u32 * 2; + let mut bytes = Vec::new(); + bytes.extend_from_slice(b"RIFF"); + bytes.extend_from_slice(&(36 + data_bytes).to_le_bytes()); + bytes.extend_from_slice(b"WAVEfmt "); + bytes.extend_from_slice(&16_u32.to_le_bytes()); + bytes.extend_from_slice(&1_u16.to_le_bytes()); + bytes.extend_from_slice(&1_u16.to_le_bytes()); + bytes.extend_from_slice(&sample_rate.to_le_bytes()); + bytes.extend_from_slice(&(sample_rate * 2).to_le_bytes()); + bytes.extend_from_slice(&2_u16.to_le_bytes()); + bytes.extend_from_slice(&16_u16.to_le_bytes()); + bytes.extend_from_slice(b"data"); + bytes.extend_from_slice(&data_bytes.to_le_bytes()); + for sample in samples { + bytes.extend_from_slice(&sample.to_le_bytes()); + } + bytes + } + + #[test] + fn resolves_qwen_audio_from_model_spec() { + let info = qwen3_asr_info(); + let support = info.audio.as_ref().expect("audio support"); + + assert_eq!( + info.placeholder_token(Modality::Audio), + Some("<|audio_pad|>") + ); + assert_eq!(support.placeholder.marker_token_id, AUDIO_PAD_ID); + assert_eq!(support.placeholder.embed_token_id, AUDIO_PAD_ID); + assert!(matches!( + support.spec.field_layouts.encoder_input, + llm_multimodal::FieldLayout::Batched + )); + + let preprocessed = PreprocessedEncoderInputs { + encoder_input: ArrayD::zeros(vec![1, 128, 4]), + feature_token_counts: vec![1], + item_sizes: vec![(128, 4)], + model_specific: HashMap::from([ + ( + "feature_attention_mask".to_string(), + ModelSpecificValue::int_2d(vec![1; 4], 1, 4), + ), + ( + "audio_feature_lengths".to_string(), + ModelSpecificValue::int_1d(vec![4]), + ), + ]), + }; + let item = item::build_batched_items( + &support.spec, + preprocessed, + vec!["".to_string()], + vec![None], + ModelDtype::Float32, + ) + .unwrap() + .pop() + .unwrap(); + + assert!(matches!( + &item.data[AUDIO_PRIMARY_KEY].field, + MmField::Batched(_) + )); + } + + #[tokio::test] + async fn tracker_processor_and_lowering_preserve_audio_contract() { + let info = qwen3_asr_info(); + let wav = wav_i16_mono(16_000, &[0; 1_600]); + let expected_hash = llm_multimodal::hasher::hash_audio(&wav); + let fetched = info + .fetch_media(vec![MediaContentPart::AudioData { + data: wav, + mime_type: Some("audio/wav".to_string()), + uuid: Some("audio-1".to_string()), + }]) + .await + .unwrap(); + + let prepared = info.prepare_audios(fetched.audios, fetched.audio_uuids).await.unwrap(); + + assert_eq!(prepared.replacements.len(), 1); + assert!( + prepared.replacements[0] + .tokens + .iter() + .all(|token| *token == AUDIO_PAD_ID as i32) + ); + let item = &prepared.items[0]; + assert_eq!(item.hash, expected_hash); + assert_eq!(item.uuid.as_deref(), Some("audio-1")); + + let features = &item.data[AUDIO_PRIMARY_KEY]; + assert!(matches!(&features.field, MmField::Batched(_))); + assert!(matches!( + features.data.as_ref(), + Some(MmKwargValue::Tensor(tensor)) + if tensor.dtype == "float32" && tensor.shape.first() == Some(&128) + )); + let lengths = &item.data["audio_feature_lengths"]; + assert!(matches!(&lengths.field, MmField::Batched(_))); + assert!(matches!( + lengths.data.as_ref(), + Some(MmKwargValue::Tensor(tensor)) + if tensor.dtype == "int64" && tensor.shape.is_empty() + )); + } +} diff --git a/rust/src/chat/src/multimodal/expand.rs b/rust/src/chat/src/multimodal/expand.rs index b679f13b079..fc4b47f4d20 100644 --- a/rust/src/chat/src/multimodal/expand.rs +++ b/rust/src/chat/src/multimodal/expand.rs @@ -418,6 +418,67 @@ mod tests { assert_bool_mask(&video_ranges[0], &[true, true, true, true]); } + const QWEN3_AUDIO_PAD_ID: u32 = 151_676; + + fn qwen3_audio_prepared(replacements: Vec) -> PreparedMedia { + prepared_media( + Modality::Audio, + "<|audio_pad|>", + QWEN3_AUDIO_PAD_ID, + QWEN3_AUDIO_PAD_ID, + replacements, + ) + } + + fn qwen3_audio_replacement(tokens: usize) -> PromptReplacement { + PromptReplacement::repeated( + Modality::Audio, + "<|audio_pad|>", + QWEN3_AUDIO_PAD_ID as TokenId, + tokens, + ) + } + + #[test] + fn expand_prompt_tokens_interleaves_audio_and_image_prepared_media() { + // Audio marker precedes the image marker: the image range offset must + // account for the earlier audio expansion growth. + let mut prompt_token_ids = vec![1, QWEN3_AUDIO_PAD_ID, 2, LLAMA4_IMAGE_ID, 3]; + let prepared = vec![ + llama4_prepared(vec![llama4_single_tile_replacement()]), + qwen3_audio_prepared(vec![qwen3_audio_replacement(3)]), + ]; + + let ranges = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap(); + + assert_eq!( + prompt_token_ids, + vec![ + 1, + QWEN3_AUDIO_PAD_ID, + QWEN3_AUDIO_PAD_ID, + QWEN3_AUDIO_PAD_ID, + 2, + LLAMA4_IMAGE_START_ID, + LLAMA4_IMAGE_ID, + LLAMA4_PATCH_ID, + LLAMA4_PATCH_ID, + LLAMA4_IMAGE_END_ID, + 3, + ] + ); + + let audio_ranges = &ranges[&Modality::Audio]; + assert_eq!(audio_ranges[0].offset, 1); + assert_eq!(audio_ranges[0].length, 3); + assert_bool_mask(&audio_ranges[0], &[true, true, true]); + + let image_ranges = &ranges[&Modality::Image]; + assert_eq!(image_ranges[0].offset, 5); + assert_eq!(image_ranges[0].length, 5); + assert_bool_mask(&image_ranges[0], &[false, false, true, true, false]); + } + #[test] fn expand_prompt_tokens_error_names_modality_with_leftover_replacements() { let mut prompt_token_ids = vec![1, QWEN3_IMAGE_PAD_ID, 2]; diff --git a/rust/src/chat/src/multimodal/image.rs b/rust/src/chat/src/multimodal/image.rs index 65f86041f73..5063ae8d2fa 100644 --- a/rust/src/chat/src/multimodal/image.rs +++ b/rust/src/chat/src/multimodal/image.rs @@ -6,17 +6,15 @@ use std::sync::Arc; -use itertools::izip; -use llm_multimodal::{FieldLayout, ImageFrame, Modality, PreprocessedEncoderInputs}; +use llm_multimodal::{ImageFrame, Modality, PreprocessedEncoderInputs}; use vllm_engine_core_client::protocol::dtype::ModelDtype; -use vllm_engine_core_client::protocol::multimodal::{ - MmBatchedField, MmField, MmFieldElem, MmFlatField, MmKwargsItem, MmSharedField, MmSlice, - SliceSpec, -}; -use super::{ModalitySupport, MultimodalModelInfo, PreparedItem, PreparedMedia, tensor}; +use super::{ModalitySupport, MultimodalModelInfo, PreparedMedia, item}; use crate::error::{Error, Result, bail_multimodal, multimodal}; +/// Forward-kwargs name of the primary image encoder input. +pub(super) const IMAGE_PRIMARY_KEY: &str = "pixel_values"; + impl MultimodalModelInfo { /// Preprocess all fetched image frames as one batch and build per-item /// features. @@ -30,9 +28,7 @@ impl MultimodalModelInfo { modality: Modality::Image.to_string(), })?; let preprocessed = self.preprocess_images(support, &frames).await?; - let replacements = - self.spec - .prompt_replacements_for(&self.context, &preprocessed, Modality::Image)?; + let replacements = support.spec.prompt_replacements_for(&self.context, &preprocessed)?; if replacements.len() != frames.len() { bail_multimodal!( "number of image prompt replacements {} does not match number of images {}", @@ -40,7 +36,9 @@ impl MultimodalModelInfo { frames.len() ); } - let items = self.build_image_items(preprocessed, &frames, uuids, model_dtype)?; + let hashes = frames.iter().map(|frame| frame.hash.clone()).collect(); + let items = + item::build_batched_items(&support.spec, preprocessed, hashes, uuids, model_dtype)?; Ok(PreparedMedia { modality: Modality::Image, @@ -70,75 +68,4 @@ impl MultimodalModelInfo { .await .map_err(|error| multimodal!("image preprocessing task failed: {error}"))? } - - /// Convert one batch of preprocessed image tensors into per-item engine - /// kwargs. - /// - /// Tensor fields are sliced per item according to the model spec's field - /// layout declarations. - fn build_image_items( - &self, - preprocessed: PreprocessedEncoderInputs, - frames: &[Arc], - uuids: Vec>, - model_dtype: ModelDtype, - ) -> Result> { - let len = frames.len(); - let tensors = tensor::collect_tensors(preprocessed, "pixel_values", model_dtype)?; - - let mut items = Vec::with_capacity(len); - for (index, (frame, uuid)) in izip!(frames, uuids).enumerate() { - let mut data = MmKwargsItem::new(); - for (key, tensor) in &tensors { - let keep_on_cpu = self.spec.keep_on_cpu_keys.contains(key); - let (value, field) = match self.spec.field_layouts.get(key) { - Some(FieldLayout::Batched) => ( - tensor.batched_value_at(index)?, - MmField::Batched(MmBatchedField { keep_on_cpu }), - ), - Some(FieldLayout::Flat { sizes_key }) => { - let sizes = tensors.get(sizes_key).ok_or_else(|| { - multimodal!("flat tensor sizes key `{sizes_key}` is missing") - })?; - let (start, end) = tensor::flat_range_for_index(sizes, sizes_key, index)?; - ( - tensor.flat_value_range(start, end)?, - MmField::Flat(MmFlatField { - slices: vec![MmSlice::Slice(SliceSpec { - start: Some(0), - stop: Some((end - start) as isize), - step: None, - })], - dim: 0, - keep_on_cpu, - }), - ) - } - None => ( - tensor.clone(), - MmField::Shared(MmSharedField { - batch_size: len, - keep_on_cpu, - }), - ), - }; - - data.insert( - key.clone(), - MmFieldElem { - data: Some(value.try_into()?), - field, - }, - ); - } - - items.push(PreparedItem { - data, - hash: frame.hash.clone(), - uuid, - }); - } - - Ok(items) - } } diff --git a/rust/src/chat/src/multimodal/item.rs b/rust/src/chat/src/multimodal/item.rs new file mode 100644 index 00000000000..d5600c56880 --- /dev/null +++ b/rust/src/chat/src/multimodal/item.rs @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +//! Shared lowering from batched preprocessor output to per-item engine kwargs. + +use itertools::izip; +use llm_multimodal::{FieldLayout, PreprocessedEncoderInputs}; +use vllm_engine_core_client::protocol::dtype::ModelDtype; +use vllm_engine_core_client::protocol::multimodal::{ + MmBatchedField, MmField, MmFieldElem, MmFlatField, MmKwargsItem, MmSharedField, MmSlice, + SliceSpec, +}; + +use super::{PreparedItem, ResolvedMultimodalSpec, tensor}; +use crate::error::{Result, bail_multimodal, multimodal}; + +/// Split one batch of preprocessed tensors into engine kwargs per media item. +pub(super) fn build_batched_items( + spec: &ResolvedMultimodalSpec, + preprocessed: PreprocessedEncoderInputs, + hashes: Vec, + uuids: Vec>, + float_dtype: ModelDtype, +) -> Result> { + let len = hashes.len(); + if uuids.len() != len { + bail_multimodal!( + "number of media UUIDs {} does not match number of media items {len}", + uuids.len() + ); + } + let tensors = tensor::collect_tensors(preprocessed, spec.primary_key(), float_dtype)?; + + let mut items = Vec::with_capacity(len); + for (index, (hash, uuid)) in izip!(hashes, uuids).enumerate() { + let mut data = MmKwargsItem::new(); + for (key, tensor) in &tensors { + let keep_on_cpu = spec.keep_on_cpu_keys.contains(key); + let (value, field) = match spec.field_layout_for(key) { + Some(FieldLayout::Batched) => ( + tensor.batched_value_at(index)?, + MmField::Batched(MmBatchedField { keep_on_cpu }), + ), + Some(FieldLayout::Flat { sizes_key }) => { + let sizes = tensors.get(sizes_key).ok_or_else(|| { + multimodal!("flat tensor sizes key `{sizes_key}` is missing") + })?; + let (start, end) = tensor::flat_range_for_index(sizes, sizes_key, index)?; + ( + tensor.flat_value_range(start, end)?, + MmField::Flat(MmFlatField { + slices: vec![MmSlice::Slice(SliceSpec { + start: Some(0), + stop: Some((end - start) as isize), + step: None, + })], + dim: 0, + keep_on_cpu, + }), + ) + } + None => ( + tensor.clone(), + MmField::Shared(MmSharedField { + batch_size: len, + keep_on_cpu, + }), + ), + }; + + data.insert( + key.clone(), + MmFieldElem { + data: Some(value.try_into()?), + field, + }, + ); + } + + items.push(PreparedItem { data, hash, uuid }); + } + + Ok(items) +} diff --git a/rust/src/chat/src/multimodal/video.rs b/rust/src/chat/src/multimodal/video.rs index af29799e785..9cc3238ac60 100644 --- a/rust/src/chat/src/multimodal/video.rs +++ b/rust/src/chat/src/multimodal/video.rs @@ -23,7 +23,7 @@ use crate::error::{Error, Result, bail_multimodal, multimodal}; /// /// Video-capable vLLM models read `pixel_values_videos` alongside /// `video_grid_thw`, mirroring the HF processor output naming. -const VIDEO_PRIMARY_KEY: &str = "pixel_values_videos"; +pub(super) const VIDEO_PRIMARY_KEY: &str = "pixel_values_videos"; impl MultimodalModelInfo { /// Preprocess fetched video clips one at a time and build per-item @@ -47,8 +47,7 @@ impl MultimodalModelInfo { for (clip, uuid) in izip!(&clips, uuids) { let preprocessed = self.preprocess_video_clip(support, Arc::clone(clip)).await?; let mut clip_replacements = - self.spec - .prompt_replacements_for(&self.context, &preprocessed, Modality::Video)?; + support.spec.prompt_replacements_for(&self.context, &preprocessed)?; if clip_replacements.len() != 1 { bail_multimodal!( "expected exactly one prompt replacement per video clip, got {}", @@ -56,7 +55,8 @@ impl MultimodalModelInfo { ); } replacements.push(clip_replacements.pop().unwrap()); - items.push(self.build_video_item( + items.push(build_video_item( + support, preprocessed, clip.hash.clone(), uuid, @@ -107,30 +107,36 @@ impl MultimodalModelInfo { .await .map_err(|error| multimodal!("video preprocessing task failed: {error}"))? } +} - /// Convert one preprocessed video clip into engine kwargs. - /// - /// The clip is a batch of one, so no per-item slicing is required: the - /// primary tensor ships as a full-range flat field (the engine re-batches - /// flat fields by concatenating along the declared dim, matching vLLM's - /// `flat_from_sizes` treatment of video patches), and batched metadata - /// tensors drop their singleton batch axis. - fn build_video_item( - &self, - preprocessed: PreprocessedEncoderInputs, - hash: String, - uuid: Option, - model_dtype: ModelDtype, - ) -> Result { - let tensors = tensor::collect_tensors(preprocessed, VIDEO_PRIMARY_KEY, model_dtype)?; +/// Convert one preprocessed video clip into engine kwargs. +/// +/// The clip is a batch of one, so no per-item slicing is required: the +/// primary tensor ships as a full-range flat field (the engine re-batches +/// flat fields by concatenating along the declared dim, matching vLLM's +/// `flat_from_sizes` treatment of video patches), and batched metadata +/// tensors drop their singleton batch axis. +fn build_video_item( + support: &ModalitySupport, + preprocessed: PreprocessedEncoderInputs, + hash: String, + uuid: Option, + model_dtype: ModelDtype, +) -> Result { + let tensors = tensor::collect_tensors(preprocessed, support.spec.primary_key(), model_dtype)?; - let mut data = MmKwargsItem::new(); - for (key, tensor) in tensors { - let keep_on_cpu = self.spec.keep_on_cpu_keys.contains(&key); - let (value, field) = if key == VIDEO_PRIMARY_KEY { + let mut data = MmKwargsItem::new(); + for (key, tensor) in tensors { + let keep_on_cpu = support.spec.keep_on_cpu_keys.contains(&key); + let (value, field) = match support.spec.field_layout_for(&key) { + Some(FieldLayout::Batched) => ( + tensor.batched_value_at(0)?, + MmField::Batched(MmBatchedField { keep_on_cpu }), + ), + Some(FieldLayout::Flat { .. }) => { let len = tensor .first_dim() - .ok_or_else(|| multimodal!("video encoder input `{key}` is not a tensor"))?; + .ok_or_else(|| multimodal!("flat video input `{key}` is not a tensor"))?; ( tensor, MmField::Flat(MmFlatField { @@ -143,35 +149,26 @@ impl MultimodalModelInfo { keep_on_cpu, }), ) - } else if matches!( - self.spec.field_layouts.get(&key), - Some(FieldLayout::Batched) - ) { - ( - tensor.batched_value_at(0)?, - MmField::Batched(MmBatchedField { keep_on_cpu }), - ) - } else { - ( - tensor, - MmField::Shared(MmSharedField { - batch_size: 1, - keep_on_cpu, - }), - ) - }; + } + None => ( + tensor, + MmField::Shared(MmSharedField { + batch_size: 1, + keep_on_cpu, + }), + ), + }; - data.insert( - key, - MmFieldElem { - data: Some(value.try_into()?), - field, - }, - ); - } - - Ok(PreparedItem { data, hash, uuid }) + data.insert( + key, + MmFieldElem { + data: Some(value.try_into()?), + field, + }, + ); } + + Ok(PreparedItem { data, hash, uuid }) } #[cfg(test)] @@ -287,14 +284,14 @@ mod tests { ]), }; - let item = info - .build_video_item( - preprocessed, - "".to_string(), - None, - ModelDtype::Float32, - ) - .unwrap(); + let item = build_video_item( + info.video.as_ref().unwrap(), + preprocessed, + "".to_string(), + None, + ModelDtype::Float32, + ) + .unwrap(); let primary = &item.data[VIDEO_PRIMARY_KEY]; assert!(matches!( diff --git a/rust/src/chat/src/renderer/hf/mod.rs b/rust/src/chat/src/renderer/hf/mod.rs index 58ee1ccb7aa..7ba2a2ccc03 100644 --- a/rust/src/chat/src/renderer/hf/mod.rs +++ b/rust/src/chat/src/renderer/hf/mod.rs @@ -42,6 +42,7 @@ pub use self::format::ChatTemplateContentFormatOption; pub struct MultimodalRenderInfo { pub image_token: Option, pub video_token: Option, + pub audio_token: Option, } /// Hugging Face chat-template renderer backed by the local Jinja chat-template @@ -263,6 +264,7 @@ enum TemplateContentPart { Text { text: String }, Image, Video, + Audio, } #[derive(Debug, Serialize)] @@ -437,6 +439,12 @@ fn to_template_openai_content( .ok_or(Error::UnsupportedMultimodalContent("video_url"))?; Ok(TemplateContentPart::Video) } + ChatContentPart::InputAudio { .. } | ChatContentPart::AudioUrl { .. } => { + multimodal + .and_then(|multimodal| multimodal.audio_token.as_ref()) + .ok_or(Error::UnsupportedMultimodalContent("audio"))?; + Ok(TemplateContentPart::Audio) + } }) .collect(), } @@ -465,6 +473,12 @@ fn to_template_string_content( .ok_or(Error::UnsupportedMultimodalContent("video_url"))?; out.push_str(video_token); } + ChatContentPart::InputAudio { .. } | ChatContentPart::AudioUrl { .. } => { + let audio_token = multimodal + .and_then(|multimodal| multimodal.audio_token.as_ref()) + .ok_or(Error::UnsupportedMultimodalContent("audio"))?; + out.push_str(audio_token); + } } } Ok(out) @@ -492,7 +506,9 @@ fn append_continue_final_message_tag(message: &mut TemplateMessage) -> Result parts.iter_mut().rev().find_map(|part| match part { TemplateContentPart::Text { text } => Some(text), - TemplateContentPart::Image | TemplateContentPart::Video => None, + TemplateContentPart::Image + | TemplateContentPart::Video + | TemplateContentPart::Audio => None, }), }; let text = text.ok_or_else(|| { @@ -603,6 +619,7 @@ mod tests { .with_multimodal(Some(MultimodalRenderInfo { image_token: Some("".to_string()), video_token: Some("