diff --git a/rust/Cargo.lock b/rust/Cargo.lock index cc055505dc4..d155e07edad 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -2167,7 +2167,7 @@ checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" [[package]] name = "llm-multimodal" version = "1.7.1" -source = "git+https://github.com/smg-project/llm-multimodal?rev=7d74582aeaf0e4086a44964382655d22f1af0686#7d74582aeaf0e4086a44964382655d22f1af0686" +source = "git+https://github.com/smg-project/llm-multimodal?rev=c8a29dcc755139fdc26185f400ea48c6d6d48273#c8a29dcc755139fdc26185f400ea48c6d6d48273" dependencies = [ "anyhow", "base64 0.22.1", @@ -2473,6 +2473,7 @@ dependencies = [ "portable-atomic", "portable-atomic-util", "rawpointer", + "serde", ] [[package]] @@ -5093,6 +5094,7 @@ dependencies = [ "llm-multimodal", "minijinja", "minijinja-contrib", + "ndarray 0.17.2", "oss-harmony", "paste", "reqwest", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 435350c0711..db9764672f9 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -53,12 +53,12 @@ hyper-util = { version = "0.1.20", features = [ indexmap = "2.13.0" itertools = "0.14.0" libc = "0.2.177" -llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "7d74582aeaf0e4086a44964382655d22f1af0686" } +llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "c8a29dcc755139fdc26185f400ea48c6d6d48273" } mimalloc = "0.1.52" minijinja = { version = "2.0", features = ["unstable_machinery", "json", "builtins", "loader", "loop_controls", "preserve_order"] } minijinja-contrib = { version = "2.0", features = ["pycompat"] } native-tls-vendored = { package = "native-tls", version = "0.2.18", features = ["vendored"] } -ndarray = { version = "0.16.1", features = ["serde"] } +ndarray = { version = "0.17", features = ["serde"] } openai-harmony = { package = "oss-harmony", git = "https://github.com/oss-harmony/harmony", tag = "v0.0.11", default-features = false } openai-protocol = "1.6.0" openssl = "0.10" diff --git a/rust/src/chat/Cargo.toml b/rust/src/chat/Cargo.toml index 95ce5ff2e42..00a6e223b77 100644 --- a/rust/src/chat/Cargo.toml +++ b/rust/src/chat/Cargo.toml @@ -42,6 +42,7 @@ anyhow.workspace = true bytes.workspace = true clap.workspace = true expect-test.workspace = true +ndarray.workspace = true paste.workspace = true rmp-serde.workspace = true serial_test.workspace = true diff --git a/rust/src/chat/src/backend/hf.rs b/rust/src/chat/src/backend/hf.rs index fdfe8620b20..47c0bfb78c8 100644 --- a/rust/src/chat/src/backend/hf.rs +++ b/rust/src/chat/src/backend/hf.rs @@ -10,7 +10,7 @@ use crate::backend::{ NewChatOutputProcessorOptions, }; use crate::error::Result; -use crate::multimodal::MultimodalModelInfo; +use crate::multimodal::{MultimodalConfigFiles, MultimodalModelInfo}; use crate::output::{ DefaultChatOutputProcessor, HarmonyChatOutputProcessor, validate_harmony_parser_overrides, }; @@ -46,8 +46,12 @@ impl HfChatBackend { MultimodalModelInfo::from_paths( model_id.clone(), (!model_type.is_empty()).then_some(model_type.to_string()), - files.config_path.as_deref(), - files.preprocessor_config_path.as_deref(), + MultimodalConfigFiles { + config: files.config_path.as_deref(), + preprocessor_config: files.preprocessor_config_path.as_deref(), + video_preprocessor_config: files.video_preprocessor_config_path.as_deref(), + processor_config: files.processor_config_path.as_deref(), + }, tokenizer.clone(), )? }; @@ -139,8 +143,11 @@ pub(super) async fn load_model_backends( fn resolve_multimodal_render_info( info: Option<&MultimodalModelInfo>, ) -> Option { + use llm_multimodal::Modality; + info.map(|info| MultimodalRenderInfo { - placeholder_token: info.placeholder_token().to_string(), + image_token: info.placeholder_token(Modality::Image).map(str::to_string), + video_token: info.placeholder_token(Modality::Video).map(str::to_string), }) } @@ -192,6 +199,8 @@ mod tests { tokenizer_config_path: Some(tokenizer_config_path), generation_config_path: None, preprocessor_config_path: None, + video_preprocessor_config_path: None, + processor_config_path: None, chat_template_path: None, config_path: Some(config_path), } diff --git a/rust/src/chat/src/error.rs b/rust/src/chat/src/error.rs index da2396c2198..8e5d4ef84f8 100644 --- a/rust/src/chat/src/error.rs +++ b/rust/src/chat/src/error.rs @@ -1,5 +1,5 @@ use thiserror::Error; -use thiserror_ext::Macro; +use thiserror_ext::{AsReport as _, Macro}; type BoxedError = Box; @@ -18,6 +18,8 @@ pub enum Error { UnsupportedMultimodalRenderer, #[error("unsupported multimodal content: {0}")] UnsupportedMultimodalContent(&'static str), + #[error("`{modality}` input is not supported by this model")] + UnsupportedModality { modality: String }, #[error("multimodal preprocessing error: {0}")] Multimodal(#[message] String), #[error("{kind} parsing is not available for model `{model_id}`")] @@ -80,11 +82,39 @@ impl Error { match self { Self::PromptTooLong { .. } => true, Self::Text(error) => error.is_request_validation_error(), + Self::UnsupportedMultimodalRenderer + | Self::UnsupportedMultimodalContent(_) + | Self::UnsupportedModality { .. } => true, + _ => false, } } } +impl From for Error { + fn from(error: llm_multimodal::MediaConnectorError) -> Self { + Self::Multimodal(error.to_report_string()) + } +} + +impl From for Error { + fn from(error: llm_multimodal::MultiModalError) -> Self { + Self::Multimodal(error.to_report_string()) + } +} + +impl From for Error { + fn from(error: llm_multimodal::TransformError) -> Self { + Self::Multimodal(error.to_report_string()) + } +} + +impl From for Error { + fn from(error: llm_multimodal::registry::ModelRegistryError) -> Self { + Self::Multimodal(error.to_report_string()) + } +} + /// Format the available-parser suffix used in user-facing error messages. fn available_parser_hint(available_names: &[String]) -> String { if available_names.is_empty() { diff --git a/rust/src/chat/src/multimodal.rs b/rust/src/chat/src/multimodal.rs index 7d950d581da..422696ae710 100644 --- a/rust/src/chat/src/multimodal.rs +++ b/rust/src/chat/src/multimodal.rs @@ -1,8 +1,8 @@ -//! Chat-layer multimodal image preparation. +//! Chat-layer multimodal media preparation. //! -//! This module owns the narrow image-only multimodal path for chat requests: -//! it extracts image parts from structured chat messages, fetches and -//! preprocesses them through `llm-multimodal`, expands rendered prompt +//! This module owns the multimodal path for chat requests: it extracts media +//! parts from structured chat messages, fetches and preprocesses them through +//! `llm-multimodal` one modality at a time, expands rendered prompt //! placeholders after tokenization, and builds the engine-facing //! `MmFeatures` payload. //! @@ -16,19 +16,15 @@ use std::sync::{Arc, LazyLock}; use itertools::izip; use llm_multimodal::{ - AsyncMultiModalTracker, FieldLayout, MediaConnector, MediaConnectorConfig, MediaContentPart, - Modality, ModelMetadata, ModelProcessorSpec, ModelRegistry, PreProcessorConfig, - PreprocessedEncoderInputs as PreprocessedImages, PromptReplacement, Tokenizer as TokenResolver, - TrackedMedia, VisionPreProcessor as ImagePreProcessor, - VisionProcessorRegistry as ImageProcessorRegistry, + AsyncMultiModalTracker, FieldLayout, ImageFrame, MediaConnector, MediaConnectorConfig, + MediaContentPart, Modality, ModelMetadata, ModelProcessorSpec, ModelRegistry, + PreProcessorConfig, PreprocessedEncoderInputs, PromptReplacement, Tokenizer as TokenResolver, + TrackedMedia, VideoClip, VisionPreProcessor, VisionProcessorRegistry, }; +use thiserror_ext::AsReport as _; use tracing::warn; use vllm_engine_core_client::protocol::dtype::ModelDtype; -use vllm_engine_core_client::protocol::multimodal::{ - MmBatchedField, MmFeatureSpec, MmFeatures, MmField, MmFieldElem, MmFlatField, MmKwargsItem, - MmSharedField, MmSlice, PlaceholderRange, SliceSpec, -}; -use vllm_engine_core_client::protocol::tensor::WireTensor; +use vllm_engine_core_client::protocol::multimodal::{MmFeatureSpec, MmFeatures, MmKwargsItem}; use vllm_text::Prompt; use vllm_text::tokenizer::{DynTokenizer, Tokenizer}; @@ -36,14 +32,20 @@ use crate::error::{Error, Result, bail_multimodal, multimodal}; use crate::renderer::RenderedPrompt; use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest}; +mod expand; +mod image; mod tensor; +mod video; + +use self::expand::expand_prompt_token_ids; /// Resolved multimodal support for one loaded model. #[derive(Clone)] pub struct MultimodalModelInfo { context: MultimodalModelContext, spec: ResolvedMultimodalSpec, - image_processor: ResolvedImageProcessor, + image: Option, + video: Option, media_connector: Arc, } @@ -75,91 +77,171 @@ impl MultimodalModelContext { REGISTRY.lookup(&self.metadata()) } - /// Resolve a static image preprocessor for one loaded model. - fn resolve_image_processor(&self) -> Option<&'static dyn ImagePreProcessor> { - static REGISTRY: LazyLock = - LazyLock::new(ImageProcessorRegistry::with_defaults); + /// Resolve a static vision preprocessor for one loaded model. + /// + /// The vision preprocessor serves both the image and video modalities. + fn resolve_vision_processor(&self) -> Option<&'static dyn VisionPreProcessor> { + static REGISTRY: LazyLock = + LazyLock::new(VisionProcessorRegistry::with_defaults); REGISTRY.find(&self.model_id, self.model_type.as_deref()) } } -/// Static model-specific prompt and tensor-layout behavior. +/// Static model-specific tensor-layout behavior shared across modalities. #[derive(Clone)] struct ResolvedMultimodalSpec { raw: &'static dyn ModelProcessorSpec, - placeholder_token: String, - placeholder_marker_token_id: u32, - placeholder_embed_token_id: u32, field_layouts: HashMap, keep_on_cpu_keys: HashSet, } impl ResolvedMultimodalSpec { - fn new(raw: &'static dyn ModelProcessorSpec, context: &MultimodalModelContext) -> Result { - let metadata = context.metadata(); - let placeholder_token = - raw.placeholder_token(&metadata).map_err(|error| multimodal!("{error}"))?; - // This is the rendered prompt marker, so resolve it from the token - // string itself. Do not use `ModelProcessorSpec::placeholder_token_id()`: - // for some specs that ID is the replacement vision/patch token, - // not necessarily the token ID of `placeholder_token`. - let placeholder_marker_token_id = - context.tokenizer().token_to_id(&placeholder_token).ok_or_else(|| { - multimodal!( - "placeholder token `{placeholder_token}` is not in the tokenizer vocabulary" - ) - })?; - let placeholder_embed_token_id = - raw.placeholder_token_id(&metadata).map_err(|error| multimodal!("{error}"))? as u32; - - Ok(Self { + fn new(raw: &'static dyn ModelProcessorSpec) -> Self { + Self { raw, - placeholder_token, - placeholder_marker_token_id, - placeholder_embed_token_id, field_layouts: raw.field_layouts(), keep_on_cpu_keys: raw.keep_on_cpu_keys().into_iter().collect(), - }) + } } - fn prompt_replacements( + fn prompt_replacements_for( &self, context: &MultimodalModelContext, - preprocessed: &PreprocessedImages, + preprocessed: &PreprocessedEncoderInputs, + modality: Modality, ) -> Result> { - self.raw - .prompt_replacements(&context.metadata(), preprocessed) - .map_err(|error| multimodal!("{error}")) + Ok(self.raw.prompt_replacements_for(&context.metadata(), preprocessed, modality)?) } } -/// Static image preprocessor plus its loaded config. +/// Resolved placeholder tokens for one modality. #[derive(Clone)] -struct ResolvedImageProcessor { - raw: &'static dyn ImagePreProcessor, +struct ResolvedPlaceholder { + token: String, + /// The token ID emitted for `token` in the rendered prompt. + marker_token_id: u32, + /// The model-declared embed token ID marked in `is_embed` masks. + embed_token_id: u32, +} + +impl ResolvedPlaceholder { + fn resolve( + raw: &'static dyn ModelProcessorSpec, + context: &MultimodalModelContext, + modality: Modality, + ) -> Result { + let metadata = context.metadata(); + let token = raw.placeholder_token_for(&metadata, modality)?; + // This is the rendered prompt marker, so resolve it from the token + // string itself. Do not use `ModelProcessorSpec::placeholder_token_id_for()`: + // for some specs that ID is the replacement vision/patch token, + // not necessarily the token ID of the placeholder token. + let marker_token_id = context.tokenizer().token_to_id(&token).ok_or_else(|| { + multimodal!("placeholder token `{token}` is not in the tokenizer vocabulary") + })?; + let embed_token_id = raw.placeholder_token_id_for(&metadata, modality)? as u32; + + Ok(Self { + token, + marker_token_id, + embed_token_id, + }) + } +} + +/// Static per-modality vision preprocessor plus its loaded config and +/// resolved placeholder tokens. +#[derive(Clone)] +struct ModalitySupport { + placeholder: ResolvedPlaceholder, + processor: &'static dyn VisionPreProcessor, config: PreProcessorConfig, } -/// Request-scoped fetched media, kept together with tracker UUID metadata. -struct FetchedImageMedia { - frames: Vec>, - uuids: Vec>, +/// Model-repo config file locations consumed by multimodal support. +#[derive(Debug, Default, Clone, Copy)] +pub struct MultimodalConfigFiles<'a> { + pub config: Option<&'a Path>, + pub preprocessor_config: Option<&'a Path>, + /// Video-specific preprocessor config (`video_preprocessor_config.json`). + pub video_preprocessor_config: Option<&'a Path>, + /// Combined processor config (`processor_config.json`), whose modality + /// sections are fallback preprocessor config sources. + pub processor_config: Option<&'a Path>, +} + +/// Load a modality's dedicated preprocessor config, falling back to its section +/// in the combined processor config. +fn load_preprocessor_config( + dedicated_path: Option<&Path>, + dedicated_name: &str, + processor_config_path: Option<&Path>, + processor_section: &str, +) -> Result> { + if let Some(path) = dedicated_path { + let text = fs::read_to_string(path) + .map_err(|error| multimodal!("failed to read {dedicated_name}: {error}"))?; + let config = PreProcessorConfig::from_json(&text) + .map_err(|error| multimodal!("failed to parse {dedicated_name}: {error}"))?; + return Ok(Some(config)); + } + + let Some(path) = processor_config_path else { + return Ok(None); + }; + let text = fs::read_to_string(path) + .map_err(|error| multimodal!("failed to read processor_config.json: {error}"))?; + let value: serde_json::Value = serde_json::from_str(&text) + .map_err(|error| multimodal!("failed to parse processor_config.json: {error}"))?; + let Some(processor) = value.get(processor_section) else { + return Ok(None); + }; + let config = PreProcessorConfig::from_value(processor.clone()).map_err(|error| { + multimodal!("failed to parse {processor_section} from processor_config.json: {error}") + })?; + Ok(Some(config)) +} + +/// Request-scoped fetched media, split per modality with tracker UUID +/// metadata preserved in request order. +struct FetchedMedia { + images: Vec>, + image_uuids: Vec>, + videos: Vec>, + video_uuids: Vec>, +} + +/// One modality's preprocessed output, ready for the shared expansion and +/// feature-assembly tail. +struct PreparedMedia { + modality: Modality, + placeholder: ResolvedPlaceholder, + /// One replacement per media item, in request order. + replacements: Vec, + /// One entry per media item, aligned with `replacements`. + items: Vec, +} + +/// One media item's complete engine kwargs plus identity metadata. +struct PreparedItem { + data: MmKwargsItem, + hash: String, + uuid: Option, } impl MultimodalModelInfo { /// Load and resolve multimodal support from model files. /// - /// Returns `Ok(Some(_))` only when both the model spec and image processor - /// are registered. File read/parse failures are real errors; unsupported - /// model families are logged and returned as `Ok(None)`. + /// Returns `Ok(Some(_))` only when the model spec is registered and at + /// least one modality resolves. File read/parse failures are real errors; + /// unsupported model families are logged and returned as `Ok(None)`. pub fn from_paths( model_id: String, model_type: Option, - config_path: Option<&Path>, - preprocessor_config_path: Option<&Path>, + files: MultimodalConfigFiles<'_>, tokenizer: DynTokenizer, ) -> Result> { - let config = match config_path { + let config = match files.config { Some(path) => { let text = fs::read_to_string(path) .map_err(|error| multimodal!("failed to read config.json: {error}"))?; @@ -168,17 +250,20 @@ impl MultimodalModelInfo { } None => serde_json::Value::Object(Default::default()), }; - let preprocessor_config = match preprocessor_config_path { - Some(path) => { - let text = fs::read_to_string(path).map_err(|error| { - multimodal!("failed to read preprocessor_config.json: {error}") - })?; - PreProcessorConfig::from_json(&text).map_err(|error| { - multimodal!("failed to parse preprocessor_config.json: {error}") - })? - } - None => PreProcessorConfig::default(), - }; + let image_preprocessor_config = load_preprocessor_config( + files.preprocessor_config, + "preprocessor_config.json", + files.processor_config, + "image_processor", + )? + .unwrap_or_default(); + let video_preprocessor_config = load_preprocessor_config( + files.video_preprocessor_config, + "video_preprocessor_config.json", + files.processor_config, + "video_processor", + )? + .unwrap_or_else(|| image_preprocessor_config.clone()); let context = MultimodalModelContext { model_id, @@ -187,7 +272,21 @@ impl MultimodalModelInfo { tokenizer: TokenizerResolver(tokenizer), }; - let Some(spec) = context.resolve_model_spec() else { + Self::from_loaded( + context, + image_preprocessor_config, + video_preprocessor_config, + ) + } + + /// Resolve multimodal support from an assembled context and parsed + /// preprocessor configs. + fn from_loaded( + context: MultimodalModelContext, + image_preprocessor_config: PreProcessorConfig, + video_preprocessor_config: PreProcessorConfig, + ) -> Result> { + let Some(raw_spec) = context.resolve_model_spec() else { warn!( model_id = context.model_id, model_type = context.model_type, @@ -195,47 +294,99 @@ impl MultimodalModelInfo { ); return Ok(None); }; - let spec = ResolvedMultimodalSpec::new(spec, &context)?; - let Some(image_processor) = context.resolve_image_processor() else { + let Some(processor) = context.resolve_vision_processor() else { warn!( model_id = context.model_id, model_type = context.model_type, - "image processor is not registered; disabling multimodal support for this model" + "vision processor is not registered; disabling multimodal support for this model" ); return Ok(None); }; - let media_connector = Arc::new( - MediaConnector::new(reqwest::Client::new(), MediaConnectorConfig::default()) - .map_err(|error| multimodal!("{error}"))?, - ); + // Warn and disable the modality if the placeholder resolution fails. + let resolve_placeholder = + |modality: Modality| match ResolvedPlaceholder::resolve(raw_spec, &context, modality) { + Ok(placeholder) => Some(placeholder), + Err(error) => { + warn!( + model_id = context.model_id, + %modality, + error = %error.as_report(), + "placeholder tokens did not resolve; disabling this modality for this model" + ); + None + } + }; + + let image = resolve_placeholder(Modality::Image).map(|placeholder| ModalitySupport { + placeholder, + processor, + config: image_preprocessor_config, + }); + + let video = resolve_placeholder(Modality::Video).and_then(|placeholder| { + // Placeholder expansion attributes markers to modalities by token + // ID, so a marker shared with the image modality is ambiguous. + let image_marker = image.as_ref().map(|image| image.placeholder.marker_token_id); + if image_marker == Some(placeholder.marker_token_id) { + warn!( + model_id = context.model_id, + token = placeholder.token, + "video placeholder token collides with the image placeholder; disabling video support for this model" + ); + None + } else { + Some(ModalitySupport { + placeholder, + processor, + config: video_preprocessor_config, + }) + } + }); + + if image.is_none() && video.is_none() { + warn!( + model_id = context.model_id, + model_type = context.model_type, + "no multimodal modality resolved; disabling multimodal support for this model" + ); + return Ok(None); + } + + let media_connector = Arc::new(MediaConnector::new( + reqwest::Client::new(), + MediaConnectorConfig::default(), + )?); Ok(Some(Self { context, - spec, - image_processor: ResolvedImageProcessor { - raw: image_processor, - config: preprocessor_config, - }, + spec: ResolvedMultimodalSpec::new(raw_spec), + image, + video, media_connector, })) } - /// Return the template-visible placeholder token for this model. + /// Return the template-visible placeholder token for one modality, when + /// this model supports it. /// - /// The HF renderer uses this token while flattening image content in string - /// content format. - pub fn placeholder_token(&self) -> &str { - &self.spec.placeholder_token + /// The HF renderer uses these tokens while flattening media content in + /// string content format. + pub fn placeholder_token(&self, modality: Modality) -> Option<&str> { + match modality { + Modality::Image => self.image.as_ref()?.placeholder.token.as_str().into(), + Modality::Video => self.video.as_ref()?.placeholder.token.as_str().into(), + _ => None, + } } } /// Finalize a rendered chat prompt into text-generation input. /// /// Text-only requests pass through unchanged as `Prompt::Text`. Multimodal -/// requests are tokenized in chat, their image placeholders are expanded, and -/// preprocessed image features are attached for engine-core transport. +/// requests are tokenized in chat, their media placeholders are expanded, and +/// preprocessed media features are attached for engine-core transport. pub(crate) async fn finalize_rendered_prompt( request: &ChatRequest, rendered: RenderedPrompt, @@ -260,7 +411,7 @@ pub(crate) async fn finalize_rendered_prompt( Ok((Prompt::TokenIds(prompt_token_ids), Some(prepared))) } -/// Extract image media parts from chat messages in message/content order. +/// Extract media parts from chat messages in message/content order. /// /// Assistant history is skipped because generated assistant blocks are already /// represented as text for prompt rendering in this crate. @@ -289,6 +440,12 @@ fn extract_media_parts(request: &ChatRequest) -> Result> { detail: *detail, uuid: uuid.clone(), }), + ChatContentPart::VideoUrl { video_url, uuid } => { + all_parts.push(MediaContentPart::VideoUrl { + url: video_url.clone(), + uuid: uuid.clone(), + }) + } } } } @@ -296,8 +453,8 @@ fn extract_media_parts(request: &ChatRequest) -> Result> { } impl MultimodalModelInfo { - /// Run media fetch, image preprocessing, prompt expansion, and feature - /// build. + /// Run media fetch, per-modality preprocessing, prompt expansion, and + /// feature build. /// /// `prompt_token_ids` is mutated in place because placeholder expansion /// changes both the final prompt and the offsets recorded in @@ -313,12 +470,47 @@ impl MultimodalModelInfo { } let media_parts_len = media_parts.len(); - let fetched = self.fetch_images(media_parts).await?; - let preprocessed = self.preprocess_images(&fetched.frames).await?; - let replacements = self.spec.prompt_replacements(&self.context, &preprocessed)?; - let ranges = self.expand_prompt_tokens(prompt_token_ids, replacements)?; + // TODO: enforce per-modality item-count limits, aligned with the + // engine's `--limit-mm-per-prompt` semantics. + let fetched = self.fetch_media(media_parts).await?; + + let mut prepared = Vec::new(); + if !fetched.images.is_empty() { + prepared + .push(self.prepare_images(fetched.images, fetched.image_uuids, model_dtype).await?); + } + if !fetched.videos.is_empty() { + prepared + .push(self.prepare_videos(fetched.videos, fetched.video_uuids, model_dtype).await?); + } + + let mut ranges = expand_prompt_token_ids(prompt_token_ids, &prepared)?; + + let mut features = Vec::with_capacity(media_parts_len); + for media in prepared { + let media_ranges = ranges.remove(&media.modality).unwrap_or_default(); + if media_ranges.len() != media.items.len() { + bail_multimodal!( + "number of expanded `{}` placeholders {} does not match number of media items {}", + media.modality, + media_ranges.len(), + media.items.len() + ); + } + for (item, range) in izip!(media.items, media_ranges) { + features.push(MmFeatureSpec { + data: Some(item.data), + modality: media.modality.to_string(), + identifier: item.uuid.unwrap_or_else(|| item.hash.clone()), + mm_position: range, + mm_hash: Some(item.hash), + }); + } + } + // Mirror the Python frontend (`argsort_mm_positions`): features are + // ordered by their placeholder position in the prompt. + features.sort_by_key(|feature| feature.mm_position.offset); - let features = self.build_features(preprocessed, fetched, ranges, model_dtype)?; if features.len() != media_parts_len { bail_multimodal!( "number of built multimodal features {} does not match number of media parts {}", @@ -329,219 +521,51 @@ impl MultimodalModelInfo { Ok(features) } - /// Fetch all image parts and preserve their request-order UUID metadata. - async fn fetch_images(&self, media_parts: Vec) -> Result { + /// Fetch all media parts and split them per modality, preserving their + /// request-order UUID metadata. + async fn fetch_media(&self, media_parts: Vec) -> Result { let mut tracker = AsyncMultiModalTracker::new(Arc::clone(&self.media_connector)); for part in media_parts { - tracker.push_part(part).map_err(|error| multimodal!("{error}"))?; + tracker.push_part(part)?; } - let tracker_output = tracker.finalize().await.map_err(|error| multimodal!("{error}"))?; - let images = tracker_output.data.get(&Modality::Image).cloned().unwrap_or_default(); - let uuids = tracker_output.uuids.get(&Modality::Image).cloned().unwrap_or_default(); + let mut tracker_output = tracker.finalize().await?; - let frames = images + let images = tracker_output + .data + .remove(&Modality::Image) + .unwrap_or_default() .into_iter() .map(|media| match media { TrackedMedia::Image(frame) => Ok(frame), - _ => Err(Error::UnsupportedMultimodalContent("non-image")), + _ => Err(multimodal!( + "tracker returned non-image media for the image modality" + )), }) .collect::>>()?; + let image_uuids = tracker_output.uuids.remove(&Modality::Image).unwrap_or_default(); - Ok(FetchedImageMedia { frames, uuids }) - } + let videos = tracker_output + .data + .remove(&Modality::Video) + .unwrap_or_default() + .into_iter() + .map(|media| match media { + TrackedMedia::Video(clip) => Ok(clip), + _ => Err(multimodal!( + "tracker returned non-video media for the video modality" + )), + }) + .collect::>>()?; + let video_uuids = tracker_output.uuids.remove(&Modality::Video).unwrap_or_default(); - /// Preprocess fetched image frames with the model's resolved image - /// processor. - /// - /// The processor work is CPU-heavy relative to request wiring, so it runs - /// in a blocking task and returns owned tensors ready for wire - /// conversion. - async fn preprocess_images( - &self, - image_frames: &[Arc], - ) -> Result { - let config = self.image_processor.config.clone(); - let processor = self.image_processor.raw; - let images = image_frames.iter().map(|frame| frame.data().clone()).collect::>(); - - // TODO: is it still necessary given that we've already in a dedicated runtime? - tokio::task::spawn_blocking(move || { - processor.preprocess(&images, &config).map_err(|error| multimodal!("{error}")) + Ok(FetchedMedia { + images, + image_uuids, + videos, + video_uuids, }) - .await - .map_err(|error| multimodal!("image preprocessing task failed: {error}"))? } - - /// Replace rendered placeholder markers with model-specific replacement - /// tokens. - /// - /// Replacements are consumed in order, matching the original media-part - /// order. The returned ranges point into the already-expanded prompt. - fn expand_prompt_tokens( - &self, - prompt_token_ids: &mut Vec, - replacements: Vec, - ) -> Result> { - expand_prompt_token_ids( - prompt_token_ids, - replacements, - self.spec.placeholder_marker_token_id, - self.spec.placeholder_embed_token_id, - &self.spec.placeholder_token, - ) - } - - /// Convert preprocessed image tensors into engine-core multimodal features. - /// - /// One `MmFeatureSpec` is produced per image. Tensor fields are - /// sliced according to the model spec's field layout declarations. - fn build_features( - &self, - preprocessed: PreprocessedImages, - images: FetchedImageMedia, - ranges: Vec, - model_dtype: ModelDtype, - ) -> Result { - let len = images.frames.len(); - let tensors = tensor::collect_tensors(preprocessed, model_dtype)?; - - let mut features = Vec::with_capacity(images.frames.len()); - for (index, (frame, uuid, range)) in izip!(images.frames, images.uuids, ranges).enumerate() - { - let mut data = MmKwargsItem::new(); - for (key, tensor) in &tensors { - let keep_on_cpu = self.spec.keep_on_cpu_keys.contains(key); - let (value, field) = match self.spec.field_layouts.get(key) { - Some(FieldLayout::Batched) => ( - tensor.batched_value_at(index)?, - MmField::Batched(MmBatchedField { keep_on_cpu }), - ), - Some(FieldLayout::Flat { sizes_key }) => { - let sizes = tensors.get(sizes_key).ok_or_else(|| { - multimodal!("flat tensor sizes key `{sizes_key}` is missing") - })?; - let (start, end) = tensor::flat_range_for_index(sizes, sizes_key, index)?; - ( - tensor.flat_value_range(start, end)?, - MmField::Flat(MmFlatField { - slices: vec![MmSlice::Slice(SliceSpec { - start: Some(0), - stop: Some((end - start) as isize), - step: None, - })], - dim: 0, - keep_on_cpu, - }), - ) - } - None => ( - tensor.clone(), - MmField::Shared(MmSharedField { - batch_size: len, - keep_on_cpu, - }), - ), - }; - - data.insert( - key.clone(), - MmFieldElem { - data: Some(value.try_into()?), - field, - }, - ); - } - - let hash = frame.hash.clone(); - features.push(MmFeatureSpec { - data: Some(data), - modality: "image".to_string(), - identifier: uuid.unwrap_or_else(|| hash.clone()), - mm_position: range, - mm_hash: Some(hash), - }); - } - - Ok(features) - } -} - -fn expand_prompt_token_ids( - prompt_token_ids: &mut Vec, - replacements: Vec, - placeholder_marker_token_id: u32, - placeholder_embed_token_id: u32, - placeholder_token: &str, -) -> Result> { - if replacements.is_empty() { - return Ok(Vec::new()); - } - - let replacement_growth = replacements.iter().fold(0usize, |total, replacement| { - total.saturating_add(replacement.tokens.len().saturating_sub(1)) - }); - let mut expanded = - Vec::with_capacity(prompt_token_ids.len().saturating_add(replacement_growth)); - let mut ranges = Vec::with_capacity(replacements.len()); - let mut cursor = 0usize; - - for replacement in replacements { - if replacement.modality != Modality::Image { - bail_multimodal!( - "unsupported prompt replacement modality `{}`", - replacement.modality - ); - } - - let offset = find_next_token(prompt_token_ids, placeholder_marker_token_id, cursor) - .ok_or_else(|| { - multimodal!( - "placeholder token `{placeholder_token}` was not found in tokenized prompt" - ) - })?; - - if replacement.tokens.is_empty() { - bail_multimodal!("placeholder token `{placeholder_token}` expanded to no tokens"); - } - - let replacement_len = replacement.tokens.len(); - let is_embed = { - let mask = replacement - .tokens - .iter() - .map(|&token| token as u32 == placeholder_embed_token_id) - .collect::>(); - WireTensor::from_bool(vec![replacement_len], mask).map_err(Error::Multimodal)? - }; - - expanded.extend_from_slice(&prompt_token_ids[cursor..offset]); - let expanded_offset = expanded.len(); - expanded.extend(replacement.tokens.into_iter().map(|token| token as u32)); - ranges.push(PlaceholderRange { - offset: expanded_offset, - length: replacement_len, - is_embed: Some(is_embed), - }); - cursor = offset + 1; - } - - expanded.extend_from_slice(&prompt_token_ids[cursor..]); - *prompt_token_ids = expanded; - - Ok(ranges) -} - -/// Find `needle` in `haystack`, starting at `start`. -/// -/// This is intentionally order-preserving rather than a global replace: each -/// image consumes the next placeholder occurrence. -fn find_next_token(haystack: &[u32], needle: u32, start: usize) -> Option { - haystack - .get(start..)? - .iter() - .position(|token| *token == needle) - .map(|offset| start + offset) } /// Adapter from the frontend tokenizer trait to `llm-multimodal`. @@ -566,18 +590,19 @@ impl TokenResolver for TokenizerResolver { mod tests { use std::sync::Arc; - use llm_multimodal::TokenId; - use vllm_engine_core_client::protocol::tensor::WireArrayData; use vllm_tokenizer::test_utils::TestTokenizer; use super::*; - const LLAMA4_IMAGE_START_ID: u32 = 200088; - const LLAMA4_IMAGE_END_ID: u32 = 200089; - const LLAMA4_IMAGE_ID: u32 = 200090; - const LLAMA4_PATCH_ID: u32 = 200092; - const LLAMA4_TILE_X_SEPARATOR_ID: u32 = 200093; - const LLAMA4_TILE_Y_SEPARATOR_ID: u32 = 200094; + pub(super) const LLAMA4_IMAGE_START_ID: u32 = 200088; + pub(super) const LLAMA4_IMAGE_END_ID: u32 = 200089; + pub(super) const LLAMA4_IMAGE_ID: u32 = 200090; + pub(super) const LLAMA4_PATCH_ID: u32 = 200092; + pub(super) const LLAMA4_TILE_X_SEPARATOR_ID: u32 = 200093; + pub(super) const LLAMA4_TILE_Y_SEPARATOR_ID: u32 = 200094; + + pub(super) const QWEN3_IMAGE_PAD_ID: u32 = 151655; + pub(super) const QWEN3_VIDEO_PAD_ID: u32 = 151656; fn llama4_tokenizer() -> TestTokenizer { TestTokenizer::new() @@ -589,33 +614,31 @@ mod tests { .with_regular_token("<|tile_y_separator|>", LLAMA4_TILE_Y_SEPARATOR_ID) } - fn test_info(model_type: &str, config: serde_json::Value) -> MultimodalModelInfo { + pub(super) fn qwen3_vl_tokenizer() -> TestTokenizer { + TestTokenizer::new() + .with_regular_token("<|image_pad|>", QWEN3_IMAGE_PAD_ID) + .with_regular_token("<|video_pad|>", QWEN3_VIDEO_PAD_ID) + } + + fn test_info( + model_type: &str, + config: serde_json::Value, + tokenizer: TestTokenizer, + ) -> MultimodalModelInfo { let context = MultimodalModelContext { model_id: format!("{model_type}-test"), model_type: Some(model_type.to_string()), config, - tokenizer: TokenizerResolver(Arc::new(llama4_tokenizer())), + tokenizer: TokenizerResolver(Arc::new(tokenizer)), }; - let spec = context - .resolve_model_spec() - .unwrap_or_else(|| panic!("{model_type} spec should match")); - let spec = ResolvedMultimodalSpec::new(spec, &context).unwrap(); - let raw_image_processor = context - .resolve_image_processor() - .unwrap_or_else(|| panic!("{model_type} image processor should match")); - let media_connector = Arc::new( - MediaConnector::new(reqwest::Client::new(), MediaConnectorConfig::default()).unwrap(), - ); - MultimodalModelInfo { + MultimodalModelInfo::from_loaded( context, - spec, - image_processor: ResolvedImageProcessor { - raw: raw_image_processor, - config: PreProcessorConfig::default(), - }, - media_connector, - } + PreProcessorConfig::default(), + PreProcessorConfig::default(), + ) + .unwrap() + .unwrap_or_else(|| panic!("{model_type} multimodal support should resolve")) } fn llama4_info() -> MultimodalModelInfo { @@ -624,173 +647,96 @@ mod tests { "image_token_index": LLAMA4_PATCH_ID, "vision_config": {"image_size": 336, "patch_size": 14} }); - test_info("llama4", config) + test_info("llama4", config, llama4_tokenizer()) } - fn llama4_single_tile_replacement() -> PromptReplacement { - PromptReplacement::sequence( - Modality::Image, - "<|image|>", - vec![ - LLAMA4_IMAGE_START_ID as TokenId, - LLAMA4_IMAGE_ID as TokenId, - LLAMA4_PATCH_ID as TokenId, - LLAMA4_PATCH_ID as TokenId, - LLAMA4_IMAGE_END_ID as TokenId, - ], + pub(super) fn qwen3_vl_info() -> MultimodalModelInfo { + let config = serde_json::json!({ + "model_type": "qwen3_vl", + "image_token_id": QWEN3_IMAGE_PAD_ID, + "video_token_id": QWEN3_VIDEO_PAD_ID, + "vision_start_token_id": 151652, + "vision_end_token_id": 151653, + "vision_config": {"patch_size": 16} + }); + test_info("qwen3_vl", config, qwen3_vl_tokenizer()) + } + + #[test] + fn from_paths_resolves_image_config_from_processor_config() { + let dir = tempfile::tempdir().unwrap(); + let config_path = dir.path().join("config.json"); + std::fs::write( + &config_path, + serde_json::json!({ + "model_type": "qwen3_vl", + "image_token_id": QWEN3_IMAGE_PAD_ID, + }) + .to_string(), ) - } - - fn llama4_multi_tile_replacement() -> PromptReplacement { - PromptReplacement::sequence( - Modality::Image, - "<|image|>", - vec![ - LLAMA4_IMAGE_START_ID as TokenId, - LLAMA4_PATCH_ID as TokenId, - LLAMA4_TILE_X_SEPARATOR_ID as TokenId, - LLAMA4_PATCH_ID as TokenId, - LLAMA4_TILE_Y_SEPARATOR_ID as TokenId, - LLAMA4_IMAGE_ID as TokenId, - LLAMA4_PATCH_ID as TokenId, - LLAMA4_IMAGE_END_ID as TokenId, - ], + .unwrap(); + let processor_config_path = dir.path().join("processor_config.json"); + std::fs::write( + &processor_config_path, + r#"{"image_processor":{"size":{"shortest_edge":64}}}"#, ) - } + .unwrap(); - fn assert_bool_mask(range: &PlaceholderRange, expected: &[bool]) { - let tensor = range.is_embed.as_ref().expect("is_embed mask"); - assert_eq!(tensor.dtype, "bool"); - assert_eq!(tensor.shape, vec![expected.len()]); - assert_eq!( - tensor.data, - WireArrayData::RawView(expected.iter().map(|value| u8::from(*value)).collect()) - ); + let info = MultimodalModelInfo::from_paths( + "qwen3-vl-test".to_string(), + Some("qwen3_vl".to_string()), + MultimodalConfigFiles { + config: Some(&config_path), + processor_config: Some(&processor_config_path), + ..Default::default() + }, + Arc::new(qwen3_vl_tokenizer()), + ) + .unwrap() + .unwrap(); + + assert_eq!(info.image.unwrap().config.get_shortest_edge(), Some(64)); } #[test] - fn expand_prompt_tokens_marks_only_llama4_patch_tokens_as_embed() { - let info = llama4_info(); - let mut prompt_token_ids = vec![1, LLAMA4_IMAGE_ID, 2]; - let replacements = vec![llama4_multi_tile_replacement()]; - - let ranges = info.expand_prompt_tokens(&mut prompt_token_ids, replacements).unwrap(); + fn qwen3_vl_resolves_image_and_video_support() { + let info = qwen3_vl_info(); assert_eq!( - prompt_token_ids, - vec![ - 1, - LLAMA4_IMAGE_START_ID, - LLAMA4_PATCH_ID, - LLAMA4_TILE_X_SEPARATOR_ID, - LLAMA4_PATCH_ID, - LLAMA4_TILE_Y_SEPARATOR_ID, - LLAMA4_IMAGE_ID, - LLAMA4_PATCH_ID, - LLAMA4_IMAGE_END_ID, - 2, - ] + info.placeholder_token(Modality::Image), + Some("<|image_pad|>") ); - assert_eq!(ranges[0].offset, 1); - assert_eq!(ranges[0].length, 8); - assert_bool_mask( - &ranges[0], - &[false, true, false, true, false, false, true, false], + assert_eq!( + info.placeholder_token(Modality::Video), + Some("<|video_pad|>") + ); + assert_ne!( + info.image.as_ref().unwrap().placeholder.marker_token_id, + info.video.as_ref().unwrap().placeholder.marker_token_id, ); } #[test] - fn expand_prompt_tokens_errors_when_placeholder_missing() { - let info = llama4_info(); - let mut prompt_token_ids = vec![1, 2, 3]; - let replacements = vec![llama4_single_tile_replacement()]; - - let error = info.expand_prompt_tokens(&mut prompt_token_ids, replacements).unwrap_err(); - - assert!(matches!(error, Error::Multimodal(message) if message.contains("not found"))); - } - - #[test] - fn expand_prompt_tokens_ignores_empty_replacements() { - let info = llama4_info(); - let mut prompt_token_ids = vec![1, LLAMA4_IMAGE_ID, 2]; - let original_prompt_token_ids = prompt_token_ids.clone(); - - let ranges = info.expand_prompt_tokens(&mut prompt_token_ids, Vec::new()).unwrap(); - - assert!(ranges.is_empty()); - assert_eq!(prompt_token_ids, original_prompt_token_ids); - } - - #[test] - fn expand_prompt_tokens_leaves_prompt_unchanged_when_later_placeholder_missing() { - let info = llama4_info(); - let mut prompt_token_ids = vec![1, LLAMA4_IMAGE_ID, 2]; - let original_prompt_token_ids = prompt_token_ids.clone(); - let replacements = vec![ - llama4_single_tile_replacement(), - llama4_single_tile_replacement(), - ]; - - let error = info.expand_prompt_tokens(&mut prompt_token_ids, replacements).unwrap_err(); - - assert!(matches!(error, Error::Multimodal(message) if message.contains("not found"))); - assert_eq!(prompt_token_ids, original_prompt_token_ids); - } - - #[test] - fn expand_prompt_tokens_errors_when_replacement_is_empty() { - let info = llama4_info(); - let mut prompt_token_ids = vec![1, LLAMA4_IMAGE_ID, 2]; - let original_prompt_token_ids = prompt_token_ids.clone(); - let replacements = vec![PromptReplacement::sequence( - Modality::Image, - "<|image|>", - Vec::new(), - )]; - - let error = info.expand_prompt_tokens(&mut prompt_token_ids, replacements).unwrap_err(); - - assert!( - matches!(error, Error::Multimodal(message) if message.contains("expanded to no tokens")) - ); - assert_eq!(prompt_token_ids, original_prompt_token_ids); - } - - #[test] - fn expand_prompt_tokens_skips_llama4_image_marker_inside_replacement() { - let info = llama4_info(); - let mut prompt_token_ids = vec![1, LLAMA4_IMAGE_ID, 2, LLAMA4_IMAGE_ID, 3]; - let replacements = vec![ - llama4_single_tile_replacement(), - llama4_single_tile_replacement(), - ]; - - let ranges = info.expand_prompt_tokens(&mut prompt_token_ids, replacements).unwrap(); + fn qwen3_vl_without_video_token_id_disables_video_support_only() { + let config = serde_json::json!({ + "model_type": "qwen3_vl", + "image_token_id": QWEN3_IMAGE_PAD_ID, + "vision_config": {"patch_size": 16} + }); + let info = test_info("qwen3_vl", config, qwen3_vl_tokenizer()); assert_eq!( - prompt_token_ids, - vec![ - 1, - LLAMA4_IMAGE_START_ID, - LLAMA4_IMAGE_ID, - LLAMA4_PATCH_ID, - LLAMA4_PATCH_ID, - LLAMA4_IMAGE_END_ID, - 2, - LLAMA4_IMAGE_START_ID, - LLAMA4_IMAGE_ID, - LLAMA4_PATCH_ID, - LLAMA4_PATCH_ID, - LLAMA4_IMAGE_END_ID, - 3, - ] + info.placeholder_token(Modality::Image), + Some("<|image_pad|>") ); - assert_eq!(ranges[0].offset, 1); - assert_eq!(ranges[0].length, 5); - assert_bool_mask(&ranges[0], &[false, false, true, true, false]); - assert_eq!(ranges[1].offset, 7); - assert_eq!(ranges[1].length, 5); - assert_bool_mask(&ranges[1], &[false, false, true, true, false]); + assert_eq!(info.placeholder_token(Modality::Video), None); + } + + #[test] + fn llama4_resolves_image_support_only() { + let info = llama4_info(); + + assert_eq!(info.placeholder_token(Modality::Image), Some("<|image|>")); + assert_eq!(info.placeholder_token(Modality::Video), None); } } diff --git a/rust/src/chat/src/multimodal/expand.rs b/rust/src/chat/src/multimodal/expand.rs new file mode 100644 index 00000000000..36a2a9bdb77 --- /dev/null +++ b/rust/src/chat/src/multimodal/expand.rs @@ -0,0 +1,446 @@ +//! Prompt placeholder expansion shared across modalities. + +use std::collections::{HashMap, VecDeque}; + +use llm_multimodal::{Modality, PromptReplacement}; +use vllm_engine_core_client::protocol::multimodal::PlaceholderRange; +use vllm_engine_core_client::protocol::tensor::WireTensor; + +use super::PreparedMedia; +use crate::error::{Error, Result, bail_multimodal}; + +/// One modality's queue of pending placeholder replacements for prompt +/// expansion. +struct ExpansionLane<'a> { + modality: Modality, + marker_token_id: u32, + embed_token_id: u32, + placeholder_token: String, + replacements: VecDeque<&'a PromptReplacement>, +} + +impl<'a> ExpansionLane<'a> { + fn from_prepared(media: &'a PreparedMedia) -> Option { + if media.replacements.is_empty() { + return None; + } + + Some(Self { + modality: media.modality, + marker_token_id: media.placeholder.marker_token_id, + embed_token_id: media.placeholder.embed_token_id, + placeholder_token: media.placeholder.token.clone(), + replacements: media.replacements.iter().collect(), + }) + } +} + +/// Replace rendered placeholder markers with model-specific replacement +/// tokens across all modalities in one left-to-right pass. +/// +/// Each prepared modality consumes its own marker occurrences in order, +/// matching the original media-part order within that modality; markers of +/// different modalities may interleave freely. +/// +/// The returned ranges point into the already-expanded prompt, grouped per +/// modality in item order. +pub(super) fn expand_prompt_token_ids( + prompt_token_ids: &mut Vec, + prepared: &[PreparedMedia], +) -> Result>> { + let mut lanes = prepared.iter().filter_map(ExpansionLane::from_prepared).collect::>(); + if lanes.is_empty() { + return Ok(HashMap::new()); + } + + let replacement_growth = lanes + .iter() + .flat_map(|lane| lane.replacements.iter()) + .fold(0usize, |total, replacement| { + total.saturating_add(replacement.tokens.len().saturating_sub(1)) + }); + let expanded_len = prompt_token_ids.len().saturating_add(replacement_growth); + + let mut expanded = Vec::with_capacity(expanded_len); + let mut ranges = HashMap::>::new(); + + for &token in prompt_token_ids.iter() { + let lane = lanes + .iter_mut() + .find(|lane| lane.marker_token_id == token && !lane.replacements.is_empty()); + let Some(lane) = lane else { + expanded.push(token); + continue; + }; + + let replacement = lane.replacements.pop_front().expect("lane queue is non-empty"); + debug_assert_eq!(replacement.modality, lane.modality); + if replacement.tokens.is_empty() { + bail_multimodal!( + "placeholder token `{}` expanded to no tokens", + lane.placeholder_token + ); + } + + let replacement_len = replacement.tokens.len(); + let is_embed = { + let mask = replacement + .tokens + .iter() + .map(|&token| token as u32 == lane.embed_token_id) + .collect::>(); + WireTensor::from_bool(vec![replacement_len], mask).map_err(Error::Multimodal)? + }; + + let expanded_offset = expanded.len(); + expanded.extend(replacement.tokens.iter().map(|&token| token as u32)); + ranges.entry(lane.modality).or_default().push(PlaceholderRange { + offset: expanded_offset, + length: replacement_len, + is_embed: Some(is_embed), + }); + } + + for lane in &lanes { + if !lane.replacements.is_empty() { + bail_multimodal!( + "placeholder token `{}` was not found in tokenized prompt for {} remaining `{}` item(s)", + lane.placeholder_token, + lane.replacements.len(), + lane.modality + ); + } + } + + *prompt_token_ids = expanded; + + Ok(ranges) +} + +#[cfg(test)] +mod tests { + use llm_multimodal::TokenId; + use vllm_engine_core_client::protocol::tensor::WireArrayData; + + use super::super::tests::{ + LLAMA4_IMAGE_END_ID, LLAMA4_IMAGE_ID, LLAMA4_IMAGE_START_ID, LLAMA4_PATCH_ID, + LLAMA4_TILE_X_SEPARATOR_ID, LLAMA4_TILE_Y_SEPARATOR_ID, QWEN3_IMAGE_PAD_ID, + QWEN3_VIDEO_PAD_ID, + }; + use super::super::{PreparedMedia, ResolvedPlaceholder}; + use super::*; + + /// Build prepared media directly from placeholder token IDs. + fn prepared_media( + modality: Modality, + placeholder_token: &str, + marker_token_id: u32, + embed_token_id: u32, + replacements: Vec, + ) -> PreparedMedia { + PreparedMedia { + modality, + placeholder: ResolvedPlaceholder { + token: placeholder_token.to_string(), + marker_token_id, + embed_token_id, + }, + replacements, + items: Vec::new(), + } + } + + /// Llama4 image prepared media: the `<|image|>` marker expands to + /// sequences whose embed positions are the `<|patch|>` tokens. + fn llama4_prepared(replacements: Vec) -> PreparedMedia { + prepared_media( + Modality::Image, + "<|image|>", + LLAMA4_IMAGE_ID, + LLAMA4_PATCH_ID, + replacements, + ) + } + + fn qwen3_image_prepared(replacements: Vec) -> PreparedMedia { + prepared_media( + Modality::Image, + "<|image_pad|>", + QWEN3_IMAGE_PAD_ID, + QWEN3_IMAGE_PAD_ID, + replacements, + ) + } + + fn qwen3_video_prepared(replacements: Vec) -> PreparedMedia { + prepared_media( + Modality::Video, + "<|video_pad|>", + QWEN3_VIDEO_PAD_ID, + QWEN3_VIDEO_PAD_ID, + replacements, + ) + } + + fn llama4_single_tile_replacement() -> PromptReplacement { + PromptReplacement::sequence( + Modality::Image, + "<|image|>", + vec![ + LLAMA4_IMAGE_START_ID as TokenId, + LLAMA4_IMAGE_ID as TokenId, + LLAMA4_PATCH_ID as TokenId, + LLAMA4_PATCH_ID as TokenId, + LLAMA4_IMAGE_END_ID as TokenId, + ], + ) + } + + fn llama4_multi_tile_replacement() -> PromptReplacement { + PromptReplacement::sequence( + Modality::Image, + "<|image|>", + vec![ + LLAMA4_IMAGE_START_ID as TokenId, + LLAMA4_PATCH_ID as TokenId, + LLAMA4_TILE_X_SEPARATOR_ID as TokenId, + LLAMA4_PATCH_ID as TokenId, + LLAMA4_TILE_Y_SEPARATOR_ID as TokenId, + LLAMA4_IMAGE_ID as TokenId, + LLAMA4_PATCH_ID as TokenId, + LLAMA4_IMAGE_END_ID as TokenId, + ], + ) + } + + fn assert_bool_mask(range: &PlaceholderRange, expected: &[bool]) { + let tensor = range.is_embed.as_ref().expect("is_embed mask"); + assert_eq!(tensor.dtype, "bool"); + assert_eq!(tensor.shape, vec![expected.len()]); + assert_eq!( + tensor.data, + WireArrayData::RawView(expected.iter().map(|value| u8::from(*value)).collect()) + ); + } + + #[test] + fn expand_prompt_tokens_marks_only_llama4_patch_tokens_as_embed() { + let mut prompt_token_ids = vec![1, LLAMA4_IMAGE_ID, 2]; + let prepared = vec![llama4_prepared(vec![llama4_multi_tile_replacement()])]; + + let ranges = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap(); + let ranges = &ranges[&Modality::Image]; + + assert_eq!( + prompt_token_ids, + vec![ + 1, + LLAMA4_IMAGE_START_ID, + LLAMA4_PATCH_ID, + LLAMA4_TILE_X_SEPARATOR_ID, + LLAMA4_PATCH_ID, + LLAMA4_TILE_Y_SEPARATOR_ID, + LLAMA4_IMAGE_ID, + LLAMA4_PATCH_ID, + LLAMA4_IMAGE_END_ID, + 2, + ] + ); + assert_eq!(ranges[0].offset, 1); + assert_eq!(ranges[0].length, 8); + assert_bool_mask( + &ranges[0], + &[false, true, false, true, false, false, true, false], + ); + } + + #[test] + fn expand_prompt_tokens_errors_when_placeholder_missing() { + let mut prompt_token_ids = vec![1, 2, 3]; + let prepared = vec![llama4_prepared(vec![llama4_single_tile_replacement()])]; + + let error = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap_err(); + + assert!(matches!(error, Error::Multimodal(message) if message.contains("not found"))); + } + + #[test] + fn expand_prompt_tokens_ignores_empty_replacements() { + let mut prompt_token_ids = vec![1, LLAMA4_IMAGE_ID, 2]; + let original_prompt_token_ids = prompt_token_ids.clone(); + let prepared = vec![llama4_prepared(Vec::new())]; + + let ranges = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap(); + + assert!(ranges.is_empty()); + assert_eq!(prompt_token_ids, original_prompt_token_ids); + } + + #[test] + fn expand_prompt_tokens_leaves_prompt_unchanged_when_later_placeholder_missing() { + let mut prompt_token_ids = vec![1, LLAMA4_IMAGE_ID, 2]; + let original_prompt_token_ids = prompt_token_ids.clone(); + let prepared = vec![llama4_prepared(vec![ + llama4_single_tile_replacement(), + llama4_single_tile_replacement(), + ])]; + + let error = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap_err(); + + assert!(matches!(error, Error::Multimodal(message) if message.contains("not found"))); + assert_eq!(prompt_token_ids, original_prompt_token_ids); + } + + #[test] + fn expand_prompt_tokens_errors_when_replacement_is_empty() { + let mut prompt_token_ids = vec![1, LLAMA4_IMAGE_ID, 2]; + let original_prompt_token_ids = prompt_token_ids.clone(); + let prepared = vec![llama4_prepared(vec![PromptReplacement::sequence( + Modality::Image, + "<|image|>", + Vec::new(), + )])]; + + let error = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap_err(); + + assert!( + matches!(error, Error::Multimodal(message) if message.contains("expanded to no tokens")) + ); + assert_eq!(prompt_token_ids, original_prompt_token_ids); + } + + #[test] + fn expand_prompt_tokens_skips_llama4_image_marker_inside_replacement() { + let mut prompt_token_ids = vec![1, LLAMA4_IMAGE_ID, 2, LLAMA4_IMAGE_ID, 3]; + let prepared = vec![llama4_prepared(vec![ + llama4_single_tile_replacement(), + llama4_single_tile_replacement(), + ])]; + + let ranges = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap(); + let ranges = &ranges[&Modality::Image]; + + assert_eq!( + prompt_token_ids, + vec![ + 1, + LLAMA4_IMAGE_START_ID, + LLAMA4_IMAGE_ID, + LLAMA4_PATCH_ID, + LLAMA4_PATCH_ID, + LLAMA4_IMAGE_END_ID, + 2, + LLAMA4_IMAGE_START_ID, + LLAMA4_IMAGE_ID, + LLAMA4_PATCH_ID, + LLAMA4_PATCH_ID, + LLAMA4_IMAGE_END_ID, + 3, + ] + ); + assert_eq!(ranges[0].offset, 1); + assert_eq!(ranges[0].length, 5); + assert_bool_mask(&ranges[0], &[false, false, true, true, false]); + assert_eq!(ranges[1].offset, 7); + assert_eq!(ranges[1].length, 5); + assert_bool_mask(&ranges[1], &[false, false, true, true, false]); + } + + #[test] + fn expand_prompt_tokens_interleaves_image_and_video_prepared_media() { + let mut prompt_token_ids = vec![ + 1, + QWEN3_IMAGE_PAD_ID, + 2, + QWEN3_VIDEO_PAD_ID, + 3, + QWEN3_IMAGE_PAD_ID, + 4, + ]; + let prepared = vec![ + qwen3_image_prepared(vec![ + PromptReplacement::repeated( + Modality::Image, + "<|image_pad|>", + QWEN3_IMAGE_PAD_ID as TokenId, + 2, + ), + PromptReplacement::repeated( + Modality::Image, + "<|image_pad|>", + QWEN3_IMAGE_PAD_ID as TokenId, + 3, + ), + ]), + qwen3_video_prepared(vec![PromptReplacement::repeated( + Modality::Video, + "<|video_pad|>", + QWEN3_VIDEO_PAD_ID as TokenId, + 4, + )]), + ]; + + let ranges = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap(); + + assert_eq!( + prompt_token_ids, + vec![ + 1, + QWEN3_IMAGE_PAD_ID, + QWEN3_IMAGE_PAD_ID, + 2, + QWEN3_VIDEO_PAD_ID, + QWEN3_VIDEO_PAD_ID, + QWEN3_VIDEO_PAD_ID, + QWEN3_VIDEO_PAD_ID, + 3, + QWEN3_IMAGE_PAD_ID, + QWEN3_IMAGE_PAD_ID, + QWEN3_IMAGE_PAD_ID, + 4, + ] + ); + + let image_ranges = &ranges[&Modality::Image]; + assert_eq!(image_ranges[0].offset, 1); + assert_eq!(image_ranges[0].length, 2); + assert_bool_mask(&image_ranges[0], &[true, true]); + assert_eq!(image_ranges[1].offset, 9); + assert_eq!(image_ranges[1].length, 3); + assert_bool_mask(&image_ranges[1], &[true, true, true]); + + let video_ranges = &ranges[&Modality::Video]; + assert_eq!(video_ranges[0].offset, 4); + assert_eq!(video_ranges[0].length, 4); + assert_bool_mask(&video_ranges[0], &[true, true, true, true]); + } + + #[test] + fn expand_prompt_tokens_error_names_modality_with_leftover_replacements() { + let mut prompt_token_ids = vec![1, QWEN3_IMAGE_PAD_ID, 2]; + let original_prompt_token_ids = prompt_token_ids.clone(); + let prepared = vec![ + qwen3_image_prepared(vec![PromptReplacement::repeated( + Modality::Image, + "<|image_pad|>", + QWEN3_IMAGE_PAD_ID as TokenId, + 2, + )]), + qwen3_video_prepared(vec![PromptReplacement::repeated( + Modality::Video, + "<|video_pad|>", + QWEN3_VIDEO_PAD_ID as TokenId, + 4, + )]), + ]; + + let error = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap_err(); + + assert!(matches!( + error, + Error::Multimodal(message) + if message.contains("<|video_pad|>") && message.contains("`video`") + )); + assert_eq!(prompt_token_ids, original_prompt_token_ids); + } +} diff --git a/rust/src/chat/src/multimodal/image.rs b/rust/src/chat/src/multimodal/image.rs new file mode 100644 index 00000000000..f71f1ee3526 --- /dev/null +++ b/rust/src/chat/src/multimodal/image.rs @@ -0,0 +1,141 @@ +//! Image-modality preparation: batch preprocessing and per-item feature +//! build. + +use std::sync::Arc; + +use itertools::izip; +use llm_multimodal::{FieldLayout, ImageFrame, Modality, PreprocessedEncoderInputs}; +use vllm_engine_core_client::protocol::dtype::ModelDtype; +use vllm_engine_core_client::protocol::multimodal::{ + MmBatchedField, MmField, MmFieldElem, MmFlatField, MmKwargsItem, MmSharedField, MmSlice, + SliceSpec, +}; + +use super::{ModalitySupport, MultimodalModelInfo, PreparedItem, PreparedMedia, tensor}; +use crate::error::{Error, Result, bail_multimodal, multimodal}; + +impl MultimodalModelInfo { + /// Preprocess all fetched image frames as one batch and build per-item + /// features. + pub(super) async fn prepare_images( + &self, + frames: Vec>, + uuids: Vec>, + model_dtype: ModelDtype, + ) -> Result { + let support = self.image.as_ref().ok_or_else(|| Error::UnsupportedModality { + modality: Modality::Image.to_string(), + })?; + let preprocessed = self.preprocess_images(support, &frames).await?; + let replacements = + self.spec + .prompt_replacements_for(&self.context, &preprocessed, Modality::Image)?; + if replacements.len() != frames.len() { + bail_multimodal!( + "number of image prompt replacements {} does not match number of images {}", + replacements.len(), + frames.len() + ); + } + let items = self.build_image_items(preprocessed, &frames, uuids, model_dtype)?; + + Ok(PreparedMedia { + modality: Modality::Image, + placeholder: support.placeholder.clone(), + replacements, + items, + }) + } + + /// Preprocess fetched image frames with the model's resolved vision + /// processor. + /// + /// The processor work is CPU-heavy relative to request wiring, so it runs + /// in a blocking task and returns owned tensors ready for wire + /// conversion. + async fn preprocess_images( + &self, + support: &ModalitySupport, + image_frames: &[Arc], + ) -> Result { + let config = support.config.clone(); + let processor = support.processor; + let images = image_frames.iter().map(|frame| frame.data().clone()).collect::>(); + + // TODO: is it still necessary given that we've already in a dedicated runtime? + tokio::task::spawn_blocking(move || Ok(processor.preprocess(&images, &config)?)) + .await + .map_err(|error| multimodal!("image preprocessing task failed: {error}"))? + } + + /// Convert one batch of preprocessed image tensors into per-item engine + /// kwargs. + /// + /// Tensor fields are sliced per item according to the model spec's field + /// layout declarations. + fn build_image_items( + &self, + preprocessed: PreprocessedEncoderInputs, + frames: &[Arc], + uuids: Vec>, + model_dtype: ModelDtype, + ) -> Result> { + let len = frames.len(); + let tensors = tensor::collect_tensors(preprocessed, "pixel_values", model_dtype)?; + + let mut items = Vec::with_capacity(len); + for (index, (frame, uuid)) in izip!(frames, uuids).enumerate() { + let mut data = MmKwargsItem::new(); + for (key, tensor) in &tensors { + let keep_on_cpu = self.spec.keep_on_cpu_keys.contains(key); + let (value, field) = match self.spec.field_layouts.get(key) { + Some(FieldLayout::Batched) => ( + tensor.batched_value_at(index)?, + MmField::Batched(MmBatchedField { keep_on_cpu }), + ), + Some(FieldLayout::Flat { sizes_key }) => { + let sizes = tensors.get(sizes_key).ok_or_else(|| { + multimodal!("flat tensor sizes key `{sizes_key}` is missing") + })?; + let (start, end) = tensor::flat_range_for_index(sizes, sizes_key, index)?; + ( + tensor.flat_value_range(start, end)?, + MmField::Flat(MmFlatField { + slices: vec![MmSlice::Slice(SliceSpec { + start: Some(0), + stop: Some((end - start) as isize), + step: None, + })], + dim: 0, + keep_on_cpu, + }), + ) + } + None => ( + tensor.clone(), + MmField::Shared(MmSharedField { + batch_size: len, + keep_on_cpu, + }), + ), + }; + + data.insert( + key.clone(), + MmFieldElem { + data: Some(value.try_into()?), + field, + }, + ); + } + + items.push(PreparedItem { + data, + hash: frame.hash.clone(), + uuid, + }); + } + + Ok(items) + } +} diff --git a/rust/src/chat/src/multimodal/tensor.rs b/rust/src/chat/src/multimodal/tensor.rs index 95259f1a93f..e26022aea03 100644 --- a/rust/src/chat/src/multimodal/tensor.rs +++ b/rust/src/chat/src/multimodal/tensor.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use half::{bf16, f16}; -use llm_multimodal::{ModelSpecificValue, PreprocessedEncoderInputs as PreprocessedImages}; +use llm_multimodal::{ModelSpecificValue, PreprocessedEncoderInputs}; use vllm_engine_core_client::protocol::dtype::ModelDtype; use vllm_engine_core_client::protocol::multimodal::MmKwargValue as ProtocolKwargValue; use vllm_engine_core_client::protocol::tensor::{ShapeExt as _, WireTensor}; @@ -25,25 +25,31 @@ pub(super) enum KwargValue { Passthrough(ProtocolKwargValue), } -/// Collect `pixel_values` and model-specific outputs into one tensor map. +/// Collect the primary encoder input and model-specific outputs into one +/// tensor map. +/// +/// `primary_key` names the encoder-input tensor as the model's forward kwargs +/// expect it (e.g. `pixel_values` for images, `pixel_values_videos` for +/// videos). pub(super) fn collect_tensors( - preprocessed: PreprocessedImages, + preprocessed: PreprocessedEncoderInputs, + primary_key: &str, float_dtype: ModelDtype, ) -> Result> { - let PreprocessedImages { + let PreprocessedEncoderInputs { encoder_input, model_specific, .. } = preprocessed; - let pixel_values = { + let primary_value = { let shape = encoder_input.shape().to_vec(); let data = encoder_input.into_iter().collect(); KwargValue::from_f32_tensor(data, shape, float_dtype)? }; let mut tensors = HashMap::new(); - tensors.insert("pixel_values".to_string(), pixel_values); + tensors.insert(primary_key.to_string(), primary_value); for (key, value) in model_specific { tensors.insert(key, KwargValue::from_model_specific(value, float_dtype)?); } @@ -124,10 +130,22 @@ impl TryFrom for ProtocolKwargValue { } impl KwargValue { - /// Extract one image from a batched tensor field. + /// First-axis length for tensor values; `None` for passthrough kwargs. + pub(super) fn first_dim(&self) -> Option { + match self { + Self::F32Tensor { shape, .. } + | Self::F16Tensor { shape, .. } + | Self::Bf16Tensor { shape, .. } + | Self::I64Tensor { shape, .. } + | Self::U32Tensor { shape, .. } => shape.first().copied(), + Self::Passthrough(_) => None, + } + } + + /// Extract one media item from a batched tensor field. /// - /// Batched fields use their first axis as image index and drop that axis in - /// the per-feature value, matching vLLM's batched-field semantics. + /// Batched fields use their first axis as media-item index and drop that + /// axis in the per-feature value, matching vLLM's batched-field semantics. pub(super) fn batched_value_at(&self, index: usize) -> Result { match self { Self::F32Tensor { data, shape } => { @@ -154,9 +172,9 @@ impl KwargValue { } } - /// Extract one image's variable-length range from a flat tensor field. + /// Extract one media item's variable-length range from a flat tensor field. /// - /// Flat fields keep the first axis as the sliced length for this image. + /// Flat fields keep the first axis as the sliced length for this item. pub(super) fn flat_value_range(&self, start: usize, end: usize) -> Result { match self { Self::F32Tensor { data, shape } => { @@ -184,10 +202,10 @@ impl KwargValue { } } -/// Compute the first-axis range for one image in a flat tensor. +/// Compute the first-axis range for one media item in a flat tensor. /// /// `sizes_key` names a companion tensor whose entries are cumulative slice -/// sizes per image. +/// sizes per media item. pub(super) fn flat_range_for_index( sizes: &KwargValue, sizes_key: &str, @@ -195,7 +213,7 @@ pub(super) fn flat_range_for_index( ) -> Result<(usize, usize)> { let sizes = tensor_as_usize_vec(sizes)?; let size = *sizes.get(index).ok_or_else(|| { - multimodal!("flat tensor sizes key `{sizes_key}` has no entry for image {index}") + multimodal!("flat tensor sizes key `{sizes_key}` has no entry for media item {index}") })?; let start = sizes[..index].iter().sum::(); Ok((start, start + size)) diff --git a/rust/src/chat/src/multimodal/video.rs b/rust/src/chat/src/multimodal/video.rs new file mode 100644 index 00000000000..482074abde6 --- /dev/null +++ b/rust/src/chat/src/multimodal/video.rs @@ -0,0 +1,316 @@ +//! Video-modality preparation: per-clip preprocessing, config resolution, +//! and per-item feature build. + +use std::sync::Arc; + +use itertools::izip; +use llm_multimodal::{FieldLayout, Modality, PreprocessedEncoderInputs, VideoClip}; +use thiserror_ext::AsReport as _; +use tracing::warn; +use vllm_engine_core_client::protocol::dtype::ModelDtype; +use vllm_engine_core_client::protocol::multimodal::{ + MmBatchedField, MmField, MmFieldElem, MmFlatField, MmKwargsItem, MmSharedField, MmSlice, + SliceSpec, +}; + +use super::{ModalitySupport, MultimodalModelInfo, PreparedItem, PreparedMedia, tensor}; +use crate::error::{Error, Result, bail_multimodal, multimodal}; + +/// Forward-kwargs name of the primary video encoder input. +/// +/// Video-capable vLLM models read `pixel_values_videos` alongside +/// `video_grid_thw`, mirroring the HF processor output naming. +const VIDEO_PRIMARY_KEY: &str = "pixel_values_videos"; + +impl MultimodalModelInfo { + /// Preprocess fetched video clips one at a time and build per-item + /// features. + /// + /// Unlike images, each clip runs through the preprocessor independently + /// (a batch of one), so its tensors are complete per item and need no + /// cross-item slicing. + pub(super) async fn prepare_videos( + &self, + clips: Vec>, + uuids: Vec>, + model_dtype: ModelDtype, + ) -> Result { + let support = self.video.as_ref().ok_or_else(|| Error::UnsupportedModality { + modality: Modality::Video.to_string(), + })?; + let mut replacements = Vec::with_capacity(clips.len()); + let mut items = Vec::with_capacity(clips.len()); + + for (clip, uuid) in izip!(&clips, uuids) { + let preprocessed = self.preprocess_video_clip(support, Arc::clone(clip)).await?; + let mut clip_replacements = + self.spec + .prompt_replacements_for(&self.context, &preprocessed, Modality::Video)?; + if clip_replacements.len() != 1 { + bail_multimodal!( + "expected exactly one prompt replacement per video clip, got {}", + clip_replacements.len() + ); + } + replacements.push(clip_replacements.pop().unwrap()); + items.push(self.build_video_item( + preprocessed, + clip.hash.clone(), + uuid, + model_dtype, + )?); + } + + Ok(PreparedMedia { + modality: Modality::Video, + placeholder: support.placeholder.clone(), + replacements, + items, + }) + } + + /// Preprocess one decoded video clip with the model's resolved vision + /// processor. + async fn preprocess_video_clip( + &self, + support: &ModalitySupport, + clip: Arc, + ) -> Result { + let config = support.config.clone(); + let processor = support.processor; + + tokio::task::spawn_blocking(move || { + // Prefer the borrowed-RGB fast path, which avoids materializing a + // `DynamicImage` per sampled frame after media decode. + if let Some(rgb_video) = clip.rgb_video() { + match rgb_video.frame_refs() { + Ok(frame_refs) => match processor.preprocess_video_rgb(&frame_refs, &config) { + Ok(preprocessed) => return Ok(preprocessed), + Err(error) => warn!( + error = %error.as_report(), + "RGB video preprocessing fast path failed; falling back to materialized frames" + ), + }, + Err(error) => warn!( + error, + "RGB video frame refs are invalid; falling back to materialized frames" + ), + } + } + + let frames = clip.materialized_frames().map_err(|error| multimodal!("{error}"))?; + Ok(processor.preprocess_video(&frames, &config)?) + }) + .await + .map_err(|error| multimodal!("video preprocessing task failed: {error}"))? + } + + /// Convert one preprocessed video clip into engine kwargs. + /// + /// The clip is a batch of one, so no per-item slicing is required: the + /// primary tensor ships as a full-range flat field (the engine re-batches + /// flat fields by concatenating along the declared dim, matching vLLM's + /// `flat_from_sizes` treatment of video patches), and batched metadata + /// tensors drop their singleton batch axis. + fn build_video_item( + &self, + preprocessed: PreprocessedEncoderInputs, + hash: String, + uuid: Option, + model_dtype: ModelDtype, + ) -> Result { + let tensors = tensor::collect_tensors(preprocessed, VIDEO_PRIMARY_KEY, model_dtype)?; + + let mut data = MmKwargsItem::new(); + for (key, tensor) in tensors { + let keep_on_cpu = self.spec.keep_on_cpu_keys.contains(&key); + let (value, field) = if key == VIDEO_PRIMARY_KEY { + let len = tensor + .first_dim() + .ok_or_else(|| multimodal!("video encoder input `{key}` is not a tensor"))?; + ( + tensor, + MmField::Flat(MmFlatField { + slices: vec![MmSlice::Slice(SliceSpec { + start: Some(0), + stop: Some(len as isize), + step: None, + })], + dim: 0, + keep_on_cpu, + }), + ) + } else if matches!( + self.spec.field_layouts.get(&key), + Some(FieldLayout::Batched) + ) { + ( + tensor.batched_value_at(0)?, + MmField::Batched(MmBatchedField { keep_on_cpu }), + ) + } else { + ( + tensor, + MmField::Shared(MmSharedField { + batch_size: 1, + keep_on_cpu, + }), + ) + }; + + data.insert( + key, + MmFieldElem { + data: Some(value.try_into()?), + field, + }, + ); + } + + Ok(PreparedItem { data, hash, uuid }) + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + use std::sync::Arc; + + use llm_multimodal::ModelSpecificValue; + use ndarray::ArrayD; + use vllm_engine_core_client::protocol::multimodal::MmKwargValue; + + use super::super::tests::{ + QWEN3_IMAGE_PAD_ID, QWEN3_VIDEO_PAD_ID, qwen3_vl_info, qwen3_vl_tokenizer, + }; + use super::super::{MultimodalConfigFiles, MultimodalModelInfo}; + use super::*; + + #[test] + fn from_paths_resolves_video_config_from_dedicated_file_or_processor_config() { + let dir = tempfile::tempdir().unwrap(); + let config_path = dir.path().join("config.json"); + std::fs::write( + &config_path, + serde_json::json!({ + "model_type": "qwen3_vl", + "image_token_id": QWEN3_IMAGE_PAD_ID, + "video_token_id": QWEN3_VIDEO_PAD_ID, + }) + .to_string(), + ) + .unwrap(); + + let info_for = |files: MultimodalConfigFiles<'_>| { + MultimodalModelInfo::from_paths( + "qwen3-vl-test".to_string(), + Some("qwen3_vl".to_string()), + files, + Arc::new(qwen3_vl_tokenizer()), + ) + }; + + // Dedicated video preprocessor config file. + let video_config_path = dir.path().join("video_preprocessor_config.json"); + std::fs::write(&video_config_path, r#"{"size":{"shortest_edge":128}}"#).unwrap(); + let info = info_for(MultimodalConfigFiles { + config: Some(&config_path), + video_preprocessor_config: Some(&video_config_path), + ..Default::default() + }) + .unwrap() + .unwrap(); + assert!(info.video.is_some()); + + // `video_processor` section of the combined processor config. + let processor_config_path = dir.path().join("processor_config.json"); + std::fs::write( + &processor_config_path, + r#"{"video_processor":{"size":{"shortest_edge":128}}}"#, + ) + .unwrap(); + let info = info_for(MultimodalConfigFiles { + config: Some(&config_path), + processor_config: Some(&processor_config_path), + ..Default::default() + }) + .unwrap() + .unwrap(); + assert!(info.video.is_some()); + + // Neither source: video support still resolves on the image config. + let info = info_for(MultimodalConfigFiles { + config: Some(&config_path), + ..Default::default() + }) + .unwrap() + .unwrap(); + assert!(info.video.is_some()); + + // Malformed dedicated file is a real error, not a silent fallback. + std::fs::write(&video_config_path, r#"{"size""#).unwrap(); + let error = match info_for(MultimodalConfigFiles { + config: Some(&config_path), + video_preprocessor_config: Some(&video_config_path), + ..Default::default() + }) { + Err(error) => error, + Ok(_) => panic!("malformed video preprocessor config should fail"), + }; + assert!(matches!( + error, + Error::Multimodal(message) + if message.contains("failed to parse video_preprocessor_config.json") + )); + } + + #[test] + fn build_video_item_names_primary_tensor_and_layouts() { + let info = qwen3_vl_info(); + // One clip flattened to 6 patches with 4 features each. + let preprocessed = PreprocessedEncoderInputs { + encoder_input: ArrayD::zeros(vec![6, 4]), + feature_token_counts: vec![6], + item_sizes: vec![(32, 32)], + model_specific: HashMap::from([ + ( + "video_grid_thw".to_string(), + ModelSpecificValue::int_2d(vec![1, 2, 3], 1, 3), + ), + ( + "patches_per_video".to_string(), + ModelSpecificValue::int_1d(vec![6]), + ), + ]), + }; + + let item = info + .build_video_item( + preprocessed, + "".to_string(), + None, + ModelDtype::Float32, + ) + .unwrap(); + + let primary = &item.data[VIDEO_PRIMARY_KEY]; + assert!(matches!( + &primary.field, + MmField::Flat(MmFlatField { slices, dim: 0, .. }) + if matches!( + slices.as_slice(), + [MmSlice::Slice(SliceSpec { start: Some(0), stop: Some(6), step: None })] + ) + )); + + // Batched metadata drops its singleton batch axis per item. + let grid = &item.data["video_grid_thw"]; + assert!(matches!(&grid.field, MmField::Batched(_))); + let MmKwargValue::Tensor(grid_tensor) = grid.data.as_ref().unwrap() else { + panic!("expected tensor value for video_grid_thw"); + }; + assert_eq!(grid_tensor.shape, vec![3]); + + assert_eq!(item.hash, ""); + } +} diff --git a/rust/src/chat/src/renderer/hf/mod.rs b/rust/src/chat/src/renderer/hf/mod.rs index d9031d73a4b..1c87e75337f 100644 --- a/rust/src/chat/src/renderer/hf/mod.rs +++ b/rust/src/chat/src/renderer/hf/mod.rs @@ -31,9 +31,14 @@ pub use template::{load_chat_template, resolve_chat_template}; pub use self::format::ChatTemplateContentFormatOption; -#[derive(Debug, Clone)] +/// Template-visible placeholder tokens per supported modality. +/// +/// A `None` token means the loaded model does not support that modality, and +/// content parts of that modality are rejected during rendering. +#[derive(Debug, Clone, Default)] pub struct MultimodalRenderInfo { - pub placeholder_token: String, + pub image_token: Option, + pub video_token: Option, } /// Hugging Face chat-template renderer backed by the local Jinja chat-template @@ -254,6 +259,7 @@ enum TemplateContent { enum TemplateContentPart { Text { text: String }, Image, + Video, } #[derive(Debug, Serialize)] @@ -417,9 +423,17 @@ fn to_template_openai_content( } // All multimodal contents are normalized to `{ "type": }`. ChatContentPart::ImageUrl { .. } => { - multimodal.ok_or(Error::UnsupportedMultimodalContent("image_url"))?; + multimodal + .and_then(|multimodal| multimodal.image_token.as_ref()) + .ok_or(Error::UnsupportedMultimodalContent("image_url"))?; Ok(TemplateContentPart::Image) } + ChatContentPart::VideoUrl { .. } => { + multimodal + .and_then(|multimodal| multimodal.video_token.as_ref()) + .ok_or(Error::UnsupportedMultimodalContent("video_url"))?; + Ok(TemplateContentPart::Video) + } }) .collect(), } @@ -437,9 +451,16 @@ fn to_template_string_content( match part { ChatContentPart::Text { text } => out.push_str(text), ChatContentPart::ImageUrl { .. } => { - let multimodal = - multimodal.ok_or(Error::UnsupportedMultimodalContent("image_url"))?; - out.push_str(&multimodal.placeholder_token); + let image_token = multimodal + .and_then(|multimodal| multimodal.image_token.as_ref()) + .ok_or(Error::UnsupportedMultimodalContent("image_url"))?; + out.push_str(image_token); + } + ChatContentPart::VideoUrl { .. } => { + let video_token = multimodal + .and_then(|multimodal| multimodal.video_token.as_ref()) + .ok_or(Error::UnsupportedMultimodalContent("video_url"))?; + out.push_str(video_token); } } } @@ -468,7 +489,7 @@ fn append_continue_final_message_tag(message: &mut TemplateMessage) -> Result parts.iter_mut().rev().find_map(|part| match part { TemplateContentPart::Text { text } => Some(text), - TemplateContentPart::Image => None, + TemplateContentPart::Image | TemplateContentPart::Video => None, }), }; let text = text.ok_or_else(|| { @@ -577,7 +598,8 @@ mod tests { ) -> Result { HfChatRenderer::new(Some(template.to_string()), HashMap::new(), content_format)? .with_multimodal(Some(MultimodalRenderInfo { - placeholder_token: "".to_string(), + image_token: Some("".to_string()), + video_token: Some("