[Rust Frontend] Integrate MM video support (#47959)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>

rust/Cargo.lock

@@ -2167,7 +2167,7 @@ checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092"
 [[package]]
 name = "llm-multimodal"
 version = "1.7.1"
-source = "git+https://github.com/smg-project/llm-multimodal?rev=7d74582aeaf0e4086a44964382655d22f1af0686#7d74582aeaf0e4086a44964382655d22f1af0686"
+source = "git+https://github.com/smg-project/llm-multimodal?rev=c8a29dcc755139fdc26185f400ea48c6d6d48273#c8a29dcc755139fdc26185f400ea48c6d6d48273"
 dependencies = [
  "anyhow",
  "base64 0.22.1",
@@ -2473,6 +2473,7 @@ dependencies = [
  "portable-atomic",
  "portable-atomic-util",
  "rawpointer",
+ "serde",
 ]
 
 [[package]]
@@ -5093,6 +5094,7 @@ dependencies = [
  "llm-multimodal",
  "minijinja",
  "minijinja-contrib",
+ "ndarray 0.17.2",
  "oss-harmony",
  "paste",
  "reqwest",

rust/Cargo.toml

@@ -53,12 +53,12 @@ hyper-util = { version = "0.1.20", features = [
 indexmap = "2.13.0"
 itertools = "0.14.0"
 libc = "0.2.177"
-llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "7d74582aeaf0e4086a44964382655d22f1af0686" }
+llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "c8a29dcc755139fdc26185f400ea48c6d6d48273" }
 mimalloc = "0.1.52"
 minijinja = { version = "2.0", features = ["unstable_machinery", "json", "builtins", "loader", "loop_controls", "preserve_order"] }
 minijinja-contrib = { version = "2.0", features = ["pycompat"] }
 native-tls-vendored = { package = "native-tls", version = "0.2.18", features = ["vendored"] }
-ndarray = { version = "0.16.1", features = ["serde"] }
+ndarray = { version = "0.17", features = ["serde"] }
 openai-harmony = { package = "oss-harmony", git = "https://github.com/oss-harmony/harmony", tag = "v0.0.11", default-features = false }
 openai-protocol = "1.6.0"
 openssl = "0.10"

rust/src/chat/Cargo.toml

@@ -42,6 +42,7 @@ anyhow.workspace = true
 bytes.workspace = true
 clap.workspace = true
 expect-test.workspace = true
+ndarray.workspace = true
 paste.workspace = true
 rmp-serde.workspace = true
 serial_test.workspace = true

rust/src/chat/src/backend/hf.rs

@@ -10,7 +10,7 @@ use crate::backend::{
     NewChatOutputProcessorOptions,
 };
 use crate::error::Result;
-use crate::multimodal::MultimodalModelInfo;
+use crate::multimodal::{MultimodalConfigFiles, MultimodalModelInfo};
 use crate::output::{
     DefaultChatOutputProcessor, HarmonyChatOutputProcessor, validate_harmony_parser_overrides,
 };
@@ -46,8 +46,12 @@ impl HfChatBackend {
             MultimodalModelInfo::from_paths(
                 model_id.clone(),
                 (!model_type.is_empty()).then_some(model_type.to_string()),
-                files.config_path.as_deref(),
-                files.preprocessor_config_path.as_deref(),
+                MultimodalConfigFiles {
+                    config: files.config_path.as_deref(),
+                    preprocessor_config: files.preprocessor_config_path.as_deref(),
+                    video_preprocessor_config: files.video_preprocessor_config_path.as_deref(),
+                    processor_config: files.processor_config_path.as_deref(),
+                },
                 tokenizer.clone(),
             )?
         };
@@ -139,8 +143,11 @@ pub(super) async fn load_model_backends(
 fn resolve_multimodal_render_info(
     info: Option<&MultimodalModelInfo>,
 ) -> Option<MultimodalRenderInfo> {
+    use llm_multimodal::Modality;
+
     info.map(|info| MultimodalRenderInfo {
-        placeholder_token: info.placeholder_token().to_string(),
+        image_token: info.placeholder_token(Modality::Image).map(str::to_string),
+        video_token: info.placeholder_token(Modality::Video).map(str::to_string),
     })
 }
 
@@ -192,6 +199,8 @@ mod tests {
             tokenizer_config_path: Some(tokenizer_config_path),
             generation_config_path: None,
             preprocessor_config_path: None,
+            video_preprocessor_config_path: None,
+            processor_config_path: None,
             chat_template_path: None,
             config_path: Some(config_path),
         }

rust/src/chat/src/error.rs

@@ -1,5 +1,5 @@
 use thiserror::Error;
-use thiserror_ext::Macro;
+use thiserror_ext::{AsReport as _, Macro};
 
 type BoxedError = Box<dyn std::error::Error + Send + Sync>;
 
@@ -18,6 +18,8 @@ pub enum Error {
     UnsupportedMultimodalRenderer,
     #[error("unsupported multimodal content: {0}")]
     UnsupportedMultimodalContent(&'static str),
+    #[error("`{modality}` input is not supported by this model")]
+    UnsupportedModality { modality: String },
     #[error("multimodal preprocessing error: {0}")]
     Multimodal(#[message] String),
     #[error("{kind} parsing is not available for model `{model_id}`")]
@@ -80,11 +82,39 @@ impl Error {
         match self {
             Self::PromptTooLong { .. } => true,
             Self::Text(error) => error.is_request_validation_error(),
+            Self::UnsupportedMultimodalRenderer
+            | Self::UnsupportedMultimodalContent(_)
+            | Self::UnsupportedModality { .. } => true,
+
             _ => false,
         }
     }
 }
 
+impl From<llm_multimodal::MediaConnectorError> for Error {
+    fn from(error: llm_multimodal::MediaConnectorError) -> Self {
+        Self::Multimodal(error.to_report_string())
+    }
+}
+
+impl From<llm_multimodal::MultiModalError> for Error {
+    fn from(error: llm_multimodal::MultiModalError) -> Self {
+        Self::Multimodal(error.to_report_string())
+    }
+}
+
+impl From<llm_multimodal::TransformError> for Error {
+    fn from(error: llm_multimodal::TransformError) -> Self {
+        Self::Multimodal(error.to_report_string())
+    }
+}
+
+impl From<llm_multimodal::registry::ModelRegistryError> for Error {
+    fn from(error: llm_multimodal::registry::ModelRegistryError) -> Self {
+        Self::Multimodal(error.to_report_string())
+    }
+}
+
 /// Format the available-parser suffix used in user-facing error messages.
 fn available_parser_hint(available_names: &[String]) -> String {
     if available_names.is_empty() {

rust/src/chat/src/multimodal/expand.rs

@@ -0,0 +1,446 @@
+//! Prompt placeholder expansion shared across modalities.
+
+use std::collections::{HashMap, VecDeque};
+
+use llm_multimodal::{Modality, PromptReplacement};
+use vllm_engine_core_client::protocol::multimodal::PlaceholderRange;
+use vllm_engine_core_client::protocol::tensor::WireTensor;
+
+use super::PreparedMedia;
+use crate::error::{Error, Result, bail_multimodal};
+
+/// One modality's queue of pending placeholder replacements for prompt
+/// expansion.
+struct ExpansionLane<'a> {
+    modality: Modality,
+    marker_token_id: u32,
+    embed_token_id: u32,
+    placeholder_token: String,
+    replacements: VecDeque<&'a PromptReplacement>,
+}
+
+impl<'a> ExpansionLane<'a> {
+    fn from_prepared(media: &'a PreparedMedia) -> Option<Self> {
+        if media.replacements.is_empty() {
+            return None;
+        }
+
+        Some(Self {
+            modality: media.modality,
+            marker_token_id: media.placeholder.marker_token_id,
+            embed_token_id: media.placeholder.embed_token_id,
+            placeholder_token: media.placeholder.token.clone(),
+            replacements: media.replacements.iter().collect(),
+        })
+    }
+}
+
+/// Replace rendered placeholder markers with model-specific replacement
+/// tokens across all modalities in one left-to-right pass.
+///
+/// Each prepared modality consumes its own marker occurrences in order,
+/// matching the original media-part order within that modality; markers of
+/// different modalities may interleave freely.
+///
+/// The returned ranges point into the already-expanded prompt, grouped per
+/// modality in item order.
+pub(super) fn expand_prompt_token_ids(
+    prompt_token_ids: &mut Vec<u32>,
+    prepared: &[PreparedMedia],
+) -> Result<HashMap<Modality, Vec<PlaceholderRange>>> {
+    let mut lanes = prepared.iter().filter_map(ExpansionLane::from_prepared).collect::<Vec<_>>();
+    if lanes.is_empty() {
+        return Ok(HashMap::new());
+    }
+
+    let replacement_growth = lanes
+        .iter()
+        .flat_map(|lane| lane.replacements.iter())
+        .fold(0usize, |total, replacement| {
+            total.saturating_add(replacement.tokens.len().saturating_sub(1))
+        });
+    let expanded_len = prompt_token_ids.len().saturating_add(replacement_growth);
+
+    let mut expanded = Vec::with_capacity(expanded_len);
+    let mut ranges = HashMap::<Modality, Vec<PlaceholderRange>>::new();
+
+    for &token in prompt_token_ids.iter() {
+        let lane = lanes
+            .iter_mut()
+            .find(|lane| lane.marker_token_id == token && !lane.replacements.is_empty());
+        let Some(lane) = lane else {
+            expanded.push(token);
+            continue;
+        };
+
+        let replacement = lane.replacements.pop_front().expect("lane queue is non-empty");
+        debug_assert_eq!(replacement.modality, lane.modality);
+        if replacement.tokens.is_empty() {
+            bail_multimodal!(
+                "placeholder token `{}` expanded to no tokens",
+                lane.placeholder_token
+            );
+        }
+
+        let replacement_len = replacement.tokens.len();
+        let is_embed = {
+            let mask = replacement
+                .tokens
+                .iter()
+                .map(|&token| token as u32 == lane.embed_token_id)
+                .collect::<Vec<_>>();
+            WireTensor::from_bool(vec![replacement_len], mask).map_err(Error::Multimodal)?
+        };
+
+        let expanded_offset = expanded.len();
+        expanded.extend(replacement.tokens.iter().map(|&token| token as u32));
+        ranges.entry(lane.modality).or_default().push(PlaceholderRange {
+            offset: expanded_offset,
+            length: replacement_len,
+            is_embed: Some(is_embed),
+        });
+    }
+
+    for lane in &lanes {
+        if !lane.replacements.is_empty() {
+            bail_multimodal!(
+                "placeholder token `{}` was not found in tokenized prompt for {} remaining `{}` item(s)",
+                lane.placeholder_token,
+                lane.replacements.len(),
+                lane.modality
+            );
+        }
+    }
+
+    *prompt_token_ids = expanded;
+
+    Ok(ranges)
+}
+
+#[cfg(test)]
+mod tests {
+    use llm_multimodal::TokenId;
+    use vllm_engine_core_client::protocol::tensor::WireArrayData;
+
+    use super::super::tests::{
+        LLAMA4_IMAGE_END_ID, LLAMA4_IMAGE_ID, LLAMA4_IMAGE_START_ID, LLAMA4_PATCH_ID,
+        LLAMA4_TILE_X_SEPARATOR_ID, LLAMA4_TILE_Y_SEPARATOR_ID, QWEN3_IMAGE_PAD_ID,
+        QWEN3_VIDEO_PAD_ID,
+    };
+    use super::super::{PreparedMedia, ResolvedPlaceholder};
+    use super::*;
+
+    /// Build prepared media directly from placeholder token IDs.
+    fn prepared_media(
+        modality: Modality,
+        placeholder_token: &str,
+        marker_token_id: u32,
+        embed_token_id: u32,
+        replacements: Vec<PromptReplacement>,
+    ) -> PreparedMedia {
+        PreparedMedia {
+            modality,
+            placeholder: ResolvedPlaceholder {
+                token: placeholder_token.to_string(),
+                marker_token_id,
+                embed_token_id,
+            },
+            replacements,
+            items: Vec::new(),
+        }
+    }
+
+    /// Llama4 image prepared media: the `<|image|>` marker expands to
+    /// sequences whose embed positions are the `<|patch|>` tokens.
+    fn llama4_prepared(replacements: Vec<PromptReplacement>) -> PreparedMedia {
+        prepared_media(
+            Modality::Image,
+            "<|image|>",
+            LLAMA4_IMAGE_ID,
+            LLAMA4_PATCH_ID,
+            replacements,
+        )
+    }
+
+    fn qwen3_image_prepared(replacements: Vec<PromptReplacement>) -> PreparedMedia {
+        prepared_media(
+            Modality::Image,
+            "<|image_pad|>",
+            QWEN3_IMAGE_PAD_ID,
+            QWEN3_IMAGE_PAD_ID,
+            replacements,
+        )
+    }
+
+    fn qwen3_video_prepared(replacements: Vec<PromptReplacement>) -> PreparedMedia {
+        prepared_media(
+            Modality::Video,
+            "<|video_pad|>",
+            QWEN3_VIDEO_PAD_ID,
+            QWEN3_VIDEO_PAD_ID,
+            replacements,
+        )
+    }
+
+    fn llama4_single_tile_replacement() -> PromptReplacement {
+        PromptReplacement::sequence(
+            Modality::Image,
+            "<|image|>",
+            vec![
+                LLAMA4_IMAGE_START_ID as TokenId,
+                LLAMA4_IMAGE_ID as TokenId,
+                LLAMA4_PATCH_ID as TokenId,
+                LLAMA4_PATCH_ID as TokenId,
+                LLAMA4_IMAGE_END_ID as TokenId,
+            ],
+        )
+    }
+
+    fn llama4_multi_tile_replacement() -> PromptReplacement {
+        PromptReplacement::sequence(
+            Modality::Image,
+            "<|image|>",
+            vec![
+                LLAMA4_IMAGE_START_ID as TokenId,
+                LLAMA4_PATCH_ID as TokenId,
+                LLAMA4_TILE_X_SEPARATOR_ID as TokenId,
+                LLAMA4_PATCH_ID as TokenId,
+                LLAMA4_TILE_Y_SEPARATOR_ID as TokenId,
+                LLAMA4_IMAGE_ID as TokenId,
+                LLAMA4_PATCH_ID as TokenId,
+                LLAMA4_IMAGE_END_ID as TokenId,
+            ],
+        )
+    }
+
+    fn assert_bool_mask(range: &PlaceholderRange, expected: &[bool]) {
+        let tensor = range.is_embed.as_ref().expect("is_embed mask");
+        assert_eq!(tensor.dtype, "bool");
+        assert_eq!(tensor.shape, vec![expected.len()]);
+        assert_eq!(
+            tensor.data,
+            WireArrayData::RawView(expected.iter().map(|value| u8::from(*value)).collect())
+        );
+    }
+
+    #[test]
+    fn expand_prompt_tokens_marks_only_llama4_patch_tokens_as_embed() {
+        let mut prompt_token_ids = vec![1, LLAMA4_IMAGE_ID, 2];
+        let prepared = vec![llama4_prepared(vec![llama4_multi_tile_replacement()])];
+
+        let ranges = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap();
+        let ranges = &ranges[&Modality::Image];
+
+        assert_eq!(
+            prompt_token_ids,
+            vec![
+                1,
+                LLAMA4_IMAGE_START_ID,
+                LLAMA4_PATCH_ID,
+                LLAMA4_TILE_X_SEPARATOR_ID,
+                LLAMA4_PATCH_ID,
+                LLAMA4_TILE_Y_SEPARATOR_ID,
+                LLAMA4_IMAGE_ID,
+                LLAMA4_PATCH_ID,
+                LLAMA4_IMAGE_END_ID,
+                2,
+            ]
+        );
+        assert_eq!(ranges[0].offset, 1);
+        assert_eq!(ranges[0].length, 8);
+        assert_bool_mask(
+            &ranges[0],
+            &[false, true, false, true, false, false, true, false],
+        );
+    }
+
+    #[test]
+    fn expand_prompt_tokens_errors_when_placeholder_missing() {
+        let mut prompt_token_ids = vec![1, 2, 3];
+        let prepared = vec![llama4_prepared(vec![llama4_single_tile_replacement()])];
+
+        let error = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap_err();
+
+        assert!(matches!(error, Error::Multimodal(message) if message.contains("not found")));
+    }
+
+    #[test]
+    fn expand_prompt_tokens_ignores_empty_replacements() {
+        let mut prompt_token_ids = vec![1, LLAMA4_IMAGE_ID, 2];
+        let original_prompt_token_ids = prompt_token_ids.clone();
+        let prepared = vec![llama4_prepared(Vec::new())];
+
+        let ranges = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap();
+
+        assert!(ranges.is_empty());
+        assert_eq!(prompt_token_ids, original_prompt_token_ids);
+    }
+
+    #[test]
+    fn expand_prompt_tokens_leaves_prompt_unchanged_when_later_placeholder_missing() {
+        let mut prompt_token_ids = vec![1, LLAMA4_IMAGE_ID, 2];
+        let original_prompt_token_ids = prompt_token_ids.clone();
+        let prepared = vec![llama4_prepared(vec![
+            llama4_single_tile_replacement(),
+            llama4_single_tile_replacement(),
+        ])];
+
+        let error = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap_err();
+
+        assert!(matches!(error, Error::Multimodal(message) if message.contains("not found")));
+        assert_eq!(prompt_token_ids, original_prompt_token_ids);
+    }
+
+    #[test]
+    fn expand_prompt_tokens_errors_when_replacement_is_empty() {
+        let mut prompt_token_ids = vec![1, LLAMA4_IMAGE_ID, 2];
+        let original_prompt_token_ids = prompt_token_ids.clone();
+        let prepared = vec![llama4_prepared(vec![PromptReplacement::sequence(
+            Modality::Image,
+            "<|image|>",
+            Vec::new(),
+        )])];
+
+        let error = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap_err();
+
+        assert!(
+            matches!(error, Error::Multimodal(message) if message.contains("expanded to no tokens"))
+        );
+        assert_eq!(prompt_token_ids, original_prompt_token_ids);
+    }
+
+    #[test]
+    fn expand_prompt_tokens_skips_llama4_image_marker_inside_replacement() {
+        let mut prompt_token_ids = vec![1, LLAMA4_IMAGE_ID, 2, LLAMA4_IMAGE_ID, 3];
+        let prepared = vec![llama4_prepared(vec![
+            llama4_single_tile_replacement(),
+            llama4_single_tile_replacement(),
+        ])];
+
+        let ranges = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap();
+        let ranges = &ranges[&Modality::Image];
+
+        assert_eq!(
+            prompt_token_ids,
+            vec![
+                1,
+                LLAMA4_IMAGE_START_ID,
+                LLAMA4_IMAGE_ID,
+                LLAMA4_PATCH_ID,
+                LLAMA4_PATCH_ID,
+                LLAMA4_IMAGE_END_ID,
+                2,
+                LLAMA4_IMAGE_START_ID,
+                LLAMA4_IMAGE_ID,
+                LLAMA4_PATCH_ID,
+                LLAMA4_PATCH_ID,
+                LLAMA4_IMAGE_END_ID,
+                3,
+            ]
+        );
+        assert_eq!(ranges[0].offset, 1);
+        assert_eq!(ranges[0].length, 5);
+        assert_bool_mask(&ranges[0], &[false, false, true, true, false]);
+        assert_eq!(ranges[1].offset, 7);
+        assert_eq!(ranges[1].length, 5);
+        assert_bool_mask(&ranges[1], &[false, false, true, true, false]);
+    }
+
+    #[test]
+    fn expand_prompt_tokens_interleaves_image_and_video_prepared_media() {
+        let mut prompt_token_ids = vec![
+            1,
+            QWEN3_IMAGE_PAD_ID,
+            2,
+            QWEN3_VIDEO_PAD_ID,
+            3,
+            QWEN3_IMAGE_PAD_ID,
+            4,
+        ];
+        let prepared = vec![
+            qwen3_image_prepared(vec![
+                PromptReplacement::repeated(
+                    Modality::Image,
+                    "<|image_pad|>",
+                    QWEN3_IMAGE_PAD_ID as TokenId,
+                    2,
+                ),
+                PromptReplacement::repeated(
+                    Modality::Image,
+                    "<|image_pad|>",
+                    QWEN3_IMAGE_PAD_ID as TokenId,
+                    3,
+                ),
+            ]),
+            qwen3_video_prepared(vec![PromptReplacement::repeated(
+                Modality::Video,
+                "<|video_pad|>",
+                QWEN3_VIDEO_PAD_ID as TokenId,
+                4,
+            )]),
+        ];
+
+        let ranges = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap();
+
+        assert_eq!(
+            prompt_token_ids,
+            vec![
+                1,
+                QWEN3_IMAGE_PAD_ID,
+                QWEN3_IMAGE_PAD_ID,
+                2,
+                QWEN3_VIDEO_PAD_ID,
+                QWEN3_VIDEO_PAD_ID,
+                QWEN3_VIDEO_PAD_ID,
+                QWEN3_VIDEO_PAD_ID,
+                3,
+                QWEN3_IMAGE_PAD_ID,
+                QWEN3_IMAGE_PAD_ID,
+                QWEN3_IMAGE_PAD_ID,
+                4,
+            ]
+        );
+
+        let image_ranges = &ranges[&Modality::Image];
+        assert_eq!(image_ranges[0].offset, 1);
+        assert_eq!(image_ranges[0].length, 2);
+        assert_bool_mask(&image_ranges[0], &[true, true]);
+        assert_eq!(image_ranges[1].offset, 9);
+        assert_eq!(image_ranges[1].length, 3);
+        assert_bool_mask(&image_ranges[1], &[true, true, true]);
+
+        let video_ranges = &ranges[&Modality::Video];
+        assert_eq!(video_ranges[0].offset, 4);
+        assert_eq!(video_ranges[0].length, 4);
+        assert_bool_mask(&video_ranges[0], &[true, true, true, true]);
+    }
+
+    #[test]
+    fn expand_prompt_tokens_error_names_modality_with_leftover_replacements() {
+        let mut prompt_token_ids = vec![1, QWEN3_IMAGE_PAD_ID, 2];
+        let original_prompt_token_ids = prompt_token_ids.clone();
+        let prepared = vec![
+            qwen3_image_prepared(vec![PromptReplacement::repeated(
+                Modality::Image,
+                "<|image_pad|>",
+                QWEN3_IMAGE_PAD_ID as TokenId,
+                2,
+            )]),
+            qwen3_video_prepared(vec![PromptReplacement::repeated(
+                Modality::Video,
+                "<|video_pad|>",
+                QWEN3_VIDEO_PAD_ID as TokenId,
+                4,
+            )]),
+        ];
+
+        let error = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap_err();
+
+        assert!(matches!(
+            error,
+            Error::Multimodal(message)
+                if message.contains("<|video_pad|>") && message.contains("`video`")
+        ));
+        assert_eq!(prompt_token_ids, original_prompt_token_ids);
+    }
+}

rust/src/chat/src/multimodal/image.rs

@@ -0,0 +1,141 @@
+//! Image-modality preparation: batch preprocessing and per-item feature
+//! build.
+
+use std::sync::Arc;
+
+use itertools::izip;
+use llm_multimodal::{FieldLayout, ImageFrame, Modality, PreprocessedEncoderInputs};
+use vllm_engine_core_client::protocol::dtype::ModelDtype;
+use vllm_engine_core_client::protocol::multimodal::{
+    MmBatchedField, MmField, MmFieldElem, MmFlatField, MmKwargsItem, MmSharedField, MmSlice,
+    SliceSpec,
+};
+
+use super::{ModalitySupport, MultimodalModelInfo, PreparedItem, PreparedMedia, tensor};
+use crate::error::{Error, Result, bail_multimodal, multimodal};
+
+impl MultimodalModelInfo {
+    /// Preprocess all fetched image frames as one batch and build per-item
+    /// features.
+    pub(super) async fn prepare_images(
+        &self,
+        frames: Vec<Arc<ImageFrame>>,
+        uuids: Vec<Option<String>>,
+        model_dtype: ModelDtype,
+    ) -> Result<PreparedMedia> {
+        let support = self.image.as_ref().ok_or_else(|| Error::UnsupportedModality {
+            modality: Modality::Image.to_string(),
+        })?;
+        let preprocessed = self.preprocess_images(support, &frames).await?;
+        let replacements =
+            self.spec
+                .prompt_replacements_for(&self.context, &preprocessed, Modality::Image)?;
+        if replacements.len() != frames.len() {
+            bail_multimodal!(
+                "number of image prompt replacements {} does not match number of images {}",
+                replacements.len(),
+                frames.len()
+            );
+        }
+        let items = self.build_image_items(preprocessed, &frames, uuids, model_dtype)?;
+
+        Ok(PreparedMedia {
+            modality: Modality::Image,
+            placeholder: support.placeholder.clone(),
+            replacements,
+            items,
+        })
+    }
+
+    /// Preprocess fetched image frames with the model's resolved vision
+    /// processor.
+    ///
+    /// The processor work is CPU-heavy relative to request wiring, so it runs
+    /// in a blocking task and returns owned tensors ready for wire
+    /// conversion.
+    async fn preprocess_images(
+        &self,
+        support: &ModalitySupport,
+        image_frames: &[Arc<ImageFrame>],
+    ) -> Result<PreprocessedEncoderInputs> {
+        let config = support.config.clone();
+        let processor = support.processor;
+        let images = image_frames.iter().map(|frame| frame.data().clone()).collect::<Vec<_>>();
+
+        // TODO: is it still necessary given that we've already in a dedicated runtime?
+        tokio::task::spawn_blocking(move || Ok(processor.preprocess(&images, &config)?))
+            .await
+            .map_err(|error| multimodal!("image preprocessing task failed: {error}"))?
+    }
+
+    /// Convert one batch of preprocessed image tensors into per-item engine
+    /// kwargs.
+    ///
+    /// Tensor fields are sliced per item according to the model spec's field
+    /// layout declarations.
+    fn build_image_items(
+        &self,
+        preprocessed: PreprocessedEncoderInputs,
+        frames: &[Arc<ImageFrame>],
+        uuids: Vec<Option<String>>,
+        model_dtype: ModelDtype,
+    ) -> Result<Vec<PreparedItem>> {
+        let len = frames.len();
+        let tensors = tensor::collect_tensors(preprocessed, "pixel_values", model_dtype)?;
+
+        let mut items = Vec::with_capacity(len);
+        for (index, (frame, uuid)) in izip!(frames, uuids).enumerate() {
+            let mut data = MmKwargsItem::new();
+            for (key, tensor) in &tensors {
+                let keep_on_cpu = self.spec.keep_on_cpu_keys.contains(key);
+                let (value, field) = match self.spec.field_layouts.get(key) {
+                    Some(FieldLayout::Batched) => (
+                        tensor.batched_value_at(index)?,
+                        MmField::Batched(MmBatchedField { keep_on_cpu }),
+                    ),
+                    Some(FieldLayout::Flat { sizes_key }) => {
+                        let sizes = tensors.get(sizes_key).ok_or_else(|| {
+                            multimodal!("flat tensor sizes key `{sizes_key}` is missing")
+                        })?;
+                        let (start, end) = tensor::flat_range_for_index(sizes, sizes_key, index)?;
+                        (
+                            tensor.flat_value_range(start, end)?,
+                            MmField::Flat(MmFlatField {
+                                slices: vec![MmSlice::Slice(SliceSpec {
+                                    start: Some(0),
+                                    stop: Some((end - start) as isize),
+                                    step: None,
+                                })],
+                                dim: 0,
+                                keep_on_cpu,
+                            }),
+                        )
+                    }
+                    None => (
+                        tensor.clone(),
+                        MmField::Shared(MmSharedField {
+                            batch_size: len,
+                            keep_on_cpu,
+                        }),
+                    ),
+                };
+
+                data.insert(
+                    key.clone(),
+                    MmFieldElem {
+                        data: Some(value.try_into()?),
+                        field,
+                    },
+                );
+            }
+
+            items.push(PreparedItem {
+                data,
+                hash: frame.hash.clone(),
+                uuid,
+            });
+        }
+
+        Ok(items)
+    }
+}

rust/src/chat/src/multimodal/tensor.rs

@@ -1,7 +1,7 @@
 use std::collections::HashMap;
 
 use half::{bf16, f16};
-use llm_multimodal::{ModelSpecificValue, PreprocessedEncoderInputs as PreprocessedImages};
+use llm_multimodal::{ModelSpecificValue, PreprocessedEncoderInputs};
 use vllm_engine_core_client::protocol::dtype::ModelDtype;
 use vllm_engine_core_client::protocol::multimodal::MmKwargValue as ProtocolKwargValue;
 use vllm_engine_core_client::protocol::tensor::{ShapeExt as _, WireTensor};
@@ -25,25 +25,31 @@ pub(super) enum KwargValue {
     Passthrough(ProtocolKwargValue),
 }
 
-/// Collect `pixel_values` and model-specific outputs into one tensor map.
+/// Collect the primary encoder input and model-specific outputs into one
+/// tensor map.
+///
+/// `primary_key` names the encoder-input tensor as the model's forward kwargs
+/// expect it (e.g. `pixel_values` for images, `pixel_values_videos` for
+/// videos).
 pub(super) fn collect_tensors(
-    preprocessed: PreprocessedImages,
+    preprocessed: PreprocessedEncoderInputs,
+    primary_key: &str,
     float_dtype: ModelDtype,
 ) -> Result<HashMap<String, KwargValue>> {
-    let PreprocessedImages {
+    let PreprocessedEncoderInputs {
         encoder_input,
         model_specific,
         ..
     } = preprocessed;
 
-    let pixel_values = {
+    let primary_value = {
         let shape = encoder_input.shape().to_vec();
         let data = encoder_input.into_iter().collect();
         KwargValue::from_f32_tensor(data, shape, float_dtype)?
     };
 
     let mut tensors = HashMap::new();
-    tensors.insert("pixel_values".to_string(), pixel_values);
+    tensors.insert(primary_key.to_string(), primary_value);
     for (key, value) in model_specific {
         tensors.insert(key, KwargValue::from_model_specific(value, float_dtype)?);
     }
@@ -124,10 +130,22 @@ impl TryFrom<KwargValue> for ProtocolKwargValue {
 }
 
 impl KwargValue {
-    /// Extract one image from a batched tensor field.
+    /// First-axis length for tensor values; `None` for passthrough kwargs.
+    pub(super) fn first_dim(&self) -> Option<usize> {
+        match self {
+            Self::F32Tensor { shape, .. }
+            | Self::F16Tensor { shape, .. }
+            | Self::Bf16Tensor { shape, .. }
+            | Self::I64Tensor { shape, .. }
+            | Self::U32Tensor { shape, .. } => shape.first().copied(),
+            Self::Passthrough(_) => None,
+        }
+    }
+
+    /// Extract one media item from a batched tensor field.
     ///
-    /// Batched fields use their first axis as image index and drop that axis in
-    /// the per-feature value, matching vLLM's batched-field semantics.
+    /// Batched fields use their first axis as media-item index and drop that
+    /// axis in the per-feature value, matching vLLM's batched-field semantics.
     pub(super) fn batched_value_at(&self, index: usize) -> Result<Self> {
         match self {
             Self::F32Tensor { data, shape } => {
@@ -154,9 +172,9 @@ impl KwargValue {
         }
     }
 
-    /// Extract one image's variable-length range from a flat tensor field.
+    /// Extract one media item's variable-length range from a flat tensor field.
     ///
-    /// Flat fields keep the first axis as the sliced length for this image.
+    /// Flat fields keep the first axis as the sliced length for this item.
     pub(super) fn flat_value_range(&self, start: usize, end: usize) -> Result<Self> {
         match self {
             Self::F32Tensor { data, shape } => {
@@ -184,10 +202,10 @@ impl KwargValue {
     }
 }
 
-/// Compute the first-axis range for one image in a flat tensor.
+/// Compute the first-axis range for one media item in a flat tensor.
 ///
 /// `sizes_key` names a companion tensor whose entries are cumulative slice
-/// sizes per image.
+/// sizes per media item.
 pub(super) fn flat_range_for_index(
     sizes: &KwargValue,
     sizes_key: &str,
@@ -195,7 +213,7 @@ pub(super) fn flat_range_for_index(
 ) -> Result<(usize, usize)> {
     let sizes = tensor_as_usize_vec(sizes)?;
     let size = *sizes.get(index).ok_or_else(|| {
-        multimodal!("flat tensor sizes key `{sizes_key}` has no entry for image {index}")
+        multimodal!("flat tensor sizes key `{sizes_key}` has no entry for media item {index}")
     })?;
     let start = sizes[..index].iter().sum::<usize>();
     Ok((start, start + size))

rust/src/chat/src/multimodal/video.rs

@@ -0,0 +1,316 @@
+//! Video-modality preparation: per-clip preprocessing, config resolution,
+//! and per-item feature build.
+
+use std::sync::Arc;
+
+use itertools::izip;
+use llm_multimodal::{FieldLayout, Modality, PreprocessedEncoderInputs, VideoClip};
+use thiserror_ext::AsReport as _;
+use tracing::warn;
+use vllm_engine_core_client::protocol::dtype::ModelDtype;
+use vllm_engine_core_client::protocol::multimodal::{
+    MmBatchedField, MmField, MmFieldElem, MmFlatField, MmKwargsItem, MmSharedField, MmSlice,
+    SliceSpec,
+};
+
+use super::{ModalitySupport, MultimodalModelInfo, PreparedItem, PreparedMedia, tensor};
+use crate::error::{Error, Result, bail_multimodal, multimodal};
+
+/// Forward-kwargs name of the primary video encoder input.
+///
+/// Video-capable vLLM models read `pixel_values_videos` alongside
+/// `video_grid_thw`, mirroring the HF processor output naming.
+const VIDEO_PRIMARY_KEY: &str = "pixel_values_videos";
+
+impl MultimodalModelInfo {
+    /// Preprocess fetched video clips one at a time and build per-item
+    /// features.
+    ///
+    /// Unlike images, each clip runs through the preprocessor independently
+    /// (a batch of one), so its tensors are complete per item and need no
+    /// cross-item slicing.
+    pub(super) async fn prepare_videos(
+        &self,
+        clips: Vec<Arc<VideoClip>>,
+        uuids: Vec<Option<String>>,
+        model_dtype: ModelDtype,
+    ) -> Result<PreparedMedia> {
+        let support = self.video.as_ref().ok_or_else(|| Error::UnsupportedModality {
+            modality: Modality::Video.to_string(),
+        })?;
+        let mut replacements = Vec::with_capacity(clips.len());
+        let mut items = Vec::with_capacity(clips.len());
+
+        for (clip, uuid) in izip!(&clips, uuids) {
+            let preprocessed = self.preprocess_video_clip(support, Arc::clone(clip)).await?;
+            let mut clip_replacements =
+                self.spec
+                    .prompt_replacements_for(&self.context, &preprocessed, Modality::Video)?;
+            if clip_replacements.len() != 1 {
+                bail_multimodal!(
+                    "expected exactly one prompt replacement per video clip, got {}",
+                    clip_replacements.len()
+                );
+            }
+            replacements.push(clip_replacements.pop().unwrap());
+            items.push(self.build_video_item(
+                preprocessed,
+                clip.hash.clone(),
+                uuid,
+                model_dtype,
+            )?);
+        }
+
+        Ok(PreparedMedia {
+            modality: Modality::Video,
+            placeholder: support.placeholder.clone(),
+            replacements,
+            items,
+        })
+    }
+
+    /// Preprocess one decoded video clip with the model's resolved vision
+    /// processor.
+    async fn preprocess_video_clip(
+        &self,
+        support: &ModalitySupport,
+        clip: Arc<VideoClip>,
+    ) -> Result<PreprocessedEncoderInputs> {
+        let config = support.config.clone();
+        let processor = support.processor;
+
+        tokio::task::spawn_blocking(move || {
+            // Prefer the borrowed-RGB fast path, which avoids materializing a
+            // `DynamicImage` per sampled frame after media decode.
+            if let Some(rgb_video) = clip.rgb_video() {
+                match rgb_video.frame_refs() {
+                    Ok(frame_refs) => match processor.preprocess_video_rgb(&frame_refs, &config) {
+                        Ok(preprocessed) => return Ok(preprocessed),
+                        Err(error) => warn!(
+                            error = %error.as_report(),
+                            "RGB video preprocessing fast path failed; falling back to materialized frames"
+                        ),
+                    },
+                    Err(error) => warn!(
+                        error,
+                        "RGB video frame refs are invalid; falling back to materialized frames"
+                    ),
+                }
+            }
+
+            let frames = clip.materialized_frames().map_err(|error| multimodal!("{error}"))?;
+            Ok(processor.preprocess_video(&frames, &config)?)
+        })
+        .await
+        .map_err(|error| multimodal!("video preprocessing task failed: {error}"))?
+    }
+
+    /// Convert one preprocessed video clip into engine kwargs.
+    ///
+    /// The clip is a batch of one, so no per-item slicing is required: the
+    /// primary tensor ships as a full-range flat field (the engine re-batches
+    /// flat fields by concatenating along the declared dim, matching vLLM's
+    /// `flat_from_sizes` treatment of video patches), and batched metadata
+    /// tensors drop their singleton batch axis.
+    fn build_video_item(
+        &self,
+        preprocessed: PreprocessedEncoderInputs,
+        hash: String,
+        uuid: Option<String>,
+        model_dtype: ModelDtype,
+    ) -> Result<PreparedItem> {
+        let tensors = tensor::collect_tensors(preprocessed, VIDEO_PRIMARY_KEY, model_dtype)?;
+
+        let mut data = MmKwargsItem::new();
+        for (key, tensor) in tensors {
+            let keep_on_cpu = self.spec.keep_on_cpu_keys.contains(&key);
+            let (value, field) = if key == VIDEO_PRIMARY_KEY {
+                let len = tensor
+                    .first_dim()
+                    .ok_or_else(|| multimodal!("video encoder input `{key}` is not a tensor"))?;
+                (
+                    tensor,
+                    MmField::Flat(MmFlatField {
+                        slices: vec![MmSlice::Slice(SliceSpec {
+                            start: Some(0),
+                            stop: Some(len as isize),
+                            step: None,
+                        })],
+                        dim: 0,
+                        keep_on_cpu,
+                    }),
+                )
+            } else if matches!(
+                self.spec.field_layouts.get(&key),
+                Some(FieldLayout::Batched)
+            ) {
+                (
+                    tensor.batched_value_at(0)?,
+                    MmField::Batched(MmBatchedField { keep_on_cpu }),
+                )
+            } else {
+                (
+                    tensor,
+                    MmField::Shared(MmSharedField {
+                        batch_size: 1,
+                        keep_on_cpu,
+                    }),
+                )
+            };
+
+            data.insert(
+                key,
+                MmFieldElem {
+                    data: Some(value.try_into()?),
+                    field,
+                },
+            );
+        }
+
+        Ok(PreparedItem { data, hash, uuid })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::collections::HashMap;
+    use std::sync::Arc;
+
+    use llm_multimodal::ModelSpecificValue;
+    use ndarray::ArrayD;
+    use vllm_engine_core_client::protocol::multimodal::MmKwargValue;
+
+    use super::super::tests::{
+        QWEN3_IMAGE_PAD_ID, QWEN3_VIDEO_PAD_ID, qwen3_vl_info, qwen3_vl_tokenizer,
+    };
+    use super::super::{MultimodalConfigFiles, MultimodalModelInfo};
+    use super::*;
+
+    #[test]
+    fn from_paths_resolves_video_config_from_dedicated_file_or_processor_config() {
+        let dir = tempfile::tempdir().unwrap();
+        let config_path = dir.path().join("config.json");
+        std::fs::write(
+            &config_path,
+            serde_json::json!({
+                "model_type": "qwen3_vl",
+                "image_token_id": QWEN3_IMAGE_PAD_ID,
+                "video_token_id": QWEN3_VIDEO_PAD_ID,
+            })
+            .to_string(),
+        )
+        .unwrap();
+
+        let info_for = |files: MultimodalConfigFiles<'_>| {
+            MultimodalModelInfo::from_paths(
+                "qwen3-vl-test".to_string(),
+                Some("qwen3_vl".to_string()),
+                files,
+                Arc::new(qwen3_vl_tokenizer()),
+            )
+        };
+
+        // Dedicated video preprocessor config file.
+        let video_config_path = dir.path().join("video_preprocessor_config.json");
+        std::fs::write(&video_config_path, r#"{"size":{"shortest_edge":128}}"#).unwrap();
+        let info = info_for(MultimodalConfigFiles {
+            config: Some(&config_path),
+            video_preprocessor_config: Some(&video_config_path),
+            ..Default::default()
+        })
+        .unwrap()
+        .unwrap();
+        assert!(info.video.is_some());
+
+        // `video_processor` section of the combined processor config.
+        let processor_config_path = dir.path().join("processor_config.json");
+        std::fs::write(
+            &processor_config_path,
+            r#"{"video_processor":{"size":{"shortest_edge":128}}}"#,
+        )
+        .unwrap();
+        let info = info_for(MultimodalConfigFiles {
+            config: Some(&config_path),
+            processor_config: Some(&processor_config_path),
+            ..Default::default()
+        })
+        .unwrap()
+        .unwrap();
+        assert!(info.video.is_some());
+
+        // Neither source: video support still resolves on the image config.
+        let info = info_for(MultimodalConfigFiles {
+            config: Some(&config_path),
+            ..Default::default()
+        })
+        .unwrap()
+        .unwrap();
+        assert!(info.video.is_some());
+
+        // Malformed dedicated file is a real error, not a silent fallback.
+        std::fs::write(&video_config_path, r#"{"size""#).unwrap();
+        let error = match info_for(MultimodalConfigFiles {
+            config: Some(&config_path),
+            video_preprocessor_config: Some(&video_config_path),
+            ..Default::default()
+        }) {
+            Err(error) => error,
+            Ok(_) => panic!("malformed video preprocessor config should fail"),
+        };
+        assert!(matches!(
+            error,
+            Error::Multimodal(message)
+                if message.contains("failed to parse video_preprocessor_config.json")
+        ));
+    }
+
+    #[test]
+    fn build_video_item_names_primary_tensor_and_layouts() {
+        let info = qwen3_vl_info();
+        // One clip flattened to 6 patches with 4 features each.
+        let preprocessed = PreprocessedEncoderInputs {
+            encoder_input: ArrayD::zeros(vec![6, 4]),
+            feature_token_counts: vec![6],
+            item_sizes: vec![(32, 32)],
+            model_specific: HashMap::from([
+                (
+                    "video_grid_thw".to_string(),
+                    ModelSpecificValue::int_2d(vec![1, 2, 3], 1, 3),
+                ),
+                (
+                    "patches_per_video".to_string(),
+                    ModelSpecificValue::int_1d(vec![6]),
+                ),
+            ]),
+        };
+
+        let item = info
+            .build_video_item(
+                preprocessed,
+                "<hash>".to_string(),
+                None,
+                ModelDtype::Float32,
+            )
+            .unwrap();
+
+        let primary = &item.data[VIDEO_PRIMARY_KEY];
+        assert!(matches!(
+            &primary.field,
+            MmField::Flat(MmFlatField { slices, dim: 0, .. })
+                if matches!(
+                    slices.as_slice(),
+                    [MmSlice::Slice(SliceSpec { start: Some(0), stop: Some(6), step: None })]
+                )
+        ));
+
+        // Batched metadata drops its singleton batch axis per item.
+        let grid = &item.data["video_grid_thw"];
+        assert!(matches!(&grid.field, MmField::Batched(_)));
+        let MmKwargValue::Tensor(grid_tensor) = grid.data.as_ref().unwrap() else {
+            panic!("expected tensor value for video_grid_thw");
+        };
+        assert_eq!(grid_tensor.shape, vec![3]);
+
+        assert_eq!(item.hash, "<hash>");
+    }
+}

rust/src/chat/src/renderer/hf/mod.rs

@@ -31,9 +31,14 @@ pub use template::{load_chat_template, resolve_chat_template};
 
 pub use self::format::ChatTemplateContentFormatOption;
 
-#[derive(Debug, Clone)]
+/// Template-visible placeholder tokens per supported modality.
+///
+/// A `None` token means the loaded model does not support that modality, and
+/// content parts of that modality are rejected during rendering.
+#[derive(Debug, Clone, Default)]
 pub struct MultimodalRenderInfo {
-    pub placeholder_token: String,
+    pub image_token: Option<String>,
+    pub video_token: Option<String>,
 }
 
 /// Hugging Face chat-template renderer backed by the local Jinja chat-template
@@ -254,6 +259,7 @@ enum TemplateContent {
 enum TemplateContentPart {
     Text { text: String },
     Image,
+    Video,
 }
 
 #[derive(Debug, Serialize)]
@@ -417,9 +423,17 @@ fn to_template_openai_content(
                 }
                 // All multimodal contents are normalized to `{ "type": <modality> }`.
                 ChatContentPart::ImageUrl { .. } => {
-                    multimodal.ok_or(Error::UnsupportedMultimodalContent("image_url"))?;
+                    multimodal
+                        .and_then(|multimodal| multimodal.image_token.as_ref())
+                        .ok_or(Error::UnsupportedMultimodalContent("image_url"))?;
                     Ok(TemplateContentPart::Image)
                 }
+                ChatContentPart::VideoUrl { .. } => {
+                    multimodal
+                        .and_then(|multimodal| multimodal.video_token.as_ref())
+                        .ok_or(Error::UnsupportedMultimodalContent("video_url"))?;
+                    Ok(TemplateContentPart::Video)
+                }
             })
             .collect(),
     }
@@ -437,9 +451,16 @@ fn to_template_string_content(
                 match part {
                     ChatContentPart::Text { text } => out.push_str(text),
                     ChatContentPart::ImageUrl { .. } => {
-                        let multimodal =
-                            multimodal.ok_or(Error::UnsupportedMultimodalContent("image_url"))?;
-                        out.push_str(&multimodal.placeholder_token);
+                        let image_token = multimodal
+                            .and_then(|multimodal| multimodal.image_token.as_ref())
+                            .ok_or(Error::UnsupportedMultimodalContent("image_url"))?;
+                        out.push_str(image_token);
+                    }
+                    ChatContentPart::VideoUrl { .. } => {
+                        let video_token = multimodal
+                            .and_then(|multimodal| multimodal.video_token.as_ref())
+                            .ok_or(Error::UnsupportedMultimodalContent("video_url"))?;
+                        out.push_str(video_token);
                     }
                 }
             }
@@ -468,7 +489,7 @@ fn append_continue_final_message_tag(message: &mut TemplateMessage) -> Result<St
         // Pick the last text part in the message.
         TemplateContent::OpenAi(parts) => parts.iter_mut().rev().find_map(|part| match part {
             TemplateContentPart::Text { text } => Some(text),
-            TemplateContentPart::Image => None,
+            TemplateContentPart::Image | TemplateContentPart::Video => None,
         }),
     };
     let text = text.ok_or_else(|| {
@@ -577,7 +598,8 @@ mod tests {
     ) -> Result<crate::RenderedPrompt> {
         HfChatRenderer::new(Some(template.to_string()), HashMap::new(), content_format)?
             .with_multimodal(Some(MultimodalRenderInfo {
-                placeholder_token: "<image>".to_string(),
+                image_token: Some("<image>".to_string()),
+                video_token: Some("<video>".to_string()),
             }))
             .render(request)
     }
@@ -590,6 +612,14 @@ mod tests {
         ])])
     }
 
+    fn video_request() -> ChatRequest {
+        sample_request(vec![ChatMessage::user(vec![
+            ChatContentPart::text("a"),
+            ChatContentPart::video_url("https://example.com/demo.mp4"),
+            ChatContentPart::text("b"),
+        ])])
+    }
+
     #[test]
     fn string_content_format_replaces_image_with_placeholder_text() {
         let rendered = render_mm(
@@ -602,6 +632,18 @@ mod tests {
         assert_eq!(rendered.prompt, Prompt::Text("a<image>b".to_string()));
     }
 
+    #[test]
+    fn string_content_format_replaces_video_with_placeholder_text() {
+        let rendered = render_mm(
+            "{{ messages[0].content }}",
+            &video_request(),
+            ChatTemplateContentFormatOption::String,
+        )
+        .unwrap();
+
+        assert_eq!(rendered.prompt, Prompt::Text("a<video>b".to_string()));
+    }
+
     #[test]
     fn openai_content_format_normalizes_image_url_for_template() {
         let rendered = render_mm(
@@ -614,6 +656,39 @@ mod tests {
         assert_eq!(rendered.prompt, Prompt::Text("a<|image_pad|>b".to_string()));
     }
 
+    #[test]
+    fn openai_content_format_normalizes_video_url_for_template() {
+        let rendered = render_mm(
+            "{% for item in messages[0].content %}{% if item.type == 'video' %}<|video_pad|>{% else %}{{ item.text }}{% endif %}{% endfor %}",
+            &video_request(),
+            ChatTemplateContentFormatOption::OpenAi,
+        )
+        .unwrap();
+
+        assert_eq!(rendered.prompt, Prompt::Text("a<|video_pad|>b".to_string()));
+    }
+
+    #[test]
+    fn video_parts_are_rejected_when_model_lacks_video_support() {
+        let error = HfChatRenderer::new(
+            Some("{{ messages[0].content }}".to_string()),
+            HashMap::new(),
+            ChatTemplateContentFormatOption::String,
+        )
+        .unwrap()
+        .with_multimodal(Some(MultimodalRenderInfo {
+            image_token: Some("<image>".to_string()),
+            video_token: None,
+        }))
+        .render(&video_request())
+        .unwrap_err();
+
+        assert!(matches!(
+            error,
+            Error::UnsupportedMultimodalContent("video_url")
+        ));
+    }
+
     #[test]
     fn chat_template_supports_pycompat_templates() {
         let request = sample_request(vec![ChatMessage::text(ChatRole::User, "<think>hello")]);

rust/src/chat/src/request.rs

@@ -36,7 +36,13 @@ pub enum ChatContentPart {
         detail: Option<ImageDetail>,
         uuid: Option<String>,
     },
+    /// One video URL/data URL content block.
+    VideoUrl {
+        video_url: String,
+        uuid: Option<String>,
+    },
     // ImageData...
+    // VideoData...
     // ImageEmbeds...
 }
 
@@ -55,12 +61,21 @@ impl ChatContentPart {
         }
     }
 
+    /// Construct one video URL content part with the given URL string.
+    pub fn video_url(video_url: impl Into<String>) -> Self {
+        Self::VideoUrl {
+            video_url: video_url.into(),
+            uuid: None,
+        }
+    }
+
     /// Return the text content of this part when it's a text block, or an
     /// "unsupported multimodal content" error otherwise.
     pub(crate) fn as_text(&self) -> Result<&str> {
         match self {
             Self::Text { text } => Ok(text),
             Self::ImageUrl { .. } => Err(Error::UnsupportedMultimodalContent("image_url")),
+            Self::VideoUrl { .. } => Err(Error::UnsupportedMultimodalContent("video_url")),
         }
     }
 
@@ -73,7 +88,7 @@ impl ChatContentPart {
     pub(crate) fn is_multimodal(&self) -> bool {
         match self {
             Self::Text { .. } => false,
-            Self::ImageUrl { .. } => true,
+            Self::ImageUrl { .. } | Self::VideoUrl { .. } => true,
         }
     }
 }
@@ -555,6 +570,26 @@ mod tests {
         assert_eq!(content, ChatContent::Text("hello".to_string()));
     }
 
+    #[test]
+    fn chat_content_video_url_part_round_trips_through_serde() {
+        let content = ChatContent::Parts(vec![ChatContentPart::VideoUrl {
+            video_url: "https://example.com/demo.mp4".to_string(),
+            uuid: Some("video-1".to_string()),
+        }]);
+
+        let value = to_value(&content).unwrap();
+        assert_eq!(
+            value,
+            json!([{
+                "type": "video_url",
+                "video_url": "https://example.com/demo.mp4",
+                "uuid": "video-1",
+            }])
+        );
+        let decoded: ChatContent = serde_json::from_value(value).unwrap();
+        assert_eq!(decoded, content);
+    }
+
     #[test]
     fn chat_content_deserializes_from_openai_text_blocks() {
         let content: ChatContent =

rust/src/server/src/routes/openai/chat_completions/convert.rs

@@ -290,7 +290,10 @@ fn convert_content(content: MessageContent) -> Result<ChatContent, ApiError> {
                     detail: image_url.detail,
                     uuid,
                 }),
-                _ => bail_invalid_request!("Only text and image_url content parts are supported."),
+                ContentPart::VideoUrl { video_url, uuid } => Ok(ChatContentPart::VideoUrl {
+                    video_url: video_url.url,
+                    uuid,
+                }),
             })
             .try_collect()
             .map(ChatContent::Parts),
@@ -772,28 +775,34 @@ mod tests {
     }
 
     #[test]
-    fn prepare_chat_request_rejects_video_content_parts() {
+    fn prepare_chat_request_accepts_video_content_parts() {
         let request = ChatCompletionRequest {
             messages: vec![ChatMessage::User {
                 content: MessageContent::Parts(vec![ContentPart::VideoUrl {
                     video_url: VideoUrl {
                         url: "https://example.com/video.mp4".to_string(),
                     },
+                    uuid: Some("video-uuid".to_string()),
                 }]),
                 name: None,
             }],
             ..base_request()
         };
 
-        let error = prepare_chat_request(
+        let prepared = prepare_chat_request(
             request,
             &served(&["Qwen/Qwen1.5-0.5B-Chat"]),
             ResolvedRequestContext::default(),
         )
-        .unwrap_err();
+        .expect("request is valid");
 
-        expect!["Only text and image_url content parts are supported."]
-            .assert_eq(&error.to_error_response().error.message);
+        assert_eq!(
+            prepared.chat_request.messages,
+            vec![VllmChatMessage::user(vec![ChatContentPart::VideoUrl {
+                video_url: "https://example.com/video.mp4".to_string(),
+                uuid: Some("video-uuid".to_string()),
+            }])]
+        );
     }
 
     #[test]

rust/src/server/src/routes/openai/utils/types.rs

@@ -91,7 +91,11 @@ pub enum ContentPart {
         uuid: Option<String>,
     },
     #[serde(rename = "video_url")]
-    VideoUrl { video_url: VideoUrl },
+    VideoUrl {
+        video_url: VideoUrl,
+        #[serde(skip_serializing_if = "Option::is_none")]
+        uuid: Option<String>,
+    },
 }
 
 #[serde_with::skip_serializing_none]

rust/src/server/src/routes/tests.rs

@@ -504,7 +504,7 @@ impl ChatRenderer for FakeChatBackend {
         let placeholder = self
             .multimodal_model_info
             .as_ref()
-            .map(|info| info.placeholder_token())
+            .and_then(|info| info.placeholder_token(llm_multimodal::Modality::Image))
             .unwrap_or("<image>");
         let mut prompt = String::new();
         for message in &request.messages {
@@ -544,7 +544,9 @@ fn render_fake_content(content: &ChatContent, placeholder: &str) -> vllm_chat::R
             for part in parts {
                 match part {
                     ChatContentPart::Text { text } => out.push_str(text),
-                    ChatContentPart::ImageUrl { .. } => out.push_str(placeholder),
+                    ChatContentPart::ImageUrl { .. } | ChatContentPart::VideoUrl { .. } => {
+                        out.push_str(placeholder)
+                    }
                 }
             }
             out
@@ -565,8 +567,10 @@ fn qwen_multimodal_model_info() -> vllm_chat::multimodal::MultimodalModelInfo {
     let info = vllm_chat::multimodal::MultimodalModelInfo::from_paths(
         "qwen2-vl-test".to_string(),
         Some("qwen2_vl".to_string()),
-        Some(&config_path),
-        None,
+        vllm_chat::multimodal::MultimodalConfigFiles {
+            config: Some(&config_path),
+            ..Default::default()
+        },
         Arc::new(fake_chat_tokenizer()),
     )
     .expect("load multimodal info")

rust/src/text/src/backend/hf/model_files.rs

@@ -41,6 +41,10 @@ pub struct ResolvedModelFiles {
     pub tokenizer_config_path: Option<PathBuf>,
     pub generation_config_path: Option<PathBuf>,
     pub preprocessor_config_path: Option<PathBuf>,
+    /// Video-specific preprocessor config, when provided by the model repo.
+    pub video_preprocessor_config_path: Option<PathBuf>,
+    /// Combined processor config, which may embed a `video_processor` section.
+    pub processor_config_path: Option<PathBuf>,
     pub chat_template_path: Option<PathBuf>,
     pub config_path: Option<PathBuf>,
 }
@@ -70,6 +74,11 @@ fn resolve_local_model_files(model_dir: &Path) -> Result<ResolvedModelFiles> {
         tokenizer_config_path,
         generation_config_path: local_file_if_exists(model_dir, "generation_config.json"),
         preprocessor_config_path: local_file_if_exists(model_dir, "preprocessor_config.json"),
+        video_preprocessor_config_path: local_file_if_exists(
+            model_dir,
+            "video_preprocessor_config.json",
+        ),
+        processor_config_path: local_file_if_exists(model_dir, "processor_config.json"),
         chat_template_path: discover_chat_template_in_dir(model_dir),
         config_path: local_file_if_exists(model_dir, "config.json"),
     })
@@ -107,6 +116,10 @@ async fn resolve_remote_model_files(model_id: &str) -> Result<ResolvedModelFiles
         download_if_present(&repo, model_id, &siblings, "generation_config.json").await?;
     let preprocessor_config_path =
         download_if_present(&repo, model_id, &siblings, "preprocessor_config.json").await?;
+    let video_preprocessor_config_path =
+        download_if_present(&repo, model_id, &siblings, "video_preprocessor_config.json").await?;
+    let processor_config_path =
+        download_if_present(&repo, model_id, &siblings, "processor_config.json").await?;
     let chat_template_name = siblings
         .contains("chat_template.json")
         .then_some("chat_template.json")
@@ -123,6 +136,8 @@ async fn resolve_remote_model_files(model_id: &str) -> Result<ResolvedModelFiles
         tokenizer_config_path,
         generation_config_path,
         preprocessor_config_path,
+        video_preprocessor_config_path,
+        processor_config_path,
         chat_template_path,
         config_path,
     })
@@ -143,6 +158,8 @@ fn resolve_cached_model_files(model_id: &str) -> Result<Option<ResolvedModelFile
     })?;
     let generation_config_path = cache_repo.get("generation_config.json");
     let preprocessor_config_path = cache_repo.get("preprocessor_config.json");
+    let video_preprocessor_config_path = cache_repo.get("video_preprocessor_config.json");
+    let processor_config_path = cache_repo.get("processor_config.json");
     let chat_template_path = discover_chat_template_in_dir(model_dir);
     let config_path = cache_repo.get("config.json");
 
@@ -151,6 +168,8 @@ fn resolve_cached_model_files(model_id: &str) -> Result<Option<ResolvedModelFile
         tokenizer_config_path,
         generation_config_path,
         preprocessor_config_path,
+        video_preprocessor_config_path,
+        processor_config_path,
         chat_template_path,
         config_path,
     }))