karylab_agents/vllm
1
0
Fork 0
forked from Karylab-cklius/vllm
Code Pull Requests 2 Actions 2 Packages Activity

[Rust Frontend] Integrate MM audio support (#48554)

Browse Source 
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Bugen Zhao
2026-07-15 15:00:17 +08:00
committed by GitHub
co-authored by Isotr0py mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent adce068118
commit e281ac663a
18 changed files with 1275 additions and 258 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rust/Cargo.lock
Generated
+403 -11
View File
@@ -182,7 +182,7 @@ dependencies = [
"hex",
"hmac",
"rand 0.9.2",
"reqwest",
"reqwest 0.12.28",
"reqwest-eventsource",
"secrecy",
"serde",
@@ -379,6 +379,12 @@ version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
[[package]]
name = "base64ct"
version = "1.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06"
[[package]]
name = "bit-set"
version = "0.5.3"
@@ -665,6 +671,35 @@ version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b"
[[package]]
name = "cookie"
version = "0.18.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747"
dependencies = [
"percent-encoding",
"time",
"version_check",
]
[[package]]
name = "cookie_store"
version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "15b2c103cf610ec6cae3da84a766285b42fd16aad564758459e6ecf128c75206"
dependencies = [
"cookie",
"document-features",
"idna",
"indexmap 2.13.0",
"log",
"serde",
"serde_derive",
"serde_json",
"time",
"url",
]
[[package]]
name = "core-foundation"
version = "0.9.4"
@@ -888,6 +923,16 @@ dependencies = [
"serde",
]
[[package]]
name = "der"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a69dedd701da44b0536442edf09c81a64b0ab97a7a4a5e3d1971f00027cbc63d"
dependencies = [
"pem-rfc7468",
"zeroize",
]
[[package]]
name = "deranged"
version = "0.5.8"
@@ -1171,6 +1216,12 @@ dependencies = [
"once_cell",
]
[[package]]
name = "extended"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365"
[[package]]
name = "fancy-regex"
version = "0.13.0"
@@ -1589,13 +1640,15 @@ dependencies = [
"indicatif",
"libc",
"log",
"native-tls",
"num_cpus",
"rand 0.9.2",
"reqwest",
"reqwest 0.12.28",
"serde",
"serde_json",
"thiserror 2.0.18",
"tokio",
"ureq",
"windows-sys 0.61.2",
]
@@ -2167,22 +2220,27 @@ checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092"
[[package]]
name = "llm-multimodal"
version = "1.7.1"
source = "git+https://github.com/smg-project/llm-multimodal?rev=c8a29dcc755139fdc26185f400ea48c6d6d48273#c8a29dcc755139fdc26185f400ea48c6d6d48273"
source = "git+https://github.com/smg-project/llm-multimodal?rev=7df38e53f99aefaebe86934f010aa8084ec99b2f#7df38e53f99aefaebe86934f010aa8084ec99b2f"
dependencies = [
"anyhow",
"base64 0.22.1",
"blake3",
"bytes",
"cc",
"fast_image_resize",
"hf-hub",
"image",
"libloading",
"ndarray 0.17.2",
"once_cell",
"reqwest",
"pkg-config",
"rayon",
"reqwest 0.13.4",
"rustfft",
"serde",
"serde_json",
"serde_with",
"symphonia",
"tempfile",
"thiserror 2.0.18",
"tokio",
@@ -2740,6 +2798,15 @@ dependencies = [
"pkg-config",
]
[[package]]
name = "pem-rfc7468"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9"
dependencies = [
"base64ct",
]
[[package]]
name = "percent-encoding"
version = "2.3.2"
@@ -3245,9 +3312,9 @@ checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3"
[[package]]
name = "rayon"
version = "1.11.0"
version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f"
checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
dependencies = [
"either",
"rayon-core",
@@ -3346,6 +3413,12 @@ dependencies = [
"regex-syntax",
]
[[package]]
name = "regex-lite"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973"
[[package]]
name = "regex-syntax"
version = "0.8.10"
@@ -3392,7 +3465,43 @@ dependencies = [
"url",
"wasm-bindgen",
"wasm-bindgen-futures",
"wasm-streams",
"wasm-streams 0.4.2",
"web-sys",
]
[[package]]
name = "reqwest"
version = "0.13.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3"
dependencies = [
"base64 0.22.1",
"bytes",
"futures-core",
"futures-util",
"http",
"http-body",
"http-body-util",
"hyper",
"hyper-tls",
"hyper-util",
"js-sys",
"log",
"native-tls",
"percent-encoding",
"pin-project-lite",
"rustls-pki-types",
"sync_wrapper",
"tokio",
"tokio-native-tls",
"tokio-util",
"tower",
"tower-http",
"tower-service",
"url",
"wasm-bindgen",
"wasm-bindgen-futures",
"wasm-streams 0.5.0",
"web-sys",
]
@@ -3408,7 +3517,7 @@ dependencies = [
"mime",
"nom",
"pin-project-lite",
"reqwest",
"reqwest 0.12.28",
"thiserror 1.0.69",
]
@@ -3526,7 +3635,9 @@ version = "0.23.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4"
dependencies = [
"log",
"once_cell",
"ring",
"rustls-pki-types",
"rustls-webpki",
"subtle",
@@ -4052,6 +4163,17 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "socks"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b"
dependencies = [
"byteorder",
"libc",
"winapi",
]
[[package]]
name = "spm_precompiled"
version = "0.1.4"
@@ -4127,6 +4249,203 @@ version = "2.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
[[package]]
name = "symphonia"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1758d6c853020a7244de03cc3e0185eaea3f58715122422dd3cc7452e6d4c16a"
dependencies = [
"lazy_static",
"symphonia-bundle-flac",
"symphonia-bundle-mp3",
"symphonia-codec-aac",
"symphonia-codec-adpcm",
"symphonia-codec-alac",
"symphonia-codec-pcm",
"symphonia-codec-vorbis",
"symphonia-core",
"symphonia-format-caf",
"symphonia-format-isomp4",
"symphonia-format-mkv",
"symphonia-format-ogg",
"symphonia-format-riff",
"symphonia-metadata",
]
[[package]]
name = "symphonia-bundle-flac"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee69ad01236a67260b82fd1ff9790dd75ead29f2f46af145e63b7e72273e0e03"
dependencies = [
"log",
"symphonia-common",
"symphonia-core",
"symphonia-metadata",
]
[[package]]
name = "symphonia-bundle-mp3"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "350f1f2f2e19ad4dd315db94304d1eb361b29af070681f94e51b8fdaad769546"
dependencies = [
"lazy_static",
"log",
"symphonia-core",
]
[[package]]
name = "symphonia-codec-aac"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1979c515a76371b186aad2feff5f23e21cbec775bf95de08bf1e3af92a2ad76"
dependencies = [
"lazy_static",
"log",
"symphonia-common",
"symphonia-core",
]
[[package]]
name = "symphonia-codec-adpcm"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ebbdfd76d6cc5a601c6292a44357c5b7c82f2cd7cdc0f171421f5c5cff0ea1f"
dependencies = [
"log",
"symphonia-core",
]
[[package]]
name = "symphonia-codec-alac"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a149cbfc7fb5c405d123a273227d31de17138419552112bf1aa7b73e65827b8"
dependencies = [
"log",
"symphonia-common",
"symphonia-core",
]
[[package]]
name = "symphonia-codec-pcm"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50baee168f0e9dcf6ba7fc06e8b57eb62072a4490cc7cf13af77e72baae5d328"
dependencies = [
"log",
"symphonia-core",
]
[[package]]
name = "symphonia-codec-vorbis"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "45b07b4423cd8e0fc472575909a5554b12c2f58e3c190b38c24f042e732fd8de"
dependencies = [
"log",
"symphonia-common",
"symphonia-core",
]
[[package]]
name = "symphonia-common"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8257891ffa7f05e02b58f4761e2abf7e5278c8744fd59e981559e050f86eef55"
dependencies = [
"log",
"symphonia-core",
"symphonia-metadata",
]
[[package]]
name = "symphonia-core"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95ec293b5f288383b72a7bffcade6b2860b642cf66f28b3bd5967349a49938b1"
dependencies = [
"bitflags",
"bytemuck",
"lazy_static",
"log",
"num-complex",
"smallvec",
]
[[package]]
name = "symphonia-format-caf"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cde3ca76633d3400ab57195456c09f8a58d775ff5452329f3f212b6efc8622f5"
dependencies = [
"log",
"symphonia-common",
"symphonia-core",
]
[[package]]
name = "symphonia-format-isomp4"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d179a01305b3505940135a9f0180d6ef4b487912748fe97554756f120fbd05e"
dependencies = [
"log",
"symphonia-common",
"symphonia-core",
"symphonia-metadata",
]
[[package]]
name = "symphonia-format-mkv"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb17713e134f5ad316c2690fa3104590ccc85842cdbcf82c3cd1a845cb08aa74"
dependencies = [
"lazy_static",
"log",
"symphonia-common",
"symphonia-core",
]
[[package]]
name = "symphonia-format-ogg"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b05a67e02b1e4fca1a261ba4fe06910a9357489ad8c36aafdd2960e9c6559433"
dependencies = [
"log",
"symphonia-common",
"symphonia-core",
"symphonia-metadata",
]
[[package]]
name = "symphonia-format-riff"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17424452a777666d3eaf09a5c651029b15b6a333812fcc5b5474f2a3f0cff3f0"
dependencies = [
"extended",
"log",
"symphonia-core",
"symphonia-metadata",
]
[[package]]
name = "symphonia-metadata"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a31acf5cd623398a6208e2225d18f4b20f761c55098a796a5247ad516a4a8681"
dependencies = [
"lazy_static",
"log",
"regex-lite",
"smallvec",
"symphonia-core",
]
[[package]]
name = "syn"
version = "1.0.109"
@@ -4982,6 +5301,42 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
[[package]]
name = "ureq"
version = "3.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0"
dependencies = [
"base64 0.22.1",
"cookie_store",
"der",
"flate2",
"log",
"native-tls",
"percent-encoding",
"rustls",
"rustls-pki-types",
"serde",
"serde_json",
"socks",
"ureq-proto",
"utf8-zero",
"webpki-root-certs",
"webpki-roots",
]
[[package]]
name = "ureq-proto"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c"
dependencies = [
"base64 0.22.1",
"http",
"httparse",
"log",
]
[[package]]
name = "url"
version = "2.5.8"
@@ -5006,6 +5361,12 @@ version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246"
[[package]]
name = "utf8-zero"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e"
[[package]]
name = "utf8_iter"
version = "1.0.4"
@@ -5097,7 +5458,7 @@ dependencies = [
"ndarray 0.17.2",
"oss-harmony",
"paste",
"reqwest",
"reqwest 0.13.4",
"rmp-serde",
"serde",
"serde-json-fmt",
@@ -5344,7 +5705,7 @@ dependencies = [
"futures",
"hf-hub",
"itertools 0.14.0",
"reqwest",
"reqwest 0.12.28",
"serde",
"serde_json",
"serde_with",
@@ -5368,7 +5729,7 @@ dependencies = [
"criterion",
"fastokens",
"hf-hub",
"reqwest",
"reqwest 0.12.28",
"riptoken",
"rustc-hash 1.1.0",
"serde",
@@ -5531,6 +5892,19 @@ dependencies = [
"web-sys",
]
[[package]]
name = "wasm-streams"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb"
dependencies = [
"futures-util",
"js-sys",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
]
[[package]]
name = "wasmparser"
version = "0.244.0"
@@ -5563,6 +5937,24 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "webpki-root-certs"
version = "1.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267"
dependencies = [
"rustls-pki-types",
]
[[package]]
name = "webpki-roots"
version = "1.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf"
dependencies = [
"rustls-pki-types",
]
[[package]]
name = "weezl"
version = "0.1.12"
rust/Cargo.toml
+2 -1
View File
@@ -53,7 +53,7 @@ hyper-util = { version = "0.1.20", features = [
indexmap = "2.13.0"
itertools = "0.14.0"
libc = "0.2.177"
llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "c8a29dcc755139fdc26185f400ea48c6d6d48273" }
llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "7df38e53f99aefaebe86934f010aa8084ec99b2f", default-features = false, features = ["native-tls"] }
mimalloc = "0.1.52"
minijinja = { version = "2.0", features = ["unstable_machinery", "json", "builtins", "loader", "loop_controls", "preserve_order"] }
minijinja-contrib = { version = "2.0", features = ["pycompat"] }
@@ -73,6 +73,7 @@ pythonize = "0.28.0"
rand = "0.9.2"
reasoning-parser = "1.2.2"
reqwest = { version = "0.12.8", default-features = false, features = ["native-tls"] }
reqwest-0-13 = { package = "reqwest", version = "0.13.4", default-features = false, features = ["native-tls"] }
riptoken = { version = "0.3.0", default-features = false }
rmp-serde = "1.3.1"
rmpv = { version = "1.3.1", features = ["with-serde"] }
rust/src/chat/Cargo.toml
+1 -1
View File
@@ -16,7 +16,7 @@ llm-multimodal.workspace = true
minijinja.workspace = true
minijinja-contrib.workspace = true
openai-harmony.workspace = true
reqwest.workspace = true
reqwest-0-13.workspace = true
serde.workspace = true
serde-json-fmt.workspace = true
serde_json.workspace = true
rust/src/chat/src/backend/hf.rs
+1
View File
@@ -151,6 +151,7 @@ fn resolve_multimodal_render_info(
info.map(|info| MultimodalRenderInfo {
image_token: info.placeholder_token(Modality::Image).map(str::to_string),
video_token: info.placeholder_token(Modality::Video).map(str::to_string),
audio_token: info.placeholder_token(Modality::Audio).map(str::to_string),
})
}
rust/src/chat/src/multimodal.rs
+203 -92
View File
@@ -5,24 +5,24 @@
//!
//! This module owns the multimodal path for chat requests: it extracts media
//! parts from structured chat messages, fetches and preprocesses them through
//! `llm-multimodal` one modality at a time, expands rendered prompt
//! placeholders after tokenization, and builds the engine-facing
//! `MmFeatures` payload.
//! `llm-multimodal`, expands rendered prompt placeholders after tokenization,
//! and builds the engine-facing `MmFeatures` payload.
//!
//! Raw media stays above `vllm-text`; this module lowers it into token IDs and
//! opaque tensor payloads before the request is handed to text generation.
use std::collections::{HashMap, HashSet};
use std::collections::HashSet;
use std::fs;
use std::path::Path;
use std::sync::{Arc, LazyLock};
use itertools::izip;
use llm_multimodal::{
AsyncMultiModalTracker, FieldLayout, ImageFrame, MediaConnector, MediaConnectorConfig,
MediaContentPart, Modality, ModelMetadata, ModelProcessorSpec, ModelRegistry,
PreProcessorConfig, PreprocessedEncoderInputs, PromptReplacement, Tokenizer as TokenResolver,
TrackedMedia, VideoClip, VisionPreProcessor, VisionProcessorRegistry,
AsyncMultiModalTracker, AudioClip, AudioPreProcessor, EncoderFieldLayouts, FieldLayout,
ImageFrame, MediaConnector, MediaConnectorConfig, MediaContentPart, Modality, ModelMetadata,
ModelProcessorSpec, ModelRegistry, PreProcessorConfig, PreprocessedEncoderInputs,
PromptReplacement, Tokenizer as TokenResolver, TrackedMedia, VideoClip, VisionPreProcessor,
VisionProcessorRegistry,
};
use thiserror_ext::AsReport as _;
use tracing::warn;
@@ -35,8 +35,10 @@ use crate::error::{Error, Result, bail_multimodal, multimodal};
use crate::renderer::RenderedPrompt;
use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest};
mod audio;
mod expand;
mod image;
mod item;
mod tensor;
mod video;
@@ -46,9 +48,9 @@ use self::expand::expand_prompt_token_ids;
#[derive(Clone)]
pub struct MultimodalModelInfo {
context: MultimodalModelContext,
spec: ResolvedMultimodalSpec,
image: Option<ModalitySupport>,
video: Option<ModalitySupport>,
audio: Option<AudioModalitySupport>,
media_connector: Arc<MediaConnector>,
}
@@ -88,22 +90,50 @@ impl MultimodalModelContext {
LazyLock::new(VisionProcessorRegistry::with_defaults);
REGISTRY.find(&self.model_id, self.model_type.as_deref())
}
/// Resolve an audio preprocessor for one loaded model.
fn resolve_audio_processor(
&self,
model_spec: &'static dyn ModelProcessorSpec,
preprocessor_config: &PreProcessorConfig,
) -> Option<Arc<dyn AudioPreProcessor>> {
model_spec.audio_processor(&self.config, preprocessor_config).map(Arc::from)
}
}
/// Static model-specific tensor-layout behavior shared across modalities.
/// Static model-specific tensor-layout behavior for one modality.
#[derive(Clone)]
struct ResolvedMultimodalSpec {
raw: &'static dyn ModelProcessorSpec,
field_layouts: HashMap<String, FieldLayout>,
modality: Modality,
field_layouts: EncoderFieldLayouts,
keep_on_cpu_keys: HashSet<String>,
}
impl ResolvedMultimodalSpec {
fn new(raw: &'static dyn ModelProcessorSpec) -> Self {
fn new(raw: &'static dyn ModelProcessorSpec, modality: Modality) -> Self {
Self {
raw,
field_layouts: raw.field_layouts(),
keep_on_cpu_keys: raw.keep_on_cpu_keys().into_iter().collect(),
modality,
field_layouts: raw.encoder_field_layouts_for(modality),
keep_on_cpu_keys: raw.keep_on_cpu_keys_for(modality).into_iter().collect(),
}
}
fn primary_key(&self) -> &'static str {
match self.modality {
Modality::Image => image::IMAGE_PRIMARY_KEY,
Modality::Video => video::VIDEO_PRIMARY_KEY,
Modality::Audio => audio::AUDIO_PRIMARY_KEY,
Modality::ImageEmbeds => unreachable!("image embeds use no encoder preprocessor"),
}
}
fn field_layout_for(&self, key: &str) -> Option<&FieldLayout> {
if key == self.primary_key() {
Some(&self.field_layouts.encoder_input)
} else {
self.field_layouts.model_specific.get(key)
}
}
@@ -111,14 +141,15 @@ impl ResolvedMultimodalSpec {
&self,
context: &MultimodalModelContext,
preprocessed: &PreprocessedEncoderInputs,
modality: Modality,
) -> Result<Vec<PromptReplacement>> {
Ok(self.raw.prompt_replacements_for(&context.metadata(), preprocessed, modality)?)
Ok(self
.raw
.prompt_replacements_for(&context.metadata(), preprocessed, self.modality)?)
}
}
/// Resolved placeholder tokens for one modality.
#[derive(Clone)]
#[derive(Debug, Clone)]
struct ResolvedPlaceholder {
token: String,
/// The token ID emitted for `token` in the rendered prompt.
@@ -152,19 +183,30 @@ impl ResolvedPlaceholder {
}
}
/// Static per-modality vision preprocessor plus its loaded config and
/// resolved placeholder tokens.
/// Static per-modality vision preprocessor plus its loaded config, resolved
/// placeholder tokens, and the model's shared tensor-layout spec.
#[derive(Clone)]
struct ModalitySupport {
spec: ResolvedMultimodalSpec,
placeholder: ResolvedPlaceholder,
processor: &'static dyn VisionPreProcessor,
config: PreProcessorConfig,
}
/// Static audio preprocessor plus the resolved model contract for its output.
#[derive(Clone)]
struct AudioModalitySupport {
spec: ResolvedMultimodalSpec,
placeholder: ResolvedPlaceholder,
processor: Arc<dyn AudioPreProcessor>,
}
/// Model-repo config file locations consumed by multimodal support.
#[derive(Debug, Default, Clone, Copy)]
pub struct MultimodalConfigFiles<'a> {
pub config: Option<&'a Path>,
/// Shared preprocessor config (`preprocessor_config.json`), consumed by
/// image/audio processors and used as the video fallback.
pub preprocessor_config: Option<&'a Path>,
/// Video-specific preprocessor config (`video_preprocessor_config.json`).
pub video_preprocessor_config: Option<&'a Path>,
@@ -212,6 +254,8 @@ struct FetchedMedia {
image_uuids: Vec<Option<String>>,
videos: Vec<Arc<VideoClip>>,
video_uuids: Vec<Option<String>>,
audios: Vec<Arc<AudioClip>>,
audio_uuids: Vec<Option<String>>,
}
/// One modality's preprocessed output, ready for the shared expansion and
@@ -253,7 +297,7 @@ impl MultimodalModelInfo {
}
None => serde_json::Value::Object(Default::default()),
};
let image_preprocessor_config = load_preprocessor_config(
let preprocessor_config = load_preprocessor_config(
files.preprocessor_config,
"preprocessor_config.json",
files.processor_config,
@@ -266,7 +310,7 @@ impl MultimodalModelInfo {
files.processor_config,
"video_processor",
)?
.unwrap_or_else(|| image_preprocessor_config.clone());
.unwrap_or_else(|| preprocessor_config.clone());
let context = MultimodalModelContext {
model_id,
@@ -275,41 +319,76 @@ impl MultimodalModelInfo {
tokenizer: TokenizerResolver(tokenizer),
};
Self::from_loaded(
context,
image_preprocessor_config,
video_preprocessor_config,
)
Self::from_loaded(context, preprocessor_config, video_preprocessor_config)
}
/// Resolve multimodal support from an assembled context and parsed
/// preprocessor configs.
fn from_loaded(
context: MultimodalModelContext,
image_preprocessor_config: PreProcessorConfig,
preprocessor_config: PreProcessorConfig,
video_preprocessor_config: PreProcessorConfig,
) -> Result<Option<Self>> {
let (image, video) = Self::resolve_vision_lanes(
&context,
preprocessor_config.clone(),
video_preprocessor_config,
);
let audio = Self::resolve_audio_lane(&context, &preprocessor_config)?;
if image.is_none() && video.is_none() && audio.is_none() {
warn!(
model_id = context.model_id,
model_type = context.model_type,
"no multimodal modality resolved; disabling multimodal support for this model"
);
return Ok(None);
}
let media_connector = Arc::new(MediaConnector::new(
reqwest_0_13::Client::new(),
MediaConnectorConfig::default(),
)?);
Ok(Some(Self {
context,
image,
video,
audio,
media_connector,
}))
}
/// Resolve the spec-backed vision lanes (image and video).
///
/// A missing model spec or vision processor disables both lanes but is
/// not an error: unsupported model families are logged and skipped.
fn resolve_vision_lanes(
context: &MultimodalModelContext,
preprocessor_config: PreProcessorConfig,
video_preprocessor_config: PreProcessorConfig,
) -> (Option<ModalitySupport>, Option<ModalitySupport>) {
let Some(raw_spec) = context.resolve_model_spec() else {
warn!(
model_id = context.model_id,
model_type = context.model_type,
"multimodal model spec is not registered; disabling multimodal support for this model"
"multimodal model spec is not registered; disabling image/video support for this model"
);
return Ok(None);
return (None, None);
};
let Some(processor) = context.resolve_vision_processor() else {
warn!(
model_id = context.model_id,
model_type = context.model_type,
"vision processor is not registered; disabling multimodal support for this model"
"vision processor is not registered; disabling image/video support for this model"
);
return Ok(None);
return (None, None);
};
// Warn and disable the modality if the placeholder resolution fails.
let resolve_placeholder =
|modality: Modality| match ResolvedPlaceholder::resolve(raw_spec, &context, modality) {
|modality: Modality| match ResolvedPlaceholder::resolve(raw_spec, context, modality) {
Ok(placeholder) => Some(placeholder),
Err(error) => {
warn!(
@@ -323,9 +402,10 @@ impl MultimodalModelInfo {
};
let image = resolve_placeholder(Modality::Image).map(|placeholder| ModalitySupport {
spec: ResolvedMultimodalSpec::new(raw_spec, Modality::Image),
placeholder,
processor,
config: image_preprocessor_config,
config: preprocessor_config.clone(),
});
let video = resolve_placeholder(Modality::Video).and_then(|placeholder| {
@@ -341,6 +421,7 @@ impl MultimodalModelInfo {
None
} else {
Some(ModalitySupport {
spec: ResolvedMultimodalSpec::new(raw_spec, Modality::Video),
placeholder,
processor,
config: video_preprocessor_config,
@@ -348,26 +429,33 @@ impl MultimodalModelInfo {
}
});
if image.is_none() && video.is_none() {
warn!(
model_id = context.model_id,
model_type = context.model_type,
"no multimodal modality resolved; disabling multimodal support for this model"
);
(image, video)
}
/// Resolve a spec-backed audio lane when the model advertises audio support.
fn resolve_audio_lane(
context: &MultimodalModelContext,
preprocessor_config: &PreProcessorConfig,
) -> Result<Option<AudioModalitySupport>> {
let Some(raw_spec) = context.resolve_model_spec() else {
return Ok(None);
};
if !raw_spec.modality_limits(&context.metadata())?.contains_key(&Modality::Audio) {
return Ok(None);
}
let media_connector = Arc::new(MediaConnector::new(
reqwest::Client::new(),
MediaConnectorConfig::default(),
)?);
Ok(Some(Self {
context,
spec: ResolvedMultimodalSpec::new(raw_spec),
image,
video,
media_connector,
let Some(processor) = context.resolve_audio_processor(raw_spec, preprocessor_config) else {
warn!(
model_id = context.model_id,
model_spec = raw_spec.name(),
"model spec does not provide an audio processor; disabling audio support for this model"
);
return Ok(None);
};
Ok(Some(AudioModalitySupport {
spec: ResolvedMultimodalSpec::new(raw_spec, Modality::Audio),
placeholder: ResolvedPlaceholder::resolve(raw_spec, context, Modality::Audio)?,
processor,
}))
}
@@ -380,7 +468,8 @@ impl MultimodalModelInfo {
match modality {
Modality::Image => self.image.as_ref()?.placeholder.token.as_str().into(),
Modality::Video => self.video.as_ref()?.placeholder.token.as_str().into(),
_ => None,
Modality::Audio => self.audio.as_ref()?.placeholder.token.as_str().into(),
Modality::ImageEmbeds => None,
}
}
}
@@ -449,12 +538,35 @@ fn extract_media_parts(request: &ChatRequest) -> Result<Vec<MediaContentPart>> {
uuid: uuid.clone(),
})
}
ChatContentPart::InputAudio { data, format, uuid } => {
all_parts.push(MediaContentPart::AudioUrl {
url: input_audio_data_url(data, format.as_deref())?,
uuid: uuid.clone(),
})
}
ChatContentPart::AudioUrl { audio_url, uuid } => {
all_parts.push(MediaContentPart::AudioUrl {
url: audio_url.clone(),
uuid: uuid.clone(),
})
}
}
}
}
Ok(all_parts)
}
/// Wrap OpenAI base64 audio in a data URL consumed by `MediaConnector`.
fn input_audio_data_url(data: &str, format: Option<&str>) -> Result<String> {
let mime_type = match format {
Some(format) if format.eq_ignore_ascii_case("wav") => "audio/wav",
Some(format) if format.eq_ignore_ascii_case("mp3") => "audio/mpeg",
Some(format) => bail_multimodal!("unsupported input_audio format `{format}`"),
None => "application/octet-stream",
};
Ok(format!("data:{mime_type};base64,{data}"))
}
impl MultimodalModelInfo {
/// Run media fetch, per-modality preprocessing, prompt expansion, and
/// feature build.
@@ -468,11 +580,10 @@ impl MultimodalModelInfo {
prompt_token_ids: &mut Vec<u32>,
model_dtype: ModelDtype,
) -> Result<MmFeatures> {
if media_parts.is_empty() {
let media_parts_len = media_parts.len();
if media_parts_len == 0 {
return Ok(Vec::new());
}
let media_parts_len = media_parts.len();
// TODO: enforce per-modality item-count limits, aligned with the
// engine's `--limit-mm-per-prompt` semantics.
let fetched = self.fetch_media(media_parts).await?;
@@ -486,6 +597,9 @@ impl MultimodalModelInfo {
prepared
.push(self.prepare_videos(fetched.videos, fetched.video_uuids, model_dtype).await?);
}
if !fetched.audios.is_empty() {
prepared.push(self.prepare_audios(fetched.audios, fetched.audio_uuids).await?);
}
let mut ranges = expand_prompt_token_ids(prompt_token_ids, &prepared)?;
@@ -524,8 +638,8 @@ impl MultimodalModelInfo {
Ok(features)
}
/// Fetch all media parts and split them per modality, preserving their
/// request-order UUID metadata.
/// Fetch all connector-backed media parts and split them per modality,
/// preserving their request-order UUID metadata.
async fn fetch_media(&self, media_parts: Vec<MediaContentPart>) -> Result<FetchedMedia> {
let mut tracker = AsyncMultiModalTracker::new(Arc::clone(&self.media_connector));
for part in media_parts {
@@ -562,11 +676,27 @@ impl MultimodalModelInfo {
.collect::<Result<Vec<_>>>()?;
let video_uuids = tracker_output.uuids.remove(&Modality::Video).unwrap_or_default();
let audios = tracker_output
.data
.remove(&Modality::Audio)
.unwrap_or_default()
.into_iter()
.map(|media| match media {
TrackedMedia::Audio(clip) => Ok(clip),
_ => Err(multimodal!(
"tracker returned non-audio media for the audio modality"
)),
})
.collect::<Result<Vec<_>>>()?;
let audio_uuids = tracker_output.uuids.remove(&Modality::Audio).unwrap_or_default();
Ok(FetchedMedia {
images,
image_uuids,
videos,
video_uuids,
audios,
audio_uuids,
})
}
}
@@ -665,42 +795,6 @@ mod tests {
test_info("qwen3_vl", config, qwen3_vl_tokenizer())
}
#[test]
fn from_paths_resolves_image_config_from_processor_config() {
let dir = tempfile::tempdir().unwrap();
let config_path = dir.path().join("config.json");
std::fs::write(
&config_path,
serde_json::json!({
"model_type": "qwen3_vl",
"image_token_id": QWEN3_IMAGE_PAD_ID,
})
.to_string(),
)
.unwrap();
let processor_config_path = dir.path().join("processor_config.json");
std::fs::write(
&processor_config_path,
r#"{"image_processor":{"size":{"shortest_edge":64}}}"#,
)
.unwrap();
let info = MultimodalModelInfo::from_paths(
"qwen3-vl-test".to_string(),
Some("qwen3_vl".to_string()),
MultimodalConfigFiles {
config: Some(&config_path),
processor_config: Some(&processor_config_path),
..Default::default()
},
Arc::new(qwen3_vl_tokenizer()),
)
.unwrap()
.unwrap();
assert_eq!(info.image.unwrap().config.get_shortest_edge(), Some(64));
}
#[test]
fn qwen3_vl_resolves_image_and_video_support() {
let info = qwen3_vl_info();
@@ -742,4 +836,21 @@ mod tests {
assert_eq!(info.placeholder_token(Modality::Image), Some("<|image|>"));
assert_eq!(info.placeholder_token(Modality::Video), None);
}
#[test]
fn input_audio_uses_connector_data_urls() {
assert_eq!(
input_audio_data_url("AAAA", Some("wav")).unwrap(),
"data:audio/wav;base64,AAAA"
);
assert_eq!(
input_audio_data_url("BBBB", Some("MP3")).unwrap(),
"data:audio/mpeg;base64,BBBB"
);
assert_eq!(
input_audio_data_url("CCCC", None).unwrap(),
"data:application/octet-stream;base64,CCCC"
);
assert!(input_audio_data_url("AAAA", Some("flac")).is_err());
}
}
rust/src/chat/src/multimodal/audio.rs
+213
View File
@@ -0,0 +1,213 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
//! Audio-modality preparation through `llm-multimodal`.
use std::sync::Arc;
use llm_multimodal::{AudioClip, Modality, PreprocessedEncoderInputs};
use vllm_engine_core_client::protocol::dtype::ModelDtype;
use super::{AudioModalitySupport, MultimodalModelInfo, PreparedMedia, item};
use crate::error::{Error, Result, bail_multimodal, multimodal};
/// Forward-kwargs name of the primary audio encoder input.
pub(super) const AUDIO_PRIMARY_KEY: &str = "input_audio_features";
impl MultimodalModelInfo {
/// Preprocess fetched audio clips as one batch and build per-item features.
pub(super) async fn prepare_audios(
&self,
clips: Vec<Arc<AudioClip>>,
uuids: Vec<Option<String>>,
) -> Result<PreparedMedia> {
let support = self.audio.as_ref().ok_or_else(|| Error::UnsupportedModality {
modality: Modality::Audio.to_string(),
})?;
let preprocessed = self.preprocess_audios(support, &clips).await?;
let replacements = support.spec.prompt_replacements_for(&self.context, &preprocessed)?;
if replacements.len() != clips.len() {
bail_multimodal!(
"number of audio prompt replacements {} does not match number of audio clips {}",
replacements.len(),
clips.len()
);
}
let hashes = clips.iter().map(|clip| clip.hash.clone()).collect();
let items = item::build_batched_items(
&support.spec,
preprocessed,
hashes,
uuids,
ModelDtype::Float32,
)?;
Ok(PreparedMedia {
modality: Modality::Audio,
placeholder: support.placeholder.clone(),
replacements,
items,
})
}
/// Run CPU-heavy audio preprocessing in a blocking task.
async fn preprocess_audios(
&self,
support: &AudioModalitySupport,
clips: &[Arc<AudioClip>],
) -> Result<PreprocessedEncoderInputs> {
let processor = Arc::clone(&support.processor);
let clips = clips.to_vec();
tokio::task::spawn_blocking(move || Ok(processor.preprocess(&clips)?))
.await
.map_err(|error| multimodal!("audio preprocessing task failed: {error}"))?
}
}
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use std::sync::Arc;
use llm_multimodal::{MediaContentPart, ModelSpecificValue, PreProcessorConfig};
use ndarray::ArrayD;
use vllm_engine_core_client::protocol::multimodal::{MmField, MmKwargValue};
use vllm_tokenizer::test_utils::TestTokenizer;
use super::super::{MultimodalModelContext, TokenizerResolver};
use super::*;
const AUDIO_PAD_ID: u32 = 151_676;
fn qwen3_asr_info() -> MultimodalModelInfo {
let context = MultimodalModelContext {
model_id: "Qwen/Qwen3-ASR-1.7B".to_string(),
model_type: Some("qwen3_asr".to_string()),
config: serde_json::json!({"model_type": "qwen3_asr"}),
tokenizer: TokenizerResolver(Arc::new(
TestTokenizer::new().with_regular_token("<|audio_pad|>", AUDIO_PAD_ID),
)),
};
MultimodalModelInfo::from_loaded(
context,
PreProcessorConfig::default(),
PreProcessorConfig::default(),
)
.unwrap()
.expect("Qwen3-ASR multimodal support")
}
fn wav_i16_mono(sample_rate: u32, samples: &[i16]) -> Vec<u8> {
let data_bytes = samples.len() as u32 * 2;
let mut bytes = Vec::new();
bytes.extend_from_slice(b"RIFF");
bytes.extend_from_slice(&(36 + data_bytes).to_le_bytes());
bytes.extend_from_slice(b"WAVEfmt ");
bytes.extend_from_slice(&16_u32.to_le_bytes());
bytes.extend_from_slice(&1_u16.to_le_bytes());
bytes.extend_from_slice(&1_u16.to_le_bytes());
bytes.extend_from_slice(&sample_rate.to_le_bytes());
bytes.extend_from_slice(&(sample_rate * 2).to_le_bytes());
bytes.extend_from_slice(&2_u16.to_le_bytes());
bytes.extend_from_slice(&16_u16.to_le_bytes());
bytes.extend_from_slice(b"data");
bytes.extend_from_slice(&data_bytes.to_le_bytes());
for sample in samples {
bytes.extend_from_slice(&sample.to_le_bytes());
}
bytes
}
#[test]
fn resolves_qwen_audio_from_model_spec() {
let info = qwen3_asr_info();
let support = info.audio.as_ref().expect("audio support");
assert_eq!(
info.placeholder_token(Modality::Audio),
Some("<|audio_pad|>")
);
assert_eq!(support.placeholder.marker_token_id, AUDIO_PAD_ID);
assert_eq!(support.placeholder.embed_token_id, AUDIO_PAD_ID);
assert!(matches!(
support.spec.field_layouts.encoder_input,
llm_multimodal::FieldLayout::Batched
));
let preprocessed = PreprocessedEncoderInputs {
encoder_input: ArrayD::zeros(vec![1, 128, 4]),
feature_token_counts: vec![1],
item_sizes: vec![(128, 4)],
model_specific: HashMap::from([
(
"feature_attention_mask".to_string(),
ModelSpecificValue::int_2d(vec![1; 4], 1, 4),
),
(
"audio_feature_lengths".to_string(),
ModelSpecificValue::int_1d(vec![4]),
),
]),
};
let item = item::build_batched_items(
&support.spec,
preprocessed,
vec!["<hash>".to_string()],
vec![None],
ModelDtype::Float32,
)
.unwrap()
.pop()
.unwrap();
assert!(matches!(
&item.data[AUDIO_PRIMARY_KEY].field,
MmField::Batched(_)
));
}
#[tokio::test]
async fn tracker_processor_and_lowering_preserve_audio_contract() {
let info = qwen3_asr_info();
let wav = wav_i16_mono(16_000, &[0; 1_600]);
let expected_hash = llm_multimodal::hasher::hash_audio(&wav);
let fetched = info
.fetch_media(vec![MediaContentPart::AudioData {
data: wav,
mime_type: Some("audio/wav".to_string()),
uuid: Some("audio-1".to_string()),
}])
.await
.unwrap();
let prepared = info.prepare_audios(fetched.audios, fetched.audio_uuids).await.unwrap();
assert_eq!(prepared.replacements.len(), 1);
assert!(
prepared.replacements[0]
.tokens
.iter()
.all(|token| *token == AUDIO_PAD_ID as i32)
);
let item = &prepared.items[0];
assert_eq!(item.hash, expected_hash);
assert_eq!(item.uuid.as_deref(), Some("audio-1"));
let features = &item.data[AUDIO_PRIMARY_KEY];
assert!(matches!(&features.field, MmField::Batched(_)));
assert!(matches!(
features.data.as_ref(),
Some(MmKwargValue::Tensor(tensor))
if tensor.dtype == "float32" && tensor.shape.first() == Some(&128)
));
let lengths = &item.data["audio_feature_lengths"];
assert!(matches!(&lengths.field, MmField::Batched(_)));
assert!(matches!(
lengths.data.as_ref(),
Some(MmKwargValue::Tensor(tensor))
if tensor.dtype == "int64" && tensor.shape.is_empty()
));
}
}
rust/src/chat/src/multimodal/expand.rs
+61
View File
@@ -418,6 +418,67 @@ mod tests {
assert_bool_mask(&video_ranges[0], &[true, true, true, true]);
}
const QWEN3_AUDIO_PAD_ID: u32 = 151_676;
fn qwen3_audio_prepared(replacements: Vec<PromptReplacement>) -> PreparedMedia {
prepared_media(
Modality::Audio,
"<|audio_pad|>",
QWEN3_AUDIO_PAD_ID,
QWEN3_AUDIO_PAD_ID,
replacements,
)
}
fn qwen3_audio_replacement(tokens: usize) -> PromptReplacement {
PromptReplacement::repeated(
Modality::Audio,
"<|audio_pad|>",
QWEN3_AUDIO_PAD_ID as TokenId,
tokens,
)
}
#[test]
fn expand_prompt_tokens_interleaves_audio_and_image_prepared_media() {
// Audio marker precedes the image marker: the image range offset must
// account for the earlier audio expansion growth.
let mut prompt_token_ids = vec![1, QWEN3_AUDIO_PAD_ID, 2, LLAMA4_IMAGE_ID, 3];
let prepared = vec![
llama4_prepared(vec![llama4_single_tile_replacement()]),
qwen3_audio_prepared(vec![qwen3_audio_replacement(3)]),
];
let ranges = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap();
assert_eq!(
prompt_token_ids,
vec![
1,
QWEN3_AUDIO_PAD_ID,
QWEN3_AUDIO_PAD_ID,
QWEN3_AUDIO_PAD_ID,
2,
LLAMA4_IMAGE_START_ID,
LLAMA4_IMAGE_ID,
LLAMA4_PATCH_ID,
LLAMA4_PATCH_ID,
LLAMA4_IMAGE_END_ID,
3,
]
);
let audio_ranges = &ranges[&Modality::Audio];
assert_eq!(audio_ranges[0].offset, 1);
assert_eq!(audio_ranges[0].length, 3);
assert_bool_mask(&audio_ranges[0], &[true, true, true]);
let image_ranges = &ranges[&Modality::Image];
assert_eq!(image_ranges[0].offset, 5);
assert_eq!(image_ranges[0].length, 5);
assert_bool_mask(&image_ranges[0], &[false, false, true, true, false]);
}
#[test]
fn expand_prompt_tokens_error_names_modality_with_leftover_replacements() {
let mut prompt_token_ids = vec![1, QWEN3_IMAGE_PAD_ID, 2];
rust/src/chat/src/multimodal/image.rs
+9 -82
View File
@@ -6,17 +6,15 @@
use std::sync::Arc;
use itertools::izip;
use llm_multimodal::{FieldLayout, ImageFrame, Modality, PreprocessedEncoderInputs};
use llm_multimodal::{ImageFrame, Modality, PreprocessedEncoderInputs};
use vllm_engine_core_client::protocol::dtype::ModelDtype;
use vllm_engine_core_client::protocol::multimodal::{
MmBatchedField, MmField, MmFieldElem, MmFlatField, MmKwargsItem, MmSharedField, MmSlice,
SliceSpec,
};
use super::{ModalitySupport, MultimodalModelInfo, PreparedItem, PreparedMedia, tensor};
use super::{ModalitySupport, MultimodalModelInfo, PreparedMedia, item};
use crate::error::{Error, Result, bail_multimodal, multimodal};
/// Forward-kwargs name of the primary image encoder input.
pub(super) const IMAGE_PRIMARY_KEY: &str = "pixel_values";
impl MultimodalModelInfo {
/// Preprocess all fetched image frames as one batch and build per-item
/// features.
@@ -30,9 +28,7 @@ impl MultimodalModelInfo {
modality: Modality::Image.to_string(),
})?;
let preprocessed = self.preprocess_images(support, &frames).await?;
let replacements =
self.spec
.prompt_replacements_for(&self.context, &preprocessed, Modality::Image)?;
let replacements = support.spec.prompt_replacements_for(&self.context, &preprocessed)?;
if replacements.len() != frames.len() {
bail_multimodal!(
"number of image prompt replacements {} does not match number of images {}",
@@ -40,7 +36,9 @@ impl MultimodalModelInfo {
frames.len()
);
}
let items = self.build_image_items(preprocessed, &frames, uuids, model_dtype)?;
let hashes = frames.iter().map(|frame| frame.hash.clone()).collect();
let items =
item::build_batched_items(&support.spec, preprocessed, hashes, uuids, model_dtype)?;
Ok(PreparedMedia {
modality: Modality::Image,
@@ -70,75 +68,4 @@ impl MultimodalModelInfo {
.await
.map_err(|error| multimodal!("image preprocessing task failed: {error}"))?
}
/// Convert one batch of preprocessed image tensors into per-item engine
/// kwargs.
///
/// Tensor fields are sliced per item according to the model spec's field
/// layout declarations.
fn build_image_items(
&self,
preprocessed: PreprocessedEncoderInputs,
frames: &[Arc<ImageFrame>],
uuids: Vec<Option<String>>,
model_dtype: ModelDtype,
) -> Result<Vec<PreparedItem>> {
let len = frames.len();
let tensors = tensor::collect_tensors(preprocessed, "pixel_values", model_dtype)?;
let mut items = Vec::with_capacity(len);
for (index, (frame, uuid)) in izip!(frames, uuids).enumerate() {
let mut data = MmKwargsItem::new();
for (key, tensor) in &tensors {
let keep_on_cpu = self.spec.keep_on_cpu_keys.contains(key);
let (value, field) = match self.spec.field_layouts.get(key) {
Some(FieldLayout::Batched) => (
tensor.batched_value_at(index)?,
MmField::Batched(MmBatchedField { keep_on_cpu }),
),
Some(FieldLayout::Flat { sizes_key }) => {
let sizes = tensors.get(sizes_key).ok_or_else(|| {
multimodal!("flat tensor sizes key `{sizes_key}` is missing")
})?;
let (start, end) = tensor::flat_range_for_index(sizes, sizes_key, index)?;
(
tensor.flat_value_range(start, end)?,
MmField::Flat(MmFlatField {
slices: vec![MmSlice::Slice(SliceSpec {
start: Some(0),
stop: Some((end - start) as isize),
step: None,
})],
dim: 0,
keep_on_cpu,
}),
)
}
None => (
tensor.clone(),
MmField::Shared(MmSharedField {
batch_size: len,
keep_on_cpu,
}),
),
};
data.insert(
key.clone(),
MmFieldElem {
data: Some(value.try_into()?),
field,
},
);
}
items.push(PreparedItem {
data,
hash: frame.hash.clone(),
uuid,
});
}
Ok(items)
}
}
rust/src/chat/src/multimodal/item.rs
+84
View File
@@ -0,0 +1,84 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
//! Shared lowering from batched preprocessor output to per-item engine kwargs.
use itertools::izip;
use llm_multimodal::{FieldLayout, PreprocessedEncoderInputs};
use vllm_engine_core_client::protocol::dtype::ModelDtype;
use vllm_engine_core_client::protocol::multimodal::{
MmBatchedField, MmField, MmFieldElem, MmFlatField, MmKwargsItem, MmSharedField, MmSlice,
SliceSpec,
};
use super::{PreparedItem, ResolvedMultimodalSpec, tensor};
use crate::error::{Result, bail_multimodal, multimodal};
/// Split one batch of preprocessed tensors into engine kwargs per media item.
pub(super) fn build_batched_items(
spec: &ResolvedMultimodalSpec,
preprocessed: PreprocessedEncoderInputs,
hashes: Vec<String>,
uuids: Vec<Option<String>>,
float_dtype: ModelDtype,
) -> Result<Vec<PreparedItem>> {
let len = hashes.len();
if uuids.len() != len {
bail_multimodal!(
"number of media UUIDs {} does not match number of media items {len}",
uuids.len()
);
}
let tensors = tensor::collect_tensors(preprocessed, spec.primary_key(), float_dtype)?;
let mut items = Vec::with_capacity(len);
for (index, (hash, uuid)) in izip!(hashes, uuids).enumerate() {
let mut data = MmKwargsItem::new();
for (key, tensor) in &tensors {
let keep_on_cpu = spec.keep_on_cpu_keys.contains(key);
let (value, field) = match spec.field_layout_for(key) {
Some(FieldLayout::Batched) => (
tensor.batched_value_at(index)?,
MmField::Batched(MmBatchedField { keep_on_cpu }),
),
Some(FieldLayout::Flat { sizes_key }) => {
let sizes = tensors.get(sizes_key).ok_or_else(|| {
multimodal!("flat tensor sizes key `{sizes_key}` is missing")
})?;
let (start, end) = tensor::flat_range_for_index(sizes, sizes_key, index)?;
(
tensor.flat_value_range(start, end)?,
MmField::Flat(MmFlatField {
slices: vec![MmSlice::Slice(SliceSpec {
start: Some(0),
stop: Some((end - start) as isize),
step: None,
})],
dim: 0,
keep_on_cpu,
}),
)
}
None => (
tensor.clone(),
MmField::Shared(MmSharedField {
batch_size: len,
keep_on_cpu,
}),
),
};
data.insert(
key.clone(),
MmFieldElem {
data: Some(value.try_into()?),
field,
},
);
}
items.push(PreparedItem { data, hash, uuid });
}
Ok(items)
}
rust/src/chat/src/multimodal/video.rs
+56 -59
View File
@@ -23,7 +23,7 @@ use crate::error::{Error, Result, bail_multimodal, multimodal};
///
/// Video-capable vLLM models read `pixel_values_videos` alongside
/// `video_grid_thw`, mirroring the HF processor output naming.
const VIDEO_PRIMARY_KEY: &str = "pixel_values_videos";
pub(super) const VIDEO_PRIMARY_KEY: &str = "pixel_values_videos";
impl MultimodalModelInfo {
/// Preprocess fetched video clips one at a time and build per-item
@@ -47,8 +47,7 @@ impl MultimodalModelInfo {
for (clip, uuid) in izip!(&clips, uuids) {
let preprocessed = self.preprocess_video_clip(support, Arc::clone(clip)).await?;
let mut clip_replacements =
self.spec
.prompt_replacements_for(&self.context, &preprocessed, Modality::Video)?;
support.spec.prompt_replacements_for(&self.context, &preprocessed)?;
if clip_replacements.len() != 1 {
bail_multimodal!(
"expected exactly one prompt replacement per video clip, got {}",
@@ -56,7 +55,8 @@ impl MultimodalModelInfo {
);
}
replacements.push(clip_replacements.pop().unwrap());
items.push(self.build_video_item(
items.push(build_video_item(
support,
preprocessed,
clip.hash.clone(),
uuid,
@@ -107,30 +107,36 @@ impl MultimodalModelInfo {
.await
.map_err(|error| multimodal!("video preprocessing task failed: {error}"))?
}
}
/// Convert one preprocessed video clip into engine kwargs.
///
/// The clip is a batch of one, so no per-item slicing is required: the
/// primary tensor ships as a full-range flat field (the engine re-batches
/// flat fields by concatenating along the declared dim, matching vLLM's
/// `flat_from_sizes` treatment of video patches), and batched metadata
/// tensors drop their singleton batch axis.
fn build_video_item(
&self,
preprocessed: PreprocessedEncoderInputs,
hash: String,
uuid: Option<String>,
model_dtype: ModelDtype,
) -> Result<PreparedItem> {
let tensors = tensor::collect_tensors(preprocessed, VIDEO_PRIMARY_KEY, model_dtype)?;
/// Convert one preprocessed video clip into engine kwargs.
///
/// The clip is a batch of one, so no per-item slicing is required: the
/// primary tensor ships as a full-range flat field (the engine re-batches
/// flat fields by concatenating along the declared dim, matching vLLM's
/// `flat_from_sizes` treatment of video patches), and batched metadata
/// tensors drop their singleton batch axis.
fn build_video_item(
support: &ModalitySupport,
preprocessed: PreprocessedEncoderInputs,
hash: String,
uuid: Option<String>,
model_dtype: ModelDtype,
) -> Result<PreparedItem> {
let tensors = tensor::collect_tensors(preprocessed, support.spec.primary_key(), model_dtype)?;
let mut data = MmKwargsItem::new();
for (key, tensor) in tensors {
let keep_on_cpu = self.spec.keep_on_cpu_keys.contains(&key);
let (value, field) = if key == VIDEO_PRIMARY_KEY {
let mut data = MmKwargsItem::new();
for (key, tensor) in tensors {
let keep_on_cpu = support.spec.keep_on_cpu_keys.contains(&key);
let (value, field) = match support.spec.field_layout_for(&key) {
Some(FieldLayout::Batched) => (
tensor.batched_value_at(0)?,
MmField::Batched(MmBatchedField { keep_on_cpu }),
),
Some(FieldLayout::Flat { .. }) => {
let len = tensor
.first_dim()
.ok_or_else(|| multimodal!("video encoder input `{key}` is not a tensor"))?;
.ok_or_else(|| multimodal!("flat video input `{key}` is not a tensor"))?;
(
tensor,
MmField::Flat(MmFlatField {
@@ -143,35 +149,26 @@ impl MultimodalModelInfo {
keep_on_cpu,
}),
)
} else if matches!(
self.spec.field_layouts.get(&key),
Some(FieldLayout::Batched)
) {
(
tensor.batched_value_at(0)?,
MmField::Batched(MmBatchedField { keep_on_cpu }),
)
} else {
(
tensor,
MmField::Shared(MmSharedField {
batch_size: 1,
keep_on_cpu,
}),
)
};
}
None => (
tensor,
MmField::Shared(MmSharedField {
batch_size: 1,
keep_on_cpu,
}),
),
};
data.insert(
key,
MmFieldElem {
data: Some(value.try_into()?),
field,
},
);
}
Ok(PreparedItem { data, hash, uuid })
data.insert(
key,
MmFieldElem {
data: Some(value.try_into()?),
field,
},
);
}
Ok(PreparedItem { data, hash, uuid })
}
#[cfg(test)]
@@ -287,14 +284,14 @@ mod tests {
]),
};
let item = info
.build_video_item(
preprocessed,
"<hash>".to_string(),
None,
ModelDtype::Float32,
)
.unwrap();
let item = build_video_item(
info.video.as_ref().unwrap(),
preprocessed,
"<hash>".to_string(),
None,
ModelDtype::Float32,
)
.unwrap();
let primary = &item.data[VIDEO_PRIMARY_KEY];
assert!(matches!(
rust/src/chat/src/renderer/hf/mod.rs
+58 -1
View File
@@ -42,6 +42,7 @@ pub use self::format::ChatTemplateContentFormatOption;
pub struct MultimodalRenderInfo {
pub image_token: Option<String>,
pub video_token: Option<String>,
pub audio_token: Option<String>,
}
/// Hugging Face chat-template renderer backed by the local Jinja chat-template
@@ -263,6 +264,7 @@ enum TemplateContentPart {
Text { text: String },
Image,
Video,
Audio,
}
#[derive(Debug, Serialize)]
@@ -437,6 +439,12 @@ fn to_template_openai_content(
.ok_or(Error::UnsupportedMultimodalContent("video_url"))?;
Ok(TemplateContentPart::Video)
}
ChatContentPart::InputAudio { .. } | ChatContentPart::AudioUrl { .. } => {
multimodal
.and_then(|multimodal| multimodal.audio_token.as_ref())
.ok_or(Error::UnsupportedMultimodalContent("audio"))?;
Ok(TemplateContentPart::Audio)
}
})
.collect(),
}
@@ -465,6 +473,12 @@ fn to_template_string_content(
.ok_or(Error::UnsupportedMultimodalContent("video_url"))?;
out.push_str(video_token);
}
ChatContentPart::InputAudio { .. } | ChatContentPart::AudioUrl { .. } => {
let audio_token = multimodal
.and_then(|multimodal| multimodal.audio_token.as_ref())
.ok_or(Error::UnsupportedMultimodalContent("audio"))?;
out.push_str(audio_token);
}
}
}
Ok(out)
@@ -492,7 +506,9 @@ fn append_continue_final_message_tag(message: &mut TemplateMessage) -> Result<St
// Pick the last text part in the message.
TemplateContent::OpenAi(parts) => parts.iter_mut().rev().find_map(|part| match part {
TemplateContentPart::Text { text } => Some(text),
TemplateContentPart::Image | TemplateContentPart::Video => None,
TemplateContentPart::Image
| TemplateContentPart::Video
| TemplateContentPart::Audio => None,
}),
};
let text = text.ok_or_else(|| {
@@ -603,6 +619,7 @@ mod tests {
.with_multimodal(Some(MultimodalRenderInfo {
image_token: Some("<image>".to_string()),
video_token: Some("<video>".to_string()),
audio_token: Some("<audio>".to_string()),
}))
.render(request)
}
@@ -623,6 +640,15 @@ mod tests {
])])
}
fn audio_request() -> ChatRequest {
sample_request(vec![ChatMessage::user(vec![
ChatContentPart::text("a"),
ChatContentPart::input_audio("dGVzdA==", Some("wav".to_string())),
ChatContentPart::audio_url("https://example.com/demo.mp3"),
ChatContentPart::text("b"),
])])
}
#[test]
fn string_content_format_replaces_image_with_placeholder_text() {
let rendered = render_mm(
@@ -647,6 +673,21 @@ mod tests {
assert_eq!(rendered.prompt, Prompt::Text("a<video>b".to_string()));
}
#[test]
fn string_content_format_replaces_audio_with_placeholder_text() {
let rendered = render_mm(
"{{ messages[0].content }}",
&audio_request(),
ChatTemplateContentFormatOption::String,
)
.unwrap();
assert_eq!(
rendered.prompt,
Prompt::Text("a<audio><audio>b".to_string())
);
}
#[test]
fn openai_content_format_normalizes_image_url_for_template() {
let rendered = render_mm(
@@ -671,6 +712,21 @@ mod tests {
assert_eq!(rendered.prompt, Prompt::Text("a<|video_pad|>b".to_string()));
}
#[test]
fn openai_content_format_normalizes_audio_for_template() {
let rendered = render_mm(
"{% for item in messages[0].content %}{% if item.type == 'audio' %}<|audio_pad|>{% else %}{{ item.text }}{% endif %}{% endfor %}",
&audio_request(),
ChatTemplateContentFormatOption::OpenAi,
)
.unwrap();
assert_eq!(
rendered.prompt,
Prompt::Text("a<|audio_pad|><|audio_pad|>b".to_string())
);
}
#[test]
fn video_parts_are_rejected_when_model_lacks_video_support() {
let error = HfChatRenderer::new(
@@ -682,6 +738,7 @@ mod tests {
.with_multimodal(Some(MultimodalRenderInfo {
image_token: Some("<image>".to_string()),
video_token: None,
audio_token: None,
}))
.render(&video_request())
.unwrap_err();
rust/src/chat/src/request.rs
+34 -1
View File
@@ -44,6 +44,17 @@ pub enum ChatContentPart {
video_url: String,
uuid: Option<String>,
},
/// One `input_audio` content block carrying base64-encoded audio bytes.
InputAudio {
data: String,
format: Option<String>,
uuid: Option<String>,
},
/// One audio URL/data URL content block.
AudioUrl {
audio_url: String,
uuid: Option<String>,
},
// ImageData...
// VideoData...
// ImageEmbeds...
@@ -72,6 +83,23 @@ impl ChatContentPart {
}
}
/// Construct one base64-encoded input-audio content part.
pub fn input_audio(data: impl Into<String>, format: Option<String>) -> Self {
Self::InputAudio {
data: data.into(),
format,
uuid: None,
}
}
/// Construct one audio URL content part with the given URL string.
pub fn audio_url(audio_url: impl Into<String>) -> Self {
Self::AudioUrl {
audio_url: audio_url.into(),
uuid: None,
}
}
/// Return the text content of this part when it's a text block, or an
/// "unsupported multimodal content" error otherwise.
pub(crate) fn as_text(&self) -> Result<&str> {
@@ -79,6 +107,8 @@ impl ChatContentPart {
Self::Text { text } => Ok(text),
Self::ImageUrl { .. } => Err(Error::UnsupportedMultimodalContent("image_url")),
Self::VideoUrl { .. } => Err(Error::UnsupportedMultimodalContent("video_url")),
Self::InputAudio { .. } => Err(Error::UnsupportedMultimodalContent("input_audio")),
Self::AudioUrl { .. } => Err(Error::UnsupportedMultimodalContent("audio_url")),
}
}
@@ -91,7 +121,10 @@ impl ChatContentPart {
pub(crate) fn is_multimodal(&self) -> bool {
match self {
Self::Text { .. } => false,
Self::ImageUrl { .. } | Self::VideoUrl { .. } => true,
Self::ImageUrl { .. }
| Self::VideoUrl { .. }
| Self::InputAudio { .. }
| Self::AudioUrl { .. } => true,
}
}
}
rust/src/server/src/routes/openai/chat_completions/convert.rs
+56 -2
View File
@@ -299,6 +299,15 @@ fn convert_content(content: MessageContent) -> Result<ChatContent, ApiError> {
video_url: video_url.url,
uuid,
}),
ContentPart::AudioUrl { audio_url, uuid } => Ok(ChatContentPart::AudioUrl {
audio_url: audio_url.url,
uuid,
}),
ContentPart::InputAudio { input_audio, uuid } => Ok(ChatContentPart::InputAudio {
data: input_audio.data,
format: input_audio.format,
uuid,
}),
})
.try_collect()
.map(ChatContent::Parts),
@@ -402,8 +411,8 @@ mod tests {
AssistantRole, ChatCompletionMessage, ChatCompletionRequest,
};
use crate::routes::openai::utils::types::{
ChatMessage, ContentPart, Function, FunctionCallResponse, ImageUrl, MessageContent,
StreamOptions, Tool, ToolCall, ToolChoice, ToolChoiceValue, VideoUrl,
AudioUrl, ChatMessage, ContentPart, Function, FunctionCallResponse, ImageUrl, InputAudio,
MessageContent, StreamOptions, Tool, ToolCall, ToolChoice, ToolChoiceValue, VideoUrl,
};
use crate::utils::{ResolvedRequestContext, resolve_request_context};
@@ -810,6 +819,51 @@ mod tests {
);
}
#[test]
fn prepare_chat_request_accepts_audio_content_parts() {
let request = ChatCompletionRequest {
model: "Qwen/Qwen3-ASR-1.7B".to_string(),
messages: vec![ChatMessage::User {
content: MessageContent::Parts(vec![
ContentPart::InputAudio {
input_audio: InputAudio {
data: "dGVzdA==".to_string(),
format: Some("wav".to_string()),
},
uuid: Some("audio-1".to_string()),
},
ContentPart::AudioUrl {
audio_url: AudioUrl {
url: "https://example.com/audio.mp3".to_string(),
},
uuid: None,
},
]),
name: None,
}],
..base_request()
};
let prepared = prepare_chat_request(
request,
&served(&["Qwen/Qwen3-ASR-1.7B"]),
ResolvedRequestContext::default(),
)
.expect("request is valid");
assert_eq!(
prepared.chat_request.messages,
vec![VllmChatMessage::user(vec![
ChatContentPart::InputAudio {
data: "dGVzdA==".to_string(),
format: Some("wav".to_string()),
uuid: Some("audio-1".to_string()),
},
ChatContentPart::audio_url("https://example.com/audio.mp3"),
])]
);
}
#[test]
fn prepare_chat_request_rejects_assistant_image_url_content_parts() {
let request = ChatCompletionRequest {
rust/src/server/src/routes/openai/utils/types.rs
+25
View File
@@ -99,6 +99,18 @@ pub enum ContentPart {
#[serde(skip_serializing_if = "Option::is_none")]
uuid: Option<String>,
},
#[serde(rename = "audio_url")]
AudioUrl {
audio_url: AudioUrl,
#[serde(skip_serializing_if = "Option::is_none")]
uuid: Option<String>,
},
#[serde(rename = "input_audio")]
InputAudio {
input_audio: InputAudio,
#[serde(skip_serializing_if = "Option::is_none")]
uuid: Option<String>,
},
}
#[serde_with::skip_serializing_none]
@@ -113,6 +125,19 @@ pub struct VideoUrl {
pub url: String,
}
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
pub struct AudioUrl {
pub url: String,
}
/// Base64-encoded audio bytes in OpenAI `input_audio` form.
#[serde_with::skip_serializing_none]
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
pub struct InputAudio {
pub data: String,
pub format: Option<String>,
}
// ============================================================================
// Streaming
// ============================================================================
rust/src/server/src/routes/tests.rs
+4 -3
View File
@@ -551,9 +551,10 @@ fn render_fake_content(content: &ChatContent, placeholder: &str) -> vllm_chat::R
for part in parts {
match part {
ChatContentPart::Text { text } => out.push_str(text),
ChatContentPart::ImageUrl { .. } | ChatContentPart::VideoUrl { .. } => {
out.push_str(placeholder)
}
ChatContentPart::ImageUrl { .. }
| ChatContentPart::VideoUrl { .. }
| ChatContentPart::InputAudio { .. }
| ChatContentPart::AudioUrl { .. } => out.push_str(placeholder),
}
}
out
vllm/model_executor/models/qwen2_5_omni_thinker.py
+28
View File
@@ -114,6 +114,23 @@ except (ImportError, ModuleNotFoundError):
logger = init_logger(__name__)
def unpad_and_flat_audio_features(
input_audio_features: torch.Tensor,
audio_feature_lengths: torch.Tensor,
) -> torch.Tensor:
"""Unpad and flatten batched audio features."""
if len(audio_feature_lengths) != input_audio_features.size(0):
raise ValueError(
"Length of audio_feature_lengths must match "
"the batch size of input_audio_features."
)
unpadded_features = [
audio_feature[..., :length]
for audio_feature, length in zip(input_audio_features, audio_feature_lengths)
]
return torch.cat(unpadded_features, dim=-1)
def check_interleaved_audio_video(
is_video: torch.Tensor,
is_audio: torch.Tensor,
@@ -887,6 +904,17 @@ class Qwen2_5OmniConditionalGenerationMixin:
if input_audio_features is None:
return None
# inputs features from rust frontend is batched and padded
# with shape [batch_size, n_mels, padded_seq_len], different
# from python's shape [n_mels, batch_size * seq_len]
if (
isinstance(input_audio_features, torch.Tensor)
and input_audio_features.dim() == 3
):
input_audio_features = unpad_and_flat_audio_features(
input_audio_features, audio_feature_lengths
)
return Qwen2_5OmniAudioFeatureInputs(
type="audio_features",
input_features=input_audio_features,
vllm/model_executor/models/qwen3_asr.py
+20 -1
View File
@@ -46,9 +46,12 @@ from vllm.model_executor.models.interfaces import (
SupportsTranscription,
)
from vllm.model_executor.models.module_mapping import MultiModelKeys
from vllm.model_executor.models.qwen2_5_omni_thinker import (
Qwen2_5OmniAudioFeatureInputs,
unpad_and_flat_audio_features,
)
from vllm.model_executor.models.qwen3 import Qwen3ForCausalLM
from vllm.model_executor.models.qwen3_omni_moe_thinker import (
Qwen2_5OmniAudioFeatureInputs,
Qwen3OmniMoeAudioEncoder,
Qwen3OmniMoeThinkerMultiModalProcessor,
)
@@ -363,6 +366,11 @@ class Qwen3ASRForConditionalGeneration(
"thinker.lm_head.": "language_model.lm_head.",
"thinker.model.": "language_model.model.",
"thinker.": "",
# HF format mapper
"model.audio_tower.": "audio_tower.",
"model.language_model.": "language_model.model.",
"model.multi_modal_projector.linear_1.": "audio_tower.proj1.",
"model.multi_modal_projector.linear_2.": "audio_tower.proj2.",
}
)
@@ -412,6 +420,17 @@ class Qwen3ASRForConditionalGeneration(
if input_audio_features is None:
return None
# inputs features from rust frontend is batched and padded
# with shape [batch_size, n_mels, padded_seq_len], different
# from python's shape [n_mels, batch_size * seq_len]
if (
isinstance(input_audio_features, torch.Tensor)
and input_audio_features.dim() == 3
):
input_audio_features = unpad_and_flat_audio_features(
input_audio_features, audio_feature_lengths
)
return Qwen2_5OmniAudioFeatureInputs(
type="audio_features",
input_features=input_audio_features,
vllm/transformers_utils/configs/qwen3_asr.py
+17 -4
View File
@@ -407,16 +407,29 @@ class Qwen3ASRConfig(PretrainedConfig):
def __init__(
self,
thinker_config=None,
audio_config=None,
text_config=None,
audio_token_id=None,
support_languages=None,
timestamp_token_id=None,
timestamp_segment_time=None,
**kwargs,
):
if thinker_config is None:
thinker_config = {}
logger.info(
"thinker_config is None. Initializing thinker model with default values"
)
thinker_config = {
key: value
for key, value in (
("audio_config", audio_config),
("text_config", text_config),
("audio_token_id", audio_token_id),
)
if value is not None
}
if not thinker_config:
logger.info(
"thinker_config is None. Initializing thinker model with "
"default values"
)
self.thinker_config = Qwen3ASRThinkerConfig(**thinker_config)
self.support_languages = support_languages