Migrate Voxtral to mistral-common 1.11.5 audio API (#46705)

Signed-off-by: Julien Denize <40604584+juliendenize@users.noreply.github.com>

examples/generate/multimodal/audio_language_offline.py

@@ -463,16 +463,15 @@ def run_ultravox(question: str, audio_count: int) -> ModelRequestData:
 # Voxtral
 # Make sure to install mistral-common[audio].
 def run_voxtral(question: str, audio_count: int) -> ModelRequestData:
-    from mistral_common.audio import Audio
     from mistral_common.protocol.instruct.chunk import (
         AudioChunk,
-        RawAudio,
         TextChunk,
     )
     from mistral_common.protocol.instruct.messages import (
         UserMessage,
     )
     from mistral_common.protocol.instruct.request import ChatCompletionRequest
+    from mistral_common.tokens.tokenizers.audio import Audio
     from mistral_common.tokens.tokenizers.mistral import MistralTokenizer
 
     model_name = "mistralai/Voxtral-Mini-3B-2507"
@@ -495,9 +494,7 @@ def run_voxtral(question: str, audio_count: int) -> ModelRequestData:
         Audio.from_file(str(audio_assets[i].get_local_path()), strict=False)
         for i in range(audio_count)
     ]
-    audio_chunks = [
-        AudioChunk(input_audio=RawAudio.from_audio(audio)) for audio in audios
-    ]
+    audio_chunks = [AudioChunk.from_audio(audio) for audio in audios]
 
     messages = [UserMessage(content=[*audio_chunks, text_chunk])]
 

requirements/common.txt

@@ -33,7 +33,7 @@ partial-json-parser # used for parsing partial JSON outputs
 jsonschema >= 4.23.0 # required for MiniMax M3 tool schema validation
 pyzmq >= 25.0.0
 msgspec
-mistral_common[image] >= 1.11.3
+mistral_common[image] >= 1.11.5
 opencv-python-headless >= 4.13.0    # required for video IO
 pyyaml
 six>=1.16.0; python_version > '3.11' # transitive dependency of pandas that needs to be the latest version for python 3.12

requirements/test/cuda.in

@@ -31,7 +31,7 @@ torchaudio==2.11.0
 torchvision==0.26.0
 transformers_stream_generator # required for qwen-vl test
 matplotlib # required for qwen-vl test
-mistral_common[image,audio] >= 1.11.3 # required for voxtral test
+mistral_common[image,audio] >= 1.11.5 # required for voxtral test
 num2words # required for smolvlm test
 open_clip_torch==2.32.0 # Required for nemotron_vl test, Nemotron Parse in test_common.py
 opencv-python-headless >= 4.13.0 # required for video test

requirements/test/cuda.txt

@@ -398,7 +398,7 @@ mbstrdecoder==1.1.3
     #   typepy
 mdurl==0.1.2
     # via markdown-it-py
-mistral-common==1.11.3
+mistral-common==1.11.5
     # via
     #   -c requirements/common.txt
     #   -r requirements/test/cuda.in

requirements/test/nightly-torch.txt

@@ -23,7 +23,7 @@ jiwer # required for audio tests
 timm # required for internvl test
 transformers_stream_generator # required for qwen-vl test
 matplotlib # required for qwen-vl test
-mistral_common[image,audio] >= 1.11.3 # required for voxtral test
+mistral_common[image,audio] >= 1.11.5 # required for voxtral test
 num2words # required for smolvlm test
 opencv-python-headless >= 4.13.0 # required for video test
 datamodel_code_generator # required for minicpm3 test

requirements/test/rocm.in

@@ -30,7 +30,7 @@ tblib # for pickling test exceptions
 timm>=1.0.17 # required for internvl and gemma3n-mm test
 transformers_stream_generator # required for qwen-vl test
 matplotlib # required for qwen-vl test
-mistral_common[image,audio]>=1.11.3 # required for voxtral test
+mistral_common[image,audio]>=1.11.5 # required for voxtral test
 num2words # required for smolvlm test
 open_clip_torch==2.32.0 # Required for nemotron_vl test, Nemotron Parse in test_common.py
 opencv-python-headless>=4.13.0 # required for video test

requirements/test/rocm.txt

@@ -499,7 +499,7 @@ mcp==1.27.0
     # via -r requirements/test/../common.txt
 mdurl==0.1.2
     # via markdown-it-py
-mistral-common==1.11.3
+mistral-common==1.11.5
     # via
     #   -c requirements/common.txt
     #   -r requirements/test/../common.txt

requirements/test/xpu.txt

@@ -264,7 +264,7 @@ mbstrdecoder==1.1.4
     #   typepy
 mdurl==0.1.2
     # via markdown-it-py
-mistral-common==1.11.3
+mistral-common==1.11.5
     # via
     #   -c requirements/common.txt
     #   -r requirements/test/xpu.in

tests/models/multimodal/generation/test_voxtral.py

@@ -4,9 +4,9 @@
 import json
 
 import pytest
-from mistral_common.audio import Audio
-from mistral_common.protocol.instruct.chunk import AudioChunk, RawAudio, TextChunk
+from mistral_common.protocol.instruct.chunk import AudioChunk, TextChunk
 from mistral_common.protocol.instruct.messages import UserMessage
+from mistral_common.tokens.tokenizers.audio import Audio
 from transformers import VoxtralForConditionalGeneration
 
 from vllm.tokenizers.mistral import MistralTokenizer
@@ -36,9 +36,7 @@ def _get_prompt(audio_assets: AudioTestAssets, question: str) -> list[int]:
         Audio.from_file(str(asset.get_local_path()), strict=False)
         for asset in audio_assets
     ]
-    audio_chunks = [
-        AudioChunk(input_audio=RawAudio.from_audio(audio)) for audio in audios
-    ]
+    audio_chunks = [AudioChunk.from_audio(audio) for audio in audios]
 
     messages = [
         UserMessage(content=[*audio_chunks, TextChunk(text=question)]).to_openai()

tests/models/multimodal/generation/test_voxtral_realtime.py

@@ -4,12 +4,11 @@ import contextlib
 
 import pytest
 import pytest_asyncio
-from mistral_common.audio import Audio
-from mistral_common.protocol.instruct.chunk import RawAudio
 from mistral_common.protocol.transcription.request import (
     StreamingMode,
     TranscriptionRequest,
 )
+from mistral_common.tokens.tokenizers.audio import Audio
 from mistral_common.tokens.tokenizers.mistral import MistralTokenizer
 from mistral_common.tokens.tokenizers.tekken import SpecialTokenPolicy
 
@@ -101,7 +100,7 @@ def test_voxtral_realtime_forward(audio_assets, tokenizer, engine):
     def from_file(file_path: str):
         audio = Audio.from_file(file_path, strict=False)
         req = TranscriptionRequest(
-            audio=RawAudio.from_audio(audio),
+            audio=audio.to_base64(audio.format),
             streaming=StreamingMode.OFFLINE,
             language=None,
         )
@@ -156,7 +155,7 @@ async def test_voxtral_realtime_generator(audio_assets, tokenizer, async_engine)
 
         req = TranscriptionRequest(
             streaming=StreamingMode.OFFLINE,
-            audio=RawAudio.from_audio(audio),
+            audio=audio.to_base64(audio.format),
             language=None,
         )
         audio_enc = tokenizer.encode_transcription(req)

vllm/model_executor/models/voxtral.py

@@ -10,11 +10,12 @@ import numpy as np
 import regex as re
 import torch
 import torch.nn as nn
-from mistral_common.audio import Audio, mel_filter_bank
-from mistral_common.protocol.instruct.chunk import AudioChunk, RawAudio, TextChunk
+from mistral_common.audio import mel_filter_bank
+from mistral_common.protocol.instruct.chunk import AudioChunk, TextChunk
 from mistral_common.protocol.instruct.messages import UserMessage
 from mistral_common.protocol.instruct.request import ChatCompletionRequest
 from mistral_common.protocol.transcription.request import TranscriptionRequest
+from mistral_common.tokens.tokenizers.audio import Audio
 from transformers import BatchFeature, WhisperConfig
 
 from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
@@ -182,7 +183,7 @@ class VoxtralDummyInputsBuilder(BaseDummyInputsBuilder[VoxtralProcessingInfo]):
                 sampling_rate=feature_extractor.sampling_rate,
                 format=format,
             )
-            chunk = AudioChunk(input_audio=RawAudio.from_audio(audio_item))
+            chunk = AudioChunk.from_audio(audio_item)
             audio_chunks.append(chunk)
 
         request = ChatCompletionRequest(
@@ -462,7 +463,7 @@ class VoxtralForConditionalGeneration(
         audio = Audio(audio, int(stt_config.sample_rate), format="wav")  # lossless
         req = TranscriptionRequest(
             model=model_config.model,
-            audio=RawAudio.from_audio(audio),
+            audio=audio.to_base64(audio.format),
             language=language,
         )
 

vllm/model_executor/models/voxtral_realtime.py

@@ -7,13 +7,11 @@ from collections.abc import AsyncGenerator, Iterable, Iterator, Mapping
 
 import numpy as np
 import torch
-from mistral_common.audio import Audio
-from mistral_common.protocol.instruct.chunk import RawAudio
 from mistral_common.protocol.transcription.request import (
     StreamingMode,
     TranscriptionRequest,
 )
-from mistral_common.tokens.tokenizers.audio import AudioConfig
+from mistral_common.tokens.tokenizers.audio import Audio, AudioConfig
 
 from vllm.compilation.decorators import support_torch_compile
 from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
@@ -477,7 +475,7 @@ class VoxtralRealtimeGeneration(VoxtralForConditionalGeneration, SupportsRealtim
 
         req = TranscriptionRequest(
             model=model_config.model,
-            audio=RawAudio.from_audio(audio),
+            audio=audio.to_base64(audio.format),
             language=language,
             streaming=StreamingMode.OFFLINE,
         )