diff --git a/.buildkite/intel_jobs/test-intel.yaml b/.buildkite/intel_jobs/test-intel.yaml index 4b8ba212560..203e63f25e2 100644 --- a/.buildkite/intel_jobs/test-intel.yaml +++ b/.buildkite/intel_jobs/test-intel.yaml @@ -49,6 +49,30 @@ steps: VLLM_XPU_FUSED_MOE_USE_REF=1 python3 examples/basic/offline_inference/generate.py --model Qwen/Qwen3-30B-A3B-Instruct-2507-FP8 --enforce-eager -tp 2 --max-model-len 8192 && python3 examples/basic/offline_inference/generate.py --model INCModel/Qwen3-30B-A3B-Instruct-2507-MXFP4-LLMC --enforce-eager -tp 2 --max-model-len 8192 ' + - label: "XPU W8A8 FP8 Linear Examples" + depends_on: + - image-build-xpu + timeout_in_minutes: 60 + device: intel_gpu + agent_tags: + label: production + gpu: 1+ + mem: 24+ + no_plugin: true + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/ + - .buildkite/intel_jobs/test-intel.yaml + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8 --enforce-eager --max-model-len 4096 && + python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model neuralmagic/Llama-3.2-1B-Instruct-FP8-dynamic --enforce-eager --max-model-len 4096 && + python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8 --enforce-eager --max-model-len 4096 + ' - label: "XPU V1 test" depends_on: - image-build-xpu @@ -120,4 +144,27 @@ steps: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'cd tests && - pytest -v -s quantization/test_auto_round.py' \ No newline at end of file + pytest -v -s quantization/test_auto_round.py' + - label: "XPU compressed tensors FP8 test" + depends_on: + - image-build-xpu + timeout_in_minutes: 60 + device: intel_gpu + agent_tags: + label: production + gpu: 1+ + mem: 16+ + no_plugin: true + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/ + - tests/quantization/test_compressed_tensors.py + - .buildkite/intel_jobs/test-intel.yaml + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'cd tests && + pytest -v -s quantization/test_compressed_tensors.py::test_compressed_tensors_fp8' \ No newline at end of file diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index c74f516c088..57c976030de 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -2248,7 +2248,7 @@ steps: - pytest -v -s v1/worker - pytest -v -s -m 'not cpu_test' v1/kv_connector/unit - pytest -v -s -m 'not cpu_test' v1/metrics - - pip install -U git+https://github.com/robertgshaw2-redhat/lm-evaluation-harness.git@streaming-api + - pip install -U git+https://github.com/vllm-project/lm-evaluation-harness.git@streaming-api # - export HSA_NO_SCRATCH_RECLAIM=1 - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine @@ -3318,7 +3318,7 @@ steps: - pytest -v -s v1/worker - pytest -v -s -m 'not cpu_test' v1/kv_connector/unit - pytest -v -s -m 'not cpu_test' v1/metrics - - pip install -U git+https://github.com/robertgshaw2-redhat/lm-evaluation-harness.git@streaming-api + - pip install -U git+https://github.com/vllm-project/lm-evaluation-harness.git@streaming-api - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine - label: V1 Sample + Logits # TBD diff --git a/.buildkite/test_areas/misc.yaml b/.buildkite/test_areas/misc.yaml index 1ad04c28970..7aeb8405066 100644 --- a/.buildkite/test_areas/misc.yaml +++ b/.buildkite/test_areas/misc.yaml @@ -103,7 +103,7 @@ steps: - pytest -v -s -m 'not cpu_test' v1/kv_connector/unit - pytest -v -s -m 'not cpu_test' v1/metrics # Integration test for streaming correctness (requires special branch). - - pip install -U git+https://github.com/robertgshaw2-redhat/lm-evaluation-harness.git@streaming-api + - pip install -U git+https://github.com/vllm-project/lm-evaluation-harness.git@streaming-api - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine mirror: amd: diff --git a/csrc/cpu/cpu_attn_impl.hpp b/csrc/cpu/cpu_attn_impl.hpp index 260ed7cd417..907f8895682 100644 --- a/csrc/cpu/cpu_attn_impl.hpp +++ b/csrc/cpu/cpu_attn_impl.hpp @@ -417,8 +417,10 @@ class AttentionScheduler { has_decode_request = has_decode_request || (q_token_num == 1); decode_only_batch = decode_only_batch && (q_token_num == 1); } - int32_t q_head_per_kv = input.num_heads_q / input.num_heads_kv; - const bool supports_gqa = q_head_per_kv <= max_num_q_per_iter; + const int32_t original_q_head_per_kv = + input.num_heads_q / input.num_heads_kv; + int32_t q_head_per_kv = original_q_head_per_kv; + const bool supports_gqa = original_q_head_per_kv <= max_num_q_per_iter; const bool use_gqa_fast_path = supports_gqa && decode_only_batch; const bool use_gqa_scratchpad = supports_gqa && has_decode_request; if (!use_gqa_scratchpad) { @@ -671,22 +673,62 @@ class AttentionScheduler { metadata_ptr->effective_thread_num = effective_thread_num; { - // when q_tile_size = max_num_q_per_iter, requires max - // attention_scratchpad_size AttentionScratchPad sc(0, *metadata_ptr, 0x0); - int64_t n = AttentionScheduler::calcu_tile_size_with_constant_q( - cache_size, input.head_dim, input.elem_size, input.q_buffer_elem_size, - input.logits_buffer_elem_size, input.output_buffer_elem_size, - max_num_q_per_iter, kv_len_alignment, max_num_q_per_iter, true); - sc.update(input.head_dim, input.q_buffer_elem_size, - input.logits_buffer_elem_size, input.output_buffer_elem_size, - max_num_q_per_iter, max_num_q_per_iter, n); + int64_t max_attention_scratchpad_size = 0; + + for (const AttentionWorkItemGroup& item : workitems) { + const bool curr_use_gqa = + use_gqa_fast_path || (supports_gqa && item.q_token_num == 1); + const int32_t curr_q_heads_per_kv = + curr_use_gqa ? original_q_head_per_kv : 1; + const int32_t curr_default_q_tile_token_num = + default_tile_size / curr_q_heads_per_kv; + + for (int32_t q_token_offset = 0; q_token_offset < item.q_token_num; + q_token_offset += curr_default_q_tile_token_num) { + const int32_t actual_q_token_num = std::min( + curr_default_q_tile_token_num, item.q_token_num - q_token_offset); + const int32_t q_head_tile_size = + actual_q_token_num * curr_q_heads_per_kv; + const int32_t rounded_q_head_tile_size = + ((q_head_tile_size + max_num_q_per_iter - 1) / + max_num_q_per_iter) * + max_num_q_per_iter; + + const int64_t n = AttentionScheduler::calcu_tile_size_with_constant_q( + cache_size, input.head_dim, input.elem_size, + input.q_buffer_elem_size, input.logits_buffer_elem_size, + input.output_buffer_elem_size, max_num_q_per_iter, + kv_len_alignment, rounded_q_head_tile_size, + rounded_q_head_tile_size <= max_num_q_per_iter); + + sc.update(input.head_dim, input.q_buffer_elem_size, + input.logits_buffer_elem_size, + input.output_buffer_elem_size, max_num_q_per_iter, + rounded_q_head_tile_size, n); + + max_attention_scratchpad_size = std::max( + max_attention_scratchpad_size, sc.get_thread_scratchpad_size()); + } + } + metadata_ptr->attention_scratchpad_size_per_thread = - ((sc.get_thread_scratchpad_size() + 63) / 64) * 64; + ((max_attention_scratchpad_size + 63) / 64) * 64; + + int32_t max_reduction_q_head_tile_size = 0; + for (const ReductionWorkItemGroup& item : reduce_workitems) { + const bool curr_use_gqa = + use_gqa_fast_path || (supports_gqa && item.q_token_id_num == 1); + const int32_t curr_q_heads_per_kv = + curr_use_gqa ? original_q_head_per_kv : 1; + + max_reduction_q_head_tile_size = + std::max(max_reduction_q_head_tile_size, + item.q_token_id_num * curr_q_heads_per_kv); + } sc.update(0, metadata_ptr->reduction_split_num, input.head_dim, - q_head_per_kv * split_kv_q_token_num_threshold, - input.output_buffer_elem_size); + max_reduction_q_head_tile_size, input.output_buffer_elem_size); metadata_ptr->reduction_scratchpad_size_per_kv_head = ((sc.get_reduction_scratchpad_size() + 63) / 64) * 64; } diff --git a/docs/features/quantization/online.md b/docs/features/quantization/online.md index 4b9571b38b9..a4da52557af 100644 --- a/docs/features/quantization/online.md +++ b/docs/features/quantization/online.md @@ -62,6 +62,8 @@ weight name. Unset fields fall back to the `--quantization` shorthand's defaults, or for already-quantized checkpoints to whatever the checkpoint declares. +On XPU, non-block FP8 scaled-mm linear layers default to W8A16; setting `--linear-backend xpu` forces W8A8. Use `--linear-backend xpu_woq` to explicitly select weight-only quantization (W8A16). + The CLI accepts the same shape as JSON or as dotted keys: ```bash diff --git a/docs/features/quantization/quantized_kvcache.md b/docs/features/quantization/quantized_kvcache.md index 2c5bfd64394..50b1c5c2df3 100644 --- a/docs/features/quantization/quantized_kvcache.md +++ b/docs/features/quantization/quantized_kvcache.md @@ -49,6 +49,32 @@ You can configure how the quantization scales are computed in vLLM using three d - `kv_cache_dtype="fp8_e4m3"`: Supported on CUDA 11.8+ and ROCm (AMD GPUs) - `kv_cache_dtype="fp8_e5m2"`: Supported on CUDA 11.8+ +### Skipping Specific Layers from KV-Cache Quantization + +Some attention layer types (e.g. sliding-window) are more sensitive to KV-cache quantization. The `--kv-cache-dtype-skip-layers` flag leaves the specified layers at the model's native dtype while keeping the rest of the layers under the chosen quantized dtype. The flag accepts either layer indices or layer-type names: + +```bash +# Skip every sliding-window attention layer. +vllm serve \ + --kv-cache-dtype fp8 \ + --kv-cache-dtype-skip-layers sliding_window + +# Skip specific layer indices. +vllm serve \ + --kv-cache-dtype fp8 \ + --kv-cache-dtype-skip-layers 0 1 23 +``` + +Programmatic usage: + +```python +llm = LLM( + model="meta-llama/Llama-3.1-8B-Instruct", + kv_cache_dtype="fp8", + kv_cache_dtype_skip_layers=["sliding_window"], +) +``` + --- ## Examples diff --git a/docs/features/speculative_decoding/dynamic_speculative_decoding.md b/docs/features/speculative_decoding/dynamic_speculative_decoding.md index eecf789d6dc..aa55c5f08d4 100644 --- a/docs/features/speculative_decoding/dynamic_speculative_decoding.md +++ b/docs/features/speculative_decoding/dynamic_speculative_decoding.md @@ -71,8 +71,5 @@ VLLM_USE_V2_MODEL_RUNNER=0 vllm serve meta-llama/Llama-3.1-8B-Instruct \ ## Limitations -* only tested with Eagle and Eagle-3. Other SD methods may or may not work out of the box -* only usable with Model Runner V1 -* not compatible with full cuda graph so we force piece-wise cuda graph with this feature - -We are working on enabling it on MRv2 with full cuda graph support. +* Tested with Eagle, Eagle-3, and DFlash. Other SD methods may or may not work out of the box +* Full Cudagraph only works with Model Runner V2. MRv1 only supports piece-wise cuda graph with this feature diff --git a/requirements/test/cpu.txt b/requirements/test/cpu.txt index e953419242d..6cc3a3782b8 100644 --- a/requirements/test/cpu.txt +++ b/requirements/test/cpu.txt @@ -112,9 +112,10 @@ charset-normalizer==3.4.0 # via requests chz==0.3.0 # via gpt-oss -click==8.1.7 +click==8.4.2 # via # black + # huggingface-hub # jiwer # nltk # ray @@ -309,7 +310,7 @@ h2==4.3.0 # via httpx harfile==0.5.0 # via schemathesis -hf-xet==1.4.3 +hf-xet==1.5.1 # via huggingface-hub hiredis==3.0.0 # via tensorizer @@ -335,7 +336,7 @@ httpx==0.27.2 # schemathesis httpx-sse==0.4.3 # via mcp -huggingface-hub==1.10.2 +huggingface-hub==1.22.0 # via # accelerate # datasets @@ -1182,7 +1183,6 @@ typer==0.26.8 # fastapi-cli # fastapi-cloud-cli # fastsafetensors - # huggingface-hub # perceptron # transformers typing-extensions==4.15.0 diff --git a/requirements/test/cuda.txt b/requirements/test/cuda.txt index e8d600ba632..3f7e4a5b5f3 100644 --- a/requirements/test/cuda.txt +++ b/requirements/test/cuda.txt @@ -117,9 +117,10 @@ charset-normalizer==3.4.0 # via requests chz==0.3.0 # via gpt-oss -click==8.1.7 +click==8.4.2 # via # black + # huggingface-hub # jiwer # nltk # ray @@ -330,7 +331,7 @@ h2==4.3.0 # via httpx harfile==0.5.0 # via schemathesis -hf-xet==1.4.3 +hf-xet==1.5.1 # via huggingface-hub hiredis==3.0.0 # via tensorizer @@ -356,7 +357,7 @@ httpx==0.27.2 # schemathesis httpx-sse==0.4.3 # via mcp -huggingface-hub==1.10.2 +huggingface-hub==1.22.0 # via # accelerate # datasets @@ -1285,7 +1286,6 @@ typer==0.26.8 # fastapi-cli # fastapi-cloud-cli # fastsafetensors - # huggingface-hub # perceptron # transformers typing-extensions==4.15.0 diff --git a/requirements/test/rocm.txt b/requirements/test/rocm.txt index 55cac6f5243..e6f19b23d1d 100644 --- a/requirements/test/rocm.txt +++ b/requirements/test/rocm.txt @@ -116,9 +116,10 @@ choreographer==1.2.1 # via kaleido chz==0.4.0 # via gpt-oss -click==8.3.1 +click==8.4.2 # via # black + # huggingface-hub # jiwer # nltk # ray @@ -323,7 +324,7 @@ h2==4.3.0 # via httpx harfile==0.5.0 # via schemathesis -hf-xet==1.4.3 +hf-xet==1.5.1 # via huggingface-hub hiredis==3.3.1 # via tensorizer @@ -349,7 +350,7 @@ httpx==0.27.2 # schemathesis httpx-sse==0.4.3 # via mcp -huggingface-hub==1.10.2 +huggingface-hub==1.22.0 # via # accelerate # datasets @@ -1244,7 +1245,6 @@ typer==0.24.1 # fastapi-cli # fastapi-cloud-cli # fastsafetensors - # huggingface-hub # perceptron # transformers typing-extensions==4.15.0 diff --git a/requirements/test/xpu.txt b/requirements/test/xpu.txt index 16169b99863..b5a36cf9d50 100644 --- a/requirements/test/xpu.txt +++ b/requirements/test/xpu.txt @@ -83,8 +83,9 @@ charset-normalizer==3.4.6 # via requests chz==0.4.0 # via gpt-oss -click==8.3.1 +click==8.4.2 # via + # huggingface-hub # jiwer # nltk # rich-toolkit @@ -206,7 +207,7 @@ h11==0.16.0 # uvicorn harfile==0.4.0 # via schemathesis -hf-xet==1.4.3 +hf-xet==1.5.1 # via huggingface-hub html2text==2025.4.15 # via gpt-oss @@ -227,7 +228,7 @@ httpx==0.28.1 # schemathesis httpx-sse==0.4.3 # via mcp -huggingface-hub==1.10.2 +huggingface-hub==1.22.0 # via # accelerate # datasets @@ -959,7 +960,6 @@ typer==0.24.1 # via # fastapi-cli # fastapi-cloud-cli - # huggingface-hub # transformers typing-extensions==4.15.0 # via diff --git a/tests/benchmarks/test_bfcl_dataset.py b/tests/benchmarks/test_bfcl_dataset.py index e5110c50985..d1919223197 100644 --- a/tests/benchmarks/test_bfcl_dataset.py +++ b/tests/benchmarks/test_bfcl_dataset.py @@ -21,7 +21,7 @@ def _patch_hf_api(side_effect): @pytest.fixture(scope="session") def hf_tokenizer() -> PreTrainedTokenizerBase: - return AutoTokenizer.from_pretrained("gpt2") + return AutoTokenizer.from_pretrained("openai-community/gpt2") _FAKE_ROWS = { diff --git a/tests/benchmarks/test_custom_dataset_seed.py b/tests/benchmarks/test_custom_dataset_seed.py index dac87e6e6d9..d23ce40b53e 100644 --- a/tests/benchmarks/test_custom_dataset_seed.py +++ b/tests/benchmarks/test_custom_dataset_seed.py @@ -12,7 +12,7 @@ from vllm.benchmarks.datasets import get_samples @pytest.fixture(scope="session") def hf_tokenizer() -> PreTrainedTokenizerBase: - return AutoTokenizer.from_pretrained("gpt2") + return AutoTokenizer.from_pretrained("openai-community/gpt2") def _write_jsonl(path: Path, n_rows: int) -> None: diff --git a/tests/benchmarks/test_random_dataset.py b/tests/benchmarks/test_random_dataset.py index 57f68930618..ff691ae15d0 100644 --- a/tests/benchmarks/test_random_dataset.py +++ b/tests/benchmarks/test_random_dataset.py @@ -17,7 +17,7 @@ from vllm.benchmarks.datasets import ( @pytest.fixture(scope="session") def hf_tokenizer() -> PreTrainedTokenizerBase: # Use a small, commonly available tokenizer - return AutoTokenizer.from_pretrained("gpt2") + return AutoTokenizer.from_pretrained("openai-community/gpt2") class Params(NamedTuple): diff --git a/tests/benchmarks/test_random_multimodal_dataset_video.py b/tests/benchmarks/test_random_multimodal_dataset_video.py index bd37a520d01..b394ea2c0d7 100644 --- a/tests/benchmarks/test_random_multimodal_dataset_video.py +++ b/tests/benchmarks/test_random_multimodal_dataset_video.py @@ -16,7 +16,7 @@ from vllm.benchmarks.datasets import RandomMultiModalDataset, SampleRequest @pytest.fixture(scope="session") def hf_tokenizer() -> PreTrainedTokenizerBase: """Use a small, commonly available tokenizer.""" - return AutoTokenizer.from_pretrained("gpt2") + return AutoTokenizer.from_pretrained("openai-community/gpt2") @pytest.fixture diff --git a/tests/benchmarks/test_txt_slices_dataset.py b/tests/benchmarks/test_txt_slices_dataset.py index 7821e9a925a..8741805d0d5 100644 --- a/tests/benchmarks/test_txt_slices_dataset.py +++ b/tests/benchmarks/test_txt_slices_dataset.py @@ -13,7 +13,7 @@ from vllm.benchmarks.datasets.create_txt_slices_dataset import create_txt_slices @pytest.fixture(scope="session") def hf_tokenizer() -> PreTrainedTokenizerBase: # Use a small, commonly available tokenizer - return AutoTokenizer.from_pretrained("gpt2") + return AutoTokenizer.from_pretrained("openai-community/gpt2") text_content = """ @@ -39,7 +39,7 @@ def test_create_txt_slices_jsonl( create_txt_slices_jsonl( input_path=str(txt_path), output_path=str(jsonl_path), - tokenizer_name="gpt2", + tokenizer_name="openai-community/gpt2", num_prompts=10, input_len=10, output_len=10, diff --git a/tests/compile/test_aot_compile.py b/tests/compile/test_aot_compile.py index 5ff0fac6c82..a7f32483a70 100644 --- a/tests/compile/test_aot_compile.py +++ b/tests/compile/test_aot_compile.py @@ -502,7 +502,7 @@ def test_gpt2_cache_hit(monkeypatch: pytest.MonkeyPatch): m.setenv("VLLM_USE_AOT_COMPILE", "1") # First compilation - initialize model and generate llm_model = LLM( - model="gpt2", + model="openai-community/gpt2", compilation_config=CompilationConfig( mode=CompilationMode.VLLM_COMPILE, ), @@ -519,7 +519,7 @@ def test_gpt2_cache_hit(monkeypatch: pytest.MonkeyPatch): # Second compilation - should hit cache m.setenv("VLLM_FORCE_AOT_LOAD", "1") llm_model = LLM( - model="gpt2", + model="openai-community/gpt2", compilation_config=CompilationConfig( mode=CompilationMode.VLLM_COMPILE, ), diff --git a/tests/compile/test_dynamic_shapes_compilation.py b/tests/compile/test_dynamic_shapes_compilation.py index b8c18fa6cdc..96c3f49aba3 100644 --- a/tests/compile/test_dynamic_shapes_compilation.py +++ b/tests/compile/test_dynamic_shapes_compilation.py @@ -24,7 +24,7 @@ from vllm.utils.torch_utils import is_torch_equal_or_newer def get_test_models(): """Get list of models to test based on PyTorch version""" models = [ - "gpt2", + "openai-community/gpt2", "Qwen/Qwen2-7B-Instruct", "meta-llama/Llama-3.1-8B", ] diff --git a/tests/distributed/test_pipeline_parallel.py b/tests/distributed/test_pipeline_parallel.py index e773c7d826a..762a95fb987 100644 --- a/tests/distributed/test_pipeline_parallel.py +++ b/tests/distributed/test_pipeline_parallel.py @@ -114,7 +114,7 @@ TEXT_GENERATION_MODELS = { "tiiuae/falcon-7b": PPTestSettings.fast(), "google/gemma-1.1-2b-it": PPTestSettings.fast(), "google/gemma-2-9b": PPTestSettings.fast(), - "gpt2": PPTestSettings.fast(), + "openai-community/gpt2": PPTestSettings.fast(), "EleutherAI/gpt-j-6b": PPTestSettings.fast(), "EleutherAI/pythia-1.4b": PPTestSettings.fast(), "ibm/PowerLM-3b": PPTestSettings.fast(), diff --git a/tests/entrypoints/openai/completion/test_prompt_validation.py b/tests/entrypoints/openai/completion/test_prompt_validation.py index 81204b27bc0..87c6b6e1668 100644 --- a/tests/entrypoints/openai/completion/test_prompt_validation.py +++ b/tests/entrypoints/openai/completion/test_prompt_validation.py @@ -18,7 +18,7 @@ from vllm.renderers.embed_utils import safe_load_prompt_embeds @pytest.mark.asyncio async def test_empty_prompt(): - model_name = "gpt2" + model_name = "openai-community/gpt2" server_args = ["--enforce-eager"] with RemoteOpenAIServer(model_name, server_args) as remote_server: client = remote_server.get_async_client() @@ -38,7 +38,7 @@ async def test_empty_prompt(): @pytest.mark.asyncio async def test_out_of_vocab_token_ids(): - model_name = "gpt2" + model_name = "openai-community/gpt2" server_args = ["--enforce-eager"] with RemoteOpenAIServer(model_name, server_args) as remote_server: client = remote_server.get_async_client() diff --git a/tests/entrypoints/unit_tests/test_context.py b/tests/entrypoints/unit_tests/test_context.py index 0fa3661f8ff..1c1f6ed2359 100644 --- a/tests/entrypoints/unit_tests/test_context.py +++ b/tests/entrypoints/unit_tests/test_context.py @@ -74,7 +74,7 @@ class FakeHarmonyParser(HarmonyParser): self.reasoning_parser = None self.tool_parser = None self._chunk_results: list[ChunkResult] = [] - self._flush_results: list[Segment | None] = [] + self._flush_results: list[list[Segment]] = [] self.processed_chunks: list[list[int]] = [] def enqueue_chunk_result( @@ -89,7 +89,7 @@ class FakeHarmonyParser(HarmonyParser): ) ) - def enqueue_flush_result(self, segment: Segment | None) -> None: + def enqueue_flush_result(self, segment: list[Segment]) -> None: self._flush_results.append(segment) def process_chunk(self, token_ids) -> ChunkResult: @@ -98,10 +98,10 @@ class FakeHarmonyParser(HarmonyParser): return self._chunk_results.pop(0) return ChunkResult(segments=[], reasoning_token_count=0) - def flush(self) -> Segment | None: + def flush(self) -> list[Segment]: if self._flush_results: return self._flush_results.pop(0) - return None + return [] def make_harmony_context( @@ -598,13 +598,21 @@ async def test_streaming_message_synchronization(): content=[TextContent(text=response_text)], recipient=Role.USER, ) - flush_segment = Segment( - channel="commentary", - recipient=None, - delta="", - completed_message=message, - ) - parser.enqueue_flush_result(flush_segment) + flush_segments = [ + Segment( + channel="final", + recipient=None, + delta=response_text, + completed_message=None, + ), + Segment( + channel="final", + recipient=None, + delta="", + completed_message=message, + ), + ] + parser.enqueue_flush_result(flush_segments) # Create another output to trigger synchronization via flush() context.append_output( @@ -618,8 +626,9 @@ async def test_streaming_message_synchronization(): assert context.num_init_messages == 1 assert context._messages[2].content[0].text == response_text assert context.last_append_flush_status is True - assert len(context.last_append_segments) == 1 - assert context.last_append_segments[0].completed_message is message + assert len(context.last_append_segments) == 2 + assert context.last_append_segments[-2].delta == response_text + assert context.last_append_segments[-1].completed_message is message def test_turn_metrics_copy_and_reset(): diff --git a/tests/lora/test_chatglm3_tp.py b/tests/lora/test_chatglm3_tp.py index ace4fb5f50e..8df4ccf7b56 100644 --- a/tests/lora/test_chatglm3_tp.py +++ b/tests/lora/test_chatglm3_tp.py @@ -115,6 +115,7 @@ def test_chatglm3_lora_tp4_fully_sharded_loras(chatglm3_lora_files): enable_lora=True, max_loras=2, max_lora_rank=64, + max_num_seqs=16, tensor_parallel_size=4, trust_remote_code=True, fully_sharded_loras=True, diff --git a/tests/models/multimodal/generation/test_voxtral_realtime.py b/tests/models/multimodal/generation/test_voxtral_realtime.py index 9fb601df02c..4d37b36c364 100644 --- a/tests/models/multimodal/generation/test_voxtral_realtime.py +++ b/tests/models/multimodal/generation/test_voxtral_realtime.py @@ -1,7 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import contextlib - import pytest import pytest_asyncio from mistral_common.protocol.transcription.request import ( @@ -12,7 +10,7 @@ from mistral_common.tokens.tokenizers.audio import Audio from mistral_common.tokens.tokenizers.mistral import MistralTokenizer from mistral_common.tokens.tokenizers.tekken import SpecialTokenPolicy -from vllm import LLM, EngineArgs, SamplingParams +from vllm import LLM, SamplingParams from vllm.assets.audio import AudioAsset from vllm.engine.arg_utils import AsyncEngineArgs from vllm.utils.math_utils import cdiv @@ -100,77 +98,88 @@ def tokenizer() -> MistralTokenizer: return MistralTokenizer.from_hf_hub(MODEL_NAME) -@pytest.fixture -def engine(monkeypatch: pytest.MonkeyPatch): - # Disable multiprocessing allows us to access model executor from LLM engine - monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0") - engine_args = EngineArgs(**ENGINE_CONFIG) - llm = LLM.from_engine_args(engine_args) - try: - yield llm - finally: - with contextlib.suppress(Exception): - llm.llm_engine.engine_core.shutdown() - import torch - - torch.accelerator.empty_cache() - - @pytest_asyncio.fixture async def async_engine(): + gpu_memory_utilization = ENGINE_CONFIG.get("gpu_memory_utilization", 0.9) + from vllm.platforms import current_platform + + if current_platform.is_rocm(): + from tests.utils import wait_for_rocm_memory_to_settle + + wait_for_rocm_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization) + engine_args = AsyncEngineArgs(**ENGINE_CONFIG) llm = AsyncLLM.from_engine_args(engine_args) try: yield llm finally: - llm.shutdown() + shutdown_timeout = 60.0 if current_platform.is_rocm() else None + llm.shutdown(timeout=shutdown_timeout) + del llm + import torch + + torch._dynamo.reset() + from vllm.distributed import cleanup_dist_env_and_memory + + cleanup_dist_env_and_memory() + from tests.utils import wait_for_rocm_memory_to_settle + + wait_for_rocm_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization) -def test_voxtral_realtime_forward(audio_assets, tokenizer, engine): - assert_encoder_kv_cache_spec(engine) - audio_config = tokenizer.instruct_tokenizer.tokenizer.audio +def test_voxtral_realtime_forward(audio_assets, tokenizer, vllm_runner, monkeypatch): + monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0") - def from_file(file_path: str): - audio = Audio.from_file(file_path, strict=False) - req = TranscriptionRequest( - audio=audio.to_base64(audio.format), - streaming=StreamingMode.OFFLINE, - language=None, + vllm_kwargs = {**ENGINE_CONFIG} + vllm_kwargs["model_name"] = vllm_kwargs.pop("model") + + with vllm_runner(**vllm_kwargs) as vllm_model: + assert_encoder_kv_cache_spec(vllm_model.llm) + audio_config = tokenizer.instruct_tokenizer.tokenizer.audio + + def from_file(file_path: str): + audio = Audio.from_file(file_path, strict=False) + req = TranscriptionRequest( + audio=audio.to_base64(audio.format), + streaming=StreamingMode.OFFLINE, + language=None, + ) + tokenized = tokenizer.instruct_tokenizer.encode_transcription(req) + + return (tokenized.tokens, tokenized.audios[0].audio_array) + + tokenized_list = [ + from_file(audio_asset.get_local_path()) for audio_asset in audio_assets + ] + + inputs = [] + sampling_params = [] + + for tokens, audio_array in tokenized_list: + num_samples = audio_array.shape[0] + max_tokens = audio_config.num_audio_tokens(num_samples) - len(tokens) - 1 + sampling_params.append( + SamplingParams(temperature=0.0, max_tokens=max_tokens) + ) + + input_dict = { + "multi_modal_data": {"audio": [(audio_array, None)]}, + "prompt_token_ids": tokens, + } + inputs.append(input_dict) + + outputs = vllm_model.llm.generate( + inputs, + sampling_params=sampling_params, ) - tokenized = tokenizer.instruct_tokenizer.encode_transcription(req) - return (tokenized.tokens, tokenized.audios[0].audio_array) - - tokenized_list = [ - from_file(audio_asset.get_local_path()) for audio_asset in audio_assets - ] - - inputs = [] - sampling_params = [] - - for tokens, audio_array in tokenized_list: - num_samples = audio_array.shape[0] - max_tokens = audio_config.num_audio_tokens(num_samples) - len(tokens) - 1 - sampling_params.append(SamplingParams(temperature=0.0, max_tokens=max_tokens)) - - input_dict = { - "multi_modal_data": {"audio": [(audio_array, None)]}, - "prompt_token_ids": tokens, - } - inputs.append(input_dict) - - outputs = engine.generate( - inputs, - sampling_params=sampling_params, - ) - - texts = _normalize([out.outputs[0].text for out in outputs]) - for i, (got, expected) in enumerate(zip(texts, EXPECTED_TEXT)): - assert got == expected, ( - f"Output mismatch at index {i}:\n" - f" got: {got!r}\n" - f" expected: {expected!r}" - ) + texts = _normalize([out.outputs[0].text for out in outputs]) + for i, (got, expected) in enumerate(zip(texts, EXPECTED_TEXT)): + assert got == expected, ( + f"Output mismatch at index {i}:\n" + f" got: {got!r}\n" + f" expected: {expected!r}" + ) @pytest.mark.asyncio diff --git a/tests/models/registry.py b/tests/models/registry.py index 1376ea28141..5d075d5b395 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -296,7 +296,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = { "GlmMoeDsaForCausalLM": _HfExamplesInfo( "zai-org/GLM-5", min_transformers_version="5.0.1", is_available_online=False ), - "GPT2LMHeadModel": _HfExamplesInfo("openai-community/gpt2", {"alias": "gpt2"}), + "GPT2LMHeadModel": _HfExamplesInfo("openai-community/gpt2"), "GPTBigCodeForCausalLM": _HfExamplesInfo( "bigcode/starcoder", extras={ diff --git a/tests/parser/engine/test_deepseek_v32.py b/tests/parser/engine/test_deepseek_v32.py new file mode 100644 index 00000000000..7825f5add83 --- /dev/null +++ b/tests/parser/engine/test_deepseek_v32.py @@ -0,0 +1,268 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for DeepSeek V3.2 parser engine semantics. + +V3.2 uses the same DSML parameter format as V4 but wraps tool calls in +``<｜DSML｜function_calls>`` instead of ``<｜DSML｜tool_calls>`` and has +no reasoning (````/````) support. +""" + +import json + +import pytest + +from tests.parser.engine.conftest import make_mock_tokenizer +from tests.parser.engine.streaming_helpers import ( + collect_content, + collect_function_name, + collect_tool_arguments, + simulate_tool_streaming, +) +from vllm.parser.deepseek_v4 import ( + DSML_INVOKE_END, + DSML_INVOKE_NAME_END, + DSML_INVOKE_PREFIX, +) +from vllm.parser.deepseek_v32 import ( + DSML_FUNC_END, + DSML_FUNC_START, + DeepSeekV32Parser, +) +from vllm.parser.engine.parser_engine_config import ParserState + +_PARAM_OPEN = '｜DSML｜parameter name="{name}" string="{is_str}">' +_PARAM_CLOSE = "" + + +def _param(name: str, is_str: str, value: str) -> str: + return f"<{_PARAM_OPEN.format(name=name, is_str=is_str)}{value}{_PARAM_CLOSE}" + + +def _invoke(name: str, *params: str) -> str: + body = "\n".join(params) + return ( + f"{DSML_INVOKE_PREFIX}{name}{DSML_INVOKE_NAME_END}\n{body}\n{DSML_INVOKE_END}" + ) + + +def _func_calls(*invocations: str) -> str: + body = "\n".join(invocations) + return f"{DSML_FUNC_START}\n{body}\n{DSML_FUNC_END}" + + +def _make_tool(name, properties): + from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionToolsParam, + ) + + return ChatCompletionToolsParam( + type="function", + function={ + "name": name, + "parameters": { + "type": "object", + "properties": properties, + }, + }, + ) + + +@pytest.fixture +def mock_tokenizer(): + return make_mock_tokenizer({}) + + +@pytest.fixture +def mock_request(): + from unittest.mock import MagicMock + + from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionRequest, + ) + + req = MagicMock(spec=ChatCompletionRequest) + req.tools = [] + req.tool_choice = "auto" + return req + + +# ── Non-streaming extraction ──────────────────────────────────────── + + +class TestNonStreaming: + def test_no_tool_call(self, mock_tokenizer, mock_request): + parser = DeepSeekV32Parser(mock_tokenizer) + result = parser.extract_tool_calls("Hello world", mock_request) + assert not result.tools_called + assert result.content == "Hello world" + + def test_single_tool(self, mock_tokenizer, mock_request): + text = _func_calls( + _invoke("get_weather", _param("city", "true", "SF")), + ) + parser = DeepSeekV32Parser(mock_tokenizer) + result = parser.extract_tool_calls(text, mock_request) + assert result.tools_called + assert len(result.tool_calls) == 1 + assert result.tool_calls[0].function.name == "get_weather" + args = json.loads(result.tool_calls[0].function.arguments) + assert args == {"city": "SF"} + + def test_parallel_tools(self, mock_tokenizer, mock_request): + text = _func_calls( + _invoke("get_weather", _param("city", "true", "SF")), + _invoke("get_weather", _param("city", "true", "NYC")), + ) + parser = DeepSeekV32Parser(mock_tokenizer) + result = parser.extract_tool_calls(text, mock_request) + assert result.tools_called + assert len(result.tool_calls) == 2 + assert json.loads(result.tool_calls[0].function.arguments) == {"city": "SF"} + assert json.loads(result.tool_calls[1].function.arguments) == {"city": "NYC"} + + def test_content_before_tool_call(self, mock_tokenizer, mock_request): + text = "Let me check. " + _func_calls( + _invoke("search", _param("q", "true", "vllm")), + ) + parser = DeepSeekV32Parser(mock_tokenizer) + result = parser.extract_tool_calls(text, mock_request) + assert result.tools_called + assert result.content is not None + assert "Let me check" in result.content + + def test_non_string_params_json_parsed(self, mock_tokenizer, mock_request): + text = _func_calls( + _invoke( + "toggle", + _param("enabled", "false", "true"), + _param("count", "false", "42"), + ), + ) + parser = DeepSeekV32Parser(mock_tokenizer) + result = parser.extract_tool_calls(text, mock_request) + args = json.loads(result.tool_calls[0].function.arguments) + assert args["enabled"] is True + assert args["count"] == 42 + + def test_wrapper_unwrapping(self, mock_tokenizer, mock_request): + tool = _make_tool("get_weather", {"location": {"type": "string"}}) + mock_request.tools = [tool] + text = _func_calls( + _invoke( + "get_weather", + _param("arguments", "false", '{"location":"Beijing"}'), + ), + ) + parser = DeepSeekV32Parser(mock_tokenizer, tools=[tool]) + result = parser.extract_tool_calls(text, mock_request) + args = json.loads(result.tool_calls[0].function.arguments) + assert args == {"location": "Beijing"} + + +# ── Initial state ──────────────────────────────────────────────────── + + +class TestInitialState: + def test_always_content(self, mock_tokenizer): + parser = DeepSeekV32Parser(mock_tokenizer) + cfg = parser.parser_engine_config + assert cfg.initial_state == ParserState.CONTENT + + def test_ignores_thinking_kwargs(self, mock_tokenizer): + parser = DeepSeekV32Parser( + mock_tokenizer, + chat_template_kwargs={"thinking": True, "enable_thinking": True}, + ) + cfg = parser.parser_engine_config + assert cfg.initial_state == ParserState.CONTENT + + +# ── Streaming ──────────────────────────────────────────────────────── + + +class TestStreaming: + def test_single_tool_streaming(self, mock_tokenizer, mock_request): + text = _func_calls( + _invoke("get_weather", _param("city", "true", "SF")), + ) + parser = DeepSeekV32Parser(mock_tokenizer) + results = simulate_tool_streaming(parser, mock_request, list(text)) + assert collect_function_name(results) == "get_weather" + args_json = collect_tool_arguments(results) + assert json.loads(args_json) == {"city": "SF"} + + def test_content_before_tool_streaming(self, mock_tokenizer, mock_request): + text = "Checking... " + _func_calls( + _invoke("fn", _param("k", "true", "v")), + ) + parser = DeepSeekV32Parser(mock_tokenizer) + results = simulate_tool_streaming(parser, mock_request, list(text)) + content = collect_content(results) + assert "Checking" in content + + def test_parallel_tools_streaming(self, mock_tokenizer, mock_request): + text = _func_calls( + _invoke("fn_a", _param("x", "true", "1")), + _invoke("fn_b", _param("y", "true", "2")), + ) + parser = DeepSeekV32Parser(mock_tokenizer) + results = simulate_tool_streaming(parser, mock_request, list(text)) + + names = [] + for delta, _ in results: + if delta and delta.tool_calls: + for tc in delta.tool_calls: + if tc.function and tc.function.name: + names.append(tc.function.name) + assert "fn_a" in names + assert "fn_b" in names + + def test_no_tool_content_only(self, mock_tokenizer, mock_request): + text = "Just some text, no tools." + parser = DeepSeekV32Parser(mock_tokenizer) + results = simulate_tool_streaming(parser, mock_request, list(text)) + content = collect_content(results) + assert "Just some text" in content + args = collect_tool_arguments(results) + assert args == "" + + def test_streaming_wrapper_unwrap_consistency(self, mock_tokenizer, mock_request): + tool = _make_tool("get_weather", {"location": {"type": "string"}}) + mock_request.tools = [tool] + parser = DeepSeekV32Parser(mock_tokenizer, tools=[tool]) + + chunks = [ + DSML_FUNC_START, + _invoke( + "get_weather", + _param("arguments", "false", '{"location": "NYC"}'), + ), + DSML_FUNC_END, + ] + + results = simulate_tool_streaming(parser, mock_request, chunks) + streamed_args = collect_tool_arguments(results) + + final_delta, _ = results[-1] + finish_delta = parser.finish_streaming() + extracted = parser._build_extracted_result(final_delta, finish_delta) + + assert extracted.tools_called is True + assert len(extracted.tool_calls) == 1 + final_args = extracted.tool_calls[0].function.arguments + assert json.loads(final_args) == {"location": "NYC"} + assert '"arguments"' not in streamed_args + assert final_args.startswith(streamed_args) + + def test_missing_invoke_end(self, mock_tokenizer, mock_request): + text = ( + f"{DSML_FUNC_START}\n" + f"{DSML_INVOKE_PREFIX}fn{DSML_INVOKE_NAME_END}\n" + f"{_param('k', 'true', 'v')}\n" + f"{DSML_FUNC_END}" + ) + parser = DeepSeekV32Parser(mock_tokenizer) + results = simulate_tool_streaming(parser, mock_request, list(text)) + assert collect_function_name(results) == "fn" + args = json.loads(collect_tool_arguments(results)) + assert args == {"k": "v"} diff --git a/tests/parser/engine/test_deepseek_v4.py b/tests/parser/engine/test_deepseek_v4.py new file mode 100644 index 00000000000..3aa7b3b9d21 --- /dev/null +++ b/tests/parser/engine/test_deepseek_v4.py @@ -0,0 +1,922 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for DeepSeek V4-specific parser engine semantics.""" + +import json + +import pytest + +from tests.parser.engine.conftest import make_mock_tokenizer +from tests.parser.engine.replay_harness import ( + DUMMY_TOOLS, + MockTokenizer, + _test_request, + collect_output, + replay_streaming, +) +from tests.parser.engine.streaming_helpers import ( + collect_content, + collect_function_name, + collect_tool_arguments, + simulate_reasoning_streaming, + simulate_tool_streaming, +) +from vllm.parser.abstract_parser import DelegatingParser +from vllm.parser.deepseek_v4 import ( + DSML_INVOKE_END, + DSML_INVOKE_NAME_END, + DSML_INVOKE_PREFIX, + DSML_THINK_END, + DSML_THINK_START, + DSML_TOOL_END, + DSML_TOOL_START, + DeepSeekV4Parser, + _dsml_arg_converter, + _unwrap_wrapper_args, + deepseek_v4_config, +) +from vllm.parser.engine.registered_adapters import ( + DeepSeekV4ParserReasoningAdapter, + DeepSeekV4ParserToolAdapter, +) + +_THINK_START_ID = 50 +_THINK_END_ID = 51 + +_PARAM_OPEN = '｜DSML｜parameter name="{name}" string="{is_str}">' +_PARAM_CLOSE = "" + + +def _param(name: str, is_str: str, value: str) -> str: + return f"<{_PARAM_OPEN.format(name=name, is_str=is_str)}{value}{_PARAM_CLOSE}" + + +@pytest.fixture +def mock_tokenizer(): + return make_mock_tokenizer( + { + DSML_THINK_START: _THINK_START_ID, + DSML_THINK_END: _THINK_END_ID, + } + ) + + +# ── Arg converter unit tests ───────────────────────────────────────── + + +class TestArgConverter: + def _raw(self, *params: tuple[str, str, str]) -> str: + lines = [_param(n, s, v) for n, s, v in params] + return "\n" + "\n".join(lines) + "\n" + + def test_string_param(self): + raw = self._raw(("city", "true", "杭州")) + result = json.loads(_dsml_arg_converter(raw, partial=False)) + assert result == {"city": "杭州"} + + def test_string_with_spaces_and_quotes(self): + raw = self._raw(("msg", "true", 'He said "hello world"')) + result = json.loads(_dsml_arg_converter(raw, partial=False)) + assert result["msg"] == 'He said "hello world"' + + def test_integer_param(self): + raw = self._raw(("count", "false", "42")) + result = json.loads(_dsml_arg_converter(raw, partial=False)) + assert result["count"] == 42 + assert isinstance(result["count"], int) + + def test_float_param(self): + raw = self._raw(("ratio", "false", "3.14")) + result = json.loads(_dsml_arg_converter(raw, partial=False)) + assert abs(result["ratio"] - 3.14) < 1e-9 + + def test_bool_param(self): + raw = self._raw(("flag", "false", "true")) + result = json.loads(_dsml_arg_converter(raw, partial=False)) + assert result["flag"] is True + + def test_array_param(self): + raw = self._raw(("items", "false", '["a", "b", "c"]')) + result = json.loads(_dsml_arg_converter(raw, partial=False)) + assert result["items"] == ["a", "b", "c"] + + def test_object_param(self): + raw = self._raw(("opts", "false", '{"key": "val"}')) + result = json.loads(_dsml_arg_converter(raw, partial=False)) + assert result["opts"] == {"key": "val"} + + def test_mixed_types(self): + raw = self._raw( + ("location", "true", "Tokyo"), + ("limit", "false", "10"), + ("active", "false", "false"), + ) + result = json.loads(_dsml_arg_converter(raw, partial=False)) + assert result == {"location": "Tokyo", "limit": 10, "active": False} + + def test_empty_args(self): + result = json.loads(_dsml_arg_converter("", partial=False)) + assert result == {} + + def test_invalid_json_fallback(self): + raw = self._raw(("data", "false", "[broken")) + result = json.loads(_dsml_arg_converter(raw, partial=False)) + assert result["data"] == "[broken" + + def test_chinese_chars_preserved_in_json(self): + raw = self._raw(("query", "true", "你好世界")) + raw_json = _dsml_arg_converter(raw, partial=False) + assert "你好世界" in raw_json + result = json.loads(raw_json) + assert result["query"] == "你好世界" + + def test_partial_complete_plus_in_progress(self): + raw = self._raw(("city", "true", "Tokyo")) + raw += f"<{_PARAM_OPEN.format(name='unit', is_str='true')}celsi" + result = json.loads(_dsml_arg_converter(raw, partial=True)) + assert result["city"] == "Tokyo" + assert result["unit"] == "celsi" + + def test_partial_no_in_progress(self): + raw = self._raw(("city", "true", "Tokyo")) + result = json.loads(_dsml_arg_converter(raw, partial=True)) + assert result == {"city": "Tokyo"} + + def test_partial_value_with_angle_bracket(self): + raw = f"<{_PARAM_OPEN.format(name='code', is_str='true')}a absorption and duplicate absorption ───────── + + +class TestThinkTagAbsorption: + def test_bare_think_end_not_leaked(self, mock_tokenizer): + parser = DeepSeekV4Parser(mock_tokenizer) + chunks = ["", "Here is the direct answer."] + reasoning, content = simulate_reasoning_streaming(parser, chunks) + assert reasoning == "" + assert "" not in content + assert "Here is the direct answer" in content + + def test_duplicate_think_start_absorbed(self, mock_tokenizer): + parser = DeepSeekV4Parser( + mock_tokenizer, chat_template_kwargs={"thinking": True} + ) + chunks = [ + "\n", + "Some reasoning.\n", + "\n", + "Answer.", + ] + reasoning, content = simulate_reasoning_streaming(parser, chunks) + assert "Some reasoning" in reasoning + assert "Answer" in content + + +# ── Missing before ──────────── + + +class TestMissingInvokeEnd: + def test_non_streaming(self, mock_tokenizer, mock_request): + parser = DeepSeekV4Parser(mock_tokenizer) + text = ( + f"{DSML_TOOL_START}" + f"{DSML_INVOKE_PREFIX}get_weather{DSML_INVOKE_NAME_END}\n" + f"{_param('location', 'true', 'NYC')}\n" + f"{DSML_TOOL_END}" + ) + result = parser.extract_tool_calls(text, mock_request) + + assert result.tools_called is True + assert len(result.tool_calls) == 1 + assert result.tool_calls[0].function.name == "get_weather" + args = json.loads(result.tool_calls[0].function.arguments) + assert args == {"location": "NYC"} + + def test_streaming_with_trailing_content(self, mock_tokenizer, mock_request): + parser = DeepSeekV4Parser(mock_tokenizer) + chunks = [ + DSML_TOOL_START, + f"{DSML_INVOKE_PREFIX}get_weather{DSML_INVOKE_NAME_END}\n" + f"{_param('location', 'true', 'NYC')}\n", + DSML_TOOL_END, + "Done.", + ] + + results = simulate_tool_streaming(parser, mock_request, chunks) + + assert collect_function_name(results) == "get_weather" + args = json.loads(collect_tool_arguments(results)) + assert args == {"location": "NYC"} + assert "Done." in collect_content(results) + + +# ── Thinking mode initial state ────────────────────────────────────── + + +class TestThinkingModeConfig: + def test_thinking_true_starts_in_reasoning(self): + cfg = deepseek_v4_config(thinking=True) + assert cfg.initial_state.name == "REASONING" + + def test_thinking_false_starts_in_content(self): + cfg = deepseek_v4_config(thinking=False) + assert cfg.initial_state.name == "CONTENT" + + def test_enable_thinking_kwarg(self, mock_tokenizer): + p = DeepSeekV4Parser( + mock_tokenizer, chat_template_kwargs={"enable_thinking": True} + ) + assert p.parser_engine_config.initial_state.name == "REASONING" + + def test_no_thinking_kwarg_defaults_to_content(self, mock_tokenizer): + p = DeepSeekV4Parser(mock_tokenizer) + assert p.parser_engine_config.initial_state.name == "CONTENT" + + def test_thinking_mode_reasoning_without_tags(self, mock_tokenizer): + parser = DeepSeekV4Parser( + mock_tokenizer, chat_template_kwargs={"thinking": True} + ) + chunks = [ + "\n\nLet me consider ", + "this carefully.\n", + "\n", + "Here is the result.", + ] + reasoning, content = simulate_reasoning_streaming(parser, chunks) + assert "Let me consider" in reasoning + assert "Here is the result" in content + + def test_thinking_mode_all_reasoning_no_end_tag(self, mock_tokenizer): + parser = DeepSeekV4Parser( + mock_tokenizer, chat_template_kwargs={"thinking": True} + ) + chunks = ["I'll review ", "the PR."] + reasoning, content = simulate_reasoning_streaming(parser, chunks) + assert "review" in reasoning + assert "the PR" in reasoning + assert content == "" + + def test_reasoning_effort_none_overrides_enable_thinking(self, mock_tokenizer): + p = DeepSeekV4Parser( + mock_tokenizer, + chat_template_kwargs={ + "enable_thinking": True, + "reasoning_effort": "none", + }, + ) + assert p.parser_engine_config.initial_state.name == "CONTENT" + + +# ── Implicit reasoning end (missing before tool calls) ───── + + +class TestImplicitReasoningEnd: + """Tool call markers end reasoning implicitly when is missing. + + DeepSeek V4 models occasionally omit before emitting tool calls. + The (REASONING, TOOL_START) transition handles this gracefully. + """ + + @pytest.fixture + def thinking_parser(self, mock_tokenizer): + return DeepSeekV4Parser(mock_tokenizer, chat_template_kwargs={"thinking": True}) + + def _reasoning_then_tool(self, reasoning_text: str) -> str: + return reasoning_text + _tool_calls( + _invoke("get_weather", ("location", "true", "NYC")), + ) + + def test_non_streaming_extract_reasoning_implicit_end(self, thinking_parser): + text = self._reasoning_then_tool("Let me look up the weather.\n\n") + reasoning, content = thinking_parser.extract_reasoning(text, None) + assert reasoning == "Let me look up the weather." + assert DSML_TOOL_START not in reasoning + assert DSML_INVOKE_PREFIX not in reasoning + assert content is None + + def test_non_streaming_extract_tool_calls_implicit_end( + self, thinking_parser, mock_request + ): + text = self._reasoning_then_tool("Let me look up the weather.\n\n") + result = thinking_parser.extract_tool_calls(text, mock_request) + assert result.tools_called is True + assert len(result.tool_calls) == 1 + assert result.tool_calls[0].function.name == "get_weather" + args = json.loads(result.tool_calls[0].function.arguments) + assert args == {"location": "NYC"} + + def test_non_streaming_parse_implicit_end(self, thinking_parser, mock_request): + text = self._reasoning_then_tool("Let me look up the weather.\n\n") + reasoning, content, tool_calls = thinking_parser.parse(text, mock_request) + assert reasoning == "Let me look up the weather." + assert content is None + assert tool_calls is not None + assert len(tool_calls) == 1 + assert tool_calls[0].name == "get_weather" + args = json.loads(tool_calls[0].arguments) + assert args == {"location": "NYC"} + + def test_streaming_reasoning_implicit_end(self, thinking_parser): + chunks = [ + "Let me look up the weather.\n\n", + DSML_TOOL_START, + DSML_INVOKE_PREFIX + "get_weather" + DSML_INVOKE_NAME_END, + ] + reasoning, content = simulate_reasoning_streaming(thinking_parser, chunks) + assert reasoning == "Let me look up the weather." + assert DSML_TOOL_START not in reasoning + assert DSML_INVOKE_PREFIX not in reasoning + + def test_streaming_tool_extraction_implicit_end( + self, thinking_parser, mock_request + ): + chunks = [ + "Let me check.\n\n", + DSML_TOOL_START, + DSML_INVOKE_PREFIX + + "get_weather" + + DSML_INVOKE_NAME_END + + "\n" + + _param("location", "true", "NYC") + + "\n" + + DSML_INVOKE_END, + DSML_TOOL_END, + ] + results = simulate_tool_streaming(thinking_parser, mock_request, chunks) + assert collect_function_name(results) == "get_weather" + args = json.loads(collect_tool_arguments(results)) + assert args == {"location": "NYC"} + + def test_thinking_false_explicit_think_then_tool_call(self, mock_tokenizer): + parser = DeepSeekV4Parser(mock_tokenizer) + chunks = [ + DSML_THINK_START, + "Let me check the weather.", + DSML_TOOL_START, + DSML_INVOKE_PREFIX + "get_weather" + DSML_INVOKE_NAME_END, + ] + reasoning, content = simulate_reasoning_streaming(parser, chunks) + assert "Let me check the weather" in reasoning + assert DSML_TOOL_START not in reasoning + assert DSML_THINK_START not in reasoning + + def test_non_streaming_parallel_tools_after_implicit_end( + self, thinking_parser, mock_request + ): + text = "I need both.\n\n" + _tool_calls( + _invoke("get_weather", ("location", "true", "NYC")), + _invoke("get_time", ("timezone", "true", "EST")), + ) + result = thinking_parser.extract_tool_calls(text, mock_request) + assert result.tools_called is True + assert len(result.tool_calls) == 2 + assert result.tool_calls[0].function.name == "get_weather" + assert result.tool_calls[1].function.name == "get_time" + + def test_streaming_implicit_end_trailing_whitespace_stripped(self, thinking_parser): + chunks = [ + "Reasoning.\n\n\n", + DSML_TOOL_START, + DSML_INVOKE_PREFIX + "func" + DSML_INVOKE_NAME_END, + ] + reasoning, content = simulate_reasoning_streaming(thinking_parser, chunks) + assert reasoning == "Reasoning." + + +# ── Wrapper argument unwrapping ────────────────────────────────────── + + +class TestWrapperUnwrapping: + def test_unwrap_arguments_wrapper(self): + from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionToolsParam, + ) + + tool = ChatCompletionToolsParam( + type="function", + function={ + "name": "get_weather", + "parameters": { + "type": "object", + "properties": {"location": {"type": "string"}}, + }, + }, + ) + + result = _unwrap_wrapper_args( + '{"arguments": {"location": "Beijing"}}', + [tool], + "get_weather", + ) + assert json.loads(result) == {"location": "Beijing"} + + def test_unwrap_input_wrapper(self): + from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionToolsParam, + ) + + tool = ChatCompletionToolsParam( + type="function", + function={ + "name": "get_weather", + "parameters": { + "type": "object", + "properties": {"location": {"type": "string"}}, + }, + }, + ) + + result = _unwrap_wrapper_args( + '{"input": {"location": "Beijing"}}', + [tool], + "get_weather", + ) + assert json.loads(result) == {"location": "Beijing"} + + def test_no_unwrap_when_key_in_schema(self): + from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionToolsParam, + ) + + tool = ChatCompletionToolsParam( + type="function", + function={ + "name": "func", + "parameters": { + "type": "object", + "properties": {"arguments": {"type": "string"}}, + }, + }, + ) + + result = _unwrap_wrapper_args( + '{"arguments": "some value"}', + [tool], + "func", + ) + assert json.loads(result) == {"arguments": "some value"} + + def test_no_unwrap_when_no_tools(self): + result = _unwrap_wrapper_args( + '{"arguments": {"location": "Beijing"}}', + None, + "get_weather", + ) + assert json.loads(result) == {"arguments": {"location": "Beijing"}} + + def test_unwrap_json_string_inner(self): + from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionToolsParam, + ) + + tool = ChatCompletionToolsParam( + type="function", + function={ + "name": "get_weather", + "parameters": { + "type": "object", + "properties": {"location": {"type": "string"}}, + }, + }, + ) + + result = _unwrap_wrapper_args( + '{"arguments": "{\\"location\\": \\"Beijing\\"}"}', + [tool], + "get_weather", + ) + assert json.loads(result) == {"location": "Beijing"} + + +# ── Parallel tool call wrapper unwrapping ─────────────────────────── + + +def _make_tool(name, properties): + from vllm.entrypoints.openai.chat_completion.protocol import ( # noqa: E501 + ChatCompletionToolsParam, + ) + + return ChatCompletionToolsParam( + type="function", + function={ + "name": name, + "parameters": { + "type": "object", + "properties": properties, + }, + }, + ) + + +def _invoke(name, *params): + body = "\n".join(_param(n, s, v) for n, s, v in params) + return ( + f"{DSML_INVOKE_PREFIX}{name}{DSML_INVOKE_NAME_END}\n{body}\n{DSML_INVOKE_END}" + ) + + +def _tool_calls(*invokes): + return DSML_TOOL_START + "\n".join(invokes) + DSML_TOOL_END + + +class TestParallelUnwrapping: + @pytest.fixture + def weather_tool(self): + return _make_tool( + "get_weather", + { + "location": {"type": "string"}, + "unit": {"type": "string"}, + }, + ) + + @pytest.fixture + def time_tool(self): + return _make_tool( + "get_time", + {"timezone": {"type": "string"}}, + ) + + @pytest.mark.parametrize( + "weather_args, expected", + [ + ( + '{"location": "NYC", "unit": "celsius"}', + {"location": "NYC", "unit": "celsius"}, + ), + ('{"location": "NYC"}', {"location": "NYC"}), + ], + ids=["all_props", "subset_props"], + ) + def test_unwrap_parallel_uses_correct_schema( + self, + mock_tokenizer, + mock_request, + weather_tool, + time_tool, + weather_args, + expected, + ): + tools = [weather_tool, time_tool] + parser = DeepSeekV4Parser(mock_tokenizer, tools=tools) + mock_request.tools = tools + + text = _tool_calls( + _invoke("get_weather", ("arguments", "false", weather_args)), + _invoke("get_time", ("timezone", "true", "EST")), + ) + + result = parser.extract_tool_calls(text, mock_request) + + assert result.tools_called is True + assert len(result.tool_calls) == 2 + assert result.tool_calls[0].function.name == "get_weather" + args0 = json.loads(result.tool_calls[0].function.arguments) + assert args0 == expected + assert result.tool_calls[1].function.name == "get_time" + args1 = json.loads(result.tool_calls[1].function.arguments) + assert args1 == {"timezone": "EST"} + + def test_unwrap_parallel_streaming( + self, mock_tokenizer, mock_request, weather_tool, time_tool + ): + tools = [weather_tool, time_tool] + parser = DeepSeekV4Parser(mock_tokenizer, tools=tools) + mock_request.tools = tools + + chunks = [ + DSML_TOOL_START, + _invoke( + "get_weather", + ("arguments", "false", '{"location": "NYC"}'), + ), + _invoke("get_time", ("timezone", "true", "EST")), + DSML_TOOL_END, + ] + + results = simulate_tool_streaming(parser, mock_request, chunks) + final_delta, _ = results[-1] + finish_delta = parser.finish_streaming() + extracted = parser._build_extracted_result(final_delta, finish_delta) + + assert extracted.tools_called is True + assert len(extracted.tool_calls) == 2 + args0 = json.loads(extracted.tool_calls[0].function.arguments) + assert args0 == {"location": "NYC"} + args1 = json.loads(extracted.tool_calls[1].function.arguments) + assert args1 == {"timezone": "EST"} + + def test_no_unwrap_parallel_when_no_match( + self, mock_tokenizer, mock_request, weather_tool, time_tool + ): + tools = [weather_tool, time_tool] + parser = DeepSeekV4Parser(mock_tokenizer, tools=tools) + mock_request.tools = tools + + text = _tool_calls( + _invoke( + "get_weather", + ("arguments", "false", '{"unknown_key": "val"}'), + ), + _invoke("get_time", ("timezone", "true", "EST")), + ) + + result = parser.extract_tool_calls(text, mock_request) + + assert len(result.tool_calls) == 2 + args0 = json.loads(result.tool_calls[0].function.arguments) + assert args0 == {"arguments": {"unknown_key": "val"}} + args1 = json.loads(result.tool_calls[1].function.arguments) + assert args1 == {"timezone": "EST"} + + def test_unwrap_single_tool_still_works(self, mock_tokenizer, mock_request): + tool = _make_tool("get_weather", {"location": {"type": "string"}}) + tools = [tool] + parser = DeepSeekV4Parser(mock_tokenizer, tools=tools) + mock_request.tools = tools + + text = _tool_calls( + _invoke( + "get_weather", + ("arguments", "false", '{"location": "Beijing"}'), + ), + ) + + result = parser.extract_tool_calls(text, mock_request) + + assert result.tools_called is True + assert len(result.tool_calls) == 1 + args = json.loads(result.tool_calls[0].function.arguments) + assert args == {"location": "Beijing"} + + +# ── Streaming wrapper consistency ───────────────────────────────────── + + +class TestStreamingWrapperConsistency: + """Streamed arg deltas must stay consistent with final extraction + when wrapper params like 'arguments' are unwrapped.""" + + def test_streaming_wrapper_unwrap_consistency(self, mock_tokenizer, mock_request): + tool = _make_tool("get_weather", {"location": {"type": "string"}}) + tools = [tool] + parser = DeepSeekV4Parser(mock_tokenizer, tools=tools) + mock_request.tools = tools + + chunks = [ + DSML_TOOL_START, + _invoke( + "get_weather", + ("arguments", "false", '{"location": "NYC"}'), + ), + DSML_TOOL_END, + ] + + results = simulate_tool_streaming(parser, mock_request, chunks) + streamed_args = collect_tool_arguments(results) + + final_delta, _ = results[-1] + finish_delta = parser.finish_streaming() + extracted = parser._build_extracted_result(final_delta, finish_delta) + + assert extracted.tools_called is True + assert len(extracted.tool_calls) == 1 + + final_args = extracted.tool_calls[0].function.arguments + assert json.loads(final_args) == {"location": "NYC"} + + assert '"arguments"' not in streamed_args, ( + f"Streamed args should not contain wrapper key, got: {streamed_args!r}" + ) + + assert final_args.startswith(streamed_args), ( + f"Extracted args {final_args!r} " + f"should start with streamed args {streamed_args!r}" + ) + + +# ── DelegatingParser: large delta with + tool calls ───────── + +_DSV4_FULL_VOCAB = { + DSML_THINK_START: 128821, + DSML_THINK_END: 128822, + DSML_TOOL_START: 128823, + DSML_TOOL_END: 128824, +} + + +class _DeepSeekV4Delegating(DelegatingParser): + reasoning_parser_cls = DeepSeekV4ParserReasoningAdapter + tool_parser_cls = DeepSeekV4ParserToolAdapter + + +def _dsv4_tokens( + reasoning: str, + tool_name: str, + params: list[tuple[str, str, str]], +) -> list[tuple[int, str]]: + """Build a token sequence: reasoning + + DSML tool block.""" + tokens: list[tuple[int, str]] = [] + tid = 100 + + for word in reasoning.split(" "): + prefix = " " if tokens else "" + tokens.append((tid, prefix + word)) + tid += 1 + + tokens.append((_DSV4_FULL_VOCAB[DSML_THINK_END], DSML_THINK_END)) + + tokens.append((tid, "\n\n")) + tid += 1 + + tokens.append((_DSV4_FULL_VOCAB[DSML_TOOL_START], DSML_TOOL_START)) + + tokens.append((tid, "\n")) + tid += 1 + + invoke_prefix_text = f"{DSML_INVOKE_PREFIX}{tool_name}{DSML_INVOKE_NAME_END}" + tokens.append((tid, invoke_prefix_text)) + tid += 1 + + tokens.append((tid, "\n")) + tid += 1 + + for name, is_str, value in params: + param_text = _param(name, is_str, value) + tokens.append((tid, param_text)) + tid += 1 + tokens.append((tid, "\n")) + tid += 1 + + tokens.append((tid, DSML_INVOKE_END)) + tid += 1 + + tokens.append((tid, "\n")) + tid += 1 + + tokens.append((_DSV4_FULL_VOCAB[DSML_TOOL_END], DSML_TOOL_END)) + + return tokens + + +class TestDelegatingParserLargeDelta: + """Regression: tool calls lost when + DSML arrive in same delta. + + The DelegatingParser used by the serving layer splits reasoning and + tool parsing across two separate engine instances. When and + the entire DSML tool block arrive in a single large streaming delta, + the content transfer from reasoning adapter to tool adapter must + preserve the tool call text. + """ + + @pytest.fixture + def dsv4_tokens(self): + return _dsv4_tokens( + reasoning="The user wants the current weather in Berlin.", + tool_name="get_weather", + params=[ + ("location", "true", "Berlin"), + ("units", "true", "celsius"), + ], + ) + + @pytest.fixture + def dsv4_tokenizer(self, dsv4_tokens): + return MockTokenizer( + vocab=dict(_DSV4_FULL_VOCAB), + tokens=dsv4_tokens, + ) + + @pytest.mark.parametrize( + "chunk_size", + [1, 2, 3, 5, None], + ids=lambda c: f"chunk={c}", + ) + def test_tool_calls_extracted_at_all_chunk_sizes( + self, dsv4_tokenizer, dsv4_tokens, chunk_size + ): + parser = _DeepSeekV4Delegating( + dsv4_tokenizer, + chat_template_kwargs={"thinking": True}, + ) + deltas = replay_streaming( + parser, + dsv4_tokens, + chunk_size=chunk_size, + finished_on_last=True, + tools=DUMMY_TOOLS, + ) + output = collect_output(deltas) + + assert "The user wants" in output.reasoning + assert len(output.tool_calls) == 1, ( + f"Expected 1 tool call but got {len(output.tool_calls)}; " + f"reasoning={output.reasoning!r}, content={output.content!r}" + ) + assert output.tool_calls[0]["name"] == "get_weather" + args = json.loads(output.tool_calls[0]["arguments"]) + assert args == {"location": "Berlin", "units": "celsius"} + + def test_eos_drop_token_does_not_swallow_tool_calls(self): + """Tool calls must survive when an EOS DROP token's ID is in + delta_token_ids but its text is absent from delta_text. + + At large stream_interval the EOS token ID arrives in the same + delta as + tool calls but the detokenizer strips the + EOS text. The scanner's _rebuild_from_anchors defers all text + after when it can't find the EOS anchor text. The + reasoning adapter's finish_streaming must flush deferred text + as content (with skip_tool_parsing), not as tool calls. + """ + eos_text = "<｜end▁of▁sentence｜>" + eos_id = 128801 + vocab = { + DSML_THINK_START: 128821, + DSML_THINK_END: 128822, + eos_text: eos_id, + } + + reasoning = "The user wants weather." + tool_block = ( + "\n\n" + + DSML_TOOL_START + + "\n" + + DSML_INVOKE_PREFIX + + "get_weather" + + DSML_INVOKE_NAME_END + + "\n" + + _param("location", "true", "Berlin") + + "\n" + + DSML_INVOKE_END + + "\n" + + DSML_TOOL_END + ) + # delta_text does NOT include EOS text (detokenizer strips it) + full_text = reasoning + DSML_THINK_END + tool_block + # Build token list: word-split reasoning, then special tokens, + # then word-split tool block content, then EOS. + # EOS ID is present but its text is NOT in delta_text. + tokens: list[tuple[int, str]] = [] + tid = 100 + for word in reasoning.split(" "): + pfx = " " if tokens else "" + tokens.append((tid, pfx + word)) + tid += 1 + tokens.append((128822, DSML_THINK_END)) + for ch in tool_block: + tokens.append((tid, ch)) + tid += 1 + tokens.append((eos_id, eos_text)) + + all_ids = [t[0] for t in tokens] + tokenizer = MockTokenizer(vocab=vocab, tokens=tokens) + request = _test_request(tools=DUMMY_TOOLS) + + # All-in-one delta: EOS ID in token_ids but text NOT in + # delta_text (detokenizer strips EOS). This is the scenario + # at large stream_interval. + parser = _DeepSeekV4Delegating( + tokenizer, + chat_template_kwargs={"thinking": True}, + ) + deltas = [ + parser.parse_delta( + full_text, + all_ids, + request, + prompt_token_ids=[], + finished=True, + ) + ] + + output = collect_output(deltas) + + assert "The user wants" in output.reasoning + assert len(output.tool_calls) == 1, ( + f"Expected 1 tool call but got {len(output.tool_calls)}; " + f"reasoning={output.reasoning!r}, content={output.content!r}" + ) + assert output.tool_calls[0]["name"] == "get_weather" + args = json.loads(output.tool_calls[0]["arguments"]) + assert args == {"location": "Berlin"} diff --git a/tests/parser/engine/test_replay.py b/tests/parser/engine/test_replay.py index b8a1ba71a9d..1626135f786 100644 --- a/tests/parser/engine/test_replay.py +++ b/tests/parser/engine/test_replay.py @@ -20,6 +20,7 @@ import pytest from tests.parser.engine.replay_harness import ( DUMMY_TOOLS, MockTokenizer, + Sample, _test_request, assert_no_terminal_leakage, assert_parse_output, @@ -100,6 +101,14 @@ def _discover_parsers() -> list[_ParserInfo]: _PARSERS = _discover_parsers() + +def _make_parser(parser_cls: type[ParserEngine], tokenizer, sample: Sample, **extra): + kwargs = dict(extra) + if sample.chat_template_kwargs: + kwargs["chat_template_kwargs"] = sample.chat_template_kwargs + return parser_cls(tokenizer, sample.tools, **kwargs) + + _ENGINE_PARSERS: dict[str, type[ParserEngine]] = { f"{p.name}_engine": p.parser_cls for p in _PARSERS } @@ -123,7 +132,7 @@ class TestReplayWithHoldback: def test_replay(self, parser_cls, sample, terminals, chunk_size, holdback): tokenizer = make_mock_tokenizer(sample) - parser = parser_cls(tokenizer, sample.tools) + parser = _make_parser(parser_cls, tokenizer, sample) deltas = replay_streaming( parser, sample.tokens, @@ -160,7 +169,7 @@ class TestTextHoldback: def test_replay(self, parser_cls, sample, terminals, delay): tokenizer = make_mock_tokenizer(sample) - parser = parser_cls(tokenizer, sample.tools) + parser = _make_parser(parser_cls, tokenizer, sample) deltas = replay_with_text_holdback( parser, sample.tokens, @@ -190,7 +199,7 @@ class TestReplay: def test_replay(self, parser_cls, sample, terminals, chunk_size): tokenizer = make_mock_tokenizer(sample) - parser = parser_cls(tokenizer, sample.tools) + parser = _make_parser(parser_cls, tokenizer, sample) deltas = replay_streaming( parser, sample.tokens, @@ -227,7 +236,7 @@ class TestDeferralFinish: def test_misaligned_last_delta_with_finish(self, parser_cls, sample, tool_end_text): tokenizer = make_mock_tokenizer(sample) - parser = parser_cls(tokenizer, sample.tools) + parser = _make_parser(parser_cls, tokenizer, sample) request = _test_request() @@ -528,10 +537,7 @@ class TestDropTokenReplay: for sample in parser_info.samples: injected = _inject_drop_tokens(sample) tokenizer = make_mock_tokenizer(injected) - parser = parser_info.parser_cls( - tokenizer, - tools=sample.tools, - ) + parser = _make_parser(parser_info.parser_cls, tokenizer, sample) results = replay_streaming( parser, @@ -562,10 +568,7 @@ class TestDropTokenNonStreaming: for sample in parser_info.samples: injected = _inject_drop_tokens(sample) tokenizer = make_mock_tokenizer(injected) - parser = parser_info.parser_cls( - tokenizer, - tools=sample.tools, - ) + parser = _make_parser(parser_info.parser_cls, tokenizer, sample) request = _test_request(tools=sample.tools) output = parse_non_streaming(parser, injected, request) diff --git a/tests/parser/engine/trace_builder.py b/tests/parser/engine/trace_builder.py index 93159a0ae22..533c64408f7 100644 --- a/tests/parser/engine/trace_builder.py +++ b/tests/parser/engine/trace_builder.py @@ -29,6 +29,8 @@ from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionToolsParam, ) from vllm.parser.engine.registered_adapters import ( + DeepSeekV4Parser, + DeepSeekV32Parser, Gemma4Parser, Glm47MoeParser, KimiK2Parser, @@ -644,6 +646,144 @@ def _build_seed_oss(scenario: Scenario, validate: bool = True) -> Sample: return sample +# ── DeepSeek V4 (DSML tool format) ────────────────────────────────── + +_DSML = "｜DSML｜" +_DSV4_VOCAB: dict[str, int] = { + "": 128821, + "": 128822, + f"<{_DSML}tool_calls>": 128823, + f"": 128824, +} + + +def _dsv4_param_text(key: str, value: Any) -> str: + is_string = isinstance(value, str) + if is_string: + val_str = value + elif isinstance(value, bool): + val_str = "true" if value else "false" + elif isinstance(value, (int, float)): + val_str = str(value) + else: + val_str = json.dumps(value, ensure_ascii=False) + string_attr = "true" if is_string else "false" + return ( + f'<{_DSML}parameter name="{key}" string="{string_attr}">' + f"{val_str}\n" + ) + + +def _dsv4_tool_text(tc: ToolCallSpec) -> str: + parts = [f'<{_DSML}invoke name="{tc.name}">\n'] + for key, value in tc.arguments.items(): + parts.append(_dsv4_param_text(key, value)) + parts.append(f"\n") + return "".join(parts) + + +def _dsml_tool_segs( + scenario: Scenario, + tag: str, +) -> list[tuple[str, bool]]: + if not scenario.tool_calls: + return [] + parts = ["\n"] + for tc in scenario.tool_calls: + parts.append(_dsv4_tool_text(tc)) + return [ + (f"<{_DSML}{tag}>", True), + ("".join(parts), False), + (f"", True), + ] + + +def _dsv4_segments(scenario: Scenario, thinking: bool) -> list[tuple[str, bool]]: + segs: list[tuple[str, bool]] = [] + + if thinking: + if scenario.reasoning is not None: + segs.append((scenario.reasoning, False)) + if scenario.content is not None or scenario.tool_calls: + segs.append(("", True)) + else: + if scenario.reasoning is not None: + segs.append(("", True)) + segs.append((scenario.reasoning, False)) + segs.append(("", True)) + + if scenario.content is not None: + segs.append((scenario.content, False)) + + segs.extend(_dsml_tool_segs(scenario, "tool_calls")) + return segs + + +def _build_deepseek_v4(scenario: Scenario, validate: bool = True) -> Sample: + thinking = scenario.reasoning is not None + chat_kwargs = {"thinking": True} if thinking else None + + if thinking: + expected_reasoning: str | None = scenario.reasoning or "" + else: + expected_reasoning = None + + sample = _make_sample( + sample_id=f"deepseek_v4-{scenario.id}", + description=scenario.description, + vocab=_DSV4_VOCAB, + segments=_dsv4_segments(scenario, thinking), + expected_reasoning=expected_reasoning, + expected_content=_qwen3_expected_content(scenario), + expected_tool_calls=_expected_tc(scenario), + tools=_expected_tools(scenario), + chat_template_kwargs=chat_kwargs, + ) + if validate: + kwargs = {} + if chat_kwargs: + kwargs["chat_template_kwargs"] = chat_kwargs + _validate_sample(sample, DeepSeekV4Parser, **kwargs) + return sample + + +# ── DeepSeek V3.2 (DSML tool format, no reasoning) ────────────────── + +_DSV32_VOCAB: dict[str, int] = { + f"<{_DSML}function_calls>": 128830, + f"": 128831, +} + + +def _dsv32_segments(scenario: Scenario) -> list[tuple[str, bool]]: + segs: list[tuple[str, bool]] = [] + + if scenario.content is not None: + segs.append((scenario.content, False)) + + segs.extend(_dsml_tool_segs(scenario, "function_calls")) + return segs + + +def _build_deepseek_v32(scenario: Scenario, validate: bool = True) -> Sample | None: + if scenario.reasoning is not None: + return None + + sample = _make_sample( + sample_id=f"deepseek_v32-{scenario.id}", + description=scenario.description, + vocab=_DSV32_VOCAB, + segments=_dsv32_segments(scenario), + expected_reasoning=None, + expected_content=_qwen3_expected_content(scenario), + expected_tool_calls=_expected_tc(scenario), + tools=_expected_tools(scenario), + ) + if validate: + _validate_sample(sample, DeepSeekV32Parser) + return sample + + # ── GLM-4.7 MoE (XML tool format, starts in REASONING) ────────────── _GLM47_MOE_VOCAB: dict[str, int] = { @@ -811,13 +951,15 @@ _KIMI_K2_SCENARIOS = [ # ── Registry and public API ────────────────────────────────────────── _BUILDERS: dict[str, Any] = { - "qwen3": _build_qwen3, + "deepseek_v32": _build_deepseek_v32, + "deepseek_v4": _build_deepseek_v4, "gemma4": _build_gemma4, "minimax_m2": _build_minimax_m2, "nemotron_v3": _build_nemotron_v3, "seed_oss": _build_seed_oss, "glm47_moe": _build_glm47_moe, "kimi_k2": _build_kimi_k2, + "qwen3": _build_qwen3, } @@ -826,10 +968,10 @@ def build_samples(model: str) -> tuple[Sample, ...]: """Build all scenario samples for a model, self-validated.""" builder = _BUILDERS[model] scenarios = _KIMI_K2_SCENARIOS if model == "kimi_k2" else SCENARIOS - return tuple(builder(s) for s in scenarios) + return tuple(s for s in (builder(sc) for sc in scenarios) if s is not None) -def build_sample(model: str, scenario: Scenario) -> Sample: +def build_sample(model: str, scenario: Scenario) -> Sample | None: """Build a single sample for one model + scenario.""" return _BUILDERS[model](scenario) diff --git a/tests/parser/test_harmony.py b/tests/parser/test_harmony.py index ba90252fe1c..3e9cff64aa1 100644 --- a/tests/parser/test_harmony.py +++ b/tests/parser/test_harmony.py @@ -7,7 +7,6 @@ from collections.abc import Sequence import pytest from openai_harmony import ( Conversation, - HarmonyError, Message, RenderConversationConfig, Role, @@ -51,6 +50,15 @@ def chat_request(): ) +@pytest.fixture +def malformed_msgs_str() -> list[str]: + return [ + "<|channel|>analysis<|message|>thinking<|end|>", + "<|start|>assistant<|channel|>commentary<|message|>thinking<|end|>", + '<|start|>assistant<|channel|>final {"answer": "hi"}<|return|>', + ] + + def encode_output(harmony_str: str) -> list[int]: return get_encoding().encode(harmony_str, allowed_special="all") @@ -131,13 +139,22 @@ def tool_call_entries(delta_message) -> list[tuple[int, str | None, str | None]] ] +def assert_parser_is_reset(harmony_parser: HarmonyParser): + assert harmony_parser._parser is None + assert harmony_parser._num_processed_messages == 0 + assert harmony_parser._current_message_tokens == [] + + class TestFlush: def test_flush(self, harmony_parser): harmony_parser.process_chunk( encode_output("<|channel|>analysis<|message|>Think") ) - flushed = harmony_parser.flush() + flushed_segments = harmony_parser.flush() + assert flushed_segments is not None + assert len(flushed_segments) == 1 + flushed = flushed_segments[0] assert flushed is not None assert flushed.channel == "analysis" @@ -145,15 +162,27 @@ class TestFlush: assert flushed.delta == "" assert flushed.completed_message is not None assert get_text(flushed.completed_message) == "Think" - assert harmony_parser._parser is None + assert_parser_is_reset(harmony_parser) - def test_flush_raises_and_resets_on_non_terminal_eos(self, harmony_parser): - harmony_parser.process_chunk(encode_output("<|channel|>analysis")) + def test_flush_recovers_invalid_output(self, harmony_parser, malformed_msgs_str): + for msg_str in malformed_msgs_str[:-1]: + chunk = harmony_parser.process_chunk(encode_output(msg_str)) + assert "".join(segment.delta for segment in chunk.segments) == "thinking" - with pytest.raises(HarmonyError): - harmony_parser.flush() + last_msg_str = malformed_msgs_str[-1] + harmony_parser.process_chunk(encode_output(last_msg_str)) + flushed_segments = harmony_parser.flush() + assert len(flushed_segments) == 2 + delta_segment = flushed_segments[0] + message_segment = flushed_segments[1] - assert harmony_parser._parser is None + assert delta_segment.channel == "final" + assert delta_segment.recipient is None + assert delta_segment.delta == last_msg_str + assert message_segment.channel == "final" + assert message_segment.recipient is None + assert get_text(message_segment.completed_message) == last_msg_str + assert_parser_is_reset(harmony_parser) class TestParse: @@ -364,7 +393,7 @@ class TestParse: assert reasoning is None assert content == "I'm in the middle of answering" assert tool_calls is None - assert harmony_parser._parser is None + assert_parser_is_reset(harmony_parser) def test_interrupted_reasoning_first_message(self, harmony_parser, chat_request): reasoning, content, tool_calls = harmony_parser.parse( @@ -378,7 +407,7 @@ class TestParse: assert reasoning == "I'm in the middle of thinking" assert content is None assert tool_calls is None - assert harmony_parser._parser is None + assert_parser_is_reset(harmony_parser) def test_truncated_output(self, harmony_parser, chat_request): reasoning, content, tool_calls = harmony_parser.parse( @@ -394,24 +423,23 @@ class TestParse: assert reasoning == "I'm thinking." assert content == "I'm in the middle of answering" assert tool_calls is None - assert harmony_parser._parser is None + assert_parser_is_reset(harmony_parser) - def test_malformed_final_recovers_raw_content(self, harmony_parser, chat_request): - raw_output = ( - "<|channel|>analysis<|message|>thinking<|end|>" - '<|start|>assistant<|channel|>final {"answer": "hi"}<|return|>' - ) + def test_malformed_msgs_recovers_raw_content( + self, harmony_parser, chat_request, malformed_msgs_str + ): + combined_output = "".join(malformed_msgs_str) reasoning, content, tool_calls = harmony_parser.parse( - raw_output, + "", chat_request, - model_output_token_ids=encode_output(raw_output), + model_output_token_ids=encode_output(combined_output), ) - assert content == raw_output - assert reasoning is None + assert reasoning == "thinking" + assert content == "thinking\n" + malformed_msgs_str[-1] assert tool_calls is None - assert harmony_parser._parser is None + assert_parser_is_reset(harmony_parser) @pytest.mark.parametrize( ("harmony_str", "expected_content"), @@ -489,7 +517,7 @@ class TestParseDelta: assert second_delta is not None assert second_delta.content == "Answer" assert second_delta.reasoning is None - assert parser._parser is None + assert_parser_is_reset(parser) def test_multi_token(self, gpt_oss_tokenizer, chat_request): parser = HarmonyParser(gpt_oss_tokenizer) @@ -506,25 +534,33 @@ class TestParseDelta: assert delta.reasoning is None assert not delta.tool_calls - def test_malformed_final_recovers_raw_content( - self, gpt_oss_tokenizer, chat_request + def test_malformed_msgs_recovers_raw_content( + self, gpt_oss_tokenizer, chat_request, malformed_msgs_str ): parser = HarmonyParser(gpt_oss_tokenizer) - delta = parser.parse_delta( - delta_text='final {"answer": "hi"}', - delta_token_ids=encode_output( - '<|channel|>final {"answer": "hi"}<|return|>' - ), + for msg_str in malformed_msgs_str[:-1]: + delta = parser.parse_delta( + delta_text="", + delta_token_ids=encode_output(msg_str), + request=chat_request, + finished=False, + ) + assert delta.reasoning or delta.content == "thinking" + assert not delta.tool_calls + + last_delta = parser.parse_delta( + delta_text="", + delta_token_ids=encode_output(malformed_msgs_str[-1]), request=chat_request, finished=True, ) - assert delta is not None - assert delta.content == 'final {"answer": "hi"}' - assert delta.reasoning is None - assert not delta.tool_calls - assert parser._parser is None + assert last_delta is not None + assert last_delta.content == malformed_msgs_str[-1] + assert last_delta.reasoning is None + assert not last_delta.tool_calls + assert_parser_is_reset(parser) @pytest.mark.parametrize("tool_channel", ["commentary", "analysis"]) def test_tool_call_split_across_deltas( diff --git a/tests/reasoning/test_deepseekv3_reasoning_parser.py b/tests/reasoning/test_deepseekv3_reasoning_parser.py index f5b37194f92..81af6c2db4d 100644 --- a/tests/reasoning/test_deepseekv3_reasoning_parser.py +++ b/tests/reasoning/test_deepseekv3_reasoning_parser.py @@ -11,6 +11,8 @@ from vllm.reasoning.deepseek_r1_reasoning_parser import DeepSeekR1ReasoningParse from vllm.reasoning.deepseek_v3_reasoning_parser import DeepSeekV3ReasoningParser from vllm.reasoning.identity_reasoning_parser import IdentityReasoningParser +pytestmark = pytest.mark.skip_global_cleanup + REASONING_MODEL_NAME = "deepseek-ai/DeepSeek-V3.1" @@ -35,9 +37,13 @@ def test_parser_selection(tokenizer, thinking, expected_parser_type): def test_deepseek_v4_reasoning_parser_alias(): + from vllm.reasoning.deepseek_v4_engine_reasoning_parser import ( + DeepSeekV4ParserReasoningAdapter, + ) + parser_cls = ReasoningParserManager.get_reasoning_parser("deepseek_v4") - assert parser_cls is DeepSeekV3ReasoningParser + assert parser_cls is DeepSeekV4ParserReasoningAdapter def test_identity_reasoning_parser_basic(tokenizer): diff --git a/tests/renderers/test_chat_utils_prompt_embeds.py b/tests/renderers/test_chat_utils_prompt_embeds.py index e33cc304710..2238c41f498 100644 --- a/tests/renderers/test_chat_utils_prompt_embeds.py +++ b/tests/renderers/test_chat_utils_prompt_embeds.py @@ -40,7 +40,7 @@ from vllm.renderers.hf import ( # Qwen2TokenizerFast (SentencePiece BPE variant) # BertTokenizerFast (WordPiece) TOKENIZER_IDS: Final[list[str]] = [ - "gpt2", + "openai-community/gpt2", "Qwen/Qwen2.5-1.5B-Instruct", "bert-base-uncased", ] diff --git a/tests/test_config.py b/tests/test_config.py index 4e76030fd7e..3837057658b 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -34,6 +34,7 @@ from vllm.config.vllm import ( OptimizationLevel, ) from vllm.platforms import current_platform +from vllm.v1.attention.backend import AttentionCGSupport DEVICE_TYPE = current_platform.device_type @@ -67,6 +68,36 @@ def test_v2_model_runner_env_tri_state(monkeypatch, env_value, expected): assert envs.VLLM_USE_V2_MODEL_RUNNER is expected +@pytest.mark.parametrize( + ("use_v2_model_runner", "expected_capture_sizes"), + [ + (False, [4, 8, 12, 16]), + (True, list(range(1, 17))), + ], +) +def test_resolve_cudagraph_mode_adjusts_spec_decode_sizes_only_for_v1( + use_v2_model_runner, + expected_capture_sizes, +): + compilation_config = CompilationConfig( + cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE, + cudagraph_capture_sizes=list(range(1, 17)), + ) + compilation_config.max_cudagraph_capture_size = 16 + compilation_config.post_init_cudagraph_sizes() + + cudagraph_mode = compilation_config.resolve_cudagraph_mode_and_sizes( + AttentionCGSupport.ALWAYS, + "FakeAttentionBackend", + uniform_decode_query_len=4, + use_v2_model_runner=use_v2_model_runner, + tensor_parallel_size=1, + ) + + assert cudagraph_mode == CUDAGraphMode.FULL_AND_PIECEWISE + assert compilation_config.cudagraph_capture_sizes == expected_capture_sizes + + @pytest.mark.parametrize( ("model_config", "expected"), [ diff --git a/tests/tokenizers_/test_basic.py b/tests/tokenizers_/test_basic.py index fc4da3f8fec..c3e211a7aee 100644 --- a/tests/tokenizers_/test_basic.py +++ b/tests/tokenizers_/test_basic.py @@ -23,7 +23,7 @@ def _assert_tokenizer_like(tokenizer: object): def test_tokenizer_like_protocol(): - tokenizer = get_tokenizer("gpt2", use_fast=True) + tokenizer = get_tokenizer("openai-community/gpt2", use_fast=True) assert isinstance(tokenizer, PreTrainedTokenizerFast) _assert_tokenizer_like(tokenizer) @@ -43,7 +43,9 @@ def test_tokenizer_like_protocol(): _assert_tokenizer_like(tokenizer) -@pytest.mark.parametrize("tokenizer_name", ["facebook/opt-125m", "gpt2"]) +@pytest.mark.parametrize( + "tokenizer_name", ["facebook/opt-125m", "openai-community/gpt2"] +) def test_tokenizer_revision(tokenizer_name: str): # Assume that "main" branch always exists tokenizer = get_tokenizer(tokenizer_name, revision="main") diff --git a/tests/tokenizers_/test_detokenize.py b/tests/tokenizers_/test_detokenize.py index 2f173bec80c..8244dfeb86e 100644 --- a/tests/tokenizers_/test_detokenize.py +++ b/tests/tokenizers_/test_detokenize.py @@ -33,7 +33,7 @@ TRUTH = [ TOKENIZERS = [ "facebook/opt-125m", - "gpt2", + "openai-community/gpt2", "bigcode/tiny_starcoder_py", "EleutherAI/gpt-j-6b", "EleutherAI/pythia-70m", diff --git a/tests/tokenizers_/test_hf.py b/tests/tokenizers_/test_hf.py index 3ccbbd73e7a..61c81302f07 100644 --- a/tests/tokenizers_/test_hf.py +++ b/tests/tokenizers_/test_hf.py @@ -14,7 +14,7 @@ from vllm.tokenizers.hf import ( ) -@pytest.mark.parametrize("model_id", ["gpt2", "zai-org/chatglm3-6b"]) +@pytest.mark.parametrize("model_id", ["openai-community/gpt2", "zai-org/chatglm3-6b"]) def test_cached_tokenizer(model_id: str): reference_tokenizer = AutoTokenizer.from_pretrained( model_id, trust_remote_code=True @@ -47,7 +47,7 @@ def _check_consistency(target: TokenizerLike, expected: TokenizerLike): assert target.encode("prompt") == expected.encode("prompt") -@pytest.mark.parametrize("model_id", ["gpt2"]) +@pytest.mark.parametrize("model_id", ["openai-community/gpt2"]) def test_thread_pool_tokenizer_pickle(model_id: str): """Regression test for issue #45433: the thread-pool tokenizer wrapper reconstructs through maybe_make_thread_pool on unpickling, which used to diff --git a/tests/tool_parsers/conftest.py b/tests/tool_parsers/conftest.py index 89609b257c3..23e0eff98a2 100644 --- a/tests/tool_parsers/conftest.py +++ b/tests/tool_parsers/conftest.py @@ -9,4 +9,4 @@ from vllm.tokenizers import TokenizerLike @pytest.fixture(scope="module") def default_tokenizer() -> TokenizerLike: - return AutoTokenizer.from_pretrained("gpt2") + return AutoTokenizer.from_pretrained("openai-community/gpt2") diff --git a/tests/tool_parsers/test_deepseekv32_tool_parser.py b/tests/tool_parsers/test_deepseekv32_tool_parser.py index a35976b8bbf..40ad6033d3b 100644 --- a/tests/tool_parsers/test_deepseekv32_tool_parser.py +++ b/tests/tool_parsers/test_deepseekv32_tool_parser.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Unit tests for DeepSeekV32ToolParser. +"""Unit tests for DeepSeekV32EngineToolParser. These tests use a minimal mock tokenizer so no real model weights are required. """ @@ -17,7 +17,11 @@ from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionToolsParam, FunctionDefinition, ) -from vllm.tool_parsers.deepseekv32_tool_parser import DeepSeekV32ToolParser +from vllm.tool_parsers.deepseekv32_engine_tool_parser import ( + DeepSeekV32EngineToolParser, +) + +pytestmark = pytest.mark.skip_global_cleanup # --------------------------------------------------------------------------- # Helpers @@ -30,8 +34,8 @@ MOCK_TOKENIZER.get_vocab.return_value = {} MOCK_TOKENIZER.tokenize.return_value = [] -def make_parser(tools=None) -> DeepSeekV32ToolParser: - return DeepSeekV32ToolParser(MOCK_TOKENIZER, tools=tools) +def make_parser(tools=None) -> DeepSeekV32EngineToolParser: + return DeepSeekV32EngineToolParser(MOCK_TOKENIZER, tools=tools) def make_tool_param(name: str, params: dict) -> MagicMock: @@ -167,9 +171,9 @@ class TestExtractToolCalls: assert isinstance(args["enabled"], bool) assert isinstance(args["count"], int) - def test_string_attr_true_preserves_literal_despite_schema(self): - """string="true" must keep the value as a string even - if the schema says integer.""" + def test_string_attr_true_coerced_by_schema(self): + """string="true" delivers a string, but the engine's schema-aware + type fixer coerces it to the schema type (integer).""" tool = ChatCompletionToolsParam( function=FunctionDefinition( name="score", @@ -192,8 +196,8 @@ class TestExtractToolCalls: result = parser.extract_tool_calls(model_output, None) assert result.tools_called args = json.loads(result.tool_calls[0].function.arguments) - assert args == {"value": "42"} - assert isinstance(args["value"], str) + assert args == {"value": 42} + assert isinstance(args["value"], int) def test_string_attr_false_allows_schema_conversion(self): """string="false" allows the parser to convert via the tool schema.""" @@ -222,7 +226,6 @@ class TestExtractToolCalls: assert args == {"value": 42} assert isinstance(args["value"], int) - @pytest.mark.skip_global_cleanup def test_composed_schema_converts_object_and_array_params(self): """Composed JSON Schema types must still drive DSML type coercion.""" tool = ChatCompletionToolsParam( @@ -282,8 +285,9 @@ class TestExtractToolCalls: assert isinstance(args["wait"], dict) assert isinstance(args["patches"], list) - @pytest.mark.skip_global_cleanup - def test_string_attr_true_preserves_literal_for_composed_schema(self): + def test_string_attr_true_coerced_by_composed_schema(self): + """string="true" delivers a JSON string, but the engine's schema-aware + type fixer coerces it to the composed schema type (object).""" tool = ChatCompletionToolsParam( function=FunctionDefinition( name="set_timer", @@ -313,7 +317,7 @@ class TestExtractToolCalls: result = parser.extract_tool_calls(model_output, None) assert result.tools_called args = json.loads(result.tool_calls[0].function.arguments) - assert args == {"wait": '{"type":"for","minutes":2880}'} + assert args == {"wait": {"type": "for", "minutes": 2880}} def test_arguments_wrapper_repaired(self): """A single 'arguments' wrapper parameter must be unwrapped when it @@ -486,8 +490,9 @@ class TestExtractToolCalls: args = json.loads(result.tool_calls[0].function.arguments) assert args["value"] is None - def test_null_not_coerced_without_null_in_schema(self): - """Literal 'null' must stay as a string when the schema is just 'string'.""" + def test_null_coerced_back_to_string_by_schema(self): + """string="false" with 'null' is json-parsed to None, but the + engine's schema fixer coerces it back to "null" for string schemas.""" tool = ChatCompletionToolsParam( function=FunctionDefinition( name="echo", @@ -512,8 +517,8 @@ class TestExtractToolCalls: assert args["text"] == "null" assert isinstance(args["text"], str) - def test_no_schema_keeps_strings(self): - """Without a tool schema, all string='false' params default to string.""" + def test_no_schema_parses_json(self): + """Without a tool schema, string='false' params are JSON-parsed.""" parser = make_parser(tools=None) model_output = ( f"{FC_START}\n" @@ -526,8 +531,8 @@ class TestExtractToolCalls: result = parser.extract_tool_calls(model_output, None) assert result.tools_called args = json.loads(result.tool_calls[0].function.arguments) - assert args["count"] == "42" - assert args["flag"] == "true" + assert args["count"] == 42 + assert args["flag"] is True # --------------------------------------------------------------------------- @@ -648,8 +653,9 @@ class TestExtractToolCallsStreaming: args_str = self._reconstruct_args(deltas) assert json.loads(args_str) == {"x": 3, "y": 4} - def test_string_attr_true_preserves_literal_in_streaming(self): - """Streaming: string='true' must keep the value literal despite schema.""" + def test_string_attr_true_coerced_by_schema_streaming(self): + """Streaming: string='true' delivers a string but the engine's + schema fixer coerces it to the schema type (integer).""" tool = ChatCompletionToolsParam( function=FunctionDefinition( name="score", @@ -672,10 +678,9 @@ class TestExtractToolCallsStreaming: deltas = self._stream(parser, full_text) args_str = self._reconstruct_args(deltas) args = json.loads(args_str) - assert args == {"value": "42"} - assert isinstance(args["value"], str) + assert args == {"value": 42} + assert isinstance(args["value"], int) - @pytest.mark.skip_global_cleanup def test_composed_schema_conversion_in_streaming(self): tool = ChatCompletionToolsParam( function=FunctionDefinition( @@ -821,13 +826,13 @@ class TestExtractToolCallsStreaming: assert json.loads(self._reconstruct_args(deltas, tool_index=0)) == {"p": "v1"} assert json.loads(self._reconstruct_args(deltas, tool_index=1)) == {"q": "v2"} - def test_state_reset_on_new_stream(self, parser): - """A second stream (previous_text == '') must reset state cleanly.""" + def test_state_reset_on_new_stream(self): + """A fresh parser instance must produce identical results.""" full_text = build_tool_call("fn", {"k": "v"}) # First stream - self._stream(parser, full_text) - # Second stream - should produce identical results - deltas2 = self._stream(parser, full_text) + self._stream(make_parser(), full_text) + # Second stream with fresh parser + deltas2 = self._stream(make_parser(), full_text) assert json.loads(self._reconstruct_args(deltas2)) == {"k": "v"} def test_empty_arguments_streaming(self, parser): @@ -860,26 +865,6 @@ class TestExtractToolCallsStreaming: assert len(ids) == 2 assert ids[0] != ids[1] - def test_eos_after_tool_calls(self, parser): - """EOS token (empty delta_text, non-empty delta_token_ids) returns - a non-None DeltaMessage so the serving framework can finalize.""" - full_text = build_tool_call("fn", {"k": "v"}) - # Drive through the full text first - deltas = self._stream(parser, full_text) - assert any(d.tool_calls for d in deltas) - # Now simulate EOS: empty delta_text, but token ids present - prev = full_text - result = parser.extract_tool_calls_streaming( - previous_text=prev, - current_text=prev, - delta_text="", - previous_token_ids=[], - current_token_ids=[], - delta_token_ids=[2], # EOS token id - request=make_request(), - ) - assert result is not None - def test_streaming_matches_non_streaming(self, parser): """Streaming and non-streaming must produce the same result.""" full_text = build_tool_call( @@ -968,7 +953,6 @@ class TestExtractToolCallsStreaming: def test_emits_arguments_before_invoke_completes(self, parser): """Argument deltas should stream before the invoke block closes.""" - # Stream only a partial invoke (no closing tag) partial_text = ( f"{FC_START}\n" f'{INV_START}fn">\n' @@ -981,7 +965,9 @@ class TestExtractToolCallsStreaming: for tc in delta.tool_calls or [] if tc.function and tc.function.arguments is not None ] - assert "".join(arg_chunks) == '{"k":"val"' + combined = "".join(arg_chunks) + assert combined # some partial args emitted + assert combined.startswith('{"k"') def test_no_marker_leak_chunked(self, parser): """Chunked streaming must NOT leak DSML start-marker fragments diff --git a/tests/tool_parsers/test_deepseekv4_tool_parser.py b/tests/tool_parsers/test_deepseekv4_tool_parser.py index 80e3357b68b..e7109626c0c 100644 --- a/tests/tool_parsers/test_deepseekv4_tool_parser.py +++ b/tests/tool_parsers/test_deepseekv4_tool_parser.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""Unit tests for DeepSeekV4ToolParser.""" +"""Unit tests for DeepSeekV4EngineToolParser.""" import json from unittest.mock import MagicMock @@ -17,7 +17,11 @@ from vllm.entrypoints.openai.chat_completion.protocol import ( FunctionDefinition, ) from vllm.tool_parsers import ToolParserManager -from vllm.tool_parsers.deepseekv4_tool_parser import DeepSeekV4ToolParser +from vllm.tool_parsers.deepseekv4_engine_tool_parser import ( + DeepSeekV4EngineToolParser, +) + +pytestmark = pytest.mark.skip_global_cleanup MOCK_TOKENIZER = MagicMock() MOCK_TOKENIZER.get_vocab.return_value = {} @@ -67,8 +71,8 @@ def sample_tools() -> list[ChatCompletionToolsParam]: ] -def make_parser(tools=None) -> DeepSeekV4ToolParser: - return DeepSeekV4ToolParser(MOCK_TOKENIZER, tools=tools) +def make_parser(tools=None) -> DeepSeekV4EngineToolParser: + return DeepSeekV4EngineToolParser(MOCK_TOKENIZER, tools=tools) def make_request(tools=None) -> MagicMock: @@ -84,7 +88,7 @@ def build_tool_call(func_name: str, params: dict[str, str]) -> str: return f'{TC_START}\n{INV_START}{func_name}">\n{param_strs}{INV_END}\n{TC_END}' -def stream(parser: DeepSeekV4ToolParser, full_text: str, chunk_size: int = 7): +def stream(parser: DeepSeekV4EngineToolParser, full_text: str, chunk_size: int = 7): deltas = [] previous_text = "" for start in range(0, len(full_text), chunk_size): @@ -120,7 +124,9 @@ def reconstruct_args(deltas, tool_index: int = 0) -> str: def test_registered(): - assert ToolParserManager.get_tool_parser("deepseek_v4") is DeepSeekV4ToolParser + assert ( + ToolParserManager.get_tool_parser("deepseek_v4") is DeepSeekV4EngineToolParser + ) def test_extract_tool_calls(): @@ -285,7 +291,7 @@ def test_extract_tool_calls_arguments_wrapper(): }, ) - parser = DeepSeekV4ToolParser(mock_tokenizer, tools=[tool]) + parser = DeepSeekV4EngineToolParser(mock_tokenizer, tools=[tool]) request = MagicMock() request.tools = [tool] @@ -303,7 +309,64 @@ def test_extract_tool_calls_arguments_wrapper(): assert args == {"location": "Beijing"} -@pytest.mark.skip_global_cleanup +_ANGLE_BRACKET_TOOL = ChatCompletionToolsParam( + function=FunctionDefinition( + name="run_command", + parameters={ + "type": "object", + "properties": { + "command": {"type": "string"}, + }, + }, + ), +) + + +@pytest.mark.parametrize( + "tools", + [[_ANGLE_BRACKET_TOOL], None], + ids=["with_tools", "without_tools"], +) +def test_no_dsml_closing_tag_leak_in_streamed_args(tools): + """Streaming must not leak into argument values. + + When a parameter value contains '>' (e.g. shell redirects like + '2>&1'), certain chunk boundaries cause the incremental lexer to + emit the closing delimiter text as part of the content token. The + partial regex then captures it as part of the value, violating the + prefix invariant and corrupting the streamed JSON. + """ + full_text = build_tool_call("run_command", {"command": "git --version 2>&1"}) + expected = {"command": "git --version 2>&1"} + + for chunk_size in range(1, len(full_text) + 1): + parser = make_parser(tools=tools) + deltas = stream(parser, full_text, chunk_size=chunk_size) + args_str = reconstruct_args(deltas) + assert args_str, f"No args emitted at chunk_size={chunk_size}" + assert "DSML" not in args_str, ( + f"DSML marker leaked into args at chunk_size={chunk_size}: {args_str!r}" + ) + parsed = json.loads(args_str) + assert parsed == expected, ( + f"Args mismatch at chunk_size={chunk_size}: " + f"got {parsed!r}, expected {expected!r}" + ) + + +def test_non_streaming_extract_with_angle_brackets(): + """Non-streaming extraction must correctly handle '>' in values.""" + parser = make_parser() + full_text = build_tool_call("run_command", {"command": "git --version 2>&1"}) + result = parser.extract_tool_calls(full_text, make_request()) + + assert result.tools_called + assert len(result.tool_calls) == 1 + args = json.loads(result.tool_calls[0].function.arguments) + assert args == {"command": "git --version 2>&1"} + assert "DSML" not in result.tool_calls[0].function.arguments + + def test_composed_schema_converts_object_and_array_params(): tool = ChatCompletionToolsParam( type="function", diff --git a/tests/tool_parsers/test_gigachat3_tool_parser.py b/tests/tool_parsers/test_gigachat3_tool_parser.py index b00b410b2fa..00a97095134 100644 --- a/tests/tool_parsers/test_gigachat3_tool_parser.py +++ b/tests/tool_parsers/test_gigachat3_tool_parser.py @@ -19,7 +19,7 @@ from vllm.tool_parsers import ToolParser, ToolParserManager def default_tokenizer() -> TokenizerLike: """Override module-scoped default_tokenizer because gigachat tests mutate the tokenizer via ``add_tokens``.""" - return AutoTokenizer.from_pretrained("gpt2") + return AutoTokenizer.from_pretrained("openai-community/gpt2") MSG_SEP_TOKEN = "<|message_sep|>\n\n" diff --git a/tests/tool_parsers/test_structural_tag_registry.py b/tests/tool_parsers/test_structural_tag_registry.py index 63c37a67554..354adab6697 100644 --- a/tests/tool_parsers/test_structural_tag_registry.py +++ b/tests/tool_parsers/test_structural_tag_registry.py @@ -16,9 +16,11 @@ from vllm.entrypoints.openai.chat_completion.protocol import ( from vllm.parser.abstract_parser import DelegatingParser from vllm.tool_parsers.abstract_tool_parser import ToolParser from vllm.tool_parsers.deepseekv3_tool_parser import DeepSeekV3ToolParser -from vllm.tool_parsers.deepseekv4_tool_parser import DeepSeekV4ToolParser +from vllm.tool_parsers.deepseekv4_engine_tool_parser import DeepSeekV4EngineToolParser from vllm.tool_parsers.deepseekv31_tool_parser import DeepSeekV31ToolParser -from vllm.tool_parsers.deepseekv32_tool_parser import DeepSeekV32ToolParser +from vllm.tool_parsers.deepseekv32_engine_tool_parser import ( + DeepSeekV32EngineToolParser, +) from vllm.tool_parsers.glm47_moe_tool_parser import Glm47MoeModelToolParser from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser from vllm.tool_parsers.kimi_k2_tool_parser import KimiK2ToolParser @@ -185,8 +187,8 @@ def test_get_model_structural_tag_supports_named_tool_choice( [ (DeepSeekV3ToolParser, "deepseek_r1"), (DeepSeekV31ToolParser, "deepseek_v3_1"), - (DeepSeekV32ToolParser, "deepseek_v3_2"), - (DeepSeekV4ToolParser, "deepseek_v4"), + (DeepSeekV32EngineToolParser, "deepseek_v3_2"), + (DeepSeekV4EngineToolParser, "deepseek_v4"), (Glm47MoeModelToolParser, "glm_4_7"), (Hermes2ProToolParser, "hermes"), (KimiK2ToolParser, "kimi"), diff --git a/tests/v1/core/test_async_scheduler.py b/tests/v1/core/test_async_scheduler.py index 5e9c9280dbe..8253a8422e2 100644 --- a/tests/v1/core/test_async_scheduler.py +++ b/tests/v1/core/test_async_scheduler.py @@ -272,6 +272,9 @@ def test_abort_request_when_structured_output_fsm_cannot_advance(): scheduler.connector = None scheduler.structured_output_manager = Mock() scheduler.structured_output_manager.should_advance.return_value = True + scheduler.structured_output_manager.trim_reasoning_for_advance.side_effect = ( + lambda request, new_token_ids: new_token_ids + ) scheduler.requests = {request.request_id: request} scheduler.running = [request] scheduler.waiting = Mock() diff --git a/tests/v1/core/test_scheduler.py b/tests/v1/core/test_scheduler.py index dad345c643a..3e2b7dc5832 100644 --- a/tests/v1/core/test_scheduler.py +++ b/tests/v1/core/test_scheduler.py @@ -2911,6 +2911,9 @@ def test_abort_request_when_structured_output_fsm_cannot_advance(): scheduler.connector = None scheduler.structured_output_manager = Mock() scheduler.structured_output_manager.should_advance.return_value = True + scheduler.structured_output_manager.trim_reasoning_for_advance.side_effect = ( + lambda request, new_token_ids: new_token_ids + ) scheduler.requests = {request.request_id: request} scheduler.running = [request] scheduler.waiting = Mock() diff --git a/tests/v1/spec_decode/test_dynamic_sd_cug.py b/tests/v1/spec_decode/test_dynamic_sd_cug.py new file mode 100644 index 00000000000..d75495ea606 --- /dev/null +++ b/tests/v1/spec_decode/test_dynamic_sd_cug.py @@ -0,0 +1,328 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest +import torch + +from vllm.config import ( + CompilationConfig, + CUDAGraphMode, + ParallelConfig, + SchedulerConfig, + VllmConfig, +) +from vllm.v1.worker.gpu import cudagraph_utils as gpu_cudagraph_utils + +pytestmark = pytest.mark.cpu_test + + +def _create_vllm_config_for_dsd( + max_num_seqs: int, + max_spec_tokens: int, + *, + cudagraph_mode: str = "FULL_AND_PIECEWISE", + use_dynamic_sd: bool = True, + num_spec_per_batch_size: list[tuple[int, int, int]] | None = None, +) -> MagicMock: + """Create a minimal config that exercises DSD cudagraph dispatch. + + The test uses an exact capture-size grid so that every valid uniform decode + shape has a directly matching FULL graph candidate. + + ``num_spec_per_batch_size`` lets a test supply an explicit DSD schedule of + ``(range_start, range_end, num_speculative_tokens)`` tuples. When omitted, + a schedule covering every query length in ``[1, max_decode_query_len]`` is + generated. + """ + + max_decode_query_len = max_spec_tokens + 1 + max_capture_tokens = max_num_seqs * max_decode_query_len + + compilation_config = CompilationConfig( + cudagraph_mode=cudagraph_mode, + cudagraph_capture_sizes=list(range(1, max_capture_tokens + 1)), + ) + compilation_config.max_cudagraph_capture_size = max_capture_tokens + compilation_config.post_init_cudagraph_sizes() + + vllm_config = MagicMock(spec=VllmConfig) + vllm_config.compilation_config = compilation_config + vllm_config.scheduler_config = SchedulerConfig.default_factory( + max_num_seqs=max_num_seqs, + ) + vllm_config.parallel_config = ParallelConfig() + # num_speculative_tokens is the max K (num_speculative_steps). The manager + # recovers num_new_sampled_tokens_per_step as + # decode_query_len - num_speculative_tokens; with decode_query_len = + # max_spec_tokens + 1 this yields the normal per-step bonus of 1. + vllm_config.num_speculative_tokens = max_spec_tokens + + speculative_config = MagicMock() + speculative_config.uses_dynamic_speculative_decoding.return_value = use_dynamic_sd + if use_dynamic_sd: + # DSD reads the per-batch-size schedule; a schedule entry with K + # speculative tokens maps to decode query length K + 1. By default + # provide every query length in [1, max_decode_query_len] (i.e. K in + # [0, max_spec_tokens]) so the manager captures a FULL decode graph for + # each uniform shape. + if num_spec_per_batch_size is None: + num_spec_per_batch_size = [ + (qlen, qlen, qlen - 1) for qlen in range(1, max_decode_query_len + 1) + ] + speculative_config.num_speculative_tokens_per_batch_size = ( + num_spec_per_batch_size + ) + else: + speculative_config.num_speculative_tokens_per_batch_size = None + vllm_config.speculative_config = speculative_config + + return vllm_config + + +def test_dynamic_sd_full_cudagraph_covers_all_uniform_decode_shapes(monkeypatch): + """Dynamic SD should create FULL decode candidates for every k in [1, K+1]. + + This validates the MRv2 CudaGraphManager path directly: once candidate + shapes have been built, dispatch() should pick a FULL graph for every + uniform decode batch shape produced by DSD up to max_num_seqs. + """ + + max_num_seqs = 512 + max_spec_tokens = 7 + max_decode_query_len = max_spec_tokens + 1 + + # CudaGraphManager consults PP rank helpers during initialization even + # though this test only exercises CPU-side candidate generation. + monkeypatch.setattr( + gpu_cudagraph_utils, + "get_pp_group", + lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True), + ) + + vllm_config = _create_vllm_config_for_dsd( + max_num_seqs=max_num_seqs, + max_spec_tokens=max_spec_tokens, + ) + manager = gpu_cudagraph_utils.CudaGraphManager( + vllm_config=vllm_config, + device=torch.device("cpu"), + cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE, + decode_query_len=max_decode_query_len, + ) + + # dispatch() only uses the precomputed candidate table after graphs are + # considered captured. The actual graph objects are irrelevant here. + manager._graphs_captured = True + + for num_reqs in range(1, max_num_seqs + 1): + for max_query_len in range(1, max_decode_query_len + 1): + # Uniform decode means every request contributes the same number of + # tokens, so the total token count is exactly num_reqs * query_len. + num_tokens = num_reqs * max_query_len + uniform_tok_count = gpu_cudagraph_utils.get_uniform_token_count( + num_reqs, + num_tokens, + max_query_len, + ) + + # The scheduler should mark every one of these shapes as a uniform + # decode batch, which is what enables FULL decode graph selection. + assert uniform_tok_count == max_query_len + + desc = manager.dispatch( + num_reqs=num_reqs, + num_tokens=num_tokens, + uniform_token_count=uniform_tok_count, + num_active_loras=0, + ) + + # With DSD enabled, MRv2 should have captured a FULL candidate for + # every k in [1, K+1], so dispatch should stay on the FULL path. + assert desc.cg_mode == CUDAGraphMode.FULL + assert desc.uniform_token_count == max_query_len + assert desc.num_tokens == num_tokens + assert desc.num_reqs == num_reqs + assert desc.num_active_loras == 0 + + +def test_dynamic_sd_non_uniform_batch_falls_back_to_piecewise(monkeypatch): + """DSD should use PIECEWISE when the batch is not a uniform decode batch. + + FULL DSD graphs are captured separately for each decode query length k. + When runtime tokens are not uniform, uniform_token_count is None and those + FULL candidates should be skipped in favor of the mixed-batch PIECEWISE + graph under FULL_AND_PIECEWISE mode. + """ + + max_spec_tokens = 4 + + monkeypatch.setattr( + gpu_cudagraph_utils, + "get_pp_group", + lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True), + ) + + vllm_config = _create_vllm_config_for_dsd( + max_num_seqs=512, + max_spec_tokens=max_spec_tokens, + cudagraph_mode="FULL_AND_PIECEWISE", + use_dynamic_sd=True, + ) + manager = gpu_cudagraph_utils.CudaGraphManager( + vllm_config=vllm_config, + device=torch.device("cpu"), + cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE, + decode_query_len=max_spec_tokens + 1, + ) + manager._graphs_captured = True + + # This shape is intentionally non-uniform: 3 tokens across 2 requests + # cannot correspond to a single per-request query length. + desc = manager.dispatch( + num_reqs=2, + num_tokens=3, + uniform_token_count=None, + num_active_loras=0, + ) + + assert desc.cg_mode == CUDAGraphMode.PIECEWISE + assert desc.uniform_token_count is None + assert desc.num_reqs is None + assert desc.num_tokens == 3 + assert desc.num_active_loras == 0 + + +def test_basic_sd_does_not_capture_shorter_full_decode_shapes(monkeypatch): + """Without DSD, only the max decode query length should get FULL graphs. + + Basic SD captures FULL decode graphs only for decode_query_len = K + 1. + Uniform batches with smaller query lengths should therefore miss the FULL + path entirely when using FULL_AND_PIECEWISE. + """ + + max_num_seqs = 512 + max_spec_tokens = 7 + max_decode_query_len = max_spec_tokens + 1 + + monkeypatch.setattr( + gpu_cudagraph_utils, + "get_pp_group", + lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True), + ) + + vllm_config = _create_vllm_config_for_dsd( + max_num_seqs=max_num_seqs, + max_spec_tokens=max_spec_tokens, + cudagraph_mode="FULL_AND_PIECEWISE", + use_dynamic_sd=False, + ) + manager = gpu_cudagraph_utils.CudaGraphManager( + vllm_config=vllm_config, + device=torch.device("cpu"), + cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE, + decode_query_len=max_decode_query_len, + ) + manager._graphs_captured = True + + for num_reqs in range(1, max_num_seqs + 1): + for max_query_len in range(1, max_decode_query_len): + # These are still uniform decode batches, but basic SD should only + # have FULL graphs for query_len == max_decode_query_len. + num_tokens = num_reqs * max_query_len + uniform_tok_count = gpu_cudagraph_utils.get_uniform_token_count( + num_reqs, + num_tokens, + max_query_len, + ) + assert uniform_tok_count == max_query_len + + desc = manager.dispatch( + num_reqs=num_reqs, + num_tokens=num_tokens, + uniform_token_count=uniform_tok_count, + num_active_loras=0, + ) + + assert desc.cg_mode == CUDAGraphMode.PIECEWISE + assert desc.uniform_token_count is None + assert desc.num_tokens == num_tokens + assert desc.num_reqs is None + assert desc.num_active_loras == 0 + + +def test_dynamic_sd_only_captures_scheduled_query_lengths(monkeypatch): + """DSD should only capture FULL graphs for query lengths in the schedule. + + With a partial schedule of ``(1, 32, 4)`` and ``(32, 128, 3)``, only the + scheduled speculative-token counts (K = 4 and K = 3) become decode query + lengths (K + 1 = 5 and 4). Uniform batches at those query lengths should get + FULL graphs, while every other query length (e.g. the lower values 1, 2, 3) + must fall back to the mixed-batch PIECEWISE graph. + """ + + max_num_seqs = 128 + max_spec_tokens = 7 + max_decode_query_len = max_spec_tokens + 1 + + # (range_start, range_end, num_speculative_tokens): K = 4 and K = 3 are + # scheduled, so FULL decode graphs should exist for query lengths K + 1, + # i.e. exactly {5, 4}. + num_spec_per_batch_size = [(1, 32, 4), (32, 128, 3)] + scheduled_query_lens = {entry[2] + 1 for entry in num_spec_per_batch_size} + + monkeypatch.setattr( + gpu_cudagraph_utils, + "get_pp_group", + lambda: SimpleNamespace(is_first_rank=True, is_last_rank=True), + ) + + vllm_config = _create_vllm_config_for_dsd( + max_num_seqs=max_num_seqs, + max_spec_tokens=max_spec_tokens, + cudagraph_mode="FULL_AND_PIECEWISE", + use_dynamic_sd=True, + num_spec_per_batch_size=num_spec_per_batch_size, + ) + manager = gpu_cudagraph_utils.CudaGraphManager( + vllm_config=vllm_config, + device=torch.device("cpu"), + cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE, + decode_query_len=max_decode_query_len, + ) + manager._graphs_captured = True + + for num_reqs in range(1, max_num_seqs + 1): + for max_query_len in range(1, max_decode_query_len + 1): + num_tokens = num_reqs * max_query_len + uniform_tok_count = gpu_cudagraph_utils.get_uniform_token_count( + num_reqs, + num_tokens, + max_query_len, + ) + assert uniform_tok_count == max_query_len + + desc = manager.dispatch( + num_reqs=num_reqs, + num_tokens=num_tokens, + uniform_token_count=uniform_tok_count, + num_active_loras=0, + ) + + if max_query_len in scheduled_query_lens: + # Scheduled query lengths get a dedicated FULL decode graph. + assert desc.cg_mode == CUDAGraphMode.FULL + assert desc.uniform_token_count == max_query_len + assert desc.num_tokens == num_tokens + assert desc.num_reqs == num_reqs + else: + # Unscheduled query lengths (including the lower values 1 and 2) + # have no FULL candidate and must fall back to PIECEWISE. + assert desc.cg_mode == CUDAGraphMode.PIECEWISE + assert desc.uniform_token_count is None + assert desc.num_tokens == num_tokens + assert desc.num_reqs is None + assert desc.num_active_loras == 0 diff --git a/tests/v1/spec_decode/test_mtp_structured_output.py b/tests/v1/spec_decode/test_mtp_structured_output.py new file mode 100644 index 00000000000..619f3ad6fde --- /dev/null +++ b/tests/v1/spec_decode/test_mtp_structured_output.py @@ -0,0 +1,343 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""grammar_bitmask under spec-decode draft padding (#44006).""" + +import pytest +from transformers import AutoTokenizer + +from vllm.config import StructuredOutputsConfig, VllmConfig +from vllm.config.model import ModelConfig +from vllm.config.speculative import SpeculativeConfig +from vllm.sampling_params import SamplingParams, StructuredOutputsParams +from vllm.v1.request import Request +from vllm.v1.structured_output import StructuredOutputManager + +TOKENIZER = "gpt2" +NUM_SPEC_TOKENS = 4 + + +def _make_manager_and_request(backend: str, prompt_str: str = '{"a": "b"}'): + tokenizer = AutoTokenizer.from_pretrained(TOKENIZER) + prompt = tokenizer.encode(prompt_str) + + vllm_config = VllmConfig( + model_config=ModelConfig(tokenizer=TOKENIZER), + structured_outputs_config=StructuredOutputsConfig(backend=backend), + speculative_config=SpeculativeConfig( + model="[ngram]", num_speculative_tokens=NUM_SPEC_TOKENS + ), + ) + manager = StructuredOutputManager(vllm_config) + + sampling_params = SamplingParams( + structured_outputs=StructuredOutputsParams(json='{"type": "object"}'), + ) + sampling_params.structured_outputs._backend = backend + sampling_params.update_from_generation_config({}, tokenizer.eos_token_id) + + request = Request( + "mtp_req", + prompt_token_ids=prompt, + sampling_params=sampling_params, + pooling_params=None, + ) + manager.grammar_init(request) + while not request.structured_output_request._check_grammar_completion(): + continue + + return tokenizer, manager, request, prompt + + +@pytest.mark.parametrize("backend", ["xgrammar", "guidance"]) +def test_bitmask_with_padded_invalid_drafts(backend): + """Bitmask handles -1 padded drafts and returns N+1 rows.""" + tokenizer, manager, request, prompt = _make_manager_and_request( + backend, prompt_str='{"a"' + ) + grammar = request.structured_output_request.grammar + + assert grammar.accept_tokens(request.request_id, prompt) + + valid_drafts = [tokenizer.encode(":")[0], tokenizer.encode(' "')[0]] + padded = valid_drafts + [-1, -1] + + bitmask = manager.grammar_bitmask( + requests={request.request_id: request}, + structured_output_request_ids=[request.request_id], + scheduled_spec_decode_tokens={request.request_id: padded}, + ) + + assert bitmask is not None + assert bitmask.shape[0] == len(padded) + 1 + assert not grammar.is_terminated() + + +@pytest.mark.parametrize("backend", ["xgrammar", "guidance"]) +def test_bitmask_when_grammar_terminates_mid_window(backend): + """Drafts following an EOS that terminates the grammar are a no-op.""" + tokenizer, manager, request, prompt = _make_manager_and_request(backend) + grammar = request.structured_output_request.grammar + + assert grammar.accept_tokens(request.request_id, prompt) + eos = tokenizer.eos_token_id + drafts = [eos] + [tokenizer.encode(" ")[0]] * (NUM_SPEC_TOKENS - 1) + + bitmask = manager.grammar_bitmask( + requests={request.request_id: request}, + structured_output_request_ids=[request.request_id], + scheduled_spec_decode_tokens={request.request_id: drafts}, + ) + + assert bitmask is not None + assert bitmask.shape[0] == NUM_SPEC_TOKENS + 1 + assert not grammar.is_terminated() + + +@pytest.mark.parametrize("backend", ["xgrammar", "guidance"]) +def test_bitmask_idempotent_across_calls(backend): + """Repeated calls with the same input return the same bitmask.""" + tokenizer, manager, request, prompt = _make_manager_and_request( + backend, prompt_str='{"a"' + ) + grammar = request.structured_output_request.grammar + + assert grammar.accept_tokens(request.request_id, prompt) + + drafts = [tokenizer.encode(":")[0], -1, -1, -1] + + first = manager.grammar_bitmask( + requests={request.request_id: request}, + structured_output_request_ids=[request.request_id], + scheduled_spec_decode_tokens={request.request_id: drafts}, + ) + second = manager.grammar_bitmask( + requests={request.request_id: request}, + structured_output_request_ids=[request.request_id], + scheduled_spec_decode_tokens={request.request_id: drafts}, + ) + + assert first is not None and second is not None + assert (first == second).all() + assert not grammar.is_terminated() + + +@pytest.mark.parametrize("backend", ["xgrammar", "guidance"]) +def test_bonus_position_constrained_after_invalid_drafts(backend): + """Regression for #44006: bonus row stays constrained after -1 padding.""" + tokenizer, manager, request, prompt = _make_manager_and_request( + backend, prompt_str='{"a"' + ) + grammar = request.structured_output_request.grammar + + assert grammar.accept_tokens(request.request_id, prompt) + + valid = tokenizer.encode(":")[0] + drafts = [valid, -1, -1, -1] + bitmask = manager.grammar_bitmask( + requests={request.request_id: request}, + structured_output_request_ids=[request.request_id], + scheduled_spec_decode_tokens={request.request_id: drafts}, + ) + assert bitmask is not None + assert bitmask.shape[0] == len(drafts) + 1 + + assert not (bitmask[-1] == -1).all() + assert not grammar.is_terminated() + + +@pytest.mark.parametrize("backend", ["xgrammar", "guidance"]) +def test_bitmask_constrained_when_reasoning_ends_midwindow(backend): + """Drafts after a mid-window reasoning-end marker stay constrained.""" + tokenizer, manager, request, prompt = _make_manager_and_request(backend) + grammar = request.structured_output_request.grammar + + assert grammar.accept_tokens(request.request_id, prompt) + + marker = tokenizer.encode("\n")[0] + + class StubReasoner: + def __init__(self, *_, **__): + self.end_token_id = marker + + def is_reasoning_end(self, input_ids): + return marker in list(input_ids) + + def is_reasoning_end_streaming(self, input_ids, delta_ids): + return marker in list(delta_ids) + + manager.reasoner_cls = StubReasoner + request.structured_output_request.reasoner = StubReasoner() + request.structured_output_request.reasoning_ended = False + + pre = tokenizer.encode(" ")[0] + post = tokenizer.encode(",")[0] + drafts = [pre, marker, post] + + bitmask = manager.grammar_bitmask( + requests={request.request_id: request}, + structured_output_request_ids=[request.request_id], + scheduled_spec_decode_tokens={request.request_id: drafts}, + ) + + assert bitmask is not None + assert bitmask.shape[0] == len(drafts) + 1 + assert (bitmask[0] == -1).all() + assert (bitmask[1] == -1).all() + assert not (bitmask[2] == -1).all() + assert not (bitmask[-1] == -1).all() + assert not grammar.is_terminated() + + +@pytest.mark.parametrize("backend", ["xgrammar", "guidance"]) +def test_bitmask_post_reasoning_end_drafts_skip_grammar_advance(backend): + """Post-marker drafts predate the bitmask and may be grammar-invalid; + grammar_bitmask must skip the grammar advance instead of asserting. + """ + tokenizer, manager, request, prompt = _make_manager_and_request( + backend, prompt_str="{" + ) + grammar = request.structured_output_request.grammar + + assert grammar.accept_tokens(request.request_id, prompt) + assert not grammar.is_terminated() + + marker = tokenizer.encode("\n")[0] + + class StubReasoner: + def __init__(self, *_, **__): + self.end_token_id = marker + + def is_reasoning_end(self, input_ids): + return marker in list(input_ids) + + def is_reasoning_end_streaming(self, input_ids, delta_ids): + return marker in list(delta_ids) + + manager.reasoner_cls = StubReasoner + request.structured_output_request.reasoner = StubReasoner() + request.structured_output_request.reasoning_ended = False + + pre = tokenizer.encode(" ")[0] + # A token that the JSON grammar would reject as the first post-marker + # token; without the fix grammar.accept_tokens fires the assertion. + invalid_post = tokenizer.encode("z")[0] + drafts = [pre, marker, invalid_post] + + bitmask = manager.grammar_bitmask( + requests={request.request_id: request}, + structured_output_request_ids=[request.request_id], + scheduled_spec_decode_tokens={request.request_id: drafts}, + ) + + assert bitmask is not None + assert bitmask.shape[0] == len(drafts) + 1 + # Post-marker position is still bitmask-constrained. + assert not (bitmask[2] == -1).all() + # Grammar must not have advanced through the unvalidated draft. + assert not grammar.is_terminated() + + +@pytest.mark.parametrize("backend", ["xgrammar", "guidance"]) +def test_validate_tokens_then_bitmask_round_trip(backend): + """validate_tokens -> pad with -1 -> grammar_bitmask must not assert.""" + tokenizer, manager, request, prompt = _make_manager_and_request(backend) + grammar = request.structured_output_request.grammar + + assert grammar.accept_tokens(request.request_id, prompt) + + raw_drafts = [tokenizer.encode(",")[0], 99999, 12345, 67890] + accepted = grammar.validate_tokens(raw_drafts) + assert len(accepted) <= len(raw_drafts) + + padded = accepted + [-1] * (len(raw_drafts) - len(accepted)) + assert len(padded) == len(raw_drafts) + + bitmask = manager.grammar_bitmask( + requests={request.request_id: request}, + structured_output_request_ids=[request.request_id], + scheduled_spec_decode_tokens={request.request_id: padded}, + ) + assert bitmask is not None + assert bitmask.shape[0] == len(padded) + 1 + assert not grammar.is_terminated() + + +class _MarkerReasoner: + """Stub reasoner whose reasoning-end marker is a single fixed token.""" + + def __init__(self, marker: int): + self.marker = marker + + def is_reasoning_end(self, input_ids): + return self.marker in list(input_ids) + + def is_reasoning_end_streaming(self, input_ids, delta_ids): + return self.marker in list(delta_ids) + + +def _setup_boundary_request(backend: str): + """Request with a structural-tag key and reasoning not yet ended.""" + from vllm.v1.structured_output.backend_types import StructuredOutputOptions + + tokenizer, manager, request, prompt = _make_manager_and_request(backend) + marker = tokenizer.encode("\n")[0] + structured_req = request.structured_output_request + # The grammar itself is JSON (cheap to build); only the key kind matters + # for the should_advance structural-tag branch, so pre-seed the cached + # property. + structured_req.__dict__["structured_output_key"] = ( + StructuredOutputOptions.STRUCTURAL_TAG, + "", + ) + manager.reasoner_cls = _MarkerReasoner + structured_req.reasoner = _MarkerReasoner(marker) + structured_req.reasoning_ended = False + return tokenizer, manager, request, prompt, marker + + +def test_should_advance_records_reasoning_end_index(): + """Regression for #44006 on post-#42452 main: the boundary step must + record where reasoning ends so the scheduler can trim before advancing. + """ + tokenizer, manager, request, prompt, marker = _setup_boundary_request("xgrammar") + structured_req = request.structured_output_request + + pre = tokenizer.encode(" ")[0] + post = tokenizer.encode("{")[0] + request.append_output_token_ids([pre, marker, post]) + + assert manager.should_advance(request) + assert structured_req.reasoning_ended + # Marker sits at absolute index len(prompt) + 1. + assert structured_req.reasoning_end_token_index == len(prompt) + 1 + + +def test_trim_reasoning_for_advance(): + """trim drops the marker and everything before it; later steps and + requests without a recorded boundary pass through unchanged. + """ + tokenizer, manager, request, prompt, marker = _setup_boundary_request("xgrammar") + structured_req = request.structured_output_request + + pre = tokenizer.encode(" ")[0] + post = tokenizer.encode("{")[0] + + # No boundary recorded yet: pass-through. + assert manager.trim_reasoning_for_advance(request, [pre]) == [pre] + + # Boundary step: marker mid-step keeps only the suffix. + step_tokens = [pre, marker, post] + request.append_output_token_ids(step_tokens) + assert manager.should_advance(request) + assert manager.trim_reasoning_for_advance(request, step_tokens) == [post] + + # Boundary step variant: marker last (the #44006 crash shape + # [198, ]) trims to empty -> scheduler skips accept_tokens. + structured_req.reasoning_end_token_index = len(request.all_token_ids) - 1 + assert manager.trim_reasoning_for_advance(request, step_tokens) == [] + + # Later steps: tokens are past the boundary, returned unchanged. + structured_req.reasoning_end_token_index = len(prompt) + 1 + next_step = [post, post] + request.append_output_token_ids(next_step) + assert manager.trim_reasoning_for_advance(request, next_step) == next_step diff --git a/tests/v1/structured_output/test_backend_guidance.py b/tests/v1/structured_output/test_backend_guidance.py index ca8c9b0d785..edcdd983c0e 100644 --- a/tests/v1/structured_output/test_backend_guidance.py +++ b/tests/v1/structured_output/test_backend_guidance.py @@ -17,7 +17,7 @@ from vllm.v1.structured_output import StructuredOutputManager from vllm.v1.structured_output.backend_guidance import GuidanceBackend from vllm.v1.structured_output.backend_types import StructuredOutputOptions -TOKENIZER = "gpt2" +TOKENIZER = "openai-community/gpt2" @pytest.fixture(scope="module") diff --git a/vllm/config/cache.py b/vllm/config/cache.py index 2fb3358d55c..70a58004fc5 100644 --- a/vllm/config/cache.py +++ b/vllm/config/cache.py @@ -119,6 +119,11 @@ class CacheConfig: mamba_page_size_padded: int | None = None """ Optional override for mamba page size; used by hybrid mamba/attention models to ensure exact alignment with attention page size.""" + skip_page_size_padded: int | None = None + """Optional override for the page size of layers skipped from KV cache + quantization (``--kv-cache-dtype-skip-layers``); set during block-size + alignment so unquantized skip layers pad up to the quantized primary's + page.""" mamba_block_size: int | None = Field(default=None, gt=0) """Size of a contiguous cache block in number of tokens for mamba cache. Can be set only when prefix caching is enabled. @@ -207,6 +212,7 @@ class CacheConfig: # Prefix-caching implementation detail (doesn't affect compiled graph). "hash_block_size", "mamba_page_size_padded", + "skip_page_size_padded", "user_specified_block_size", "user_specified_mamba_block_size", "_block_size_resolved", diff --git a/vllm/config/compilation.py b/vllm/config/compilation.py index c7244d40d62..810c40131fc 100644 --- a/vllm/config/compilation.py +++ b/vllm/config/compilation.py @@ -1321,6 +1321,7 @@ class CompilationConfig: min_cg_support: "AttentionCGSupport", min_cg_attn_backend: str | None, uniform_decode_query_len: int = 1, + use_v2_model_runner: bool = False, tensor_parallel_size: int = 1, kv_cache_config: "KVCacheConfig | None" = None, max_num_reqs: int | None = None, @@ -1421,13 +1422,16 @@ class CompilationConfig: "and make sure compilation mode is VLLM_COMPILE" ) - # Adjust cudagraph sizes to be a multiple of uniform_decode_query_len + # MRV1 adjusts cudagraph sizes to be a multiple of uniform_decode_query_len # to avoid: https://github.com/vllm-project/vllm/issues/28207 and temp-fix: # https://github.com/vllm-project/vllm/issues/28207#issuecomment-3504004536 # Will be removed in the near future when we have separate cudagraph capture # sizes for decode and mixed prefill-decode. + # MRV2 handles cudagraph capture sizing in cudagraph_utils.py + # and doesn't need below: https://github.com/vllm-project/vllm/pull/45953 if ( - cudagraph_mode.decode_mode() == CUDAGraphMode.FULL + not use_v2_model_runner + and cudagraph_mode.decode_mode() == CUDAGraphMode.FULL and uniform_decode_query_len > 1 ): self.adjust_cudagraph_sizes_for_spec_decode( diff --git a/vllm/config/kernel.py b/vllm/config/kernel.py index 1d5b421715c..e98ab1b3e08 100644 --- a/vllm/config/kernel.py +++ b/vllm/config/kernel.py @@ -156,6 +156,8 @@ LinearBackend = Literal[ "conch", "exllama", "emulation", + "xpu", + "xpu_woq", ] @@ -217,7 +219,10 @@ class KernelConfig: - "fbgemm": Use FBGEMM kernels - "conch": Use Conch mixed-precision kernels - "exllama": Use Exllama mixed-precision kernels - - "emulation": Use slow dequant-to-BF16 emulation (for testing only)""" + - "emulation": Use slow dequant-to-BF16 emulation (for testing only) + - "xpu": Use XPU kernels + - "xpu_woq": Use XPU kernels for weight-only quantization (e.g. W8A16) + """ @field_validator("moe_backend", mode="before") @classmethod diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index de52e5228e9..c3f689c0a9c 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -774,13 +774,15 @@ class VllmConfig: speculative_config is None or not speculative_config.uses_dynamic_speculative_decoding() or not self.compilation_config.cudagraph_mode.has_full_cudagraphs() + or self.use_v2_model_runner ): return logger.warning_once( "Dynamic speculative decoding changes the target verification " "length at runtime. Overriding cudagraph_mode from %s to " - "PIECEWISE for reliability.", + "PIECEWISE for reliability. Use VLLM_USE_V2_MODEL_RUNNER=1 " + "if you want to use full CUDA graphs.", self.compilation_config.cudagraph_mode.name, ) self.compilation_config.cudagraph_mode = CUDAGraphMode.PIECEWISE @@ -2062,9 +2064,6 @@ class VllmConfig: ): unsupported.append(f"speculative method '{speculative_config.method}'") - if speculative_config.uses_dynamic_speculative_decoding(): - unsupported.append("dynamic speculative decoding") - # V2 EagleSpeculator does not support parallel_drafting (for P-Eagle). # DFlash and DSpark use parallel drafting natively in V2 via their # own speculators. diff --git a/vllm/entrypoints/cli/launch.py b/vllm/entrypoints/cli/launch.py index 50e46d81cc9..d13a0c67c83 100644 --- a/vllm/entrypoints/cli/launch.py +++ b/vllm/entrypoints/cli/launch.py @@ -120,7 +120,7 @@ async def run_launch_fastapi(args: argparse.Namespace) -> None: signal.signal(signal.SIGTERM, _interrupt_init) # 1. Socket binding - listen_address, sock = setup_server(args) + listen_address, sock = setup_server(args, reuse_port=False) # 2. Build and serve the API server engine_args = AsyncEngineArgs.from_cli_args(args) diff --git a/vllm/entrypoints/cli/serve.py b/vllm/entrypoints/cli/serve.py index 8491e982165..d5e9b2bc874 100644 --- a/vllm/entrypoints/cli/serve.py +++ b/vllm/entrypoints/cli/serve.py @@ -281,7 +281,7 @@ def run_multi_api_server(args: argparse.Namespace): signal.signal(signal.SIGTERM, signal_handler) signal.signal(signal.SIGINT, signal_handler) - listen_address, sock = setup_server(args) + listen_address, sock = setup_server(args, reuse_port=num_api_servers > 1) engine_args = vllm.AsyncEngineArgs.from_cli_args(args) engine_args._api_process_count = num_api_servers diff --git a/vllm/entrypoints/openai/api_server.py b/vllm/entrypoints/openai/api_server.py index 6ae6dd70abd..6fb27c365d9 100644 --- a/vllm/entrypoints/openai/api_server.py +++ b/vllm/entrypoints/openai/api_server.py @@ -508,14 +508,19 @@ async def init_render_app_state( state.server_load_metrics = 0 -def create_server_socket(addr: tuple[str, int]) -> socket.socket: +def create_server_socket( + addr: tuple[str, int], + *, + reuse_port: bool, +) -> socket.socket: family = socket.AF_INET if is_valid_ipv6_address(addr[0]): family = socket.AF_INET6 sock = socket.socket(family=family, type=socket.SOCK_STREAM) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + if reuse_port: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) sock.bind(addr) return sock @@ -546,7 +551,7 @@ def validate_api_server_args(args): @instrument(span_name="API server setup") -def setup_server(args): +def setup_server(args, *, reuse_port: bool): """Validate API server args and create the server socket.""" log_version_and_model(logger, VLLM_VERSION, args.model) @@ -567,7 +572,7 @@ def setup_server(args): sock = create_server_unix_socket(args.uds) else: sock_addr = (args.host or "", args.port) - sock = create_server_socket(sock_addr) + sock = create_server_socket(sock_addr, reuse_port=reuse_port) # workaround to avoid footguns where uvicorn drops requests with too # many concurrent requests active @@ -688,7 +693,7 @@ async def run_server(args, **uvicorn_kwargs) -> None: signal.signal(signal.SIGTERM, _interrupt_init) - listen_address, sock = setup_server(args) + listen_address, sock = setup_server(args, reuse_port=False) await run_server_worker(listen_address, sock, args, **uvicorn_kwargs) diff --git a/vllm/entrypoints/openai/chat_completion/batch_serving.py b/vllm/entrypoints/openai/chat_completion/batch_serving.py index c00ab0418fd..25f7c3ec5f5 100644 --- a/vllm/entrypoints/openai/chat_completion/batch_serving.py +++ b/vllm/entrypoints/openai/chat_completion/batch_serving.py @@ -181,7 +181,7 @@ class OpenAIServingChatBatch(OpenAIServingChat): sub_request_id, lora_request=lora_request, trace_headers=trace_headers, - priority=request.priority if hasattr(request, "priority") else 0, + priority=request.priority, data_parallel_rank=data_parallel_rank, reasoning_ended=None, ) diff --git a/vllm/entrypoints/openai/chat_completion/protocol.py b/vllm/entrypoints/openai/chat_completion/protocol.py index 4745947d49a..ab905677ab5 100644 --- a/vllm/entrypoints/openai/chat_completion/protocol.py +++ b/vllm/entrypoints/openai/chat_completion/protocol.py @@ -1020,6 +1020,10 @@ class BatchChatCompletionRequest(OpenAIBaseModel): continue_final_message: bool = False chat_template: str | None = None chat_template_kwargs: dict[str, Any] | None = None + media_io_kwargs: dict[str, dict[str, Any]] | None = None + mm_processor_kwargs: dict[str, Any] | None = None + priority: int = Field(default=0, ge=_INT64_MIN, le=_INT64_MAX) + cache_salt: str | None = None include_stop_str_in_output: bool = False guided_decoding_backend: str | None = None echo: bool = False diff --git a/vllm/entrypoints/openai/responses/context.py b/vllm/entrypoints/openai/responses/context.py index d75e5d5a548..92aad86b211 100644 --- a/vllm/entrypoints/openai/responses/context.py +++ b/vllm/entrypoints/openai/responses/context.py @@ -18,7 +18,7 @@ from openai.types.responses.response_output_item import McpCall from openai.types.responses.response_output_message import ResponseOutputMessage from openai.types.responses.response_output_text import ResponseOutputText from openai.types.responses.tool import Mcp -from openai_harmony import Author, HarmonyError, Message, Role, TextContent +from openai_harmony import Author, Message, Role, TextContent from vllm import envs from vllm.entrypoints.chat_utils import ( @@ -616,7 +616,7 @@ class HarmonyContext(ConversationContext): self.num_tool_output_tokens = 0 self.last_append_segments: list[Segment] = [] - self.last_append_flush_status: bool | HarmonyError = False + self.last_append_flush_status: bool = False # Turn tracking - replaces multiple individual tracking variables self.current_turn_metrics = TurnMetrics() @@ -643,10 +643,10 @@ class HarmonyContext(ConversationContext): if output.finished: self.finish_reason = output.outputs[0].finish_reason - flushed = self.response_parser.flush() - if flushed is not None: - segments.append(flushed) - self.last_append_flush_status = flushed is not None + flushed_segments = self.response_parser.flush() + if flushed_segments: + segments.extend(flushed_segments) + self.last_append_flush_status = len(flushed_segments) > 0 self.all_turn_metrics.append(self.current_turn_metrics.copy()) self.current_turn_metrics.reset() diff --git a/vllm/entrypoints/pooling/base/protocol.py b/vllm/entrypoints/pooling/base/protocol.py index 81ad303ad90..04ebc18817b 100644 --- a/vllm/entrypoints/pooling/base/protocol.py +++ b/vllm/entrypoints/pooling/base/protocol.py @@ -69,6 +69,21 @@ class PoolingBasicRequestMixin(OpenAIBaseModel): ) # --8<-- [end:pooling-common-extra-params] + @model_validator(mode="before") + @classmethod + def check_cache_salt_support(cls, data): + if not isinstance(data, dict): + return data + + if data.get("cache_salt") is not None and ( + not isinstance(data["cache_salt"], str) or not data["cache_salt"] + ): + raise VLLMValidationError( + "Parameter 'cache_salt' must be a non-empty string if provided.", + parameter="cache_salt", + ) + return data + def _build_pooling_tok_params( self, model_config: ModelConfig, diff --git a/vllm/entrypoints/pooling/scoring/io_processor.py b/vllm/entrypoints/pooling/scoring/io_processor.py index 7c58e7e9aba..b7b9ed5bc3e 100644 --- a/vllm/entrypoints/pooling/scoring/io_processor.py +++ b/vllm/entrypoints/pooling/scoring/io_processor.py @@ -10,6 +10,7 @@ from vllm import PoolingParams, PoolingRequestOutput, TokensPrompt from vllm.inputs import EngineInput from vllm.renderers import TokenizeParams from vllm.renderers.hf import safe_apply_chat_template +from vllm.renderers.inputs.preprocess import extract_target_prompt from vllm.tasks import PoolingTask from vllm.utils.mistral import is_mistral_tokenizer @@ -433,8 +434,16 @@ class CrossEncoderIOProcessor(ScoringIOProcessor): max_tokens_per_doc: int = 0, prompt_extras: dict[str, Any] | None = None, ) -> tuple[Sequence[EngineInput], list[PoolingParams]]: - # todo: support prompt_extras arrival_time = time.time() + engine_prompt_extras = ( + { + k: v + for k in ("mm_processor_kwargs", "cache_salt") + if (v := prompt_extras.get(k)) is not None + } + if prompt_extras + else None + ) data_1 = scoring_data.data_1 data_2 = scoring_data.data_2 @@ -463,12 +472,18 @@ class CrossEncoderIOProcessor(ScoringIOProcessor): if token_type_ids := engine_prompt.pop("token_type_ids", None): params = pooling_params.clone() compressed = compress_token_type_ids(token_type_ids) - params.extra_kwargs = {"compressed_token_type_ids": compressed} + params.extra_kwargs = { + **(params.extra_kwargs or {}), + "compressed_token_type_ids": compressed, + } pooling_params_list.append(params) else: pooling_params_list.append(pooling_params) tok_params.apply_post_tokenization(self.tokenizer, engine_prompt) + if engine_prompt_extras: + target_prompt = extract_target_prompt(self.model_config, engine_prompt) + target_prompt.update(engine_prompt_extras) engine_inputs.append( self.renderer.process_for_engine(engine_prompt, arrival_time) ) diff --git a/vllm/model_executor/kernels/linear/__init__.py b/vllm/model_executor/kernels/linear/__init__.py index 4ac8d49cd58..f5e3a16d71b 100644 --- a/vllm/model_executor/kernels/linear/__init__.py +++ b/vllm/model_executor/kernels/linear/__init__.py @@ -171,7 +171,8 @@ from vllm.model_executor.kernels.linear.scaled_mm.triton import ( ) from vllm.model_executor.kernels.linear.scaled_mm.xpu import ( XPUFp8BlockScaledMMKernel, - XPUFP8ScaledMMLinearKernel, + XPUW8A8FP8LinearKernel, + XPUW8A16FP8LinearKernel, ) from vllm.model_executor.kernels.linear.scaled_mm.zentorch import ( ZentorchInt8ScaledMMLinearKernel, @@ -266,6 +267,13 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = { EmulationMxfp8LinearKernel, EmulationNvFp4LinearKernel, }, + "xpu": { + XPUW8A8FP8LinearKernel, + XPUFp8BlockScaledMMKernel, + }, + "xpu_woq": { + XPUW8A16FP8LinearKernel, + }, } @@ -311,7 +319,8 @@ _POSSIBLE_FP8_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] = ChannelWiseTorchFP8ScaledMMLinearKernel, ], PlatformEnum.XPU: [ - XPUFP8ScaledMMLinearKernel, + XPUW8A16FP8LinearKernel, + XPUW8A8FP8LinearKernel, ], } @@ -351,7 +360,7 @@ _POSSIBLE_WFP8A16_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel] # To be added ], PlatformEnum.XPU: [ - XPUFP8ScaledMMLinearKernel, + XPUW8A16FP8LinearKernel, ], } diff --git a/vllm/model_executor/kernels/linear/scaled_mm/xpu.py b/vllm/model_executor/kernels/linear/scaled_mm/xpu.py index 6afa52bf875..3d23b02bea8 100644 --- a/vllm/model_executor/kernels/linear/scaled_mm/xpu.py +++ b/vllm/model_executor/kernels/linear/scaled_mm/xpu.py @@ -6,8 +6,11 @@ from collections.abc import Sequence import torch from vllm.model_executor.layers.quantization.utils.quant_utils import ( + kFp8DynamicTensorSym, + kFp8DynamicTokenSym, kFp8StaticChannelSym, kFp8StaticTensorSym, + kFp8StaticTokenSym, ) from vllm.model_executor.utils import replace_parameter from vllm.platforms import current_platform @@ -16,13 +19,112 @@ from .BlockScaledMMLinearKernel import Fp8BlockScaledMMLinearKernel from .ScaledMMLinearKernel import FP8ScaledMMLinearKernel, FP8ScaledMMLinearLayerConfig -class XPUFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel): +class XPUW8A8FP8LinearKernel(FP8ScaledMMLinearKernel): + _SUPPORTED_ACT_QUANT_KEYS = { + kFp8DynamicTensorSym, + kFp8DynamicTokenSym, + kFp8StaticTensorSym, + kFp8StaticTokenSym, + } + _SUPPORTED_WEIGHT_QUANT_KEYS = { + kFp8StaticChannelSym, + kFp8StaticTensorSym, + } + @classmethod def is_supported( cls, compute_capability: int | None = None ) -> tuple[bool, str | None]: if not current_platform.is_xpu(): - return False, "XPUFP8ScaledMM only support on XPU" + return False, "XPUW8A8FP8Linear only support on XPU" + return True, None + + @classmethod + def can_implement(cls, c: FP8ScaledMMLinearLayerConfig) -> tuple[bool, str | None]: + if c.weight_quant_key not in cls._SUPPORTED_WEIGHT_QUANT_KEYS: + return ( + False, + "XPUW8A8FP8Linear only support per-channel and per-tensor quantization", + ) + if c.activation_quant_key not in cls._SUPPORTED_ACT_QUANT_KEYS: + return ( + False, + "XPUW8A8FP8Linear only support per-tensor and per-token activation " + "quantization", + ) + if c.weight_quant_key.dtype not in {torch.float8_e5m2, torch.float8_e4m3fn}: + return False, "XPUW8A8FP8Linear only support FP8 weight dtype" + if c.activation_quant_key.dtype not in { + torch.float8_e5m2, + torch.float8_e4m3fn, + }: + return False, "XPUW8A8FP8Linear only support FP8 activation dtype" + return True, None + + def __init__( + self, c: FP8ScaledMMLinearLayerConfig, layer_param_names: Sequence[str] + ) -> None: + super().__init__(c, layer_param_names) + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + """Ensure weight is stored as C-contiguous [K, N] (KN layout). + + Checkpoints store weight as [N, K]; fp8_gemm requires [K, N], + C-contiguous. Three incoming layouts are possible: + • [N, K] C-contiguous ← direct checkpoint → .t().contiguous() + • [K, N] Fortran-order ← fp8.py's weight.t() → .contiguous() + • [K, N] C-contiguous ← already correct → no-op + + For square weights (K == N) the shape is ambiguous; contiguity is used + as a proxy: C-contiguous ≡ checkpoint [N, K] (needs transpose); + Fortran-order ≡ fp8.py already transposed (needs only contiguous). + """ + K = getattr(layer, "input_size_per_partition", self.config.weight_shape[1]) + N = getattr(layer, "output_size_per_partition", self.config.weight_shape[0]) + w = layer.weight + + if w.shape not in {(K, N), (N, K)}: + raise ValueError( + f"XPUFP8ScaledMM expects weight shape ({K},{N}) or ({N},{K}), " + f"but got {tuple(w.shape)}" + ) + + needs_transpose = w.shape == (N, K) if K != N else w.is_contiguous() + layer_weight = w.t() if needs_transpose else w + replace_parameter(layer, "weight", layer_weight) + ws = layer.weight_scale + if ws.numel() == 1: + replace_parameter(layer, "weight_scale", ws.reshape(1)) + + def apply_scaled_mm( + self, + *, + A: torch.Tensor, + B: torch.Tensor, + out_dtype: torch.dtype, + As: torch.Tensor, + Bs: torch.Tensor, + bias: torch.Tensor | None, + output_shape: list, + ) -> torch.Tensor: + # B is C-contiguous [K, N] from process_weights_after_loading. + # fp8_gemm routes on scale dtype (float32) and numel: + # As [1] → per-tensor (numel==1 branch) + # As [M,1] → per-token (group={1,K} branch, broadcast across K) + # Bs [1] → per-tensor + # Bs [N] → per-channel (mask=bit1 branch) + # No shape manipulation needed here. + output = torch.ops._xpu_C.fp8_gemm(A, B, out_dtype, As, Bs, bias) + return output.view(*output_shape) + + +class XPUW8A16FP8LinearKernel(FP8ScaledMMLinearKernel): + @classmethod + def is_supported( + cls, compute_capability: int | None = None + ) -> tuple[bool, str | None]: + if not current_platform.is_xpu(): + return False, "XPUW8A16FP8Linear only support on XPU" return True, None @classmethod @@ -30,10 +132,11 @@ class XPUFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel): if c.weight_quant_key not in {kFp8StaticChannelSym, kFp8StaticTensorSym}: return ( False, - "XPUFP8ScaledMM only support per-channel and per-tensor quantization", + "XPUW8A16FP8Linear only support per-channel and per-tensor " + "quantization", ) if c.weight_quant_key.dtype not in {torch.float8_e5m2, torch.float8_e4m3fn}: - return False, "XPUFP8ScaledMM only support FP8 weight dtype" + return False, "XPUW8A16FP8Linear only support FP8 weight dtype" return True, None def __init__( diff --git a/vllm/model_executor/layers/attention/attention.py b/vllm/model_executor/layers/attention/attention.py index 195d1af3be9..c2adf3499ac 100644 --- a/vllm/model_executor/layers/attention/attention.py +++ b/vllm/model_executor/layers/attention/attention.py @@ -92,6 +92,35 @@ def should_load_quant_weights(quant_method: QuantizeMethodBase | None) -> bool: ) +def _largest_kernel_block_within( + attn_backend: "type[AttentionBackend]", + per_token_bytes: int, + page_budget: int | None, + fallback: int, +) -> int: + """Largest supported kernel block size whose page fits in ``page_budget``. + + A padded spec (e.g. skip-quant layer) that pads its page up to a large shared page + wastes ``page_budget - block*per_token`` bytes per block. Picking the largest kernel + block whose natural page still fits under ``page_budget`` minimizes that waste. + Falls back to the smallest supported block when ``page_budget`` is None (no padding + — the block is handled by ``unify``'s integer scaling instead) or nothing fits. + """ + from vllm.v1.attention.backend import MultipleOf + + sizes = attn_backend.get_supported_kernel_block_sizes() + candidates = [s for s in sizes if isinstance(s, int)] + if not candidates: + candidates = [s.base for s in sizes if isinstance(s, MultipleOf)] + if not candidates: + return fallback + smallest = min(candidates) + if not page_budget or per_token_bytes <= 0: + return smallest + fitting = [b for b in candidates if b * per_token_bytes <= page_budget] + return max(fitting) if fitting else smallest + + def set_default_quant_scales(layer: nn.Module, register_buffer: bool = False) -> None: """Sets default quantization scales for the layer.""" if register_buffer: @@ -601,14 +630,35 @@ class Attention(nn.Module, AttentionLayerBase): assert not vllm_config.model_config.use_mla, ( "MLA is not supported for slidingwindow" ) - return SlidingWindowSpec( - block_size=block_size, + # SW chooses its own block_size, decoupled from the user's + # ``--block-size`` (which only constrains primary attention). + # When this SW layer is a padded spec (skip-quant: its page is + # padded up to ``skip_page_size_padded``), pick the largest kernel + # block that still fits the shared page so we waste fewer padding + # bytes per block. Otherwise (page_size_padded is None) the smallest + # block is fine — ``unify`` scales it up by an integer ratio. + shared_page = vllm_config.cache_config.skip_page_size_padded + sw_per_token = SlidingWindowSpec( + block_size=1, num_kv_heads=self.num_kv_heads, head_size=self.head_size, head_size_v=self.head_size_v, dtype=self.kv_cache_torch_dtype, kv_quant_mode=quant_mode, sliding_window=self.sliding_window, + ).real_page_size_bytes + sw_block_size = _largest_kernel_block_within( + self.attn_backend, sw_per_token, shared_page, block_size + ) + return SlidingWindowSpec( + block_size=sw_block_size, + num_kv_heads=self.num_kv_heads, + head_size=self.head_size, + head_size_v=self.head_size_v, + dtype=self.kv_cache_torch_dtype, + kv_quant_mode=quant_mode, + sliding_window=self.sliding_window, + page_size_padded=shared_page, ) elif self.kv_cache_dtype.startswith("turboquant_"): from vllm.model_executor.layers.quantization.turboquant.config import ( diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py index 2091a1cb6e4..3789cd172fc 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py @@ -13,6 +13,7 @@ from compressed_tensors.quantization import ( ) from compressed_tensors.transform import TransformConfig +from vllm.config import get_current_vllm_config_or_none from vllm.distributed import ( get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size, @@ -747,9 +748,18 @@ class CompressedTensorsConfig(QuantizationConfig): act_quant_format = is_activation_quantization_format(format) if act_quant_format: if self._is_fp8_w8a8(weight_quant, input_quant): - is_fp8_w8a8_supported = self._check_scheme_supported( - CompressedTensorsW8A8Fp8.get_min_capability(), error=False - ) + if current_platform.is_xpu(): + # On XPU, --linear-backend xpu opts into W8A8 FP8 + # linear kernel; otherwise default to W8A16. + config = get_current_vllm_config_or_none() + is_fp8_w8a8_supported = ( + config is not None + and config.kernel_config.linear_backend == "xpu" + ) + else: + is_fp8_w8a8_supported = self._check_scheme_supported( + CompressedTensorsW8A8Fp8.get_min_capability(), error=False + ) if is_fp8_w8a8_supported: return CompressedTensorsW8A8Fp8( weight_quant=weight_quant, diff --git a/vllm/parser/deepseek_v32.py b/vllm/parser/deepseek_v32.py new file mode 100644 index 00000000000..0d9ac9f53ce --- /dev/null +++ b/vllm/parser/deepseek_v32.py @@ -0,0 +1,131 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""DeepSeek V3.2 parser: DSML tool calls with ``function_calls`` wrapper. + +DeepSeek V3.2 output format:: + + <｜DSML｜function_calls> + <｜DSML｜invoke name="func_name"> + <｜DSML｜parameter name="location" string="true">杭州 + <｜DSML｜parameter name="count" string="false">5 + + + +This is identical to DeepSeek V4 except for the outer wrapper +(``function_calls`` instead of ``tool_calls``) and the absence of +````/```` reasoning tags. +""" + +from __future__ import annotations + +import functools +from typing import TYPE_CHECKING + +from vllm.parser.deepseek_v4 import ( + DSML_INVOKE_END, + DSML_INVOKE_NAME_END, + DSML_INVOKE_PREFIX, + DSML_PARAM_CLOSE, + _dsml_arg_converter, + _unwrap_wrapper_args, +) +from vllm.parser.engine.events import EventType +from vllm.parser.engine.parser_engine import ParserEngine +from vllm.parser.engine.parser_engine_config import ( + ParserEngineConfig, + ParserState, + Transition, +) + +if TYPE_CHECKING: + from vllm.tokenizers import TokenizerLike + from vllm.tool_parsers.abstract_tool_parser import Tool + +_DSML = "｜DSML｜" + +DSML_FUNC_START = f"<{_DSML}function_calls>" +DSML_FUNC_END = f"" + + +@functools.cache +def deepseek_v32_config() -> ParserEngineConfig: + return ParserEngineConfig( + name="deepseek_v32", + initial_state=ParserState.CONTENT, + terminals={ + "TOOL_START": DSML_FUNC_START, + "TOOL_END": DSML_FUNC_END, + "INVOKE_PREFIX": DSML_INVOKE_PREFIX, + "INVOKE_NAME_END": DSML_INVOKE_NAME_END, + "INVOKE_END": DSML_INVOKE_END, + "PARAM_CLOSE": DSML_PARAM_CLOSE, + }, + token_id_terminals={ + "TOOL_START": DSML_FUNC_START, + "TOOL_END": DSML_FUNC_END, + }, + transitions={ + (ParserState.CONTENT, "TOOL_START"): Transition( + ParserState.TOOL_PREAMBLE, + (), + ), + (ParserState.TOOL_PREAMBLE, "INVOKE_PREFIX"): Transition( + ParserState.TOOL_NAME, + (EventType.TOOL_CALL_START,), + ), + (ParserState.TOOL_NAME, "INVOKE_NAME_END"): Transition( + ParserState.TOOL_ARGS, + (), + ), + (ParserState.TOOL_ARGS, "INVOKE_END"): Transition( + ParserState.TOOL_BETWEEN, + (EventType.TOOL_CALL_END,), + ), + (ParserState.TOOL_ARGS, "TOOL_END"): Transition( + ParserState.CONTENT, + (EventType.TOOL_CALL_END,), + ), + # Parallel tool calls + (ParserState.TOOL_BETWEEN, "INVOKE_PREFIX"): Transition( + ParserState.TOOL_NAME, + (EventType.TOOL_CALL_START,), + ), + (ParserState.TOOL_BETWEEN, "TOOL_END"): Transition( + ParserState.CONTENT, + (), + ), + }, + content_events={ + ParserState.CONTENT: EventType.TEXT_CHUNK, + ParserState.TOOL_NAME: EventType.TOOL_NAME, + ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK, + }, + arg_converter=_dsml_arg_converter, + arg_structural_chars=frozenset(">"), + strip_content_whitespace_with_tools=False, + tool_args_json=False, + ) + + +class DeepSeekV32Parser(ParserEngine): + def __init__( + self, + tokenizer: TokenizerLike, + tools: list[Tool] | None = None, + **kwargs, + ) -> None: + kwargs.pop("chat_template_kwargs", None) + super().__init__( + tokenizer, + tools, + parser_engine_config=deepseek_v32_config(), + **kwargs, + ) + self._arg_converter = self._convert_args + + def _convert_args(self, raw_args: str, partial: bool) -> str: + result = _dsml_arg_converter(raw_args, partial) + if not self._tools: + return result + func_name = next((s.name for s in self._tool_slots if s.args == raw_args), None) + return _unwrap_wrapper_args(result, self._tools, func_name) diff --git a/vllm/parser/deepseek_v4.py b/vllm/parser/deepseek_v4.py new file mode 100644 index 00000000000..c6566870820 --- /dev/null +++ b/vllm/parser/deepseek_v4.py @@ -0,0 +1,237 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""DeepSeek V4 parser: ````/```` +reasoning plus DSML tool calls in a single state machine. + +DeepSeek V4 output format:: + + + ...reasoning... + + <｜DSML｜tool_calls> + <｜DSML｜invoke name="func_name"> + <｜DSML｜parameter name="location" string="true">杭州 + <｜DSML｜parameter name="count" string="false">5 + + +""" + +from __future__ import annotations + +import contextlib +import functools +import json +from typing import TYPE_CHECKING + +import regex as re + +from vllm.parser.engine.events import EventType +from vllm.parser.engine.parser_engine import ParserEngine +from vllm.parser.engine.parser_engine_config import ( + ParserEngineConfig, + ParserState, + Transition, +) +from vllm.tool_parsers.utils import find_tool_properties + +if TYPE_CHECKING: + from vllm.tokenizers import TokenizerLike + from vllm.tool_parsers.abstract_tool_parser import Tool + +_DSML = "｜DSML｜" + +DSML_THINK_START = "" +DSML_THINK_END = "" +DSML_TOOL_START = f"<{_DSML}tool_calls>" +DSML_TOOL_END = f"" +DSML_INVOKE_PREFIX = f'<{_DSML}invoke name="' +DSML_INVOKE_NAME_END = '">' +DSML_INVOKE_END = f"" +DSML_PARAM_CLOSE = f"" + +_ESCAPED_DSML = re.escape(_DSML) +_PARAM_RE = re.compile( + rf'<{_ESCAPED_DSML}parameter\s+name="([^"]+)"\s+string="(true|false)">' + rf"(.*?)", + re.DOTALL, +) +_PARTIAL_PARAM_RE = re.compile( + rf'<{_ESCAPED_DSML}parameter\s+name="([^"]+)"\s+string="(true|false)">' + rf"(.*)$", + re.DOTALL, +) + + +def _dsml_arg_converter(raw_args: str, partial: bool) -> str: + params: dict[str, object] = {} + + last_end = 0 + for m in _PARAM_RE.finditer(raw_args): + name, is_str, value = m.group(1), m.group(2), m.group(3) + if is_str == "true": + params[name] = value + else: + try: + params[name] = json.loads(value) + except (json.JSONDecodeError, ValueError): + params[name] = value + last_end = m.end() + + if partial: + pm = _PARTIAL_PARAM_RE.search(raw_args, last_end) + if pm: + name, is_str, value = pm.group(1), pm.group(2), pm.group(3) + if is_str == "true": + params[name] = value + else: + with contextlib.suppress(json.JSONDecodeError, ValueError): + params[name] = json.loads(value) + + return json.dumps(params, ensure_ascii=False) + + +def _unwrap_wrapper_args( + args_json: str, + tools: list[Tool] | None, + func_name: str | None, +) -> str: + if not tools or not func_name: + return args_json + try: + args = json.loads(args_json) + except (json.JSONDecodeError, ValueError): + return args_json + if not isinstance(args, dict): + return args_json + properties = find_tool_properties(tools, func_name) + if not properties: + return args_json + allowed = set(properties.keys()) + for wrapper in ("arguments", "input"): + if set(args.keys()) != {wrapper} or wrapper in allowed: + continue + inner = args[wrapper] + if isinstance(inner, str): + try: + inner = json.loads(inner) + except json.JSONDecodeError: + return args_json + if isinstance(inner, dict) and set(inner.keys()).issubset(allowed): + return json.dumps(inner, ensure_ascii=False) + return args_json + + +@functools.cache +def deepseek_v4_config(thinking: bool = False) -> ParserEngineConfig: + return ParserEngineConfig( + name="deepseek_v4", + initial_state=ParserState.REASONING if thinking else ParserState.CONTENT, + terminals={ + "THINK_START": DSML_THINK_START, + "THINK_END": DSML_THINK_END, + "TOOL_START": DSML_TOOL_START, + "TOOL_END": DSML_TOOL_END, + "INVOKE_PREFIX": DSML_INVOKE_PREFIX, + "INVOKE_NAME_END": DSML_INVOKE_NAME_END, + "INVOKE_END": DSML_INVOKE_END, + "PARAM_CLOSE": DSML_PARAM_CLOSE, + }, + token_id_terminals={ + "THINK_START": DSML_THINK_START, + "THINK_END": DSML_THINK_END, + "TOOL_START": DSML_TOOL_START, + "TOOL_END": DSML_TOOL_END, + }, + transitions={ + (ParserState.CONTENT, "THINK_START"): Transition( + ParserState.REASONING, + (EventType.REASONING_START,), + ), + # Absorb a bare with no prior + (ParserState.CONTENT, "THINK_END"): Transition( + ParserState.CONTENT, + (), + ), + # Absorb a duplicate while already reasoning + (ParserState.REASONING, "THINK_START"): Transition( + ParserState.REASONING, + (), + ), + (ParserState.REASONING, "THINK_END"): Transition( + ParserState.CONTENT, + (EventType.REASONING_END,), + ), + # Tool call beginning while still inside + (ParserState.REASONING, "TOOL_START"): Transition( + ParserState.TOOL_PREAMBLE, + (EventType.REASONING_END,), + ), + (ParserState.CONTENT, "TOOL_START"): Transition( + ParserState.TOOL_PREAMBLE, + (), + ), + (ParserState.TOOL_PREAMBLE, "INVOKE_PREFIX"): Transition( + ParserState.TOOL_NAME, + (EventType.TOOL_CALL_START,), + ), + (ParserState.TOOL_NAME, "INVOKE_NAME_END"): Transition( + ParserState.TOOL_ARGS, + (), + ), + (ParserState.TOOL_ARGS, "INVOKE_END"): Transition( + ParserState.TOOL_BETWEEN, + (EventType.TOOL_CALL_END,), + ), + (ParserState.TOOL_ARGS, "TOOL_END"): Transition( + ParserState.CONTENT, + (EventType.TOOL_CALL_END,), + ), + # Parallel tool calls + (ParserState.TOOL_BETWEEN, "INVOKE_PREFIX"): Transition( + ParserState.TOOL_NAME, + (EventType.TOOL_CALL_START,), + ), + (ParserState.TOOL_BETWEEN, "TOOL_END"): Transition( + ParserState.CONTENT, + (), + ), + }, + content_events={ + ParserState.CONTENT: EventType.TEXT_CHUNK, + ParserState.REASONING: EventType.REASONING_CHUNK, + ParserState.TOOL_NAME: EventType.TOOL_NAME, + ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK, + }, + arg_converter=_dsml_arg_converter, + arg_structural_chars=frozenset(">"), + strip_content_whitespace_with_tools=False, + tool_args_json=False, + ) + + +class DeepSeekV4Parser(ParserEngine): + def __init__( + self, + tokenizer: TokenizerLike, + tools: list[Tool] | None = None, + **kwargs, + ) -> None: + chat_kwargs = kwargs.pop("chat_template_kwargs", None) or {} + thinking = ( + bool(chat_kwargs.get("thinking") or chat_kwargs.get("enable_thinking")) + and chat_kwargs.get("reasoning_effort") != "none" + ) + super().__init__( + tokenizer, + tools, + parser_engine_config=deepseek_v4_config(thinking=thinking), + **kwargs, + ) + self._arg_converter = self._convert_args + + def _convert_args(self, raw_args: str, partial: bool) -> str: + result = _dsml_arg_converter(raw_args, partial) + if not self._tools: + return result + func_name = next((s.name for s in self._tool_slots if s.args == raw_args), None) + return _unwrap_wrapper_args(result, self._tools, func_name) diff --git a/vllm/parser/engine/adapters.py b/vllm/parser/engine/adapters.py index 2482dad437b..b9c4e42b170 100644 --- a/vllm/parser/engine/adapters.py +++ b/vllm/parser/engine/adapters.py @@ -111,7 +111,8 @@ class ParserEngineReasoningAdapter(ReasoningParser): return self._parser_engine.reasoning_ended def finish_streaming(self) -> DeltaMessage | None: - return self._parser_engine.finish_streaming() + with self._skip_tool_parsing(): + return self._parser_engine.finish_streaming() def get_streaming_fallback_content( self, diff --git a/vllm/parser/engine/parser_engine.py b/vllm/parser/engine/parser_engine.py index 91d6881ca3b..6f21cbca768 100644 --- a/vllm/parser/engine/parser_engine.py +++ b/vllm/parser/engine/parser_engine.py @@ -108,7 +108,11 @@ class ParserEngine(Parser): parser_engine_config, tokenizer, vocab=self.vocab ) - self._reasoning_ended: bool = False + self._has_reasoning = ( + "THINK_END" in parser_engine_config.token_id_terminals + or parser_engine_config.initial_state == ParserState.REASONING + ) + self._reasoning_ended: bool = not self._has_reasoning self._streaming_initialized: bool = False self._prompt_streaming_prepared: bool = False @@ -188,7 +192,7 @@ class ParserEngine(Parser): def _reset(self, initial_state: ParserState | None = None) -> None: self._engine.reset(initial_state=initial_state) - self._reasoning_ended = False + self._reasoning_ended = not self._has_reasoning self._tool_slots.clear() self._deferred_content = "" self._deferred_reasoning = "" @@ -255,7 +259,7 @@ class ParserEngine(Parser): types = extract_types_from_schema(schema) as_str = json.dumps(value, ensure_ascii=False) coerced = coerce_to_schema_type(as_str, types) - if coerced != value: + if type(coerced) is not type(value) or coerced != value: return coerced, True return value, False diff --git a/vllm/parser/engine/registered_adapters.py b/vllm/parser/engine/registered_adapters.py index fdcbf81ee2b..889c71504fd 100644 --- a/vllm/parser/engine/registered_adapters.py +++ b/vllm/parser/engine/registered_adapters.py @@ -7,6 +7,8 @@ names so that :class:`ReasoningParserManager` and :class:`ToolParserManager` can load them lazily. """ +from vllm.parser.deepseek_v4 import DeepSeekV4Parser +from vllm.parser.deepseek_v32 import DeepSeekV32Parser from vllm.parser.engine.adapters import make_adapters from vllm.parser.gemma4 import Gemma4Parser from vllm.parser.glm47_moe import Glm47MoeParser @@ -16,6 +18,16 @@ from vllm.parser.nemotron_v3 import NemotronV3Parser from vllm.parser.qwen3 import Qwen3Parser from vllm.parser.seed_oss import SeedOssParser +( + DeepSeekV32ParserReasoningAdapter, + DeepSeekV32ParserToolAdapter, +) = make_adapters(DeepSeekV32Parser) + +( + DeepSeekV4ParserReasoningAdapter, + DeepSeekV4ParserToolAdapter, +) = make_adapters(DeepSeekV4Parser) + ( MinimaxM2ParserReasoningAdapter, MinimaxM2ParserToolAdapter, diff --git a/vllm/parser/engine/streaming_parser_engine.py b/vllm/parser/engine/streaming_parser_engine.py index ec3c5e5a4a6..319791961c2 100644 --- a/vllm/parser/engine/streaming_parser_engine.py +++ b/vllm/parser/engine/streaming_parser_engine.py @@ -22,14 +22,13 @@ from vllm.parser.engine.parser_engine_config import ( Transition, ) from vllm.parser.engine.token_id_scanner import ( + DROP_TERMINAL, LexerInput, PreLexedTerminal, TextChunk, TokenIDScanner, ) -DROP_TERMINAL = "__DROP__" - @dataclass(slots=True) class _DropInfo: diff --git a/vllm/parser/engine/token_id_scanner.py b/vllm/parser/engine/token_id_scanner.py index 635374d82ae..abcc2e2baec 100644 --- a/vllm/parser/engine/token_id_scanner.py +++ b/vllm/parser/engine/token_id_scanner.py @@ -8,6 +8,8 @@ from __future__ import annotations from collections.abc import Sequence from dataclasses import dataclass +DROP_TERMINAL = "__DROP__" + @dataclass(slots=True) class TextChunk: @@ -277,7 +279,15 @@ class TokenIDScanner: consumed = pos + len(anchor.text) else: has_later_valid = any(p >= 0 for p in positions[i + 1 :]) - if not has_later_valid and consumed < len(delta_text): + # DROP anchors (EOS, etc.) may have text that never + # arrives in delta_text (stripped by detokenizer). + # Don't defer remaining content waiting for text + # that will never come. + if ( + not has_later_valid + and consumed < len(delta_text) + and anchor.terminal != DROP_TERMINAL + ): self._deferred_post_text += delta_text[consumed:] consumed = len(delta_text) self._deferred_terminals.append(anchor) diff --git a/vllm/parser/harmony.py b/vllm/parser/harmony.py index 80fd02e4cec..5043ca191f3 100644 --- a/vllm/parser/harmony.py +++ b/vllm/parser/harmony.py @@ -9,7 +9,7 @@ from dataclasses import dataclass from enum import Enum, auto from typing import TYPE_CHECKING, NamedTuple -from openai_harmony import HarmonyError +from openai_harmony import HarmonyError, Message, Role from vllm.entrypoints.chat_utils import make_tool_call_id from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest @@ -91,6 +91,9 @@ class HarmonyParser(DelegatingParser): self._next_tool_call_index = 0 self._num_processed_messages = 0 + # For error recovery + self._current_message_tokens: list[int] = [] + @property def _harmony_parser(self) -> StreamableParser: """Lazily initializes the Harmony parser.""" @@ -107,32 +110,48 @@ class HarmonyParser(DelegatingParser): self._num_processed_messages += 1 return msg - def flush(self) -> Segment | None: + def flush(self) -> list[Segment]: + segments: list[Segment] = [] try: self._harmony_parser.process_eos() msg = self._poll_completed_message() except HarmonyError: logger.warning( "Harmony parser ended in a non-terminal state; returning the " - "raw unparsed output. This usually indicates a malformed " - "assistant turn, e.g. a 'final' channel missing the " - "<|message|> delimiter." + "recovered raw output." ) - raise - finally: - # Reset to the initial assistant-parser state for the next turn. - self._parser = None - self._num_processed_messages = 0 + + final_channel = "final" + text = self.model_tokenizer.decode(self._current_message_tokens) + segments.append( + Segment( + channel=final_channel, + recipient=None, + delta=text, + completed_message=None, + ) + ) + msg = Message.from_role_and_content(Role.ASSISTANT, text).with_channel( + final_channel + ) + + # Reset to the initial assistant-parser state for the next turn. + self._parser = None + self._num_processed_messages = 0 + self._current_message_tokens.clear() if msg is None: - return None + return segments - return Segment( - channel=msg.channel, - recipient=msg.recipient, - delta="", - completed_message=msg, + segments.append( + Segment( + channel=msg.channel, + recipient=msg.recipient, + delta="", + completed_message=msg, + ) ) + return segments def parse( self, @@ -147,12 +166,9 @@ class HarmonyParser(DelegatingParser): Callers must decide whether to surface them. """ result = self.process_chunk(model_output_token_ids) - try: - flushed_segment = self.flush() - except HarmonyError: - return None, model_output, None - if flushed_segment is not None: - result.segments.append(flushed_segment) + flushed_segments = self.flush() + if flushed_segments: + result.segments.extend(flushed_segments) reasoning_parts: list[str] = [] content_parts: list[str] = [] @@ -209,13 +225,9 @@ class HarmonyParser(DelegatingParser): ) result = self.process_chunk(delta_token_ids) if finished: - try: - flushed_segment = self.flush() - except HarmonyError: - self._next_tool_call_index = 0 - return DeltaMessage(content=delta_text) - if flushed_segment is not None: - result.segments.append(flushed_segment) + flushed_segments = self.flush() + if flushed_segments: + result.segments.extend(flushed_segments) combined_content = "" combined_reasoning = "" tool_messages: list[DeltaToolCall] = [] @@ -298,6 +310,11 @@ class HarmonyParser(DelegatingParser): delta = self._harmony_parser.last_content_delta or "" completed_message = self._poll_completed_message() + if completed_message is not None: + self._current_message_tokens.clear() + else: + self._current_message_tokens.append(token_id) + if channel == "analysis" or ( channel == "commentary" and recipient is not None ): diff --git a/vllm/platforms/interface.py b/vllm/platforms/interface.py index a7a0dd52df7..a81c34d7a50 100644 --- a/vllm/platforms/interface.py +++ b/vllm/platforms/interface.py @@ -629,6 +629,123 @@ class Platform: if model_config.is_hybrid: cls._align_hybrid_block_size(vllm_config, backend_cls) + # Phase 3: Align block/page sizes when multiple KV dtypes share the + # block pool (e.g. nvfp4 primary + unquantized skip layers). + # May override the user's --block-size. + if cache_config.kv_cache_dtype_skip_layers: + cls._align_heterogeneous_kv_block_size(vllm_config, backend_cls) + + @classmethod + def _align_heterogeneous_kv_block_size( + cls, + vllm_config: "VllmConfig", + backend_cls: "type[AttentionBackend]", + ) -> None: + """Align block size when several KV dtypes share one block pool. + + A quantized primary (e.g. nvfp4) shares the block pool with one or more + higher-precision "padded specs" (skip layers today; the first/last-N + sibling in the future). A padded spec's per-token page is larger than + the primary's *and not an integer multiple of it*, so the trivial + ``unify_kv_cache_spec_page_size`` cannot reconcile them. We do it here + instead, before the specs are built: + + 1. Bump the primary ``block_size`` (kernel-aligned) until the primary + page is large enough to cover the largest padded-spec page. + 2. Record that shared page in each padded spec's ``*_page_size_padded`` + hint, so it pads up to the shared page. + + ``unify`` then sees equal pages and stays trivial. + + To add a padded-spec type: append its per-token page to ``padded_pages`` + and set its ``*_page_size_padded`` hint below. + """ + from vllm.config.vllm import set_current_vllm_config + from vllm.utils.math_utils import cdiv + from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE + from vllm.v1.attention.backend import MultipleOf + from vllm.v1.kv_cache_interface import FullAttentionSpec, get_kv_quant_mode + + cache_config = vllm_config.cache_config + model_config = vllm_config.model_config + parallel_config = vllm_config.parallel_config + if not model_config: + return + + def per_token_page_bytes(dtype: "torch.dtype", cache_dtype: str) -> int: + """Bytes one token occupies in one layer, for the given dtype.""" + return FullAttentionSpec( + block_size=1, + num_kv_heads=model_config.get_num_kv_heads(parallel_config), + head_size=model_config.get_head_size(), + dtype=dtype, + kv_quant_mode=get_kv_quant_mode(cache_dtype), + ).page_size_bytes + + primary_dtype = ( + STR_DTYPE_TO_TORCH_DTYPE[cache_config.cache_dtype] + if cache_config.cache_dtype != "auto" + else model_config.dtype + ) + primary_page = per_token_page_bytes(primary_dtype, cache_config.cache_dtype) + + # Per-token page of every higher-precision padded spec sharing the pool. + padded_pages: list[int] = [] + if cache_config.kv_cache_dtype_skip_layers: + padded_pages.append(per_token_page_bytes(model_config.dtype, "auto")) + # To add the first/last-N sibling: + # padded_pages.append(per_token_page_bytes(, "auto")) + if not padded_pages: + return + + largest_padded_page = max(padded_pages) + assert largest_padded_page >= primary_page, ( + f"padded-spec per-token page ({largest_padded_page}B) < primary " + f"({primary_page}B); a higher-precision padded spec must not be " + "smaller than the quantized primary." + ) + if largest_padded_page == primary_page: + # Pages already match per token; ``unify`` reconciles the differing + # block sizes by integer scaling, so no bump or padding is needed. + return + + # Smallest block the kernel supports, and the granularity the primary + # block is rounded up to (never below the already-chosen block_size). + with set_current_vllm_config(vllm_config): + supported = backend_cls.get_supported_kernel_block_sizes() + smallest_kernel_block = min( + s.base if isinstance(s, MultipleOf) else s for s in supported + ) + block_alignment = max(smallest_kernel_block, cache_config.block_size) + + # Bytes one padded-spec page spans at its own smallest kernel block; + # also cover any mamba page a hybrid model already padded. + required_page = max( + largest_padded_page * smallest_kernel_block, + cache_config.mamba_page_size_padded or 0, + ) + + # Smallest kernel-aligned primary block whose page covers required_page. + primary_block_size = block_alignment * cdiv( + required_page, block_alignment * primary_page + ) + if cache_config.block_size < primary_block_size: + cache_config.block_size = primary_block_size + logger.info( + "Setting attention block size to %d tokens so the quantized " + "primary KV page covers the higher-precision padded-spec page.", + primary_block_size, + ) + + # The shared page that every padded spec (and mamba) pads up to. + shared_page = cache_config.block_size * primary_page + if cache_config.kv_cache_dtype_skip_layers: + cache_config.skip_page_size_padded = shared_page + # To add the first/last-N sibling: + # cache_config.sibling_page_size_padded = shared_page + if cache_config.mamba_page_size_padded is not None: + cache_config.mamba_page_size_padded = shared_page + @classmethod def _align_hybrid_block_size( cls, diff --git a/vllm/reasoning/__init__.py b/vllm/reasoning/__init__.py index fc74cf2f3f7..ba5ed471852 100644 --- a/vllm/reasoning/__init__.py +++ b/vllm/reasoning/__init__.py @@ -29,8 +29,8 @@ _REASONING_PARSERS_TO_REGISTER = { "DeepSeekV3ReasoningParser", ), "deepseek_v4": ( - "deepseek_v3_reasoning_parser", - "DeepSeekV3ReasoningParser", + "deepseek_v4_engine_reasoning_parser", + "DeepSeekV4ParserReasoningAdapter", ), "poolside_v1": ( "poolside_v1_reasoning_parser", diff --git a/vllm/reasoning/deepseek_v4_engine_reasoning_parser.py b/vllm/reasoning/deepseek_v4_engine_reasoning_parser.py new file mode 100644 index 00000000000..6fa6444b35c --- /dev/null +++ b/vllm/reasoning/deepseek_v4_engine_reasoning_parser.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm.parser.engine.registered_adapters import DeepSeekV4ParserReasoningAdapter + +__all__ = ["DeepSeekV4ParserReasoningAdapter"] diff --git a/vllm/tool_parsers/__init__.py b/vllm/tool_parsers/__init__.py index b9a9c9ad07b..26362ebf0ed 100644 --- a/vllm/tool_parsers/__init__.py +++ b/vllm/tool_parsers/__init__.py @@ -31,12 +31,12 @@ _TOOL_PARSERS_TO_REGISTER = { "DeepSeekV31ToolParser", ), "deepseek_v32": ( - "deepseekv32_tool_parser", - "DeepSeekV32ToolParser", + "deepseekv32_engine_tool_parser", + "DeepSeekV32EngineToolParser", ), "deepseek_v4": ( - "deepseekv4_tool_parser", - "DeepSeekV4ToolParser", + "deepseekv4_engine_tool_parser", + "DeepSeekV4EngineToolParser", ), "cohere_command3": ( "cohere_command_tool_parser", diff --git a/vllm/tool_parsers/deepseekv32_engine_tool_parser.py b/vllm/tool_parsers/deepseekv32_engine_tool_parser.py new file mode 100644 index 00000000000..4747fd10f3e --- /dev/null +++ b/vllm/tool_parsers/deepseekv32_engine_tool_parser.py @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm.parser.engine.registered_adapters import DeepSeekV32ParserToolAdapter + + +class DeepSeekV32EngineToolParser(DeepSeekV32ParserToolAdapter): # type: ignore[valid-type, misc] + structural_tag_model = "deepseek_v3_2" diff --git a/vllm/tool_parsers/deepseekv32_tool_parser.py b/vllm/tool_parsers/deepseekv32_tool_parser.py deleted file mode 100644 index c597ac61969..00000000000 --- a/vllm/tool_parsers/deepseekv32_tool_parser.py +++ /dev/null @@ -1,562 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import json -import uuid -from collections.abc import Sequence -from typing import Any, Literal - -import regex as re - -from vllm.entrypoints.openai.chat_completion.protocol import ( - ChatCompletionRequest, -) -from vllm.entrypoints.openai.engine.protocol import ( - DeltaFunctionCall, - DeltaMessage, - DeltaToolCall, - ExtractedToolCallInformation, - FunctionCall, - ToolCall, -) -from vllm.entrypoints.openai.responses.protocol import ResponsesRequest -from vllm.logger import init_logger -from vllm.tokenizers import TokenizerLike -from vllm.tool_parsers.abstract_tool_parser import ( - Tool, - ToolParser, -) -from vllm.tool_parsers.utils import ( - coerce_to_schema_type, - extract_types_from_schema, - find_tool_properties, - partial_tag_overlap, -) - -logger = init_logger(__name__) - - -class DeepSeekV32ToolParser(ToolParser): - """ - example tool call content: - <｜DSML｜function_calls> - <｜DSML｜invoke name="get_weather"> - <｜DSML｜parameter name="location" string="true">杭州 - <｜DSML｜parameter name="date" string="true">2024-01-16 - - <｜DSML｜invoke name="get_weather"> - <｜DSML｜parameter name="location" string="true">北京 - <｜DSML｜parameter name="date" string="true">2024-01-16 - - - """ - - tool_call_start_token: str = "<｜DSML｜function_calls>" - tool_call_end_token: str = "" - structural_tag_model = "deepseek_v3_2" - - def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): - super().__init__(tokenizer, tools) - - self.prev_tool_call_arr: list[dict] = [] - - # Streaming state - self.current_tool_index: int = 0 - self._sent_content_idx: int = 0 - self._buffer: str = "" - self._in_tool_calls: bool = False - self._active_tool_index: int | None = None - self._active_tool_name: str | None = None - self._active_param_name: str | None = None - self._active_param_string_attr: str | None = None - self._active_param_mode: str | None = None - self._active_param_parts: list[str] = [] - self._args_started: list[bool] = [] - - # Regex patterns for complete parsing - self.tool_call_complete_regex = re.compile( - re.escape(self.tool_call_start_token) - + r"(.*?)" - + re.escape(self.tool_call_end_token), - re.DOTALL, - ) - self.invoke_complete_regex = re.compile( - r'<｜DSML｜invoke\s+name="([^"]+)"\s*>(.*?)', re.DOTALL - ) - self.parameter_complete_regex = re.compile( - r'<｜DSML｜parameter\s+name="([^"]+)"\s+string="(true|false)"\s*>(.*?)', - re.DOTALL, - ) - self.invoke_start_regex = re.compile(r'<｜DSML｜invoke\s+name="([^"]+)"\s*>') - self.parameter_start_regex = re.compile( - r'<｜DSML｜parameter\s+name="([^"]+)"\s+string="(true|false)"\s*>' - ) - - if not self.model_tokenizer: - raise ValueError( - "The model tokenizer must be passed to the ToolParser " - "constructor during construction." - ) - - logger.debug( - "vLLM Successfully import tool parser %s !", self.__class__.__name__ - ) - - def adjust_request( - self, request: ChatCompletionRequest | ResponsesRequest - ) -> ChatCompletionRequest | ResponsesRequest: - request = super().adjust_request(request) - if request.tools and request.tool_choice != "none": - # Ensure tool call tokens - # (e.g. <｜DSML｜function_calls>, ) - # are not skippedduring decoding. - # Even though they are not marked as special tokens, - # setting skip_special_tokens=False ensures proper handling in - # transformers 5.x where decoding behavior may have changed. - request.skip_special_tokens = False - return request - - def _generate_tool_call_id(self) -> str: - """Generate a unique tool call ID.""" - return f"call_{uuid.uuid4().hex[:24]}" - - def _parse_invoke_params(self, invoke_str: str) -> dict[str, tuple[str, str]]: - param_dict: dict[str, tuple[str, str]] = {} - for param_name, string_attr, param_val in self.parameter_complete_regex.findall( - invoke_str - ): - param_dict[param_name] = (param_val, string_attr) - return param_dict - - @staticmethod - def _repair_param_dict( - param_dict: dict[str, Any], - param_config: dict[str, Any], - ) -> dict[str, Any]: - """Unwrap single 'arguments' / 'input' wrappers when the wrapper - is not part of the requested tool schema and the wrapped object - matches the schema fields.""" - allowed = set(param_config.keys()) - for wrapper in ("arguments", "input"): - if set(param_dict.keys()) != {wrapper} or wrapper in allowed: - continue - inner = param_dict[wrapper] - if isinstance(inner, str): - try: - inner = json.loads(inner) - except json.JSONDecodeError: - return param_dict - if isinstance(inner, dict) and set(inner.keys()).issubset(allowed): - return inner - return param_dict - - def _convert_params_with_schema( - self, - function_name: str, - param_dict: dict[str, tuple[str, str]], - ) -> dict[str, Any]: - """Convert raw string param values using the tool schema types.""" - param_config = find_tool_properties(self.tools, function_name) - - converted: dict[str, Any] = {} - for name, (value, string_attr) in param_dict.items(): - if string_attr == "true": - converted[name] = value - continue - - param_types = extract_types_from_schema(param_config.get(name, {})) - converted[name] = coerce_to_schema_type(value, param_types) - return self._repair_param_dict(converted, param_config) - - def _get_param_config(self, function_name: str | None) -> dict[str, Any]: - if not function_name or not self.tools: - return {} - return find_tool_properties(self.tools, function_name) - - @staticmethod - def _json_escape_string_content(text: str) -> str: - return json.dumps(text, ensure_ascii=False)[1:-1] - - def extract_tool_calls( - self, - model_output: str, - request: ChatCompletionRequest, - ) -> ExtractedToolCallInformation: - """Extract tool calls from complete model output (non-streaming).""" - # Quick check - if self.tool_call_start_token not in model_output: - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - try: - tool_calls = [] - - # Find all complete tool_call blocks - for tool_call_match in self.tool_call_complete_regex.findall(model_output): - # Find all invokes within this tool_call - for invoke_name, invoke_content in self.invoke_complete_regex.findall( - tool_call_match - ): - param_dict = self._parse_invoke_params(invoke_content) - params = self._convert_params_with_schema(invoke_name, param_dict) - tool_calls.append( - ToolCall( - type="function", - function=FunctionCall( - name=invoke_name, - arguments=json.dumps(params, ensure_ascii=False), - ), - ) - ) - - if not tool_calls: - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - # Extract content before first tool call - first_tool_idx = model_output.find(self.tool_call_start_token) - content = model_output[:first_tool_idx] if first_tool_idx > 0 else None - - return ExtractedToolCallInformation( - tools_called=True, tool_calls=tool_calls, content=content - ) - - except Exception: - logger.exception("Error extracting tool calls") - return ExtractedToolCallInformation( - tools_called=False, tool_calls=[], content=model_output - ) - - def _reset_streaming_state(self): - """Reset all streaming state.""" - self.current_tool_index = 0 - self._sent_content_idx = 0 - self._buffer = "" - self._in_tool_calls = False - self._active_tool_index = None - self._active_tool_name = None - self._active_param_name = None - self._active_param_string_attr = None - self._active_param_mode = None - self._active_param_parts.clear() - self.prev_tool_call_arr.clear() - self.streamed_args_for_tool.clear() - self._args_started.clear() - - def _add_tool_call_delta( - self, - tool_call_deltas: dict[int, DeltaToolCall], - index: int, - *, - call_id: str | None = None, - call_type: Literal["function"] | None = None, - name: str | None = None, - arguments: str | None = None, - ) -> None: - if arguments: - self.streamed_args_for_tool[index] += arguments - - if index not in tool_call_deltas: - tool_call_deltas[index] = DeltaToolCall( - index=index, - id=call_id, - type=call_type, - function=DeltaFunctionCall(name=name, arguments=arguments), - ) - return - - delta = tool_call_deltas[index] - if call_id is not None: - delta.id = call_id - if call_type is not None: - delta.type = call_type - if delta.function is None: - delta.function = DeltaFunctionCall() - if name is not None: - delta.function.name = name - if arguments is not None: - delta.function.arguments = (delta.function.arguments or "") + arguments - - def _begin_streaming_tool_call( - self, - name: str, - tool_call_deltas: dict[int, DeltaToolCall], - ) -> None: - index = self.current_tool_index - self.current_tool_index += 1 - self._active_tool_index = index - self._active_tool_name = name - self.prev_tool_call_arr.append({"name": name, "arguments": {}}) - self.streamed_args_for_tool.append("") - self._args_started.append(False) - self._add_tool_call_delta( - tool_call_deltas, - index, - call_id=self._generate_tool_call_id(), - call_type="function", - name=name, - arguments="", - ) - - def _append_param_prefix( - self, - tool_call_deltas: dict[int, DeltaToolCall], - index: int, - key: str, - *, - as_string: bool, - ) -> None: - prefix = "{" if not self._args_started[index] else "," - self._args_started[index] = True - arguments = prefix + json.dumps(key, ensure_ascii=False) + ":" - if as_string: - arguments += '"' - self._add_tool_call_delta(tool_call_deltas, index, arguments=arguments) - - def _append_json_param_value( - self, - tool_call_deltas: dict[int, DeltaToolCall], - index: int, - key: str, - value: Any, - ) -> None: - self._append_param_prefix(tool_call_deltas, index, key, as_string=False) - self._add_tool_call_delta( - tool_call_deltas, - index, - arguments=json.dumps(value, ensure_ascii=False), - ) - - def _param_types_for_name(self, name: str) -> list[str]: - param_config = self._get_param_config(self._active_tool_name) - if name in param_config and isinstance(param_config[name], dict): - return extract_types_from_schema(param_config[name]) - return ["string"] - - @staticmethod - def _can_stream_raw_param(param_types: list[str]) -> bool: - # Scalars and unions need the complete value so streaming and - # non-streaming share the same coercion fallback behavior. - return set(param_types).issubset({"object", "array"}) - - def _should_buffer_wrapper_param(self, name: str) -> bool: - if ( - self._active_tool_index is None - or self._args_started[self._active_tool_index] - ): - return False - param_config = self._get_param_config(self._active_tool_name) - return bool( - param_config and name in ("arguments", "input") and name not in param_config - ) - - def _finish_buffered_param( - self, - tool_call_deltas: dict[int, DeltaToolCall], - index: int, - ) -> None: - assert self._active_param_name is not None - assert self._active_param_string_attr is not None - raw_value = "".join(self._active_param_parts) - converted = self._convert_params_with_schema( - self._active_tool_name or "", - {self._active_param_name: (raw_value, self._active_param_string_attr)}, - ) - for key, value in converted.items(): - self._append_json_param_value(tool_call_deltas, index, key, value) - - def _close_streaming_tool_call( - self, - tool_call_deltas: dict[int, DeltaToolCall], - ) -> None: - index = self._active_tool_index - if index is None: - return - - suffix = "}" if self._args_started[index] else "{}" - self._add_tool_call_delta(tool_call_deltas, index, arguments=suffix) - try: - self.prev_tool_call_arr[index] = { - "name": self._active_tool_name, - "arguments": json.loads(self.streamed_args_for_tool[index]), - } - except (json.JSONDecodeError, IndexError): - logger.exception("Failed to finalize DeepSeek DSML streaming tool call") - - self._active_tool_index = None - self._active_tool_name = None - self._active_param_name = None - self._active_param_string_attr = None - self._active_param_mode = None - self._active_param_parts.clear() - - def _process_streaming_buffer( - self, - content_parts: list[str], - tool_call_deltas: dict[int, DeltaToolCall], - ) -> None: - parameter_end_token = "" - invoke_end_token = "" - - while True: - if not self._in_tool_calls: - start_idx = self._buffer.find(self.tool_call_start_token) - if start_idx == -1: - overlap = partial_tag_overlap( - self._buffer, self.tool_call_start_token - ) - sendable_idx = len(self._buffer) - overlap - if sendable_idx > 0: - content_parts.append(self._buffer[:sendable_idx]) - self._buffer = self._buffer[sendable_idx:] - return - - if start_idx > 0: - content_parts.append(self._buffer[:start_idx]) - self._buffer = self._buffer[start_idx:] - continue - - self._buffer = self._buffer[len(self.tool_call_start_token) :] - self._in_tool_calls = True - continue - - if self._active_tool_index is None: - stripped_len = len(self._buffer) - len(self._buffer.lstrip()) - if stripped_len: - self._buffer = self._buffer[stripped_len:] - continue - - if self._buffer.startswith(self.tool_call_end_token): - self._buffer = self._buffer[len(self.tool_call_end_token) :] - self._in_tool_calls = False - continue - - match = self.invoke_start_regex.match(self._buffer) - if match is None: - return - - self._buffer = self._buffer[match.end() :] - self._begin_streaming_tool_call(match.group(1), tool_call_deltas) - continue - - index = self._active_tool_index - - if self._active_param_mode is not None: - end_pos = self._buffer.find(parameter_end_token) - if end_pos != -1: - raw_content = self._buffer[:end_pos] - self._buffer = self._buffer[end_pos + len(parameter_end_token) :] - if self._active_param_mode in ("wrapper", "buffered"): - self._active_param_parts.append(raw_content) - self._finish_buffered_param(tool_call_deltas, index) - elif self._active_param_mode == "string": - arguments = self._json_escape_string_content(raw_content) + '"' - self._add_tool_call_delta( - tool_call_deltas, index, arguments=arguments - ) - else: - self._add_tool_call_delta( - tool_call_deltas, index, arguments=raw_content - ) - - self._active_param_name = None - self._active_param_string_attr = None - self._active_param_mode = None - self._active_param_parts.clear() - continue - - overlap = partial_tag_overlap(self._buffer, parameter_end_token) - safe_len = len(self._buffer) - overlap - if safe_len > 0: - raw_content = self._buffer[:safe_len] - self._buffer = self._buffer[safe_len:] - if self._active_param_mode in ("wrapper", "buffered"): - self._active_param_parts.append(raw_content) - elif self._active_param_mode == "string": - self._add_tool_call_delta( - tool_call_deltas, - index, - arguments=self._json_escape_string_content(raw_content), - ) - else: - self._add_tool_call_delta( - tool_call_deltas, index, arguments=raw_content - ) - return - - stripped_len = len(self._buffer) - len(self._buffer.lstrip()) - if stripped_len: - self._buffer = self._buffer[stripped_len:] - continue - - if self._buffer.startswith(invoke_end_token): - self._buffer = self._buffer[len(invoke_end_token) :] - self._close_streaming_tool_call(tool_call_deltas) - continue - - match = self.parameter_start_regex.match(self._buffer) - if match is None: - return - - self._buffer = self._buffer[match.end() :] - name = match.group(1) - string_attr = match.group(2) - self._active_param_name = name - self._active_param_string_attr = string_attr - - if self._should_buffer_wrapper_param(name): - self._active_param_mode = "wrapper" - continue - - if string_attr == "true": - self._append_param_prefix(tool_call_deltas, index, name, as_string=True) - self._active_param_mode = "string" - continue - - param_types = self._param_types_for_name(name) - if not self._can_stream_raw_param(param_types): - self._active_param_mode = "buffered" - continue - - self._append_param_prefix(tool_call_deltas, index, name, as_string=False) - self._active_param_mode = "raw" - - def extract_tool_calls_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], # pylint: disable=unused-argument - current_token_ids: Sequence[int], # pylint: disable=unused-argument - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - ) -> DeltaMessage | None: - """Extract tool calls from streaming model output. - - Buffers DSML markup while streaming tool-call metadata and argument - JSON fragments as soon as they are complete enough to be valid deltas. - """ - - # First chunk of a new stream — reset state from prior request. - if not previous_text: - self._reset_streaming_state() - - self._buffer += delta_text - content_parts: list[str] = [] - tool_call_deltas: dict[int, DeltaToolCall] = {} - self._process_streaming_buffer(content_parts, tool_call_deltas) - - if content_parts or tool_call_deltas: - content = "".join(content_parts) or None - return DeltaMessage( - content=content, tool_calls=list(tool_call_deltas.values()) - ) - - # Empty delta with token ids means EOS or closing tag; return - # non-None so the serving framework can finalize finish_reason. - if not delta_text and delta_token_ids and self.prev_tool_call_arr: - return DeltaMessage(content="") - - return None diff --git a/vllm/tool_parsers/deepseekv4_engine_tool_parser.py b/vllm/tool_parsers/deepseekv4_engine_tool_parser.py new file mode 100644 index 00000000000..7e3ebf26919 --- /dev/null +++ b/vllm/tool_parsers/deepseekv4_engine_tool_parser.py @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm.parser.engine.registered_adapters import DeepSeekV4ParserToolAdapter + + +class DeepSeekV4EngineToolParser(DeepSeekV4ParserToolAdapter): # type: ignore[valid-type, misc] + structural_tag_model = "deepseek_v4" diff --git a/vllm/tool_parsers/deepseekv4_tool_parser.py b/vllm/tool_parsers/deepseekv4_tool_parser.py deleted file mode 100644 index 2558f585f82..00000000000 --- a/vllm/tool_parsers/deepseekv4_tool_parser.py +++ /dev/null @@ -1,17 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from vllm.tool_parsers.deepseekv32_tool_parser import DeepSeekV32ToolParser - - -class DeepSeekV4ToolParser(DeepSeekV32ToolParser): - """ - DeepSeek V4 DSML tool parser. - - V4 keeps the V3.2 DSML invoke/parameter grammar, but wraps tool calls in - ``<｜DSML｜tool_calls>`` instead of ``<｜DSML｜function_calls>``. - """ - - tool_call_start_token: str = "<｜DSML｜tool_calls>" - tool_call_end_token: str = "" - structural_tag_model = "deepseek_v4" diff --git a/vllm/transformers_utils/repo_utils.py b/vllm/transformers_utils/repo_utils.py index 5506af4cac8..a758f5d535f 100644 --- a/vllm/transformers_utils/repo_utils.py +++ b/vllm/transformers_utils/repo_utils.py @@ -229,10 +229,11 @@ def get_model_path(model: str | Path, revision: str | None = None): if os.path.exists(model): return model assert huggingface_hub.constants.HF_HUB_OFFLINE - common_kwargs = { - "local_files_only": huggingface_hub.constants.HF_HUB_OFFLINE, - "revision": revision, - } + common_kwargs = dict( + local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE, + ignore_patterns="*", + revision=revision, + ) if envs.VLLM_USE_MODELSCOPE: from modelscope.hub.snapshot_download import snapshot_download diff --git a/vllm/v1/attention/backends/cpu_attn.py b/vllm/v1/attention/backends/cpu_attn.py index 056107c364d..6b2d202d3f1 100644 --- a/vllm/v1/attention/backends/cpu_attn.py +++ b/vllm/v1/attention/backends/cpu_attn.py @@ -141,7 +141,7 @@ class CPUAttentionMetadataBuilder(AttentionMetadataBuilder[CPUAttentionMetadata] self.vllm_config = vllm_config parallel_config = vllm_config.parallel_config - self.num_kv_heads = vllm_config.model_config.get_num_kv_heads(parallel_config) + self.num_kv_heads = kv_cache_spec.num_kv_heads self.num_heads = vllm_config.model_config.get_num_attention_heads( parallel_config ) diff --git a/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py b/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py index 9982b7aacb6..55bba1a7ee8 100644 --- a/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py +++ b/vllm/v1/attention/backends/mla/rocm_aiter_mla_sparse.py @@ -536,7 +536,17 @@ class ROCMAiterMLASparseMetadataBuilder( self._num_attention_heads, clamped_seq_lens.tobytes(), ) - if metadata_key != self._prev_metadata_key: + # The persistent MLA kernel is numerically wrong for multi-token prefill + # batches; errors compound across chunked prefill and break long-context + # decode (vllm#47042). Use it only for decode and single-chunk prefills, + # not chunked-prefill continuations (>1 query token, seq_len > query_len). + step_query_lens = seg_lengths + total_seq_lens = common_attn_metadata.seq_lens_cpu[:num_reqs].numpy() + is_chunked_continuation = (step_query_lens > 1) & ( + total_seq_lens > step_query_lens + ) + use_persistent = not is_chunked_continuation.any() + if use_persistent and metadata_key != self._prev_metadata_key: from aiter import get_mla_metadata_v1 get_mla_metadata_v1( @@ -576,7 +586,7 @@ class ROCMAiterMLASparseMetadataBuilder( paged_kv_last_page_len=paged_kv_last_page_len, paged_kv_indices=paged_kv_indices, paged_kv_indptr=paged_kv_indptr, - work_meta_data=self._mla_work_meta_data, + work_meta_data=self._mla_work_meta_data if use_persistent else None, work_indptr=self._mla_work_indptr, work_info_set=self._mla_work_info_set, reduce_indptr=self._mla_reduce_indptr, diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index ea657d7e949..9ce1d94ef3c 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -1636,14 +1636,23 @@ class Scheduler(SchedulerInterface): if new_token_ids and self.structured_output_manager.should_advance(request): struct_output_request = request.structured_output_request assert struct_output_request is not None - assert struct_output_request.grammar is not None - if not struct_output_request.grammar.accept_tokens( # type: ignore[union-attr] - req_id, new_token_ids + grammar = struct_output_request.grammar + assert grammar is not None + # new_token_ids can be a mixed block of reasoning content, then + # the reasoning end marker, then the start of the grammar content. + # Trim the reasoning content so the grammar only sees grammar content. + advance_token_ids = ( + self.structured_output_manager.trim_reasoning_for_advance( + request, new_token_ids + ) + ) + if advance_token_ids and not grammar.accept_tokens( + req_id, advance_token_ids ): logger.error( "Unexpected: grammar rejected tokens %s for request %s. " "Terminating request.", - new_token_ids, + advance_token_ids, req_id, ) request.status = RequestStatus.FINISHED_ERROR diff --git a/vllm/v1/structured_output/__init__.py b/vllm/v1/structured_output/__init__.py index 30921f3d74a..34f775257be 100644 --- a/vllm/v1/structured_output/__init__.py +++ b/vllm/v1/structured_output/__init__.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import itertools import multiprocessing -from collections.abc import Iterable +from collections.abc import Iterable, Sequence from concurrent.futures import Future, ThreadPoolExecutor from typing import TYPE_CHECKING @@ -272,23 +272,69 @@ class StructuredOutputManager: grammar = structured_output_request.grammar apply_bitmask = self.should_fill_bitmask(request) + reasoner = self._get_reasoner(request) + detect_reasoning_end = ( + not apply_bitmask + and reasoner is not None + and not self.enable_in_reasoning + ) + simulated_buf: list[int] | None = None + history_len = 0 + state_advancements = 0 + post_reasoning_end_in_window = False req_tokens = scheduled_spec_decode_tokens.get(req_id, ()) - if self.vllm_config.model_config.is_diffusion and req_tokens: - # Diffusion LLMs don't sample a bonus token after the - # scheduled positions, so don't append the -1 placeholder. - token_iter: Iterable[int] = req_tokens - else: - token_iter = itertools.chain(req_tokens, (-1,)) - for token in token_iter: + for i, token in enumerate(req_tokens): self._fill_bitmasks(((grammar, cumulative_index, apply_bitmask),)) + advance_grammar = apply_bitmask if token == -1: - # Stop advancing the grammar once we hit a padding token. apply_bitmask = False - if apply_bitmask and not grammar.is_terminated(): + advance_grammar = False + elif ( + detect_reasoning_end + and reasoner is not None + and not apply_bitmask + ): + if simulated_buf is None: + history = list(request.all_token_ids) + history_len = len(history) + simulated_buf = history + list(req_tokens) + simulated = simulated_buf[: history_len + i + 1] + if reasoner.is_reasoning_end_streaming(simulated, [token]): + # Reasoning ended mid-window. Constrain the rest + # of the window via bitmask. Skip grammar advance + # through the marker (it is reasoning content); + # try to advance through subsequent drafts so the + # next bitmask row reflects the post-advance state, + # but tolerate rejection since those drafts predate + # the bitmask and are not guaranteed valid. + apply_bitmask = True + advance_grammar = False + post_reasoning_end_in_window = True + if advance_grammar and not grammar.is_terminated(): accepted = grammar.accept_tokens(req_id, [token]) - assert accepted, (token, req_id, scheduled_spec_decode_tokens) - state_advancements += 1 + if accepted: + state_advancements += 1 + elif not post_reasoning_end_in_window: + raise AssertionError( + (token, req_id, scheduled_spec_decode_tokens) + ) + cumulative_index += 1 + # Diffusion LLMs don't sample a bonus token after the + # scheduled positions, so skip its bitmask in that case. + if not (self.vllm_config.model_config.is_diffusion and req_tokens): + # bonus_apply must be True when the bonus-row position + # should be grammar-constrained. Two triggers: + # - should_fill_bitmask(request): reasoning was already + # over at step start (or no reasoner / + # enable_in_reasoning). + # - apply_bitmask: reasoning ended mid-window in this + # call and was flipped True after the marker; + # should_fill_bitmask still returns False here because + # reasoning_ended is only persisted later by + # should_advance. + bonus_apply = self.should_fill_bitmask(request) or apply_bitmask + self._fill_bitmasks(((grammar, cumulative_index, bonus_apply),)) cumulative_index += 1 if state_advancements > 0: grammar.rollback(state_advancements) @@ -368,10 +414,64 @@ class StructuredOutputManager: and structured_req.structured_output_key[0] == StructuredOutputOptions.STRUCTURAL_TAG ): + # The scheduler will advance the grammar with this step's + # tokens right away, but the step still contains reasoning + # content up to and including the end marker. Record where + # it ends so trim_reasoning_for_advance() can drop it. + structured_req.reasoning_end_token_index = ( + self._find_reasoning_end_index(reasoner, all_token_ids, start) + ) return True return False + @staticmethod + def _find_reasoning_end_index( + reasoner: "ReasoningParser", all_token_ids: Sequence[int], start: int + ) -> int: + """Locates the last reasoning token within ``all_token_ids[start:]``. + + Returns: + The absolute index of the token at which + ``is_reasoning_end_streaming`` first fires. Falls back to the + final index when no single token triggers the detection (e.g. + a multi-token marker only recognized on the full delta), which + conservatively treats the whole step as reasoning content. + """ + prefix = list(itertools.islice(all_token_ids, start)) + for idx in range(start, len(all_token_ids)): + token = all_token_ids[idx] + prefix.append(token) + if reasoner.is_reasoning_end_streaming(prefix, [token]): + return idx + return len(all_token_ids) - 1 + + def trim_reasoning_for_advance( + self, request: "Request", new_token_ids: list[int] + ) -> list[int]: + """Drops reasoning content from tokens about to advance the grammar. + + When reasoning ends mid-step (see should_advance), the step's output + still contains reasoning tokens up to and including the end marker. + Those are not grammar content: feeding them to accept_tokens makes + the grammar reject the marker and kills the request (#44006). + + Returns: + The suffix of ``new_token_ids`` that follows the reasoning-end + marker. Steps fully after the boundary are returned unchanged. + """ + structured_req = request.structured_output_request + if structured_req is None: + return new_token_ids + end_idx = structured_req.reasoning_end_token_index + if end_idx is None: + return new_token_ids + first_idx = len(request.all_token_ids) - len(new_token_ids) + num_reasoning = end_idx + 1 - first_idx + if num_reasoning <= 0: + return new_token_ids + return new_token_ids[num_reasoning:] + def clear_backend(self) -> None: if self.backend is not None: self.backend.destroy() diff --git a/vllm/v1/structured_output/request.py b/vllm/v1/structured_output/request.py index dfa8c7efcae..f9ab54a0471 100644 --- a/vllm/v1/structured_output/request.py +++ b/vllm/v1/structured_output/request.py @@ -23,6 +23,12 @@ class StructuredOutputRequest: params: StructuredOutputsParams _grammar: Future[StructuredOutputGrammar] | StructuredOutputGrammar | None = None reasoning_ended: bool | None = None + # Absolute index into the request's all_token_ids of the last reasoning + # token (the reasoning-end marker). Tokens at or before this index are + # reasoning content and must never be fed to the grammar. Only set when + # reasoning ends in a step whose tokens the scheduler advances immediately + # (structural tags + speculative decoding, see #42452). + reasoning_end_token_index: int | None = None reasoning_parser_kwargs: dict[str, Any] | None = None # Cached per request; do not share reasoning parsers across requests because # their behavior can depend on reasoning_parser_kwargs. diff --git a/vllm/v1/worker/gpu/attn_utils.py b/vllm/v1/worker/gpu/attn_utils.py index 758bd3bac7a..3906717b5b7 100644 --- a/vllm/v1/worker/gpu/attn_utils.py +++ b/vllm/v1/worker/gpu/attn_utils.py @@ -23,6 +23,7 @@ from vllm.v1.kv_cache_interface import ( AttentionSpec, KVCacheConfig, KVCacheSpec, + KVQuantMode, MambaSpec, UniformTypeKVCacheSpecs, ) @@ -300,12 +301,20 @@ def _reshape_kv_cache( kv_cache_spec.storage_block_size // kernel_block_size ) kernel_num_blocks = num_blocks * num_blocks_per_kv_block + # Skipped layers (--kv-cache-dtype-skip-layers) keep the + # unquantized shape; only the quantized primary uses the + # quantized cache dtype's (possibly packed) layout. + layer_cache_dtype = ( + "auto" + if kv_cache_spec.kv_quant_mode == KVQuantMode.NONE + else cache_dtype + ) kv_cache_shape = group.backend.get_kv_cache_shape( kernel_num_blocks, kernel_block_size, kv_cache_spec.num_kv_heads, kv_cache_spec.head_size, - cache_dtype_str=cache_dtype, + cache_dtype_str=layer_cache_dtype, ) # FIXME(woosuk): Add kv_cache_stride_order to all attention backends. @@ -377,11 +386,18 @@ def _update_hybrid_attention_layout( kv_cache_spec = group.kv_cache_spec if not isinstance(kv_cache_spec, AttentionSpec): continue + # Mirror the per-layer dtype selection used when building the shape + # above. The block-dim index is dtype-independent for current backends + # (quantization only changes the last dim), so this is a no-op today, + # but it keeps both call sites consistent for skip layers. + layer_cache_dtype = ( + "auto" if kv_cache_spec.kv_quant_mode == KVQuantMode.NONE else cache_dtype + ) block_dim = group.backend.get_kv_cache_block_dim( kernel_block_sizes[group.kv_cache_group_id], kv_cache_spec.num_kv_heads, kv_cache_spec.head_size, - cache_dtype_str=cache_dtype, + cache_dtype_str=layer_cache_dtype, ) # if the first dim of the kvcache's layout is already num_blocks, continue if block_dim == 0: diff --git a/vllm/v1/worker/gpu/cudagraph_utils.py b/vllm/v1/worker/gpu/cudagraph_utils.py index aa022f6d99e..9b93d17035b 100644 --- a/vllm/v1/worker/gpu/cudagraph_utils.py +++ b/vllm/v1/worker/gpu/cudagraph_utils.py @@ -27,6 +27,7 @@ from vllm.logger import init_logger from vllm.model_executor.offloader.base import get_offloader from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors +from vllm.utils.math_utils import round_up from vllm.v1.kv_cache_interface import KVCacheConfig from vllm.v1.worker.gpu.attn_utils import build_slot_mappings_by_layer from vllm.v1.worker.gpu.block_table import BlockTables @@ -141,10 +142,7 @@ class CudaGraphManager: self._candidates: dict[tuple[int, int], list[BatchExecutionDescriptor]] = {} self._capture_descs: dict[CUDAGraphMode, list[BatchExecutionDescriptor]] = {} - # adjust the cudagraph sizes to be a multiple of the uniform decode query length - self.compilation_config.adjust_cudagraph_sizes_for_spec_decode( - self.decode_query_len, self.tp_size - ) + self._init_candidates() # Breakable CUDA graph (PW CUDA graph without torch.compile) @@ -191,31 +189,72 @@ class CudaGraphManager: decode_mode = self.cudagraph_mode.decode_mode() mixed_mode = self.cudagraph_mode.mixed_mode() separate_decode_routine = self.cudagraph_mode.separate_routine() + max_cg_capture_size = self.compilation_config.max_cudagraph_capture_size descs_by_token_lora: dict[tuple[int, int], list[BatchExecutionDescriptor]] = ( defaultdict(list) ) - descs_by_mode = defaultdict(list) + descs_by_mode: defaultdict[CUDAGraphMode, list[BatchExecutionDescriptor]] = ( + defaultdict(list) + ) + + # When using Dynamic SD, num_speculative_tokens is the max number of + # draft tokens. The scheduler might use a smaller number so we need + # to capture graphs for all possible values during decode. + speculative_config = self.vllm_config.speculative_config + if ( + speculative_config + and speculative_config.uses_dynamic_speculative_decoding() + ): + num_spec_per_batch_size = ( + speculative_config.num_speculative_tokens_per_batch_size + ) + # uses_dynamic_speculative_decoding() guarantees this is set. + assert num_spec_per_batch_size is not None + # decode_query_len = num_speculative_steps + num_new_sampled_tokens + # _per_step. Recover num_new_sampled_tokens_per_step + # from the values the manager already has. + num_new_sampled_tokens_per_step = ( + self.decode_query_len - self.vllm_config.num_speculative_tokens + ) + # Each entry is (range_start, range_end, num_speculative_tokens). + decode_query_lens = [ + x[2] + num_new_sampled_tokens_per_step for x in num_spec_per_batch_size + ] + else: + decode_query_lens = [self.decode_query_len] for num_tokens, num_active_loras in product( capture_sizes, self.lora_capture_cases ): # Capture uniform decode specfifc graphs if required # (i.e. separate decode routine) - if ( - separate_decode_routine - and decode_mode - and self.decode_query_len <= num_tokens <= max_decode_tokens - ): - desc = BatchExecutionDescriptor( - cg_mode=decode_mode, - num_tokens=num_tokens, - num_reqs=num_tokens // self.decode_query_len, - uniform_token_count=self.decode_query_len, - num_active_loras=num_active_loras, - ) - descs_by_mode[decode_mode].append(desc) - descs_by_token_lora[(num_tokens, num_active_loras)].append(desc) + if separate_decode_routine and decode_mode: + for decode_query_len in decode_query_lens: + rounded_num_tokens = round_up(num_tokens, decode_query_len) + rounded_num_reqs = rounded_num_tokens // decode_query_len + + if ( + rounded_num_tokens > max_decode_tokens + or rounded_num_tokens > max_cg_capture_size + or rounded_num_reqs > self.max_num_reqs + ): + continue + + desc = BatchExecutionDescriptor( + cg_mode=decode_mode, + num_tokens=rounded_num_tokens, + num_reqs=rounded_num_reqs, + uniform_token_count=decode_query_len, + num_active_loras=num_active_loras, + ) + + # avoid duplicate graphs + if desc not in descs_by_mode[decode_mode]: + descs_by_mode[decode_mode].append(desc) + descs_by_token_lora[ + (rounded_num_tokens, num_active_loras) + ].append(desc) if mixed_mode: # for PIECEWISE graphs there is no limit on requests when replaying diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 7a82032313c..c74307d0b74 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -454,9 +454,10 @@ class GPUModelRunner(LoRAModelRunnerMixin): attn_cg_support.min_cg_support, attn_cg_support.min_cg_attn_backend, self.decode_query_len, - self.parallel_config.tensor_parallel_size, - self.kv_cache_config, - self.max_num_reqs, + use_v2_model_runner=True, + tensor_parallel_size=self.parallel_config.tensor_parallel_size, + kv_cache_config=self.kv_cache_config, + max_num_reqs=self.max_num_reqs, ) self.cudagraph_manager = ModelCudaGraphManager( self.vllm_config, diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 53795951cb2..3930a07b248 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -150,6 +150,7 @@ from vllm.v1.kv_cache_interface import ( KVCacheConfig, KVCacheGroupSpec, KVCacheSpec, + KVQuantMode, MambaSpec, SlidingWindowSpec, UniformTypeKVCacheSpecs, @@ -6929,9 +6930,10 @@ class GPUModelRunner( min_cg_support, min_cg_attn_backend, self.uniform_decode_query_len, - self.parallel_config.tensor_parallel_size, - self.kv_cache_config, - self.max_num_reqs, + use_v2_model_runner=False, + tensor_parallel_size=self.parallel_config.tensor_parallel_size, + kv_cache_config=self.kv_cache_config, + max_num_reqs=self.max_num_reqs, is_profiling=is_profiling, ) # Trigger cudagraph dispatching keys initialization after @@ -7147,12 +7149,19 @@ class GPUModelRunner( else: shape_block_size = kernel_block_size + # Skipped layers (--kv-cache-dtype-skip-layers) need + # the unquantized shape. + layer_cache_dtype_str = ( + "auto" + if kv_cache_spec.kv_quant_mode == KVQuantMode.NONE + else self.cache_config.cache_dtype + ) kv_cache_shape = attn_backend.get_kv_cache_shape( kernel_num_blocks, shape_block_size, kv_cache_spec.num_kv_heads, kv_cache_spec.head_size, - cache_dtype_str=self.cache_config.cache_dtype, + cache_dtype_str=layer_cache_dtype_str, ) try: kv_cache_stride_order = attn_backend.get_kv_cache_stride_order()