[Core] Enable extensible KV cache for all attention backends and Mamba

Port of internal D110967544. The extensible KV cache flow (reserve KV
virtual address space up front, capture CUDA graphs first, then size and
commit the KV cache from post-capture free memory) previously required a
block-major attention backend and rejected Mamba models. This enables it
for every backend layout and for Mamba / linear attention:

- ExtensibleTensor gains num_segments: the reservation is divided into
  equal segments that grow in lockstep, with committed bytes forming a
  prefix of each segment. Physical pages are mapped at
  allocation-granularity granules and deduped across overlapping ranges,
  so a granule straddling a segment boundary is mapped exactly once.
  resize_per_segment_(bytes, zero_new=True) zeroes only the newly
  committed logical range of each segment.
- Each KV cache buffer keeps its layers' physical layout and is committed
  as one prefix per layout segment. The segment count is derived from the
  backend's get_kv_cache_shape / get_kv_cache_block_dim / stride order:
  K/V-split layouts (e.g. FlashAttention) get one prefix per half,
  block-major layouts (e.g. FlashInfer, MLA) a single prefix. Mamba state
  pages are block-major per layer, and hybrid-model attention caches are
  re-strided to block-major, so both use a single segment.
- Removed the supports_extensible_kv_cache gate plumbing from EngineCore,
  Executor, Worker, WorkerBase and GPUModelRunner; a CUDA platform check
  remains in EngineCore.
- enable_extensible_kv_cache is reported as unsupported by the V2 model
  runner so V2-default models fall back to the V1 runner (which implements
  the flow); also fixed initialize_kv_cache being called with the
  extensible kwarg on runners that do not accept it, which broke every
  default V2-runner boot on this branch.

Tested on H100:
- tests/utils_/test_extensible_tensor.py (5 passed, incl. new segmented
  lockstep-grow/zero, granule-dedup and invalid-usage tests)
- tests/v1/worker/test_extensible_kv_cache.py (new, 6 passed: segment
  derivation, split grows both halves, block-major, legacy full commit,
  Mamba per-layer growth, hybrid attention+Mamba)
- E2E Qwen3-0.6B greedy with VLLM_ATTENTION_BACKEND=FLASH_ATTN (a K/V-split
  backend the old gate rejected): extensible generations byte-identical to
  the legacy path; log shows reserve then "Extended KV cache to 34663
  blocks". V2->V1 auto-fallback path verified as well.

.buildkite/intel_jobs/basic_correctness.yaml

@@ -23,4 +23,5 @@ steps:
       bash .buildkite/scripts/hardware_ci/run-intel-test.sh
       'cd tests &&
       export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
+      pytest -v -s basic_correctness/test_cpu_offload.py &&
       pytest -v -s basic_correctness/test_mem.py::test_end_to_end'

.buildkite/intel_jobs/lora_intel.yaml

@@ -81,7 +81,7 @@ steps:
       'cd tests &&
       export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
       set -o pipefail &&
-      pytest -v -s lora/test_punica_ops.py --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[expand-0-xpu:0-dtype0-3-43264-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype1-1-2049-64-128-16]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-128-1-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-256-1-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-256-8-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels[expand-0-xpu:0-dtype0-3-2049-128-8-16]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-128-8-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels[expand-0-xpu:0-dtype1-1-2049-256-128-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype0-3-64256-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype1-2-29696-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype1-3-49408-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype0-2-16384-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[expand-0-xpu:0-dtype0-2-51328-32-4-4]"'
+      pytest -v -s lora/test_punica_ops.py --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[expand-0-xpu:0-dtype0-3-43264-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype1-1-2049-64-128-16]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-128-1-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-256-1-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-256-8-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels[expand-0-xpu:0-dtype0-3-2049-128-8-16]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-128-8-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels[expand-0-xpu:0-dtype1-1-2049-256-128-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype0-3-64256-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype1-2-29696-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype1-3-49408-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype0-2-16384-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[expand-0-xpu:0-dtype0-2-51328-32-4-4]" --deselect="tests/lora/test_kernels_hidden_size[shrink-0-xpu:0-dtype0-3-32000-32-4-4]" --deselect="tests/lora/test_kernels_hidden_size[shrink-0-xpu:0-dtype0-3-32000-32-4-4]"'
 
 - label: LoRA Punica FP8/XPU Ops
   timeout_in_minutes: 45
@@ -128,10 +128,10 @@ steps:
       bash .buildkite/scripts/hardware_ci/run-intel-test.sh
       'cd tests &&
       export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
-      (pytest -v -s lora/test_mixtral.py --deselect="tests/lora/test_mixtral.py::test_mixtral_lora[4]" || true) &&
       pytest -v -s lora/test_quant_model.py --deselect="tests/lora/test_quant_model.py::test_quant_model_lora[model0]" --deselect="tests/lora/test_quant_model.py::test_quant_model_lora[model1]" --deselect="tests/lora/test_quant_model.py::test_quant_model_tp_equality[model0]" &&
       pytest -v -s lora/test_transformers_model.py &&
       pytest -v -s lora/test_chatglm3_tp.py &&
+      pytest -v -s lora/test_llama_tp.py::test_llama_lora &&
       pytest -s -v lora/test_minicpmv_tp.py'
 
 - label: LoRA Multimodal

.buildkite/intel_jobs/misc_intel.yaml

@@ -103,6 +103,31 @@ steps:
       pytest -v -s v1/kv_offload &&
       pytest -v -s v1/kv_connector/unit/test_offloading_connector.py'
 
+- label: NixlConnector PD accuracy (2 GPUs)
+  timeout_in_minutes: 60
+  num_devices: 2
+  device: intel_gpu
+  agent_tags:
+    label: production
+    gpu: 2+
+    mem: 24+
+  no_plugin: true
+  working_dir: "."
+  env:
+    REGISTRY: "public.ecr.aws/q9t5s3a7"
+    REPO: "vllm-ci-test-repo"
+    VLLM_TEST_DEVICE: "xpu"
+  source_file_dependencies:
+    - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
+    - vllm/v1/worker/kv_connector_model_runner_mixin.py
+    - tests/v1/kv_connector/nixl_integration/
+    - vllm/platforms/xpu.py
+  commands:
+    - >-
+      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
+      'cd tests &&
+      bash v1/kv_connector/nixl_integration/run_xpu_disagg_accuracy_test.sh'
+
 - label: Regression
   key: regression
   timeout_in_minutes: 30

.buildkite/intel_jobs/models_distributed_intel.yaml

@@ -0,0 +1,27 @@
+group: Models - Distributed
+depends_on: 
+  - image-build-xpu
+steps:
+- label: Distributed Model Tests (2 GPUs)
+  key: distributed-model-tests-2-gpus
+  timeout_in_minutes: 50
+  device: intel_gpu
+  agent_tags:
+    label: production
+    gpu: 2+
+    mem: 24+
+  no_plugin: true
+  working_dir: "."
+  env:
+    REGISTRY: "public.ecr.aws/q9t5s3a7"
+    REPO: "vllm-ci-test-repo"
+    VLLM_TEST_DEVICE: "xpu"
+  source_file_dependencies:
+  - vllm/model_executor/model_loader/sharded_state_loader.py
+  - vllm/model_executor/models/
+  - tests/model_executor/model_loader/test_sharded_state_loader.py
+  commands:
+    - >-
+      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
+      'cd tests &&
+      pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m "not slow_test"'

.buildkite/intel_jobs/models_multimodal_intel.yaml

@@ -22,7 +22,7 @@ steps:
   commands:
     - >-
       bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-      'pip install av git+https://github.com/TIGER-AI-Lab/Mantis.git &&
+      'pip install av &&
       cd tests &&
       pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2" &&
       pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model'
@@ -47,8 +47,7 @@ steps:
   commands:
     - >-
       bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-      'pip install git+https://github.com/TIGER-AI-Lab/Mantis.git &&
-      cd tests &&
+      'cd tests &&
       pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model'
 
 - label: "Multi-Modal Models (Standard) 3: llava + qwen2_vl"
@@ -71,8 +70,7 @@ steps:
   commands:
     - >-
       bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-      'pip install git+https://github.com/TIGER-AI-Lab/Mantis.git &&
-      cd tests &&
+      'cd tests &&
       pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma" &&
       pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model'
 
@@ -96,7 +94,7 @@ steps:
   commands:
     - >-
       bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-      'pip install av git+https://github.com/TIGER-AI-Lab/Mantis.git &&
+      'pip install av &&
       cd tests &&
       pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py  --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/processing'
 
@@ -121,11 +119,9 @@ steps:
   commands:
     - >-
       bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-      'pip install av matplotlib ftfy git+https://github.com/TIGER-AI-Lab/Mantis.git &&
+      'pip install av matplotlib ftfy &&
       pip install open-clip-torch --no-deps &&
       cd tests &&
       pytest -v -s models/multimodal/processing/test_tensor_schema.py
-      --deselect "tests/models/multimodal/processing/test_tensor_schema.py::test_model_tensor_schema[mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4]"
-      --deselect "tests/models/multimodal/processing/test_tensor_schema.py::test_model_tensor_schema[Qwen/Qwen2.5-Omni-7B-AWQ]"
       --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB'
   parallelism: 4

.buildkite/intel_jobs/test-intel.yaml

@@ -68,7 +68,6 @@ steps:
       - >-
         bash .buildkite/scripts/hardware_ci/run-intel-test.sh
         'cd tests &&
-        bash v1/kv_connector/nixl_integration/run_xpu_disagg_accuracy_test.sh &&
         pytest -v -s v1/core --ignore=v1/core/test_reset_prefix_cache_e2e.py --ignore=v1/core/test_scheduler_e2e.py &&
         pytest -v -s v1/engine --ignore=v1/engine/test_output_processor.py &&
         pytest -v -s v1/sample --ignore=v1/sample/test_logprobs.py --ignore=v1/sample/test_logprobs_e2e.py -k "not test_topk_only and not test_topp_only and not test_topk_and_topp" &&

.buildkite/scripts/hardware_ci/run-amd-test.sh

@@ -534,6 +534,20 @@ else
   echo "--- Single-node job"
   echo "Render devices: $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES"
 
+  ulimit_core_hard=$(ulimit -H -c)
+  if [[ "$ulimit_core_hard" == "unlimited" ]]; then
+    # docker run can't pass "unlimited" to --ulimit
+    ulimit_core_hard="-1"
+  fi
+   # Disable core dumps in the ROCm test container unless the ROCm debug agent is enabled
+  coredump_flags="--ulimit core=0:$ulimit_core_hard"
+  if [[ "$commands" == *"ROCm debug agent enabled"* ]]; then
+    # Works around https://github.com/rocm/rocm-systems/issues/6206
+    coredump_flags='-e HSA_COREDUMP_PATTERN="/tmp/gpucore.%p"'
+  else
+    echo "ROCm debug agent not enabled, coredumps are disabled in the test container."
+  fi
+
   docker run \
     --device /dev/kfd $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES \
     $RDMA_FLAGS \
@@ -541,6 +555,7 @@ else
     --shm-size=16gb \
     --group-add "$render_gid" \
     --rm \
+    $coredump_flags \
     -e HF_TOKEN \
     -e "HF_HUB_DOWNLOAD_TIMEOUT=${HF_HUB_DOWNLOAD_TIMEOUT}" \
     -e "HF_HUB_ETAG_TIMEOUT=${HF_HUB_ETAG_TIMEOUT}" \

.buildkite/test-amd.yaml

@@ -114,26 +114,6 @@ steps:
 
 #----------------------------------------------------------  mi250 · compile  ----------------------------------------------------------#
 
-- label: PyTorch Compilation Unit Tests # TBD
-  timeout_in_minutes: 180
-  mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
-  agent_pool: mi250_1
-  torch_nightly: true
-  optional: true
-  working_dir: "/vllm-workspace/tests"
-  source_file_dependencies:
-  - vllm/compilation/
-  - vllm/model_executor/layers/
-  - vllm/v1/worker/
-  - vllm/v1/attention/
-  - vllm/v1/cudagraph_dispatcher.py
-  - vllm/config/compilation.py
-  - csrc/
-  - tests/compile
-  - vllm/platforms/rocm.py
-  commands:
-  - "find compile/ -maxdepth 1 -name 'test_*.py' -print0 | xargs -0 -n1 -I{} pytest -s -v '{}'"
-
 - label: PyTorch Fullgraph Smoke Test # TBD
   timeout_in_minutes: 180
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
@@ -259,7 +239,6 @@ steps:
   - vllm/
   - tests/models/multimodal
   commands:
-  - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
   - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma"
   - pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model
 
@@ -438,7 +417,7 @@ steps:
 #-----------------------------------------------------  mi300 · basic_correctness  -----------------------------------------------------#
 
 - label: Basic Correctness # TBD
-  timeout_in_minutes: 50
+  timeout_in_minutes: 95
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
   agent_pool: mi300_1
   fast_check: true
@@ -456,7 +435,7 @@ steps:
   - pytest -v -s basic_correctness/test_cpu_offload.py
 
 - label: Distributed Model Tests (2 GPUs) # TBD
-  timeout_in_minutes: 65
+  timeout_in_minutes: 110
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
   agent_pool: mi300_2
   num_gpus: 2
@@ -498,6 +477,26 @@ steps:
 
 #----------------------------------------------------------  mi300 · compile  ----------------------------------------------------------#
 
+- label: PyTorch Compilation Unit Tests # TBD
+  timeout_in_minutes: 180
+  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  agent_pool: mi300_1
+  torch_nightly: true
+  optional: true
+  working_dir: "/vllm-workspace/tests"
+  source_file_dependencies:
+  - vllm/compilation/
+  - vllm/model_executor/layers/
+  - vllm/v1/worker/
+  - vllm/v1/attention/
+  - vllm/v1/cudagraph_dispatcher.py
+  - vllm/config/compilation.py
+  - csrc/
+  - tests/compile
+  - vllm/platforms/rocm.py
+  commands:
+  - "find compile/ -maxdepth 1 -name 'test_*.py' -print0 | xargs -0 -n1 -I{} pytest -s -v '{}'"
+
 - label: Fusion E2E Config Sweep (H100-MI300) # TBD
   timeout_in_minutes: 180
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
@@ -678,7 +677,7 @@ steps:
   - pytest -v -s distributed/test_eplb_spec_decode.py
 
 - label: Distributed Tests (2xH100-2xMI300) # TBD
-  timeout_in_minutes: 30
+  timeout_in_minutes: 75
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
   agent_pool: mi300_2
   num_gpus: 2
@@ -845,10 +844,12 @@ steps:
   source_file_dependencies:
   - vllm/
   - tests/entrypoints/serve
+  - tests/entrypoints/scale_out
   commands:
   - export VLLM_WORKER_MULTIPROC_METHOD=spawn
   - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc
   - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc
+  - pytest -v -s entrypoints/scale_out
 
 - label: Entrypoints Integration (API Server OpenAI - Part 1) # TBD
   timeout_in_minutes: 180
@@ -1196,10 +1197,31 @@ steps:
   - export VLLM_WORKER_MULTIPROC_METHOD=spawn
   - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx.txt
 
+- label: ROCm LM Eval Large Models (8 GPUs) # TBD
+  timeout_in_minutes: 180
+  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  agent_pool: mi300_8
+  optional: true
+  num_gpus: 8
+  working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
+  source_file_dependencies:
+  - vllm/model_executor/models/
+  - vllm/model_executor/model_loader/
+  - vllm/model_executor/layers/quantization/
+  - vllm/v1/attention/backends/
+  - vllm/v1/attention/selector.py
+  - vllm/model_executor/layers/layernorm.py
+  - csrc/
+  - vllm/_aiter_ops.py
+  - vllm/platforms/rocm.py
+  commands:
+  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+  - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm.txt --tp-size=8
+
 #---------------------------------------------------------  mi300 · examples  ----------------------------------------------------------#
 
 - label: Examples # TBD
-  timeout_in_minutes: 45
+  timeout_in_minutes: 90
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
   agent_pool: mi300_1
   optional: true
@@ -1235,7 +1257,7 @@ steps:
 #----------------------------------------------------------  mi300 · kernels  ----------------------------------------------------------#
 
 - label: Kernels Attention Test %N # TBD
-  timeout_in_minutes: 55
+  timeout_in_minutes: 100
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
   agent_pool: mi300_1
   optional: true
@@ -1269,7 +1291,7 @@ steps:
   - pytest -v -s kernels/core --ignore=kernels/core/test_minimax_reduce_rms.py  kernels/test_concat_mla_q.py kernels/test_top_k_per_row.py
 
 - label: Kernels MoE Test %N # TBD
-  timeout_in_minutes: 50
+  timeout_in_minutes: 95
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
   agent_pool: mi300_1
   optional: true
@@ -1351,8 +1373,6 @@ steps:
   - tests/lora
   - vllm/platforms/rocm.py
   commands:
-  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-  - export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
   - pytest -v -s -x lora/test_chatglm3_tp.py
   - pytest -v -s -x lora/test_llama_tp.py
   - pytest -v -s -x lora/test_qwen3_with_multi_loras.py
@@ -1416,7 +1436,7 @@ steps:
   - pytest -v -s models/test_initialization.py::test_can_initialize_small_subset
 
 - label: Basic Models Tests (Other) # TBD
-  timeout_in_minutes: 45
+  timeout_in_minutes: 90
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
   agent_pool: mi300_1
   optional: true
@@ -1493,7 +1513,6 @@ steps:
   - tests/models/multimodal/generation
   - tests/models/multimodal/test_mapping.py
   commands:
-  - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
   - pytest -v -s models/multimodal/generation -m 'not core_model' --ignore models/multimodal/generation/test_common.py
   - pytest -v -s models/multimodal/test_mapping.py
 
@@ -1507,7 +1526,6 @@ steps:
   - vllm/
   - tests/models/multimodal/generation
   commands:
-  - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
   - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=0) and not core_model'
 
 
@@ -1521,7 +1539,6 @@ steps:
   - vllm/
   - tests/models/multimodal/generation
   commands:
-  - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
   - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model'
 
 - label: "Multi-Modal Models (Standard) 1: qwen2" # TBD
@@ -1535,7 +1552,6 @@ steps:
   - vllm/
   - tests/models/multimodal
   commands:
-  - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
   - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2"
   - pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model
 
@@ -1551,7 +1567,6 @@ steps:
   - tests/models/multimodal/generation
   - tests/models/multimodal/test_mapping.py
   commands:
-  - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
   - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma"
   - pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model
 
@@ -1566,7 +1581,6 @@ steps:
   - tests/models/multimodal/generation
   - tests/models/multimodal/test_mapping.py
   commands:
-  - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
   - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py  --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/processing
   - pytest -v -s models/multimodal/generation/test_memory_leak.py -m core_model
   - cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model
@@ -1582,7 +1596,6 @@ steps:
   - tests/models/multimodal
   - tests/models/registry.py
   commands:
-  - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
   - pytest -v -s models/multimodal/processing/test_tensor_schema.py
 
 - label: Multi-Modal Processor (CPU) %N # TBD
@@ -1598,7 +1611,6 @@ steps:
   - tests/models/multimodal
   - tests/models/registry.py
   commands:
-  - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
   - pytest -v -s models/multimodal/processing --ignore models/multimodal/processing/test_tensor_schema.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
 
 #-----------------------------------------------------  mi300 · models / quantized  -----------------------------------------------------#
@@ -1880,7 +1892,7 @@ steps:
   - pytest -v -s v1/e2e/spec_decode -k "draft_model or no_sync or batch_inference"
 
 - label: Spec Decode Eagle # TBD
-  timeout_in_minutes: 45
+  timeout_in_minutes: 90
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
   agent_pool: mi300_1
   optional: true
@@ -2096,7 +2108,7 @@ steps:
   - DP_SIZE=2 pytest -v -s entrypoints/openai/test_multi_api_servers.py
 
 - label: Metrics, Tracing (2 GPUs) # TBD
-  timeout_in_minutes: 20
+  timeout_in_minutes: 65
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
   agent_pool: mi300_2
   optional: true
@@ -2249,7 +2261,7 @@ steps:
 #------------------------------------------------------  mi300 · weight_loading  -------------------------------------------------------#
 
 - label: Weight Loading Multiple GPU # TBD
-  timeout_in_minutes: 30
+  timeout_in_minutes: 75
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
   agent_pool: mi300_2
   num_gpus: 2
@@ -2261,7 +2273,7 @@ steps:
   - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-amd.txt
 
 - label: Weight Loading Multiple GPU - Large Models # TBD
-  timeout_in_minutes: 30
+  timeout_in_minutes: 75
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
   agent_pool: mi300_2
   num_gpus: 2
@@ -2390,27 +2402,6 @@ steps:
   - export VLLM_USE_DEEP_GEMM=0
   - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm-fp8.txt --tp-size=4
 
-- label: ROCm LM Eval Large Models (8 GPUs) # TBD
-  timeout_in_minutes: 180
-  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
-  agent_pool: mi325_8
-  optional: true
-  num_gpus: 8
-  working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
-  source_file_dependencies:
-  - vllm/model_executor/models/
-  - vllm/model_executor/model_loader/
-  - vllm/model_executor/layers/quantization/
-  - vllm/v1/attention/backends/
-  - vllm/v1/attention/selector.py
-  - vllm/model_executor/layers/layernorm.py
-  - csrc/
-  - vllm/_aiter_ops.py
-  - vllm/platforms/rocm.py
-  commands:
-  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-  - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm.txt --tp-size=8
-
 #-----------------------------------------------------  mi325 · models / language  -----------------------------------------------------#
 
 - label: Language Models Test (Extended Generation) # TBD
@@ -2467,7 +2458,6 @@ steps:
   - vllm/
   - tests/models/multimodal
   commands:
-  - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
   - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen3 or gemma"
   - pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model
 
@@ -2559,10 +2549,12 @@ steps:
   source_file_dependencies:
   - vllm/
   - tests/entrypoints/serve
+  - tests/entrypoints/scale_out
   commands:
   - export VLLM_WORKER_MULTIPROC_METHOD=spawn
   - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc
   - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc
+  - pytest -v -s entrypoints/scale_out
 
 - label: Entrypoints Integration (API Server OpenAI - Part 1) # TBD
   timeout_in_minutes: 180
@@ -2824,9 +2816,10 @@ steps:
   - rocm-smi
   - python3 examples/basic/offline_inference/chat.py --attention-backend TRITON_ATTN
   - pytest -v -s tests/kernels/attention/test_attention_selector.py
+  - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py
 
 - label: Kernels Attention Test %N # TBD
-  timeout_in_minutes: 60
+  timeout_in_minutes: 100
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
   agent_pool: mi355_1
   parallelism: 2
@@ -2975,7 +2968,6 @@ steps:
   - tests/models/multimodal/generation
   - tests/models/multimodal/test_mapping.py
   commands:
-  - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
   - pytest -v -s models/multimodal/generation -m 'not core_model' --ignore models/multimodal/generation/test_common.py
   - pytest -v -s models/multimodal/test_mapping.py
 
@@ -2989,7 +2981,6 @@ steps:
   - vllm/
   - tests/models/multimodal/generation
   commands:
-  - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
   - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model'
 
 - label: Multi-Modal Models (Extended Pooling) # TBD
@@ -3015,7 +3006,6 @@ steps:
   - vllm/
   - tests/models/multimodal
   commands:
-  - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
   - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2"
   - pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model
 
@@ -3030,7 +3020,6 @@ steps:
   - vllm/
   - tests/models/multimodal/generation
   commands:
-  - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
   - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py  --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/processing
   - pytest -v -s models/multimodal/generation/test_memory_leak.py -m core_model
   - cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model
@@ -3173,7 +3162,7 @@ steps:
 #------------------------------------------------------  mi355 · weight_loading  -------------------------------------------------------#
 
 - label: Weight Loading Multiple GPU # TBD
-  timeout_in_minutes: 30
+  timeout_in_minutes: 75
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
   agent_pool: mi355_2
   num_gpus: 2
@@ -3185,7 +3174,7 @@ steps:
   - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-amd.txt
 
 - label: Weight Loading Multiple GPU - Large Models # TBD
-  timeout_in_minutes: 30
+  timeout_in_minutes: 75
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
   agent_pool: mi355_2
   working_dir: "/vllm-workspace/tests"

.buildkite/test_areas/distributed.yaml

@@ -233,7 +233,7 @@ steps:
   num_devices: 2
   commands:
     - pytest -v -s tests/distributed/test_context_parallel.py
-    - pytest -v -s tests/distributed/test_nccl_symm_mem_allreduce.py
+    - pytest -v -s tests/distributed/test_nccl_symm_mem.py
     - pytest -v -s tests/v1/distributed/test_dbo.py
     - pytest -v -s tests/distributed/test_mnnvl_alltoall.py
 

.buildkite/test_areas/entrypoints.yaml

@@ -29,6 +29,8 @@ steps:
   mirror:
     amd:
       device: mi325_1
+      # TODO(akaratza): Test after Torch >= 2.12 bump
+      soft_fail: true
       depends_on:
       - image-build-amd
 
@@ -40,10 +42,12 @@ steps:
   source_file_dependencies:
   - vllm/
   - tests/entrypoints/serve
+  - tests/entrypoints/scale_out
   commands:
   - export VLLM_WORKER_MULTIPROC_METHOD=spawn
   - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc
   - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc
+  - pytest -v -s entrypoints/scale_out
   mirror:
     amd:
       device: mi325_1

.buildkite/test_areas/lm_eval.yaml

@@ -54,8 +54,8 @@ steps:
     - export VLLM_USE_DEEP_GEMM=0  # We found Triton is faster than DeepGEMM for H100
     - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-hopper.txt --tp-size=4
 
-- label: LM Eval Small Models (2xB200)
-  key: lm-eval-small-models-2xb200
+- label: LM Eval Small Models (1xB200)
+  key: lm-eval-small-models-1xb200
   timeout_in_minutes: 120
   device: b200-k8s
   optional: true
@@ -65,9 +65,10 @@ steps:
   commands:
   - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell.txt
 
-- label: LM Eval Small Models (2xL4)
-  key: lm-eval-small-models-tp
-  timeout_in_minutes: 10
+- label: LM Eval Small Models Distributed (2xB200)
+  key: lm-eval-small-models-distributed-2xb200
+  timeout_in_minutes: 120
+  device: b200-k8s
   num_devices: 2
   optional: true
   source_file_dependencies:

.buildkite/test_areas/models_multimodal.yaml

@@ -10,7 +10,6 @@ steps:
   - vllm/
   - tests/models/multimodal
   commands:
-    - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
     - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2"
     - pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model
   mirror:
@@ -27,7 +26,6 @@ steps:
   - vllm/
   - tests/models/multimodal
   commands:
-    - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
     - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen3 or gemma"
     - pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model
   mirror:
@@ -44,7 +42,6 @@ steps:
   - vllm/
   - tests/models/multimodal
   commands:
-    - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
     - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma"
     - pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model
   mirror:
@@ -61,7 +58,6 @@ steps:
   - vllm/
   - tests/models/multimodal
   commands:
-    - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
     - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py  --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/generation/test_vit_cudagraph.py --ignore models/multimodal/processing
     - pytest -v -s models/multimodal/generation/test_vit_cudagraph.py -m core_model
     - pytest models/multimodal/generation/test_memory_leak.py -m core_model
@@ -83,7 +79,6 @@ steps:
   - tests/models/registry.py
   device: cpu-medium
   commands:
-    - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
     - pytest -v -s models/multimodal/processing --ignore models/multimodal/processing/test_tensor_schema.py
 
 - label: Multi-Modal Processor # 44min
@@ -95,7 +90,6 @@ steps:
   - tests/models/multimodal
   - tests/models/registry.py
   commands:
-    - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
     - pytest -v -s models/multimodal/processing/test_tensor_schema.py
 
 - label: Multi-Modal Accuracy Eval (Small Models) # 50min
@@ -129,7 +123,6 @@ steps:
   - tests/models/multimodal/generation
   - tests/models/multimodal/test_mapping.py
   commands:
-    - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
     - pytest -v -s models/multimodal/generation -m 'not core_model' --ignore models/multimodal/generation/test_common.py
     - pytest -v -s models/multimodal/test_mapping.py
   mirror:
@@ -146,7 +139,6 @@ steps:
   - vllm/
   - tests/models/multimodal/generation
   commands:
-    - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
     - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=0) and not core_model'
 
 - label: Multi-Modal Models (Extended Generation 3)
@@ -157,7 +149,6 @@ steps:
   - vllm/
   - tests/models/multimodal/generation
   commands:
-    - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
     - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model'
 
 - label: Multi-Modal Models (Extended Pooling)

.buildkite/test_areas/rust_frontend.yaml

@@ -46,7 +46,7 @@ steps:
   - vllm/v1/engine/
   - tests/utils.py
   # - tests/entrypoints/serve/dev/rpc/test_collective_rpc.py
-  - tests/entrypoints/serve/disagg/test_serving_tokens.py
+  - tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py
   - tests/entrypoints/serve/instrumentator/test_basic.py
   - tests/entrypoints/serve/instrumentator/test_metrics.py
   # - tests/entrypoints/serve/dev/test_sleep.py
@@ -55,7 +55,7 @@ steps:
   - export VLLM_WORKER_MULTIPROC_METHOD=spawn
   # - pytest -v -s entrypoints/serve/dev/rpc/test_collective_rpc.py
   - pytest -v -s entrypoints/serve/instrumentator/test_basic.py -k "not show_version and not server_load"
-  - pytest -v -s entrypoints/serve/disagg/test_serving_tokens.py -k "not stream and not lora and not test_generate_logprobs and not stop_string_workflow"
+  - pytest -v -s entrypoints/scale_out/token_in_token_out/test_serving_tokens.py -k "not stream and not lora and not test_generate_logprobs and not stop_string_workflow"
   - pytest -v -s entrypoints/serve/instrumentator/test_metrics.py -k "text and not show and not run_batch and not test_metrics_counts and not test_metrics_exist"
   # - pytest -v -s entrypoints/serve/dev/test_sleep.py
 

.buildkite/test_areas/spec_decode.yaml

@@ -94,6 +94,8 @@ steps:
     amd:
       device: mi325_1
       timeout_in_minutes: 65
+      # TODO(akaratza): Test after Torch >= 2.12 bump
+      soft_fail: true
       depends_on:
       - image-build-amd
       source_file_dependencies:

.github/workflows/issue_autolabel.yml

@@ -327,7 +327,7 @@ jobs:
                 message: 'CC {users} for ROCm-related issue',
               },
               mistral: {
-                users: ['patrickvonplaten', 'juliendenize', 'andylolu2'],
+                users: ['patrickvonplaten', 'juliendenize', 'andylolu2', 'NickLucche'],
                 message: 'CC {users} for Mistral-related issue',
               },
               // Add more label -> user mappings here

.github/workflows/macos-smoke-test.yml

@@ -27,7 +27,7 @@ jobs:
     timeout-minutes: 30
 
     steps:
-      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
 
       - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
         with:

.github/workflows/pre-commit.yml

@@ -48,7 +48,7 @@ jobs:
     if: always() && (needs.pre-run-check.result == 'success' || needs.pre-run-check.result == 'skipped')
     runs-on: [self-hosted, linux, x64, vllm-runners]
     steps:
-    - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+    - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
     - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
       with:
         python-version: "3.12"

.pre-commit-config.yaml

@@ -131,6 +131,19 @@ repos:
         --python-version, "3.12",
       ]
       files: ^requirements/(common|xpu|test/xpu)\.(in|txt)$
+    - id: pip-compile
+      alias: pip-compile-cpu
+      name: pip-compile-cpu
+      args: [
+        requirements/test/cuda.in,
+        -o, requirements/test/cpu.txt,
+        --index-strategy, unsafe-best-match,
+        --torch-backend, cpu,
+        --python-platform, x86_64-manylinux_2_28,
+        --python-version, "3.12",
+      ]
+      files: ^requirements/(common|cpu|test/(cuda|cpu))\.(in|txt)$
+      exclude: ^requirements/test/cuda\.txt$
     - id: pip-compile
       alias: pip-compile-docs
       name: pip-compile-docs

CMakeLists.txt

@@ -140,6 +140,21 @@ if(Python_VERSION VERSION_GREATER_EQUAL "3.11")
     WITH_SOABI)
 endif()
 
+#
+# fs_io extension (pure CXX; must stay above the non-CUDA device branch
+# so CPU builds define the target before the early return).
+# GIL-releasing filesystem helpers for FileSystemTierManager.
+#
+if(Python_VERSION VERSION_GREATER_EQUAL "3.11")
+  define_extension_target(
+    fs_io_C
+    DESTINATION vllm
+    LANGUAGE CXX
+    SOURCES csrc/fs_io.cpp
+    USE_SABI 3.11
+    WITH_SOABI)
+endif()
+
 #
 # Forward the non-CUDA device extensions to external CMake scripts.
 #

benchmarks/benchmark_topk_topp.py

@@ -132,10 +132,8 @@ def benchmark_function(
     reset_memory_stats()
 
     # Benchmark
-    start_events = [
-        torch.cuda.Event(enable_timing=True) for _ in range(benchmark_iters)
-    ]
-    end_events = [torch.cuda.Event(enable_timing=True) for _ in range(benchmark_iters)]
+    start_events = [torch.Event(enable_timing=True) for _ in range(benchmark_iters)]
+    end_events = [torch.Event(enable_timing=True) for _ in range(benchmark_iters)]
 
     for i in range(benchmark_iters):
         logits_copy = logits.clone()

benchmarks/kernels/benchmark_moe_defaults.py

@@ -134,8 +134,8 @@ def benchmark_config(
     torch.accelerator.synchronize()
 
     # Benchmark
-    start = torch.cuda.Event(enable_timing=True)
-    end = torch.cuda.Event(enable_timing=True)
+    start = torch.Event(enable_timing=True)
+    end = torch.Event(enable_timing=True)
     start.record()
     for _ in range(num_iters):
         with override_config(config):

benchmarks/kernels/benchmark_selective_state_update.py

@@ -170,8 +170,8 @@ def benchmark_config(
                 graph.replay()
             torch.accelerator.synchronize()
 
-            start = torch.cuda.Event(enable_timing=True)
-            end = torch.cuda.Event(enable_timing=True)
+            start = torch.Event(enable_timing=True)
+            end = torch.Event(enable_timing=True)
             latencies: list[float] = []
             for _ in range(num_iters):
                 start.record()

cmake/cpu_extension.cmake

@@ -427,6 +427,7 @@ if (ASIMD_FOUND AND NOT APPLE_SILICON_FOUND)
     set(VLLM_EXT_SRC
         "csrc/cpu/shm.cpp"
         "csrc/cpu/activation_lut_bf16.cpp"
+        "csrc/cpu/cpu_tanhf_neon.hpp"
         "csrc/cpu/cpu_fused_moe.cpp"
         ${VLLM_EXT_SRC})
 endif()

cmake/external_projects/vllm_flash_attn.cmake

@@ -39,7 +39,7 @@ else()
   FetchContent_Declare(
           vllm-flash-attn
           GIT_REPOSITORY https://github.com/vllm-project/flash-attention.git
-          GIT_TAG b3964b1d8b95d8e8447435668ab169a2700bab65
+          GIT_TAG 2c839c33742309ec41e620bf837495ec9926c56e
           GIT_PROGRESS TRUE
           # Don't share the vllm-flash-attn build between build types
           BINARY_DIR ${CMAKE_BINARY_DIR}/vllm-flash-attn

csrc/cpu/activation.cpp

@@ -126,6 +126,18 @@ void gelu_tanh_and_mul(torch::Tensor& out,    // [..., d]
       });
 }
 
+void gelu_tanh(torch::Tensor& out, torch::Tensor& input) {
+  int num_tokens = input.numel() / input.size(-1);
+  int d = input.size(-1);
+
+  VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "gelu_tanh_impl", [&] {
+    CPU_KERNEL_GUARD_IN(gelu_tanh_impl)
+    activation_kernel<scalar_t, gelu_tanh_act, false>(
+        num_tokens, d, input.data_ptr<scalar_t>(), out.data_ptr<scalar_t>());
+    CPU_KERNEL_GUARD_OUT(gelu_tanh_impl)
+  });
+}
+
 void gelu_new(torch::Tensor& out, torch::Tensor& input) {
   int num_tokens = input.numel() / input.size(-1);
   int d = input.size(-1);

csrc/cpu/cpu_tanhf_neon.hpp

@@ -0,0 +1,128 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+#ifndef CPU_TANHF_NEON_HPP
+#define CPU_TANHF_NEON_HPP
+
+#include <cstdint>
+#include <arm_neon.h>
+
+namespace vec_op {
+
+namespace {
+
+struct TanhfConstants {
+  float32x4_t special_bound;
+  float32x4_t two;
+  float32x4_t c0;
+  float32x4_t c2;
+  int32x4_t exponent_bias;
+  float c1;
+  float c3;
+  float two_over_ln2;
+  float c4;
+  float ln2_hi;
+  float ln2_lo;
+};
+
+const TanhfConstants kTanhfConstants = {
+    // 9.01, above which tanhf rounds to 1 (or -1 for  negative).
+    .special_bound = vdupq_n_f32(0x1.205966p+3f),
+    .two = vdupq_n_f32(0x1.0p+1f),
+    .c0 = vdupq_n_f32(0x1.fffffep-2f),
+    .c2 = vdupq_n_f32(0x1.555736p-5f),
+    .exponent_bias = vdupq_n_s32(0x3f800000),
+    .c1 = 0x1.5554aep-3f,
+    .c3 = 0x1.12287cp-7f,
+    .two_over_ln2 = 0x1.715476p+1f,
+    .c4 = 0x1.6b55a2p-10f,
+    .ln2_hi = 0x1.62e4p-1f,
+    .ln2_lo = 0x1.7f7d1cp-20f,
+};
+
+// Return the ptr but hide it's value from the compiler so accesses
+// through it can't be optimised based on contents.
+template <typename T>
+inline const T* ptr_barrier(const T* ptr) {
+  const T* opaque_ptr = ptr;
+  __asm__("" : "+r"(opaque_ptr));
+  return opaque_ptr;
+}
+
+// Check whether any lanes in the mask are set
+inline bool any_u32(uint32x4_t x) { return vmaxvq_u32(x) != 0; }
+
+// e^2x - 1 inline helper
+inline float32x4_t e2xm1f_inline(float32x4_t x, const TanhfConstants* d) {
+  float32x2_t ln2 = vld1_f32(&d->ln2_hi);
+  float32x4_t lane_consts = vld1q_f32(&d->c1);
+
+  // Reduce argument: f in [-ln2/2, ln2/2], i is exact.
+  float32x4_t j = vrndaq_f32(vmulq_laneq_f32(x, lane_consts, 2));
+  int32x4_t i = vcvtq_s32_f32(j);
+  float32x4_t f = vaddq_f32(x, x);
+  f = vfmsq_lane_f32(f, j, ln2, 0);
+  f = vfmsq_lane_f32(f, j, ln2, 1);
+
+  // Approximate expm1(f) with polynomial P, expm1(f) ~= f + f^2 * P(f)
+  float32x4_t f2 = vmulq_f32(f, f);
+  float32x4_t f4 = vmulq_f32(f2, f2);
+  float32x4_t p01 = vfmaq_laneq_f32(d->c0, f, lane_consts, 0);
+  float32x4_t p23 = vfmaq_laneq_f32(d->c2, f, lane_consts, 1);
+  float32x4_t poly = vfmaq_f32(p01, f2, p23);
+  poly = vfmaq_laneq_f32(poly, f4, lane_consts, 3);
+  poly = vfmaq_f32(f, f2, poly);
+
+  // scale = 2^i
+  int32x4_t u = vaddq_s32(vshlq_n_s32(i, 23), d->exponent_bias);
+  float32x4_t scale = vreinterpretq_f32_s32(u);
+  return vfmaq_f32(vsubq_f32(scale, vdupq_n_f32(1.0f)), poly, scale);
+}
+
+// Calculate the result tanh(x) = q / (q+2) and set special lanes to ±1
+inline float32x4_t special_case(float32x4_t x, float32x4_t q,
+                                uint32x4_t special) {
+  const TanhfConstants* d = ptr_barrier(&kTanhfConstants);
+
+  float32x4_t y = vdivq_f32(q, vaddq_f32(q, d->two));
+  uint32x4_t ix = vreinterpretq_u32_f32(x);
+  uint32x4_t one_bits = vreinterpretq_u32_s32(d->exponent_bias);
+  uint32x4_t sign_mask = vdupq_n_u32(0x80000000u);
+  uint32x4_t special_bits = vbslq_u32(sign_mask, ix, one_bits);
+  float32x4_t special_y = vreinterpretq_f32_u32(special_bits);
+  return vbslq_f32(special, special_y, y);
+}
+
+}  // namespace
+
+// Implementation of tanhf adapted from Arm Optimized Routines (tanhf
+// AdvSIMD)
+// https://github.com/ARM-software/optimized-routines/blob/master/math/aarch64/advsimd/tanhf.c
+//
+// Approximation for single-precision vector tanh(x), using a simplified
+// version of expm1f. The maximum error is 2.08 + 0.5 ULP:
+// _ZGVnN4v_tanhf (0x1.fa5eep-5) got 0x1.f9ba02p-5 want 0x1.f9ba08p-5.
+inline float32x4_t fast_tanhf_f32x4(float32x4_t x) {
+  const TanhfConstants* d = ptr_barrier(&kTanhfConstants);
+
+  // tanh(x) = (e^2x - 1) / (e^2x + 1)
+  // q = e^2x -1
+  float32x4_t q = e2xm1f_inline(x, d);
+
+  // Check for special cases
+  uint32x4_t special = vcagtq_f32(x, d->special_bound);
+
+  // Fall back to vectorised special case for any lanes which would cause
+  // expm1 to overflow
+  if (any_u32(special)) {
+    return special_case(x, q, special);
+  }
+
+  // Complete fast path if no special lanes
+  // tanh(x) = q / (q+2)
+  return vdivq_f32(q, vaddq_f32(q, d->two));
+}
+
+}  // namespace vec_op
+
+#endif  // CPU_TANHF_NEON_HPP

csrc/cpu/cpu_types_arm.hpp

@@ -3,6 +3,8 @@
 
 #include <arm_neon.h>
 
+#include "cpu/cpu_tanhf_neon.hpp"
+
 #include <torch/all.h>
 #include <ATen/cpu/vec/functional.h>
 #include <ATen/cpu/vec/vec.h>
@@ -345,6 +347,10 @@ struct FP32Vec4 : public VectorizedRegWrapper<FP32Vec4, 1, float> {
   explicit FP32Vec4(float32x4_t data) : Base(VectorizedT(data)) {};
 
   explicit FP32Vec4(const FP32Vec4& data) : Base(data) {};
+
+  FORCE_INLINE FP32Vec4 tanh() const {
+    return FP32Vec4(fast_tanhf_f32x4(reg.val[0]));
+  }
 };
 
 struct FP32Vec8 : public VectorizedRegWrapper<FP32Vec8, 2, float> {
@@ -391,6 +397,13 @@ struct FP32Vec8 : public VectorizedRegWrapper<FP32Vec8, 2, float> {
     reg.val[1] = Vectorized<float>(data.val[1]);
   }
 
+  FORCE_INLINE FP32Vec8 tanh() const {
+    FP32Vec8 r(uninit);
+    r.reg.val[0] = Vectorized<float>(fast_tanhf_f32x4(reg.val[0]));
+    r.reg.val[1] = Vectorized<float>(fast_tanhf_f32x4(reg.val[1]));
+    return r;
+  }
+
   FORCE_INLINE float reduce_sum() const noexcept {
     float answer = 0;
     std::plus<VectorizedT> add;
@@ -497,6 +510,15 @@ struct FP32Vec16 : public VectorizedRegWrapper<FP32Vec16, 4, float> {
     reg.val[3] = Vectorized<float>(vcvt_f32_f16(vget_high_f16(v.reg.val[1])));
   };
 
+  FORCE_INLINE FP32Vec16 tanh() const {
+    FP32Vec16 r(uninit);
+    r.reg.val[0] = Vectorized<float>(fast_tanhf_f32x4(reg.val[0]));
+    r.reg.val[1] = Vectorized<float>(fast_tanhf_f32x4(reg.val[1]));
+    r.reg.val[2] = Vectorized<float>(fast_tanhf_f32x4(reg.val[2]));
+    r.reg.val[3] = Vectorized<float>(fast_tanhf_f32x4(reg.val[3]));
+    return r;
+  }
+
   static FORCE_INLINE void load_even_odd(const float* ptr, FP32Vec16& even,
                                          FP32Vec16& odd) noexcept {
     const float32x4x2_t x01 = vuzpq_f32(vld1q_f32(ptr), vld1q_f32(ptr + 4));

csrc/cpu/torch_bindings.cpp

@@ -298,6 +298,10 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
   ops.def("gelu_tanh_and_mul(Tensor! out, Tensor input) -> ()");
   ops.impl("gelu_tanh_and_mul", torch::kCPU, &gelu_tanh_and_mul);
 
+  // GELU tanh implementation.
+  ops.def("gelu_tanh(Tensor! out, Tensor input) -> ()");
+  ops.impl("gelu_tanh", torch::kCPU, &gelu_tanh);
+
   // GELU implementation used in GPT-2.
   ops.def("gelu_new(Tensor! out, Tensor input) -> ()");
   ops.impl("gelu_new", torch::kCPU, &gelu_new);

csrc/fs_io.cpp

@@ -0,0 +1,69 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+#include <Python.h>
+
+#include <unistd.h>
+
+#include <vector>
+
+extern "C" {
+
+static void _batch_lookup(const std::vector<const char*>& paths,
+                          std::vector<int>& exists_flags) {
+  for (size_t i = 0; i < paths.size(); i++) {
+    exists_flags[i] = (access(paths[i], F_OK) == 0) ? 1 : 0;
+  }
+}
+
+/// @brief Check file existence for a batch of paths.
+/// @param paths list[str] – absolute paths to check.
+/// @return list[bool] – True if the corresponding path exists, False otherwise.
+/// @note Releases the GIL for the entire batch. File existence via access(2).
+static PyObject* batch_lookup(PyObject* /*self*/, PyObject* args) {
+  PyObject* path_list;
+  if (!PyArg_ParseTuple(args, "O!", &PyList_Type, &path_list)) {
+    return nullptr;
+  }
+
+  const Py_ssize_t n = PyList_Size(path_list);
+  std::vector<const char*> paths(n);
+  for (Py_ssize_t i = 0; i < n; i++) {
+    paths[i] = PyUnicode_AsUTF8AndSize(PyList_GetItem(path_list, i), nullptr);
+    if (paths[i] == nullptr) {
+      return nullptr;
+    }
+  }
+
+  std::vector<int> exists_flags(n);
+  {
+    Py_BEGIN_ALLOW_THREADS _batch_lookup(paths, exists_flags);
+    Py_END_ALLOW_THREADS
+  }
+
+  PyObject* result = PyList_New(n);
+  if (result == nullptr) {
+    return nullptr;
+  }
+  for (Py_ssize_t i = 0; i < n; i++) {
+    PyList_SetItem(result, i, PyBool_FromLong(exists_flags[i]));
+  }
+  return result;
+}
+
+static PyMethodDef fs_io_C_methods[] = {
+    {"batch_lookup", batch_lookup, METH_VARARGS,
+     "batch_lookup(paths: list[str]) -> list[bool]\n"
+     "\n"
+     "Check file existence for a batch of paths."},
+    {nullptr, nullptr, 0, nullptr},
+};
+
+static struct PyModuleDef fs_io_C_module = {
+    PyModuleDef_HEAD_INIT, "fs_io_C", "Filesystem helpers for KV offload", -1,
+    fs_io_C_methods,
+};
+
+PyMODINIT_FUNC PyInit_fs_io_C(void) { return PyModule_Create(&fs_io_C_module); }
+
+}  // extern "C"

csrc/libtorch_stable/minimax_reduce_rms_kernel.cu

@@ -804,35 +804,6 @@ void minimax_reduce_rms_op(MiniMaxReduceRMSParams const& params) {
 }  // namespace tensorrt_llm
 }  // namespace vllm
 
-torch::stable::Tensor minimax_allreduce_rms(
-    torch::stable::Tensor const& input,
-    torch::stable::Tensor const& norm_weight, torch::stable::Tensor workspace,
-    int64_t const rank, int64_t const nranks, double const eps) {
-  const torch::stable::accelerator::DeviceGuard device_guard(
-      input.get_device_index());
-  auto allreduce_params = vllm::tensorrt_llm::MiniMaxReduceRMSParams();
-
-  allreduce_params.nranks = static_cast<int>(nranks);
-  allreduce_params.rank = static_cast<int>(rank);
-  allreduce_params.dtype = input.scalar_type();
-  allreduce_params.size_q = static_cast<int>(input.numel());
-  allreduce_params.hidden_dim = static_cast<int>(input.size(-1));
-  allreduce_params.stride_q = allreduce_params.hidden_dim;
-  allreduce_params.workspace =
-      reinterpret_cast<void**>(workspace.mutable_data_ptr());
-  allreduce_params.allreduce_in = const_cast<void*>(input.const_data_ptr());
-  allreduce_params.rms_gamma = const_cast<void*>(norm_weight.const_data_ptr());
-  allreduce_params.rms_eps = static_cast<float>(eps);
-  allreduce_params.stream = get_current_cuda_stream(input.get_device_index());
-
-  torch::stable::Tensor rms_norm_out = torch::stable::empty_like(input);
-  allreduce_params.rms_norm_out = rms_norm_out.mutable_data_ptr();
-
-  vllm::tensorrt_llm::minimax_reduce_rms_op(allreduce_params);
-
-  return rms_norm_out;
-}
-
 std::tuple<torch::stable::Tensor, torch::stable::Tensor>
 minimax_allreduce_rms_qk(torch::stable::Tensor qkv,
                          torch::stable::Tensor const& norm_weight_q,

csrc/libtorch_stable/ops.h

@@ -288,10 +288,6 @@ void fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_fp8_insert(
     int64_t cache_block_size);
 
 #ifndef USE_ROCM
-torch::stable::Tensor minimax_allreduce_rms(
-    torch::stable::Tensor const& input,
-    torch::stable::Tensor const& norm_weight, torch::stable::Tensor workspace,
-    int64_t const rank, int64_t const nranks, double const eps);
 std::tuple<torch::stable::Tensor, torch::stable::Tensor>
 minimax_allreduce_rms_qk(torch::stable::Tensor qkv,
                          torch::stable::Tensor const& norm_weight_q,

csrc/libtorch_stable/torch_bindings.cpp

@@ -449,10 +449,6 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
       "int cache_block_size) -> ()");
 
 #ifndef USE_ROCM
-  ops.def(
-      "minimax_allreduce_rms("
-      "Tensor input, Tensor norm_weight, Tensor workspace, "
-      "int rank, int nranks, float eps) -> Tensor");
   ops.def(
       "minimax_allreduce_rms_qk("
       "Tensor qkv, Tensor norm_weight_q, Tensor norm_weight_k, "
@@ -705,7 +701,6 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
       "fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_fp8_insert",
       TORCH_BOX(&fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_fp8_insert));
 #ifndef USE_ROCM
-  ops.impl("minimax_allreduce_rms", TORCH_BOX(&minimax_allreduce_rms));
   ops.impl("minimax_allreduce_rms_qk", TORCH_BOX(&minimax_allreduce_rms_qk));
 #endif
   ops.impl("fused_minimax_m3_qknorm_rope_kv_insert",

csrc/ops.h

@@ -35,6 +35,8 @@ void gelu_and_mul(torch::Tensor& out, torch::Tensor& input);
 
 void gelu_tanh_and_mul(torch::Tensor& out, torch::Tensor& input);
 
+void gelu_tanh(torch::Tensor& out, torch::Tensor& input);
+
 void gelu_new(torch::Tensor& out, torch::Tensor& input);
 
 void gelu_fast(torch::Tensor& out, torch::Tensor& input);

docker/Dockerfile

@@ -793,7 +793,7 @@ RUN --mount=type=cache,target=/opt/uv/cache \
 # Install FlashInfer JIT cache (requires CUDA-version-specific index URL)
 # https://docs.flashinfer.ai/installation.html
 # From versions.json: .flashinfer.version
-ARG FLASHINFER_VERSION=0.6.12
+ARG FLASHINFER_VERSION=0.6.13
 RUN --mount=type=cache,target=/opt/uv/cache \
     uv pip install --system flashinfer-jit-cache==${FLASHINFER_VERSION} \
         --index-url https://flashinfer.ai/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')

docker/Dockerfile.cpu

@@ -193,26 +193,16 @@ FROM base AS vllm-test-deps
 
 WORKDIR /vllm-workspace
 
-# Copy test requirements
-COPY requirements/test/cuda.in requirements/test/cpu.in
+# Test requirements are compiled from requirements/test/cuda.in into
+# requirements/test/cpu.txt by the pip-compile-cpu pre-commit hook, which
+# resolves CPU wheels via uv's --torch-backend cpu.
+COPY requirements/test/cpu.txt requirements/test/cpu.txt
 
-RUN \
-    sed -i '/mamba_ssm/d' requirements/test/cpu.in && \
-    remove_packages_not_supported_on_aarch64() { \
-    case "$(uname -m)" in \
-    aarch64|arm64) \
-    sed -i '/decord/d' requirements/test/cpu.in; \
-    sed -i '/terratorch/d' requirements/test/cpu.in; \
-    ;; \
-    esac; \
-    }; \
-    remove_packages_not_supported_on_aarch64 && \
-    sed -i 's/^torch==.*/torch==2.11.0/g' requirements/test/cpu.in && \
-    sed -i 's/torchaudio.*/torchaudio/g' requirements/test/cpu.in && \
-    sed -i 's/torchvision.*/torchvision/g' requirements/test/cpu.in && \
-    # Related issue: https://github.com/vllm-project/vllm/pull/38800#issuecomment-4228314305
-    sed -i 's/^sentence-transformers.*/sentence-transformers==5.3.0/g' requirements/test/cpu.in && \
-    uv pip compile requirements/test/cpu.in -o requirements/test/cpu.txt --index-strategy unsafe-best-match --torch-backend cpu
+# cpu.txt is compiled for x86_64, so platform markers are resolved away. Drop
+# packages unavailable on aarch64 (decord, terratorch) for arm builds.
+RUN case "$(uname -m)" in \
+    aarch64|arm64) sed -i '/^decord==/d; /^terratorch==/d' requirements/test/cpu.txt ;; \
+    esac
 
 RUN --mount=type=cache,target=/root/.cache/uv \
     uv pip install -r requirements/test/cpu.txt

docker/Dockerfile.nightly_torch

@@ -257,13 +257,13 @@ RUN pip install setuptools==75.6.0 packaging==23.2 ninja==1.11.1.3 build==1.2.2.
 
 
 # build flashinfer for torch nightly from source around 10 mins
-# release version: v0.6.12
+# release version: v0.6.13
 # todo(elainewy): cache flashinfer build result for faster build
 ENV CCACHE_DIR=/root/.cache/ccache
 RUN --mount=type=cache,target=/root/.cache/ccache \
     --mount=type=cache,target=/root/.cache/uv \
     echo "git clone flashinfer..." \
-    && git clone --depth 1 --branch v0.6.12 --recursive https://github.com/flashinfer-ai/flashinfer.git \
+    && git clone --depth 1 --branch v0.6.13 --recursive https://github.com/flashinfer-ai/flashinfer.git \
     && cd flashinfer \
     && git submodule update --init --recursive \
     && echo "finish git clone flashinfer..." \

docker/versions.json

@@ -68,7 +68,7 @@
       "default": "true"
     },
     "FLASHINFER_VERSION": {
-      "default": "0.6.12"
+      "default": "0.6.13"
     },
     "GDRCOPY_CUDA_VERSION": {
       "default": "12.8"

docs/design/attention_backends.md

@@ -167,6 +167,7 @@ Priority is **1 = highest** (tried first).
 | `FLASH_ATTN` | FA4* | fp16, bf16 | `auto`, `float16`, `bfloat16` | %16 | Any | ✅ | ✅ | ❌ | ✅ | All | ≥10.0 |
 | `FLASH_ATTN_DIFFKV` | | fp16, bf16 | `auto` | Any | Any | ❌ | ❌ | ❌ | ✅ | Decoder | Any |
 | `FLEX_ATTENTION` | | fp16, bf16, fp32 | `auto`, `float16`, `bfloat16` | %16 | Any | ❌ | ✅ | ✅ | ❌ | Decoder, Encoder Only | Any |
+| `HPC_ATTN` | | fp16, bf16 | `auto`, `fp8_e4m3` | 64 | 128 | ❌ | ❌ | ❌ | ❌ | Decoder | ≥9.0 |
 | `ROCM_AITER_FA` | | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | 16, 32 | 64, 128, 256 | ✅ | ✅ | ❌ | ❌ | Decoder | N/A |
 | `ROCM_AITER_UNIFIED_ATTN` | | bf16 | `auto`, `bfloat16`, `fp8`, `fp8_e4m3` | %16 | Any | ✅ | ❌ | ✅ | ❌ | All | N/A |
 | `ROCM_ATTN` | | fp16, bf16, fp32 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | %16 | 32, 64, 80, 96, 128, 160, 192, 224, 256 | ❌ | ✅ | ✅ | ❌ | Decoder, Encoder, Encoder Only | N/A |
@@ -220,8 +221,8 @@ MLA decode backends are selected using the standard
 | Backend | Dtypes | KV Dtypes | Block Sizes | Head Sizes | Sink | Non-Causal | Sparse | MM Prefix | DCP | Attention Types | Compute Cap. |
 | ------- | ------ | --------- | ----------- | ---------- | ---- | ---------- | ------ | --------- | --- | --------------- | ------------ |
 | `CUTLASS_MLA` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 128 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 10.x |
-| `FLASHINFER_MLA` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 32, 64 | Any | ❌ | ❌ | ❌ | ❌ | ❌ | Decoder | 10.x |
-| `FLASHINFER_MLA_SPARSE` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 32, 64 | Any | ❌ | ❌ | ❌ | ❌ | ❌ | Decoder | 10.x |
+| `FLASHINFER_MLA` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 32, 64 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 10.x |
+| `FLASHINFER_MLA_SPARSE` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 32, 64 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 10.x |
 | `FLASHINFER_MLA_SPARSE_SM120` | bf16 | `auto`, `fp8`, `fp8_e4m3`, `fp8_ds_mla` | 64, 256 | Any | ❌ | ❌ | ❌ | ❌ | ❌ | Decoder | 12.x |
 | `FLASHMLA` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 64 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 9.x-10.x |
 | `FLASHMLA_SPARSE` | bf16 | `auto`, `bfloat16`, `fp8_ds_mla` | 64 | 576 | ❌ | ❌ | ✅ | ❌ | ❌ | Decoder | 9.x-10.x |

docs/design/moe_kernel_features.md

@@ -89,7 +89,7 @@ To be used with a particular `FusedMoEPrepareAndFinalizeModular` subclass, MoE k
 | gpt oss triton | standard | N/A | N/A | <sup>5</sup> | Y | Y | [`triton_kernel_fused_experts`][vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe.triton_kernel_fused_experts],</br>[`OAITritonExperts`][vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe.OAITritonExperts] |
 | marlin | standard,</br>batched | <sup>3</sup> / N/A | <sup>3</sup> / N/A | silu,</br>swigluoai | Y | Y | [`fused_marlin_moe`][vllm.model_executor.layers.fused_moe.experts.marlin_moe.fused_marlin_moe],</br>[`MarlinExperts`][vllm.model_executor.layers.fused_moe.experts.marlin_moe.MarlinExperts],</br>[`BatchedMarlinExperts`][vllm.model_executor.layers.fused_moe.experts.marlin_moe.BatchedMarlinExperts] |
 | trtllm | standard | mxfp4,</br>nvfp4 | G(16),G(32) | <sup>5</sup> | N | Y | [`TrtLlmMxfp4ExpertsMonolithic`][vllm.model_executor.layers.fused_moe.experts.trtllm_mxfp4_moe.TrtLlmMxfp4ExpertsMonolithic],</br>[`TrtLlmMxfp4ExpertsModular`][vllm.model_executor.layers.fused_moe.experts.trtllm_mxfp4_moe.TrtLlmMxfp4ExpertsModular],</br>[`TrtLlmNvFp4ExpertsMonolithic`][vllm.model_executor.layers.fused_moe.experts.trtllm_nvfp4_moe.TrtLlmNvFp4ExpertsMonolithic],</br>[`TrtLlmNvfp4ExpertsModular`][vllm.model_executor.layers.fused_moe.experts.trtllm_nvfp4_moe.TrtLlmNvFp4ExpertsModular] |
-| hpc | standard | fp8 | G(128),T | silu | Y | Y | [`HPCExperts`][vllm.model_executor.layers.fused_moe.experts.hpc.HPCExperts] |
+| hpc | standard | fp8 | G(128),T | silu | Y | Y | [`HPCExperts`][vllm.model_executor.layers.fused_moe.hpc_moe.HPCExperts] |
 | rocm aiter moe | standard | mxfp4,</br>fp8 | G(32),G(128),A,T | silu, gelu,</br>swigluoai | Y | N | `rocm_aiter_fused_experts`,</br>`AiterExperts` |
 | cpu_fused_moe | standard | N/A | N/A | silu | N | N | [`CPUFusedMOE`][vllm.model_executor.layers.fused_moe.cpu_fused_moe.CPUFusedMOE] |
 | naive batched<sup>4</sup> | batched | int8,</br>fp8 | G,A,T | silu, gelu | <sup>6</sup> | Y | [`NaiveBatchedExperts`][vllm.model_executor.layers.fused_moe.experts.fused_batched_moe.NaiveBatchedExperts] |

docs/examples/README.md

@@ -13,5 +13,6 @@ vLLM's examples are organized into the following categories:
 - **[`rl/`](../../examples/rl)** – Reinforcement learning examples.
 - **[`deployment/`](../../examples/deployment)** – Examples for deploying vLLM in production.
 - **[`ray_serving/`](../../examples/ray_serving)** – Scalable serving using Ray.
-- **[`disaggregated/`](../../examples/disaggregated)** – Examples for disaggregated serving (separate prefill and decode), including various kv cache connectors (LMCache, Mooncake, FlexKV, P2P NCCL) and failure recovery.
+- **[`disaggregated/`](../../examples/disaggregated)** – Examples for Disaggregated P/D (Prefill/Decoding) inference, including various kv cache connectors (LMCache, Mooncake, FlexKV, P2P NCCL) and failure recovery.
+- **[`scale_out/`](../../examples/scale_out)** – Examples for Token In <> Token Out API Server.
 - **[`observability/`](../../examples/observability)** – Metrics, logging, tracing (OpenTelemetry), and dashboards (Grafana, Perses).

docs/features/kv_offloading_usage.md

@@ -120,6 +120,20 @@ To enable KV cache sharing between multiple vLLM instances using the same `root_
 PYTHONHASHSEED=0 vllm serve ...
 ```
 
+### P2P (Including P/D)
+
+The P2P tier (`type: "p2p"`) shares completed KV blocks between vLLM instances over RDMA via NIXL. Each instance binds a control socket on `host:port` and exchanges blocks directly with peers — no shared filesystem required.
+
+| Key | Required | Default | Notes |
+| --- | --- | --- | --- |
+| `type` | yes | — | Must be `p2p`. |
+| `host` | no | `0.0.0.0` | Address the control socket binds to. |
+| `port` | no | `7777` | Port for the control socket. Must be reachable from peers. |
+| `backends` | no | `["UCX"]` | NIXL transport backends. See [NixlConnector Usage Guide](nixl_connector_usage.md#selecting-a-nixl-transport-backend-plugin) for available backends and selection guidance. |
+| `num_threads` | no | `4` | NIXL agent worker threads. Only used when `backends` is UCX-only; ignored when any non-UCX backend is requested. |
+
+The `backends` and `num_threads` options mirror the conditional logic used by [`NixlConnector`](nixl_connector_usage.md#selecting-a-nixl-transport-backend-plugin): when any non-UCX backend is configured, NIXL is initialised with `backends=...`; otherwise it falls back to a UCX-only agent with the configured `num_threads`. This lets the P2P tier use a different transport (e.g. `MOONCAKE`, `GDS_MT`, `LIBFABRIC`) than the main `NixlConnector` running in the same process.
+
 ## Tuning Tips
 
 - `cpu_bytes_to_use`: a bigger CPU tier means fewer trips to slower secondary tiers and a higher hit rate. The value is total across all workers, not per-worker. Leave headroom for the rest of the host workload.

docs/models/supported_models.md

@@ -405,7 +405,6 @@ th {
 | `Glm4MoeLiteForCausalLM` | GLM-4.7-Flash | `zai-org/GLM-4.7-Flash`, etc. | ✅︎ | ✅︎ |
 | `GlmMoeDsaForCausalLM` | GLM-5, GLM-5.1, GLM-5.2 | `zai-org/GLM-5`, etc. | ✅︎ | ✅︎ |
 | `GPT2LMHeadModel` | GPT-2 | `openai-community/gpt2`, `openai-community/gpt2-xl`, etc. | | ✅︎ |
-| `GPTBigCodeForCausalLM` | StarCoder, SantaCoder, WizardCoder | `bigcode/starcoder`, `bigcode/gpt_bigcode-santacoder`, `WizardLM/WizardCoder-15B-V1.0`, etc. | ✅︎ | ✅︎ |
 | `GPTJForCausalLM` | GPT-J | `EleutherAI/gpt-j-6b`, `nomic-ai/gpt4all-j`, etc. | | ✅︎ |
 | `GPTNeoXForCausalLM` | GPT-NeoX, Pythia, OpenAssistant, Dolly V2, StableLM | `EleutherAI/gpt-neox-20b`, `EleutherAI/pythia-12b`, `OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5`, `databricks/dolly-v2-12b`, `stabilityai/stablelm-tuned-alpha-7b`, etc. | | ✅︎ |
 | `GptOssForCausalLM` | GPT-OSS | `openai/gpt-oss-120b`, `openai/gpt-oss-20b` | ✅︎ | ✅︎ |
@@ -477,7 +476,6 @@ th {
 | `SolarForCausalLM` | Solar Pro | `upstage/solar-pro-preview-instruct`, etc. | ✅︎ | ✅︎ |
 | `StableLmForCausalLM` | StableLM | `stabilityai/stablelm-3b-4e1t`, `stabilityai/stablelm-base-alpha-7b-v2`, etc. | | |
 | `StableLMEpochForCausalLM` | StableLM Epoch | `stabilityai/stablelm-zephyr-3b`, etc. | | ✅︎ |
-| `Starcoder2ForCausalLM` | Starcoder2 | `bigcode/starcoder2-3b`, `bigcode/starcoder2-7b`, `bigcode/starcoder2-15b`, etc. | | ✅︎ |
 | `Step1ForCausalLM` | Step-Audio | `stepfun-ai/Step-Audio-EditX`, etc. | ✅︎ | ✅︎ |
 | `Step3p5ForCausalLM` | Step-3.5-flash | `stepfun-ai/Step-3.5-Flash`, etc. | | ✅︎ |
 | `TeleChatForCausalLM` | TeleChat | `chuhac/TeleChat2-35B`, etc. | ✅︎ | ✅︎ |
@@ -490,7 +488,9 @@ Some models are supported only via the [Transformers modeling backend](#transfor
 
 | Architecture | Models | Example HF Models | [LoRA](../features/lora.md) | [PP](../serving/parallelism_scaling.md) |
 | ------------ | ------ | ----------------- | -------------------- | ------------------------- |
+| `GPTBigCodeForCausalLM` | StarCoder, SantaCoder, WizardCoder | `bigcode/starcoder`, `bigcode/gpt_bigcode-santacoder`, `WizardLM/WizardCoder-15B-V1.0`, etc. | ✅︎ | |
 | `SmolLM3ForCausalLM` | SmolLM3 | `HuggingFaceTB/SmolLM3-3B` | ✅︎ | ✅︎ |
+| `Starcoder2ForCausalLM` | Starcoder2 | `bigcode/starcoder2-3b`, `bigcode/starcoder2-7b`, `bigcode/starcoder2-15b`, etc. | ✅︎ | ✅︎ |
 
 !!! note
     Currently, the ROCm version of vLLM supports Mistral and Mixtral only for context lengths up to 4096.
@@ -532,7 +532,6 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
 | ------------ | ------ | ------ | ----------------- | -------------------- | ------------------------- |
 | `AriaForConditionalGeneration` | Aria | T + I<sup>+</sup> | `rhymes-ai/Aria` | | |
 | `AudioFlamingo3ForConditionalGeneration` | AudioFlamingo3 | T + A | `nvidia/audio-flamingo-3-hf`, `nvidia/music-flamingo-hf` | ✅︎ | ✅︎ |
-| `AyaVisionForConditionalGeneration` | Aya Vision | T + I<sup>+</sup> | `CohereLabs/aya-vision-8b`, `CohereLabs/aya-vision-32b`, etc. | | ✅︎ |
 | `BagelForConditionalGeneration` | BAGEL | T + I<sup>+</sup> | `ByteDance-Seed/BAGEL-7B-MoT` | ✅︎ | ✅︎ |
 | `BeeForConditionalGeneration` | Bee-8B | T + I<sup>E+</sup> | `Open-Bee/Bee-8B-RL`, `Open-Bee/Bee-8B-SFT` | | ✅︎ |
 | `Blip2ForConditionalGeneration` | BLIP-2 | T + I<sup>E</sup> | `Salesforce/blip2-opt-2.7b`, `Salesforce/blip2-opt-6.7b`, etc. | ✅︎ | ✅︎ |
@@ -579,14 +578,14 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
 | `Lfm2VlForConditionalGeneration` | LFM2-VL | T + I<sup>+</sup> | `LiquidAI/LFM2-VL-450M`, `LiquidAI/LFM2-VL-3B`, `LiquidAI/LFM2-VL-8B-A1B`, etc. | ✅︎ | ✅︎ |
 | `Llama4ForConditionalGeneration` | Llama 4 | T + I<sup>+</sup> | `meta-llama/Llama-4-Scout-17B-16E-Instruct`, `meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8`, `meta-llama/Llama-4-Maverick-17B-128E-Instruct`, etc. | ✅︎ | ✅︎ |
 | `Llama_Nemotron_Nano_VL` | Llama Nemotron Nano VL | T + I<sup>E+</sup> | `nvidia/Llama-3.1-Nemotron-Nano-VL-8B-V1` | ✅︎ | ✅︎ |
-| `LlavaForConditionalGeneration` | LLaVA-1.5, Pixtral (HF Transformers) | T + I<sup>E+</sup> | `llava-hf/llava-1.5-7b-hf`, `TIGER-Lab/Mantis-8B-siglip-llama3` (see note), `mistral-community/pixtral-12b`, etc. | ✅︎ | ✅︎ |
+| `LlavaForConditionalGeneration` | LLaVA-1.5, Pixtral (HF Transformers) | T + I<sup>E+</sup> | `llava-hf/llava-1.5-7b-hf`, `mistral-community/pixtral-12b`, etc. | ✅︎ | ✅︎ |
 | `LlavaNextForConditionalGeneration` | LLaVA-NeXT, Granite Vision | T + I<sup>E+</sup> | `llava-hf/llava-v1.6-mistral-7b-hf`, `llava-hf/llava-v1.6-vicuna-7b-hf`, `ibm-granite/granite-vision-3.3-2b`, etc. | | ✅︎ |
 | `LlavaNextVideoForConditionalGeneration` | LLaVA-NeXT-Video | T + V | `llava-hf/LLaVA-NeXT-Video-7B-hf`, etc. | | ✅︎ |
 | `LlavaOnevisionForConditionalGeneration` | LLaVA-Onevision | T + I<sup>+</sup> + V<sup>+</sup> | `llava-hf/llava-onevision-qwen2-7b-ov-hf`, `llava-hf/llava-onevision-qwen2-0.5b-ov-hf`, etc. | | ✅︎ |
 | `MiDashengLMModel` | MiDashengLM | T + A<sup>+</sup> | `mispeech/midashenglm-7b` | | ✅︎ |
 | `MiMoV2OmniForCausalLM` | MiMo-V2.5-Omni | T + I<sup>E+</sup> + V<sup>E+</sup> + A<sup>+</sup> | `XiaomiMiMo/MiMo-V2.5-Omni` | | ✅︎ |
 | `MiniCPMO` | MiniCPM-O | T + I<sup>E+</sup> + V<sup>E+</sup> + A<sup>E+</sup> | `openbmb/MiniCPM-o-2_6`, etc. | ✅︎ | ✅︎ |
-| `MiniCPMV` | MiniCPM-V | T + I<sup>E+</sup> + V<sup>E+</sup> | `openbmb/MiniCPM-V-2` (see note), `openbmb/MiniCPM-Llama3-V-2_5`, `openbmb/MiniCPM-V-2_6`, `openbmb/MiniCPM-V-4`, `openbmb/MiniCPM-V-4_5`, etc. | ✅︎ | |
+| `MiniCPMV` | MiniCPM-V | T + I<sup>E+</sup> + V<sup>E+</sup> | `openbmb/MiniCPM-V-2` (see note), `openbmb/MiniCPM-Llama3-V-2_5`, `openbmb/MiniCPM-V-2_6`, `openbmb/MiniCPM-V-4`, `openbmb/MiniCPM-V-4_5`, `openbmb/MiniCPM-V-4_6`, etc. | ✅︎ | |
 | `MiniMaxM3SparseForConditionalGeneration` | MiniMax-M3 | T + I<sup>+</sup> + V<sup>+</sup> | `MiniMaxAI/MiniMax-M3`, `MiniMaxAI/MiniMax-M3-MXFP8`, etc. | | ✅︎ |
 | `MiniMaxVL01ForConditionalGeneration` | MiniMax-VL | T + I<sup>E+</sup> | `MiniMaxAI/MiniMax-VL-01`, etc. | | ✅︎ |
 | `Mistral3ForConditionalGeneration` | Mistral3 (HF Transformers) | T + I<sup>+</sup> | `mistralai/Mistral-Small-3.1-24B-Instruct-2503`, etc. | ✅︎ | ✅︎ |
@@ -594,7 +593,6 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
 | `Molmo2ForConditionalGeneration` | Molmo2 | T + I<sup>+</sup> / V | `allenai/Molmo2-4B`, `allenai/Molmo2-8B`, `allenai/Molmo2-O-7B`, `allenai/MolmoWeb-4B`<sup>^</sup>, `allenai/MolmoWeb-8B`<sup>^</sup> | ✅︎ | ✅︎ |
 | `MossAudioModel` | MOSS-Audio | T + A<sup>+</sup> | `OpenMOSS-Team/MOSS-Audio-4B-Instruct`, `OpenMOSS-Team/MOSS-Audio-4B-Thinking`, `OpenMOSS-Team/MOSS-Audio-8B-Instruct`, `OpenMOSS-Team/MOSS-Audio-8B-Thinking` | ✅︎ | ✅︎ |
 | `Moondream3ForCausalLM` | Moondream3 | T + I | `moondream/moondream3-preview` | | ✅︎ |
-| `MusicFlamingoForConditionalGeneration` | MusicFlamingo | T + A | `nvidia/music-flamingo-2601-hf`, `nvidia/music-flamingo-think-2601-hf` | ✅︎ | ✅︎ |
 | `NVLM_D_Model` | NVLM-D 1.0 | T + I<sup>+</sup> | `nvidia/NVLM-D-72B`, etc. | | ✅︎ |
 | `OpenCUAForConditionalGeneration` | OpenCUA-7B | T + I<sup>E+</sup> | `xlangai/OpenCUA-7B` | ✅︎ | ✅︎ |
 | `OpenPanguVLForConditionalGeneration` | openpangu-VL | T + I<sup>E+</sup> + V<sup>E+</sup> | `FreedomIntelligence/openPangu-VL-7B` | ✅︎ | ✅︎ |
@@ -626,8 +624,6 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
 | `Step3VLForConditionalGeneration` | Step3-VL | T + I<sup>+</sup> | `stepfun-ai/step3` | | ✅︎ |
 | `StepVLForConditionalGeneration` | Step3-VL-10B | T + I<sup>+</sup> | `stepfun-ai/Step3-VL-10B` | | ✅︎ |
 | `Step3p7ForConditionalGeneration` | Step-3.7-Flash | T + I<sup>+</sup> | `stepfun-ai/Step-3.7-Flash` | | ✅︎ |
-| `TarsierForConditionalGeneration` | Tarsier | T + I<sup>E+</sup> | `omni-search/Tarsier-7b`, `omni-search/Tarsier-34b` | | ✅︎ |
-| `Tarsier2ForConditionalGeneration`<sup>^</sup> | Tarsier2 | T + I<sup>E+</sup> + V<sup>E+</sup> | `omni-research/Tarsier2-Recap-7b`, `omni-research/Tarsier2-7b-0115` | | ✅︎ |
 | `UltravoxModel` | Ultravox | T + A<sup>E+</sup> | `fixie-ai/ultravox-v0_5-llama-3_2-1b` | ✅︎ | ✅︎ |
 | `UnlimitedOCRForCausalLM` | Unlimited-OCR | T + I<sup>+</sup> | `baidu/Unlimited-OCR`, etc. | ✅︎ | ✅︎ |
 
@@ -680,9 +676,6 @@ Some models are supported only via the [Transformers modeling backend](#transfor
     coordinate decoding and are not exposed by this vLLM implementation.
     See [Moondream3 prompt recipes](../features/multimodal_inputs.md#moondream3-prompt-recipes).
 
-!!! note
-    To use `TIGER-Lab/Mantis-8B-siglip-llama3`, you have to pass `--hf_overrides '{"architectures": ["MantisForConditionalGeneration"]}'` when running vLLM.
-
 !!! note
     The official `openbmb/MiniCPM-V-2` doesn't work yet, so we need to use a fork (`HwwwH/MiniCPM-V-2`) for now.
     For more details, please see: <https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630>

docs/serving/online_serving/README.md

@@ -119,9 +119,9 @@ For further details on profiling vLLM, please refer to [this page](../../contrib
 - `/ping` - SageMaker health check
 - `/invocations` - SageMaker-compatible endpoint (routes to the same inference functions as `/v1` endpoints)
 
-## Disaggregated Everything
+## Scale-Out APIs
 
-### Tokens IN <> Tokens OUT
+### Tokens IN <> Tokens OUT APIs
 
 - `/inference/v1/generate` - Generate completions
 - `/abort_requests` - Abort in-flight requests (only when `--tokens-only` is also set)

docs/training/layerwise.md

@@ -58,7 +58,7 @@ class Fp8PerTensorOnlineLinearMethod(LinearMethodBase):
 
 ### High Level Weight Transfer API
 
-The layerwise reloading system is integrated with the post-training weight transfer system. To use layerwise reloading in conjunction to the weight transfer system, follow the examples found [here](../../examples/rl/). Layerwise reloading is controlled by the `WeightTransferUpdateInfo.is_checkpoint_format` flag and is set to `True` by default.
+The layerwise reloading system is integrated with the post-training weight transfer system. To use layerwise reloading in conjunction to the weight transfer system, follow the examples found [here](../../examples/rl/). Checkpoint-format weight transfer engines (e.g. the NCCL and IPC backends) run layerwise reloading automatically inside their `start_weight_update`/`finish_weight_update` lifecycle.
 
 ### Mid Level `reload_weights` API
 

docs/training/weight_transfer/README.md

@@ -17,6 +17,7 @@ The weight transfer system follows a **four-phase protocol** with a pluggable ba
 | ------- | --------- | -------- |
 | [NCCL](nccl.md) | NCCL broadcast | Separate GPUs for training and inference |
 | [IPC](ipc.md) | CUDA IPC handles | Colocated training and inference on same GPU |
+| [sparse_nccl](nccl.md#sparse-nccl) | NCCL broadcast | Sparse flat-index weight patches (TP=1/PP=1) |
 
 ## Configuration
 
@@ -41,7 +42,7 @@ vllm serve my-model \
     --weight-transfer-config '{"backend": "nccl"}'
 ```
 
-The `backend` field accepts `"nccl"` (default) or `"ipc"`.
+The `backend` field accepts `"nccl"` (default), `"ipc"`, or `"sparse_nccl"`.
 
 ## API Endpoints
 
@@ -69,7 +70,7 @@ Both backends provide static methods that the trainer calls to send weights. The
 EngineClass.trainer_init(init_info)
 
 # 2. Start weight update on inference side
-llm.start_weight_update(is_checkpoint_format=True)
+llm.start_weight_update()
 
 # 3. Send weights to inference workers
 EngineClass.trainer_send_weights(

docs/training/weight_transfer/base.md

@@ -11,15 +11,23 @@ The `WeightTransferEngine` is a generic abstract class parameterized by two data
 
 ### Abstract Methods
 
-Subclasses must implement these four methods:
+Subclasses must implement these methods:
 
 | Method | Side | Description |
 | ------ | ---- | ----------- |
 | `init_transfer_engine(init_info)` | Inference | Initialize the communication channel on each inference worker |
-| `receive_weights(update_info, load_weights)` | Inference | Receive weights and call `load_weights` incrementally |
+| `start_weight_update()` | Inference | Prepare for an update (e.g. begin layerwise reload); no-op for in-place engines |
+| `finish_weight_update()` | Inference | Finalize the update (e.g. finalize layerwise reload); no-op for in-place engines |
+| `receive_weights(update_info)` | Inference | Receive weights and load them into `self.model` |
 | `shutdown()` | Inference | Clean up resources |
 | `trainer_send_weights(iterator, trainer_args)` | Trainer | Static method to send weights from the trainer process |
 
+The base class provides two methods:
+
+1. `__init__` : Engines receive `config` (`WeightTransferConfig`),  `vllm_config` (`VllmConfig`), `device` (`torch.device`) and  `model` (`nn.Module`)  
+2. `update_weights(update_info_dict)`:  Thin wrapper for `receive_weights`: parses
+the dict into user-specified data type, calls `receive_weights`, and synchronizes the device. Subclasses implement `receive_weights`.
+
 ### Request Classes
 
 The API-level request classes provide backend-agnostic serialization using plain dictionaries. The engine's `parse_init_info` and `parse_update_info` methods convert these dictionaries into typed dataclasses.
@@ -81,7 +89,7 @@ class MyUpdateInfo(WeightTransferUpdateInfo):
 ### 2. Implement the Engine
 
 ```python
-from collections.abc import Callable, Iterator
+from collections.abc import Iterator
 from typing import Any
 import torch
 
@@ -93,18 +101,25 @@ class MyWeightTransferEngine(WeightTransferEngine[MyInitInfo, MyUpdateInfo]):
         # Set up connection to trainer using init_info.endpoint, etc.
         ...
 
-    def receive_weights(
-        self,
-        update_info: MyUpdateInfo,
-        load_weights: Callable[[list[tuple[str, torch.Tensor]]], None],
-    ) -> None:
-        # Receive each weight and call load_weights incrementally
+    def start_weight_update(self) -> None:
+        # Checkpoint-format engines: run initialize_layerwise_reload(self.model).
+        # In-place engines: no-op
+        ...
+
+    def finish_weight_update(self) -> None:
+        # Checkpoint-format engines: run finalize_layerwise_reload(...).
+        # In-place engines: no-op
+        ...
+
+    def receive_weights(self, update_info: MyUpdateInfo) -> None:
+        weights = []
         for name, dtype_name, shape in zip(
             update_info.names, update_info.dtype_names, update_info.shapes
         ):
             dtype = getattr(torch, dtype_name)
             weight = self._fetch_weight(name, shape, dtype)
-            load_weights([(name, weight)])
+            weights.append((name, weight))
+        self.model.load_weights(weights)
 
     def shutdown(self) -> None:
         # Clean up resources
@@ -121,9 +136,6 @@ class MyWeightTransferEngine(WeightTransferEngine[MyInitInfo, MyUpdateInfo]):
             ...
 ```
 
-!!! important
-    The `load_weights` callable passed to `receive_weights` should be called **incrementally** (one or a few weights at a time) rather than accumulating all weights first. This avoids GPU out-of-memory errors with large models.
-
 ### 3. Register with the Factory
 
 ```python
@@ -147,7 +159,7 @@ Once registered, users can select your backend via `WeightTransferConfig(backend
 
 ## WeightTransferEngineFactory
 
-The factory uses a registry pattern with lazy loading. Built-in engines (`nccl` and `ipc`) are registered at import time but their modules are only loaded when the backend is actually requested. This avoids importing heavy dependencies (like NCCL communicators) when they aren't needed.
+The factory uses a registry pattern with lazy loading. Built-in engines (`nccl`, `ipc`, and `sparse_nccl`) are registered at import time but their modules are only loaded when the backend is actually requested. This avoids importing heavy dependencies (like NCCL communicators) when they aren't needed.
 
 ```python
 from vllm.distributed.weight_transfer.factory import WeightTransferEngineFactory
@@ -155,7 +167,8 @@ from vllm.distributed.weight_transfer.factory import WeightTransferEngineFactory
 # Create an engine from config
 engine = WeightTransferEngineFactory.create_engine(
     config=weight_transfer_config,
-    parallel_config=parallel_config,
+    vllm_config=vllm_config,
+    device=device,
     model=model,
 )
 ```

docs/training/weight_transfer/ipc.md

@@ -55,7 +55,7 @@ trainer_args = IPCTrainerSendWeightsArgs(
     llm_handle=llm_actor_handle,
 )
 # start
-ray.get(llm_actor_handle.start_weight_update.remote(is_checkpoint_format=True))
+ray.get(llm_actor_handle.start_weight_update.remote())
 # send weights
 IPCWeightTransferEngine.trainer_send_weights(
     iterator=model.named_parameters(),
@@ -80,7 +80,7 @@ trainer_args = IPCTrainerSendWeightsArgs(
 # start
 base_url = "http://localhost:8000"
 url = f"{base_url}/start_weight_update"
-response = requests.post(url, json={"is_checkpoint_format": True}, timeout=60)
+response = requests.post(url, json={}, timeout=60)
 response.raise_for_status()
 # send weights
 IPCWeightTransferEngine.trainer_send_weights(

docs/training/weight_transfer/nccl.md

@@ -11,7 +11,7 @@ The NCCL weight transfer engine uses [NCCL](https://developer.nvidia.com/nccl) b
 ## How It Works
 
 1. The trainer and all inference workers join a shared NCCL process group using `StatelessProcessGroup` (vLLM's torch.distributed-independent group abstraction).
-2. The trainer broadcasts weights to all workers simultaneously. Each worker receives and loads weights incrementally.
+2. The trainer broadcasts weights to all workers simultaneously. Each worker receives and loads the weights.
 3. Optionally, **packed tensor broadcasting** batches multiple small tensors into larger buffers with double/triple buffering and CUDA stream overlap for higher throughput. This implementation is based on [NeMo-RL's packed tensor](https://github.com/NVIDIA-NeMo/RL/blob/main/nemo_rl/utils/packed_tensor.py).
 
 ## Initialization
@@ -93,7 +93,7 @@ remaining three steps are:
 from vllm.distributed.weight_transfer.base import WeightTransferUpdateRequest
 
 # 1. Start the weight update
-llm.start_weight_update(is_checkpoint_format=True)
+llm.start_weight_update()
 
 # 2. Receive weights (can be called multiple times for chunked transfers)
 llm.update_weights(
@@ -116,19 +116,22 @@ must match the order in which the trainer iterates over its parameters.
 
 `start_weight_update` must be called before `update_weights`, and
 `finish_weight_update` must be called after all weight chunks have been
-transferred. The `is_checkpoint_format` flag controls whether layerwise reload
-processing is applied (`True` for checkpoint-format weights, `False` for
-pre-processed kernel-format weights).
+transferred. The NCCL engine receives checkpoint-format weights and applies
+layerwise reload processing automatically inside `start_weight_update` /
+`finish_weight_update`.
 
-Sparse NCCL patches still use `update_kind="sparse_flat"` inside
-`update_info`, but they should be wrapped in
-`start_weight_update(is_checkpoint_format=False)` because sparse patches apply
-directly to runtime/kernel-format parameters. The current sparse MVP requires
-`TP=1` and `PP=1`.
+## Sparse NCCL
+
+Sparse, flat-index weight patches use a separate backend,
+`WeightTransferConfig(backend="sparse_nccl")`, implemented by
+`SparseNCCLWeightTransferEngine`. It shares only NCCL process-group
+initialization with the dense engine; patches are applied directly in place to
+existing parameters (no layerwise reload). The current sparse MVP requires
+`TP=1` and `PP=1`. See the example below.
 
 ## Examples
 
 - [RLHF with NCCL weight syncing (offline, Ray)](../../../examples/rl/rlhf_nccl.py) - Trainer on one GPU, 2x tensor-parallel vLLM engine on two others, with packed NCCL weight broadcast
-- [RLHF with sparse NCCL weight syncing (offline, Ray)](../../../examples/rl/rlhf_sparse_nccl.py) - Dense-vs-sparse equivalence demo with a real model on a 2-GPU trainer/inference setup; sparse patches use `start_weight_update(is_checkpoint_format=False)` and currently require `TP=1` and `PP=1`
+- [RLHF with sparse NCCL weight syncing (offline, Ray)](../../../examples/rl/rlhf_sparse_nccl.py) - Dense-vs-sparse equivalence demo with a real model on a 2-GPU trainer/inference setup; sparse patches use `backend="sparse_nccl"` and currently require `TP=1` and `PP=1`
 - [RLHF with async weight syncing (offline, Ray)](../../../examples/rl/rlhf_async_new_apis.py) - Async generation with mid-flight pause, weight sync, resume, and validation against a fresh model
 - [RLHF with NCCL weight syncing (online serving, HTTP)](../../../examples/rl/rlhf_http_nccl.py) - Weight transfer with a running vLLM HTTP server using HTTP control plane and NCCL data plane

docs/usage/security.md

@@ -85,6 +85,21 @@ significantly reduce the attack surface for these types of abuse.
 Also, consider setting `VLLM_MEDIA_URL_ALLOW_REDIRECTS=0` to prevent HTTP
 redirects from being followed to bypass domain restrictions.
 
+### 5. **Restrict Media Decode Sizes:**
+
+Compressed media files can expand into gigabytes of memory during decoding. vLLM
+enforces decode-size limits to prevent out-of-memory denial of service:
+
+| Environment Variable | Default | Description |
+| --- | --- | --- |
+| `VLLM_MAX_IMAGE_PIXELS` | `178956970` (~179M pixels) | Maximum decoded image size in pixels. Images exceeding this are rejected before raster memory is allocated. Default matches PIL's built-in 2x decompression-bomb threshold (~680 MB for RGB). |
+| `VLLM_MAX_AUDIO_CLIP_FILESIZE_MB` | `25` | Maximum filesize in MB for a single audio file. |
+| `VLLM_MAX_AUDIO_DECODE_DURATION_S` | `600` | Maximum decoded audio duration in seconds. Prevents compressed audio from expanding into gigabytes of float32 PCM. |
+
+Setting any of these to `0` disables the corresponding limit. This is **not
+recommended** for deployments exposed to untrusted users, as it removes the
+protection against resource-exhaustion attacks.
+
 ## Security and Firewalls: Protecting Exposed vLLM Systems
 
 While vLLM is designed to allow unsafe network services to be isolated to
@@ -311,6 +326,27 @@ vLLM supports dynamically loading and unloading LoRA adapters at runtime via the
 
 **Warning:** Dynamic LoRA loading is not a secure operation and should not be enabled in deployments exposed to untrusted clients. If you must enable dynamic LoRA loading, restrict access to the `/v1/load_lora_adapter` and `/v1/unload_lora_adapter` endpoints to trusted administrators only, using a reverse proxy or network-level access controls. Do not expose these endpoints to end users. For details on configuring LoRA adapters, see the [LoRA Adapters documentation](../features/lora.md).
 
+## gRPC Interface
+
+vLLM provides an optional gRPC Generate service on a separate TCP port, enabled via the `--grpc-port` flag. When not specified, no gRPC server is started. The gRPC listener binds to the same host address as the HTTP server.
+
+**Warning:** The gRPC interface is **insecure by default** — it does not implement authentication, authorization, or encryption. It should be considered a private, internal interface intended for use only between co-located services within a trusted network. Do not expose the gRPC port to the public internet or untrusted clients. If you enable the gRPC interface, protect it via network-level access controls such as firewall rules, network segmentation, or deployment on an isolated private network.
+
+### Security Implications
+
+An attacker who can reach the gRPC port can:
+
+1. **Run arbitrary inference** via the `Generate` and `GenerateStream` RPCs without any credentials
+2. **Consume GPU and compute resources** by submitting unbounded generation requests
+3. **Cause Denial of Service** by exploiting bugs in the gRPC interface that can crash vLLM.
+
+### Recommendations
+
+- Only enable `--grpc-port` when you have a specific need for gRPC-based inference
+- Ensure the gRPC port is only accessible from trusted hosts or services
+- Use firewall rules to block external access to the gRPC port
+- Consider deploying the gRPC interface on a dedicated internal network interface
+
 ## Cache Directory Security
 
 vLLM assumes that its cache directories are **private and trusted**. Cache contents are loaded without cryptographic integrity verification, including formats that support arbitrary code execution. If an untrusted user or process can write to vLLM's cache directories, they may be able to crash vLLM or cause it to execute arbitrary code.

examples/disaggregated/disaggregated_serving/moriio_toy_proxy_server.py

@@ -327,6 +327,9 @@ async def handle_request(api: str, request: Request):
         session, decode_response = await decode_request_task
         stream_generator = stream_decode_response(session, decode_response, request_id)
         response = await make_response(stream_generator)
+        response.headers["Content-Type"] = decode_response.headers.get(
+            "Content-Type", "application/json"
+        )
         return response
     except Exception as e:
         logger.exception("An error occurred while handling the request: %s", e)

examples/generate/multimodal/audio_language_offline.py

@@ -91,44 +91,6 @@ def run_cohere_asr(question: str, audio_count: int) -> ModelRequestData:
     )
 
 
-# MusicFlamingo
-def run_musicflamingo(question: str, audio_count: int) -> ModelRequestData:
-    model_name = "nvidia/music-flamingo-2601-hf"
-    engine_args = EngineArgs(
-        model=model_name,
-        max_model_len=4096,
-        max_num_seqs=2,
-        limit_mm_per_prompt={"audio": audio_count},
-        enforce_eager=True,
-    )
-
-    # MusicFlamingo prompt placeholders use <sound>; vLLM's MusicFlamingo
-    # multimodal processor expands each one into <|sound_bos|> + audio tokens +
-    # <|sound_eos|> based on extracted audio feature lengths.
-    audio_placeholder = "<sound>" * audio_count
-    system_prompt = (
-        "You are Music Flamingo, a multimodal assistant for language and music. "
-        "On each turn you receive an audio clip which contains music and optional "
-        "text, you will receive at least one or both; use your world knowledge and "
-        "reasoning to help the user with any task. Interpret the entirety of the "
-        "content any input music--regardlenss of whether the user calls it audio, "
-        "music, or sound."
-    )
-
-    prompt = (
-        "<|im_start|>system\n"
-        f"{system_prompt}<|im_end|>\n"
-        "<|im_start|>user\n"
-        f"{audio_placeholder}{question}<|im_end|>\n"
-        "<|im_start|>assistant\n"
-    )
-
-    return ModelRequestData(
-        engine_args=engine_args,
-        prompt=prompt,
-    )
-
-
 # Gemma3N
 def run_gemma3n(question: str, audio_count: int) -> ModelRequestData:
     model_name = "google/gemma-3n-E2B-it"
@@ -565,7 +527,6 @@ model_example_map = {
     "kimi_audio": run_kimi_audio,
     "midashenglm": run_midashenglm,
     "minicpmo": run_minicpmo,
-    "musicflamingo": run_musicflamingo,
     "phi4_mm": run_phi4mm,
     "qwen2_audio": run_qwen2_audio,
     "qwen2_5_omni": run_qwen2_5_omni,

examples/generate/multimodal/vision_language_multi_image_offline.py

@@ -74,39 +74,6 @@ def load_aria(question: str, image_urls: list[str]) -> ModelRequestData:
     )
 
 
-def load_aya_vision(question: str, image_urls: list[str]) -> ModelRequestData:
-    model_name = "CohereLabs/aya-vision-8b"
-
-    engine_args = EngineArgs(
-        model=model_name,
-        max_num_seqs=2,
-        limit_mm_per_prompt={"image": len(image_urls)},
-    )
-
-    placeholders = [{"type": "image", "image": url} for url in image_urls]
-    messages = [
-        {
-            "role": "user",
-            "content": [
-                *placeholders,
-                {"type": "text", "text": question},
-            ],
-        }
-    ]
-
-    processor = AutoProcessor.from_pretrained(model_name)
-
-    prompt = processor.apply_chat_template(
-        messages, tokenize=False, add_generation_prompt=True
-    )
-
-    return ModelRequestData(
-        engine_args=engine_args,
-        prompt=prompt,
-        image_data=[fetch_image(url) for url in image_urls],
-    )
-
-
 def load_bee(question: str, image_urls: list[str]) -> ModelRequestData:
     model_name = "Open-Bee/Bee-8B-RL"
 
@@ -1275,55 +1242,6 @@ def load_step_vl(question: str, image_urls: list[str]) -> ModelRequestData:
     )
 
 
-def load_tarsier(question: str, image_urls: list[str]) -> ModelRequestData:
-    model_name = "omni-research/Tarsier-7b"
-
-    engine_args = EngineArgs(
-        model=model_name,
-        trust_remote_code=True,
-        max_model_len=4096,
-        limit_mm_per_prompt={"image": len(image_urls)},
-    )
-
-    prompt = f"USER: {'<image>' * len(image_urls)}\n{question}\n ASSISTANT:"
-    image_data = [fetch_image(url) for url in image_urls]
-
-    return ModelRequestData(
-        engine_args=engine_args,
-        prompt=prompt,
-        image_data=image_data,
-    )
-
-
-def load_tarsier2(question: str, image_urls: list[str]) -> ModelRequestData:
-    model_name = "omni-research/Tarsier2-Recap-7b"
-
-    engine_args = EngineArgs(
-        model=model_name,
-        trust_remote_code=True,
-        max_model_len=32768,
-        limit_mm_per_prompt={"image": len(image_urls)},
-        hf_overrides={
-            "architectures": ["Tarsier2ForConditionalGeneration"],
-            "model_type": "tarsier2",
-        },
-    )
-
-    prompt = (
-        "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
-        f"<|im_start|>user\n<|vision_start|>{'<|image_pad|>' * len(image_urls)}"
-        f"<|vision_end|>{question}<|im_end|>\n"
-        "<|im_start|>assistant\n"
-    )
-    image_data = [fetch_image(url) for url in image_urls]
-
-    return ModelRequestData(
-        engine_args=engine_args,
-        prompt=prompt,
-        image_data=image_data,
-    )
-
-
 # GLM-4.1V
 def load_glm4_1v(question: str, image_urls: list[str]) -> ModelRequestData:
     model_name = "zai-org/GLM-4.1V-9B-Thinking"
@@ -1469,7 +1387,6 @@ def load_molmo2(question: str, image_urls: list[str]) -> ModelRequestData:
 
 model_example_map = {
     "aria": load_aria,
-    "aya_vision": load_aya_vision,
     "bee": load_bee,
     "command_a_vision": load_command_a_vision,
     "deepseek_vl_v2": load_deepseek_vl2,
@@ -1507,8 +1424,6 @@ model_example_map = {
     "smolvlm": load_smolvlm,
     "step3": load_step3,
     "stepvl": load_step_vl,
-    "tarsier": load_tarsier,
-    "tarsier2": load_tarsier2,
     "glm4_1v": load_glm4_1v,
     "glm4_5v": load_glm4_5v,
     "glm4_5v_fp8": load_glm4_5v_fp8,

examples/generate/multimodal/vision_language_offline.py

@@ -68,28 +68,6 @@ def run_aria(questions: list[str], modality: str) -> ModelRequestData:
     )
 
 
-# Aya Vision
-def run_aya_vision(questions: list[str], modality: str) -> ModelRequestData:
-    assert modality == "image"
-    model_name = "CohereLabs/aya-vision-8b"
-
-    engine_args = EngineArgs(
-        model=model_name,
-        max_model_len=2048,
-        max_num_seqs=2,
-        mm_processor_kwargs={"crop_to_patches": True},
-        limit_mm_per_prompt={modality: 1},
-    )
-    prompts = [
-        f"<|START_OF_TURN_TOKEN|><|USER_TOKEN|><image>{question}<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>"
-        for question in questions
-    ]
-    return ModelRequestData(
-        engine_args=engine_args,
-        prompts=prompts,
-    )
-
-
 # Bee-8B
 def run_bee(questions: list[str], modality: str) -> ModelRequestData:
     assert modality == "image"
@@ -1377,28 +1355,6 @@ def run_llava_onevision(questions: list[str], modality: str) -> ModelRequestData
     )
 
 
-# Mantis
-def run_mantis(questions: list[str], modality: str) -> ModelRequestData:
-    assert modality == "image"
-
-    llama3_template = "<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n"  # noqa: E501
-    prompts = [llama3_template.format(f"{question}\n<image>") for question in questions]
-
-    engine_args = EngineArgs(
-        model="TIGER-Lab/Mantis-8B-siglip-llama3",
-        max_model_len=4096,
-        hf_overrides={"architectures": ["MantisForConditionalGeneration"]},
-        limit_mm_per_prompt={modality: 1},
-    )
-    stop_token_ids = [128009]
-
-    return ModelRequestData(
-        engine_args=engine_args,
-        prompts=prompts,
-        stop_token_ids=stop_token_ids,
-    )
-
-
 # MiniCPM-V
 def run_minicpmv_base(questions: list[str], modality: str, model_name):
     assert modality in ["image", "video", "image+video"]
@@ -2347,68 +2303,8 @@ def run_step_vl(questions: list[str], modality: str) -> ModelRequestData:
     )
 
 
-# omni-research/Tarsier-7b
-def run_tarsier(questions: list[str], modality: str) -> ModelRequestData:
-    assert modality == "image"
-    model_name = "omni-research/Tarsier-7b"
-
-    engine_args = EngineArgs(
-        model=model_name,
-        trust_remote_code=True,
-        max_model_len=4096,
-        limit_mm_per_prompt={modality: 1},
-    )
-    prompts = [(f"USER: <image>\n{question} ASSISTANT:") for question in questions]
-
-    return ModelRequestData(
-        engine_args=engine_args,
-        prompts=prompts,
-    )
-
-
-def run_tarsier2(questions: list[str], modality: str) -> ModelRequestData:
-    model_name = "omni-research/Tarsier2-Recap-7b"
-
-    mm_limit = {"image": 1, "video": 1} if modality == "image+video" else {modality: 1}
-    engine_args = EngineArgs(
-        model=model_name,
-        max_model_len=4096,
-        hf_overrides={
-            "architectures": ["Tarsier2ForConditionalGeneration"],
-            "model_type": "tarsier2",
-        },
-        limit_mm_per_prompt=mm_limit,
-    )
-
-    image_placeholder = "<|vision_start|><|image_pad|><|vision_end|>"
-    video_placeholder = "<|vision_start|><|video_pad|><|vision_end|>"
-
-    if modality == "image":
-        placeholder = image_placeholder
-    elif modality == "video":
-        placeholder = video_placeholder
-    elif modality == "image+video":
-        placeholder = image_placeholder + video_placeholder
-
-    prompts = [
-        (
-            "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
-            f"<|im_start|>user\n{placeholder}"
-            f"{question}<|im_end|>\n"
-            "<|im_start|>assistant\n"
-        )
-        for question in questions
-    ]
-
-    return ModelRequestData(
-        engine_args=engine_args,
-        prompts=prompts,
-    )
-
-
 model_example_map = {
     "aria": run_aria,
-    "aya_vision": run_aya_vision,
     "bagel": run_bagel,
     "cheers": run_cheers,
     "bee": run_bee,
@@ -2449,7 +2345,6 @@ model_example_map = {
     "llava-next": run_llava_next,
     "llava-next-video": run_llava_next_video,
     "llava-onevision": run_llava_onevision,
-    "mantis": run_mantis,
     "minicpmo": run_minicpmo,
     "minicpmv": run_minicpmv,
     "mistral3": run_mistral3,
@@ -2479,8 +2374,6 @@ model_example_map = {
     "smolvlm": run_smolvlm,
     "step3": run_step3,
     "stepvl": run_step_vl,
-    "tarsier": run_tarsier,
-    "tarsier2": run_tarsier2,
 }
 
 

examples/rl/rlhf_async_new_apis.py

@@ -306,7 +306,7 @@ gen_futures = [
 
 ray.get(llm.pause_after_n_tokens.remote())
 
-ray.get(llm.start_weight_update.remote(is_checkpoint_format=True))
+ray.get(llm.start_weight_update.remote())
 
 inference_handle = llm.update_weights.remote(
     WeightTransferUpdateRequest(

examples/rl/rlhf_http_ipc.py

@@ -80,14 +80,10 @@ def init_weight_transfer_engine(base_url: str) -> None:
     response.raise_for_status()
 
 
-def start_weight_update(
-    base_url: str,
-    is_checkpoint_format: bool = True,
-) -> None:
+def start_weight_update(base_url: str) -> None:
     """Start a weight update via HTTP endpoint."""
     url = f"{base_url}/start_weight_update"
-    payload = {"is_checkpoint_format": is_checkpoint_format}
-    response = requests.post(url, json=payload, timeout=60)
+    response = requests.post(url, json={}, timeout=60)
     response.raise_for_status()
 
 
@@ -170,7 +166,7 @@ def main():
     pause_generation(BASE_URL)
 
     # Start weight update, broadcast via IPC, then finish
-    start_weight_update(BASE_URL, is_checkpoint_format=False)
+    start_weight_update(BASE_URL)
 
     print("Broadcasting weights via CUDA IPC (HTTP)...")
     trainer_args = IPCTrainerSendWeightsArgs(send_mode="http", url=BASE_URL)

examples/rl/rlhf_http_nccl.py

@@ -83,14 +83,10 @@ def init_weight_transfer_engine(
     response.raise_for_status()
 
 
-def start_weight_update(
-    base_url: str,
-    is_checkpoint_format: bool = True,
-) -> None:
+def start_weight_update(base_url: str) -> None:
     """Start a weight update via HTTP endpoint."""
     url = f"{base_url}/start_weight_update"
-    payload = {"is_checkpoint_format": is_checkpoint_format}
-    response = requests.post(url, json=payload, timeout=60)
+    response = requests.post(url, json={}, timeout=60)
     response.raise_for_status()
 
 
@@ -223,7 +219,7 @@ def main():
         shapes.append(list(p.shape))
 
     # Start weight update
-    start_weight_update(BASE_URL, is_checkpoint_format=True)
+    start_weight_update(BASE_URL)
 
     # Start the update_weights call in a separate thread since it will block
     # waiting for NCCL broadcasts

examples/rl/rlhf_ipc.py

@@ -139,7 +139,7 @@ ray.get(llm.sleep.remote(level=0))
 
 ray.get(train_model.init_weight_transfer.remote())
 # Start weight update, sync weights, then finish
-ray.get(llm.start_weight_update.remote(is_checkpoint_format=True))
+ray.get(llm.start_weight_update.remote())
 ray.get(train_model.broadcast_weights.remote(llm))
 ray.get(llm.finish_weight_update.remote())
 

examples/rl/rlhf_ipc_fsdp_ep.py

@@ -277,15 +277,8 @@ class DataParallelInferenceEngine:
             ]
         )
 
-    def start_weight_update(self, is_checkpoint_format: bool = True):
-        ray.get(
-            [
-                actor.start_weight_update.remote(
-                    is_checkpoint_format=is_checkpoint_format
-                )
-                for actor in self.llm_actors
-            ]
-        )
+    def start_weight_update(self):
+        ray.get([actor.start_weight_update.remote() for actor in self.llm_actors])
 
     def finish_weight_update(self):
         ray.get([actor.finish_weight_update.remote() for actor in self.llm_actors])
@@ -392,7 +385,7 @@ def main():
     ray.get(inference_engine.wake_up.remote(tags=["weights"]))
 
     print("[sync] Starting weight update...")
-    ray.get(inference_engine.start_weight_update.remote(is_checkpoint_format=True))
+    ray.get(inference_engine.start_weight_update.remote())
 
     print("[sync] Packed IPC transfer FSDP → vLLM...")
     ray.get(

examples/rl/rlhf_nccl.py

@@ -202,7 +202,7 @@ ray.get([train_handle, inference_handle])
 names, dtype_names, shapes = ray.get(train_model.get_weight_metadata.remote())
 
 # Start weight update
-ray.get(llm.start_weight_update.remote(is_checkpoint_format=True))
+ray.get(llm.start_weight_update.remote())
 
 # Issue update_weights call with NCCL-specific update info
 # packed=True enables efficient batched tensor broadcasting

examples/rl/rlhf_nccl_fsdp_ep.py

@@ -299,7 +299,7 @@ async def main():
     print(f"[sync] Got metadata for {len(names)} parameters.")
 
     print("[sync] Starting weight update...")
-    await engine.start_weight_update(is_checkpoint_format=True)
+    await engine.start_weight_update()
 
     print("[sync] Broadcasting weights from FSDP → vLLM...")
     broadcast_handles = [

examples/rl/rlhf_sparse_nccl.py

@@ -44,11 +44,14 @@ from transformers import AutoModelForCausalLM, AutoTokenizer
 
 from vllm import LLM, SamplingParams
 from vllm.config import WeightTransferConfig
-from vllm.distributed.weight_transfer.base import SparseWeightPatch
 from vllm.distributed.weight_transfer.nccl_engine import (
     NCCLTrainerSendWeightsArgs,
     NCCLWeightTransferEngine,
 )
+from vllm.distributed.weight_transfer.sparse_nccl_engine import (
+    SparseNCCLWeightTransferEngine,
+    SparseWeightPatch,
+)
 from vllm.utils.network_utils import get_ip, get_open_port
 
 MODEL_NAME = "Qwen/Qwen2.5-0.5B-Instruct"
@@ -244,7 +247,6 @@ class TrainModel:
             dtype_names=[str(self.patched_param.dtype).split(".")[-1]],
             shapes=[list(self.patched_param.shape)],
             num_updates_list=[flat_indices.numel()],
-            update_kind="sparse_flat",
         )
         return update_info, selected_token_ids, patch_digest, sparse_payload_bytes
 
@@ -271,7 +273,7 @@ class TrainModel:
             raise RuntimeError("Sparse patch has not been prepared")
 
         start = time.perf_counter()
-        NCCLWeightTransferEngine.trainer_send_sparse_weights(
+        SparseNCCLWeightTransferEngine.trainer_send_weights(
             iter(self.pending_sparse_patches),
             NCCLTrainerSendWeightsArgs(group=self.model_update_group),
         )
@@ -282,6 +284,7 @@ class TrainModel:
 
 def launch_llm(
     scheduling_inference: PlacementGroupSchedulingStrategy,
+    backend: str = "nccl",
 ):
     return ray.remote(
         num_cpus=0,
@@ -293,7 +296,7 @@ def launch_llm(
         tensor_parallel_size=1,
         distributed_executor_backend="ray",
         gpu_memory_utilization=0.7,
-        weight_transfer_config=WeightTransferConfig(backend="nccl"),
+        weight_transfer_config=WeightTransferConfig(backend=backend),
     )
 
 
@@ -332,7 +335,7 @@ def run_dense_phase(
     scheduling_inference: PlacementGroupSchedulingStrategy,
 ) -> dict[str, object]:
     ray.get(train_model.reset_model.remote())
-    llm = launch_llm(scheduling_inference)
+    llm = launch_llm(scheduling_inference, backend="nccl")
     try:
         dense_before = collect_vllm_generations(llm)
 
@@ -351,7 +354,7 @@ def run_dense_phase(
         )
         trainer_init = train_model.init_weight_transfer_group.remote(world_size)
         ray.get([trainer_init, inference_init])
-        ray.get(llm.start_weight_update.remote(is_checkpoint_format=True))
+        ray.get(llm.start_weight_update.remote())
 
         dense_update_info, dense_payload_bytes = ray.get(
             train_model.get_dense_update_info.remote()
@@ -391,7 +394,7 @@ def run_sparse_phase(
     scheduling_inference: PlacementGroupSchedulingStrategy,
 ) -> dict[str, object]:
     ray.get(train_model.reset_model.remote())
-    llm = launch_llm(scheduling_inference)
+    llm = launch_llm(scheduling_inference, backend="sparse_nccl")
     try:
         sparse_before = collect_vllm_generations(llm)
 
@@ -410,7 +413,7 @@ def run_sparse_phase(
         )
         trainer_init = train_model.init_weight_transfer_group.remote(world_size)
         ray.get([trainer_init, inference_init])
-        ray.get(llm.start_weight_update.remote(is_checkpoint_format=False))
+        ray.get(llm.start_weight_update.remote())
 
         sparse_update_info, selected_token_ids, patch_digest, sparse_payload_bytes = (
             ray.get(train_model.prepare_sparse_patch.remote(PROMPTS))

requirements/cuda.txt

@@ -8,10 +8,10 @@ torch==2.11.0
 torchaudio==2.11.0
 # These must be updated alongside torch
 torchvision==0.26.0 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version
-PyNvVideoCodec==2.0.4
+PyNvVideoCodec==2.1.0
 # FlashInfer should be updated together with the Dockerfile
-flashinfer-python==0.6.12
-flashinfer-cubin==0.6.12
+flashinfer-python==0.6.13
+flashinfer-cubin==0.6.13
 apache-tvm-ffi==0.1.9
 tilelang==0.1.9
 nvidia-cudnn-frontend>=1.19.1

requirements/test/cuda.in

@@ -1,3 +1,5 @@
+-r ../common.txt
+
 # testing
 pytest
 tensorizer==2.10.1
@@ -13,7 +15,6 @@ albumentations # required for Nemotron Parse in test_common.py
 av  # required for audio_in_video tests
 backoff # required for phi4mm test
 blobfile # required for kimi-vl test
-einops # required for MPT, qwen-vl
 httpx
 librosa # required for audio tests
 vector_quantize_pytorch # required for minicpmo_26 test
@@ -34,7 +35,6 @@ matplotlib # required for qwen-vl test
 mistral_common[image,audio] >= 1.11.5 # required for voxtral test
 num2words # required for smolvlm test
 open_clip_torch==2.32.0 # Required for nemotron_vl test, Nemotron Parse in test_common.py
-opencv-python-headless >= 4.13.0 # required for video test
 datamodel_code_generator # required for minicpm3 test
 lm-eval[api]>=0.4.12 # required for model evaluation test
 mteb[bm25s]>=2, <3 # required for mteb test
@@ -55,11 +55,9 @@ grpcio-reflection==1.78.0
 
 arctic-inference == 0.1.1; platform_machine == "x86_64" # Required for suffix decoding test
 numba == 0.65.0 # Required for N-gram speculative decoding
-numpy
 runai-model-streamer[s3,gcs,azure]==0.15.7
 fastsafetensors>=0.3.2
 instanttensor>=0.1.5; platform_machine == "x86_64"
-pydantic>=2.12 # 2.11 leads to error on python 3.13
 decord==0.6.0; platform_machine == "x86_64"
 # terratorch is temporarily disabled while PyPI has the `lightning` package
 # in `quarantined` status (every published terratorch version transitively

requirements/test/cuda.txt

@@ -9,6 +9,7 @@ aiohappyeyeballs==2.6.1
 aiohttp==3.13.3
     # via
     #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
     #   aiohttp-cors
     #   datasets
     #   fsspec
@@ -24,17 +25,34 @@ albumentations==1.4.6
 alembic==1.16.4
     # via optuna
 annotated-doc==0.0.4
-    # via fastapi
+    # via
+    #   fastapi
+    #   typer
 annotated-types==0.7.0
     # via pydantic
-anyio==4.6.2.post1
+anthropic==0.112.0
     # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
+anyio==4.14.1
+    # via
+    #   anthropic
     #   httpx
+    #   mcp
+    #   openai
+    #   sse-starlette
     #   starlette
+    #   watchfiles
+apache-tvm-ffi==0.1.9
+    # via
+    #   -c requirements/cuda.txt
+    #   xgrammar
 arctic-inference==0.1.1
     # via -r requirements/test/cuda.in
 argcomplete==3.5.1
     # via datamodel-code-generator
+astor==0.8.1
+    # via depyf
 attrs==24.2.0
     # via
     #   aiohttp
@@ -59,6 +77,8 @@ bitsandbytes==0.49.2
     # via -r requirements/test/cuda.in
 black==24.10.0
     # via datamodel-code-generator
+blake3==1.0.9
+    # via -r requirements/test/../common.txt
 blobfile==3.0.0
     # via -r requirements/test/cuda.in
 bm25s==0.2.13
@@ -76,12 +96,17 @@ bounded-pool-executor==0.0.3
 buildkite-test-collector==0.1.9
     # via -r requirements/test/cuda.in
 cachetools==5.5.2
-    # via google-auth
+    # via
+    #   -r requirements/test/../common.txt
+    #   google-auth
+cbor2==6.1.2
+    # via -r requirements/test/../common.txt
 certifi==2024.8.30
     # via
     #   httpcore
     #   httpx
     #   requests
+    #   sentry-sdk
 cffi==2.0.0
     # via
     #   cryptography
@@ -98,9 +123,11 @@ click==8.1.7
     #   jiwer
     #   nltk
     #   ray
+    #   rich-toolkit
     #   schemathesis
-    #   typer
     #   uvicorn
+cloudpickle==3.1.2
+    # via -r requirements/test/../common.txt
 cohere-melody==0.9.0
     # via -r requirements/test/cuda.in
 colorama==0.4.6
@@ -111,6 +138,10 @@ colorful==0.5.6
     # via ray
 colorlog==6.10.1
     # via optuna
+compressed-tensors==0.17.0
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 contourpy==1.3.0
     # via matplotlib
 coverage==7.10.6
@@ -149,30 +180,49 @@ decorator==5.1.1
     # via librosa
 decord==0.6.0
     # via -r requirements/test/cuda.in
+depyf==0.20.0
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
+detect-installer==0.1.0
+    # via fastapi-cloud-cli
 dill==0.3.8
     # via
     #   datasets
+    #   depyf
     #   evaluate
     #   lm-eval
     #   multiprocess
+diskcache==5.6.3
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 distlib==0.3.9
     # via virtualenv
+distro==1.9.0
+    # via
+    #   anthropic
+    #   openai
 dnspython==2.7.0
     # via email-validator
 docker==7.1.0
     # via gpt-oss
 docopt==0.6.2
     # via num2words
+docstring-parser==0.18.0
+    # via anthropic
 einops==0.8.1
     # via
-    #   -r requirements/test/cuda.in
+    #   -r requirements/test/../common.txt
     #   encodec
     #   vector-quantize-pytorch
     #   vocos
 einx==0.3.0
     # via vector-quantize-pytorch
 email-validator==2.2.0
-    # via pydantic
+    # via
+    #   fastapi
+    #   pydantic
 encodec==0.1.1
     # via vocos
 et-xmlfile==2.0.0
@@ -182,7 +232,17 @@ evaluate==0.4.3
 fastapi==0.136.3
     # via
     #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
     #   gpt-oss
+    #   model-hosting-container-standards
+fastapi-cli==0.0.27
+    # via fastapi
+fastapi-cloud-cli==0.21.0
+    # via fastapi-cli
+fastar==0.11.0
+    # via
+    #   fastapi
+    #   fastapi-cloud-cli
 fastparquet==2024.11.0
     # via genai-perf
 fastrlock==0.8.2
@@ -194,6 +254,7 @@ fastsafetensors==0.3.2
 filelock==3.16.1
     # via
     #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
     #   blobfile
     #   datasets
     #   huggingface-hub
@@ -243,7 +304,10 @@ google-crc32c==1.7.1
 google-resumable-media==2.7.2
     # via google-cloud-storage
 googleapis-common-protos==1.70.0
-    # via google-api-core
+    # via
+    #   google-api-core
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
 gpt-oss==0.0.8
     # via -r requirements/test/cuda.in
 graphql-core==3.2.6
@@ -254,6 +318,7 @@ grpcio==1.78.0
     # via
     #   -r requirements/test/cuda.in
     #   grpcio-reflection
+    #   opentelemetry-exporter-otlp-proto-grpc
     #   ray
 grpcio-reflection==1.78.0
     # via -r requirements/test/cuda.in
@@ -275,12 +340,22 @@ html2text==2025.4.15
     # via gpt-oss
 httpcore==1.0.6
     # via httpx
+httptools==0.8.0
+    # via uvicorn
 httpx==0.27.2
     # via
     #   -r requirements/test/cuda.in
+    #   anthropic
+    #   fastapi
+    #   fastapi-cloud-cli
     #   huggingface-hub
+    #   mcp
+    #   model-hosting-container-standards
+    #   openai
     #   perceptron
     #   schemathesis
+httpx-sse==0.4.3
+    # via mcp
 huggingface-hub==1.10.2
     # via
     #   accelerate
@@ -314,6 +389,8 @@ idna==3.10
     #   httpx
     #   requests
     #   yarl
+ijson==3.5.0
+    # via -r requirements/test/../common.txt
 imagehash==4.3.2
     # via -r requirements/test/cuda.in
 imageio==2.37.0
@@ -326,6 +403,8 @@ iniconfig==2.0.0
     # via pytest
 instanttensor==0.1.5
     # via -r requirements/test/cuda.in
+interegular==0.3.3
+    # via lm-format-enforcer
 isodate==0.7.2
     # via azure-storage-blob
 isort==5.13.2
@@ -333,15 +412,21 @@ isort==5.13.2
 jinja2==3.1.6
     # via
     #   datamodel-code-generator
+    #   fastapi
     #   genai-perf
     #   lm-eval
     #   torch
+jiter==0.15.0
+    # via
+    #   anthropic
+    #   openai
 jiwer==3.0.5
     # via -r requirements/test/cuda.in
 jmespath==1.0.1
     # via
     #   boto3
     #   botocore
+    #   model-hosting-container-standards
 joblib==1.4.2
     # via
     #   librosa
@@ -350,7 +435,9 @@ joblib==1.4.2
 jsonschema==4.23.0
     # via
     #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
     #   hypothesis-jsonschema
+    #   mcp
     #   mistral-common
     #   ray
 jsonschema-rs==0.46.5
@@ -365,6 +452,10 @@ kaleido==0.2.1
     # via genai-perf
 kiwisolver==1.4.7
     # via matplotlib
+lark==1.2.2
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 lazy-loader==0.4
     # via
     #   librosa
@@ -373,10 +464,20 @@ libnacl==2.1.0
     # via tensorizer
 librosa==0.10.2.post1
     # via -r requirements/test/cuda.in
+llguidance==1.7.6
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 llvmlite==0.47.0
     # via numba
 lm-eval==0.4.12
     # via -r requirements/test/cuda.in
+lm-format-enforcer==0.11.3
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
+loguru==0.7.3
+    # via compressed-tensors
 lxml==5.3.0
     # via
     #   blobfile
@@ -398,12 +499,19 @@ mbstrdecoder==1.1.3
     #   dataproperty
     #   pytablewriter
     #   typepy
+mcp==1.28.1
+    # via -r requirements/test/../common.txt
 mdurl==0.1.2
     # via markdown-it-py
 mistral-common==1.11.5
     # via
     #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
     #   -r requirements/test/cuda.in
+model-hosting-container-standards==0.1.16
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 more-itertools==10.5.0
     # via lm-eval
 mpmath==1.3.0
@@ -418,6 +526,8 @@ msgpack==1.1.0
     # via
     #   librosa
     #   ray
+msgspec==0.21.1
+    # via -r requirements/test/../common.txt
 mteb==2.8.3
     # via -r requirements/test/cuda.in
 multidict==6.1.0
@@ -434,6 +544,8 @@ networkx==3.2.1
     # via
     #   scikit-image
     #   torch
+ninja==1.13.0
+    # via -r requirements/test/../common.txt
 nltk==3.9.1
     # via rouge-score
 num2words==0.5.14
@@ -445,7 +557,7 @@ numba==0.65.0
     #   librosa
 numpy==2.2.6
     # via
-    #   -r requirements/test/cuda.in
+    #   -r requirements/test/../common.txt
     #   accelerate
     #   albumentations
     #   bitsandbytes
@@ -489,6 +601,7 @@ numpy==2.2.6
     #   transformers
     #   tritonclient
     #   vocos
+    #   xgrammar
 nvidia-cublas==13.1.0.3
     # via
     #   cuda-toolkit
@@ -530,9 +643,14 @@ nvidia-nvtx==13.0.85
     # via cuda-toolkit
 open-clip-torch==2.32.0
     # via -r requirements/test/cuda.in
+openai==2.44.0
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 openai-harmony==0.0.4
     # via
     #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
     #   gpt-oss
 opencensus==0.11.4
     # via ray
@@ -541,7 +659,7 @@ opencensus-context==0.1.3
 opencv-python-headless==4.13.0.90
     # via
     #   -c requirements/common.txt
-    #   -r requirements/test/cuda.in
+    #   -r requirements/test/../common.txt
     #   albumentations
     #   mistral-common
 openpyxl==3.1.5
@@ -549,24 +667,54 @@ openpyxl==3.1.5
 opentelemetry-api==1.35.0
     # via
     #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
     #   opentelemetry-exporter-prometheus
     #   opentelemetry-sdk
     #   opentelemetry-semantic-conventions
+opentelemetry-exporter-otlp==1.35.0
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
+opentelemetry-exporter-otlp-proto-common==1.35.0
+    # via
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+opentelemetry-exporter-otlp-proto-grpc==1.35.0
+    # via opentelemetry-exporter-otlp
+opentelemetry-exporter-otlp-proto-http==1.35.0
+    # via opentelemetry-exporter-otlp
 opentelemetry-exporter-prometheus==0.56b0
     # via ray
 opentelemetry-proto==1.35.0
-    # via ray
+    # via
+    #   opentelemetry-exporter-otlp-proto-common
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+    #   ray
 opentelemetry-sdk==1.35.0
     # via
     #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
     #   opentelemetry-exporter-prometheus
     #   ray
 opentelemetry-semantic-conventions==0.56b0
     # via opentelemetry-sdk
+opentelemetry-semantic-conventions-ai==0.4.13
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 optuna==3.6.1
     # via genai-perf
 orjson==3.11.5
     # via genai-perf
+outlines-core==0.2.14
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 packaging==24.2
     # via
     #   accelerate
@@ -578,6 +726,7 @@ packaging==24.2
     #   fastparquet
     #   huggingface-hub
     #   lazy-loader
+    #   lm-format-enforcer
     #   matplotlib
     #   optuna
     #   peft
@@ -597,6 +746,8 @@ pandas==2.2.3
     #   fastparquet
     #   genai-perf
     #   statsmodels
+partial-json-parser==0.2.1.1.post7
+    # via -r requirements/test/../common.txt
 pathspec==0.12.1
     # via black
 pathvalidate==3.2.1
@@ -611,6 +762,7 @@ perf-analyzer==0.1.0
     # via genai-perf
 pillow==10.4.0
     # via
+    #   -r requirements/test/../common.txt
     #   genai-perf
     #   imagehash
     #   imageio
@@ -644,8 +796,14 @@ pqdm==0.2.0
 prometheus-client==0.22.0
     # via
     #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
     #   opentelemetry-exporter-prometheus
+    #   prometheus-fastapi-instrumentator
     #   ray
+prometheus-fastapi-instrumentator==8.0.2
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 propcache==0.2.0
     # via
     #   aiohttp
@@ -655,6 +813,7 @@ proto-plus==1.26.1
 protobuf==6.33.6
     # via
     #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
     #   google-api-core
     #   googleapis-common-protos
     #   grpcio-reflection
@@ -664,11 +823,14 @@ protobuf==6.33.6
     #   tensorizer
 psutil==6.1.0
     # via
+    #   -r requirements/test/../common.txt
     #   accelerate
     #   peft
     #   tensorizer
 py==1.11.0
     # via pytest-forked
+py-cpuinfo==9.0.0
+    # via -r requirements/test/../common.txt
 py-spy==0.4.0
     # via ray
 pyarrow==23.0.0
@@ -681,6 +843,8 @@ pyasn1==0.6.1
     #   rsa
 pyasn1-modules==0.4.2
     # via google-auth
+pybase64==1.4.3
+    # via -r requirements/test/../common.txt
 pycountry==24.6.1
     # via pydantic-extra-types
 pycparser==2.22
@@ -690,26 +854,43 @@ pycryptodomex==3.22.0
 pydantic==2.12.0
     # via
     #   -c requirements/common.txt
-    #   -r requirements/test/cuda.in
+    #   -r requirements/test/../common.txt
     #   albumentations
+    #   anthropic
+    #   compressed-tensors
     #   datamodel-code-generator
     #   fastapi
+    #   fastapi-cloud-cli
     #   gpt-oss
+    #   lm-format-enforcer
+    #   mcp
     #   mistral-common
+    #   model-hosting-container-standards
     #   mteb
+    #   openai
     #   openai-harmony
     #   pydantic-extra-types
+    #   pydantic-settings
     #   ray
+    #   xgrammar
 pydantic-core==2.41.1
     # via pydantic
 pydantic-extra-types==2.10.5
-    # via mistral-common
+    # via
+    #   fastapi
+    #   mistral-common
+pydantic-settings==2.14.2
+    # via
+    #   fastapi
+    #   mcp
 pygments==2.18.0
     # via
     #   pytest
     #   rich
 pyjwt==2.11.0
-    # via msal
+    # via
+    #   mcp
+    #   msal
 pyparsing==3.2.0
     # via matplotlib
 pyrate-limiter==4.4.0
@@ -751,6 +932,16 @@ python-dateutil==2.9.0.post0
     #   matplotlib
     #   pandas
     #   typepy
+python-dotenv==1.2.2
+    # via
+    #   pydantic-settings
+    #   uvicorn
+python-json-logger==4.1.0
+    # via -r requirements/test/../common.txt
+python-multipart==0.0.32
+    # via
+    #   fastapi
+    #   mcp
 python-rapidjson==1.20
     # via tritonclient
 pytrec-eval-terrier==0.5.7
@@ -763,12 +954,14 @@ pywavelets==1.9.0
     # via imagehash
 pyyaml==6.0.2
     # via
+    #   -r requirements/test/../common.txt
     #   accelerate
     #   albumentations
     #   datamodel-code-generator
     #   datasets
     #   genai-perf
     #   huggingface-hub
+    #   lm-format-enforcer
     #   optuna
     #   peft
     #   ray
@@ -776,7 +969,12 @@ pyyaml==6.0.2
     #   schemathesis
     #   timm
     #   transformers
+    #   uvicorn
     #   vocos
+pyzmq==27.1.0
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 rapidfuzz==3.12.1
     # via jiwer
 ray==2.48.0
@@ -789,6 +987,7 @@ referencing==0.35.1
     #   jsonschema-specifications
 regex==2026.2.28
     # via
+    #   -r requirements/test/../common.txt
     #   nltk
     #   open-clip-torch
     #   sacrebleu
@@ -797,6 +996,7 @@ regex==2026.2.28
 requests==2.32.3
     # via
     #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
     #   azure-core
     #   buildkite-test-collector
     #   datasets
@@ -809,6 +1009,7 @@ requests==2.32.3
     #   mistral-common
     #   msal
     #   mteb
+    #   opentelemetry-exporter-otlp-proto-http
     #   pooch
     #   ray
     #   responses
@@ -822,8 +1023,15 @@ rich==13.9.4
     #   genai-perf
     #   mteb
     #   perceptron
+    #   rich-toolkit
     #   schemathesis
     #   typer
+rich-toolkit==0.20.1
+    # via
+    #   fastapi-cli
+    #   fastapi-cloud-cli
+rignore==0.7.6
+    # via fastapi-cloud-cli
 rouge-score==0.1.2
     # via lm-eval
 rpds-py==0.20.1
@@ -847,6 +1055,7 @@ sacrebleu==2.4.3
 safetensors==0.7.0
     # via
     #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
     #   accelerate
     #   open-clip-torch
     #   peft
@@ -882,9 +1091,17 @@ sentence-transformers==5.2.0
     # via
     #   -r requirements/test/cuda.in
     #   mteb
+sentencepiece==0.2.1
+    # via -r requirements/test/../common.txt
+sentry-sdk==2.63.0
+    # via fastapi-cloud-cli
+setproctitle==1.3.7
+    # via -r requirements/test/../common.txt
 setuptools==77.0.3
     # via
     #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
+    #   model-hosting-container-standards
     #   pytablewriter
     #   torch
 shellingham==1.5.4
@@ -894,6 +1111,7 @@ shellingham==1.5.4
 six==1.16.0
     # via
     #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
     #   junit-xml
     #   opencensus
     #   python-dateutil
@@ -902,8 +1120,9 @@ smart-open==7.1.0
     # via ray
 sniffio==1.3.1
     # via
-    #   anyio
+    #   anthropic
     #   httpx
+    #   openai
 sortedcontainers==2.4.0
     # via hypothesis
 soundfile==0.12.1
@@ -922,10 +1141,17 @@ sqlalchemy==2.0.41
     #   optuna
 sqlitedict==2.1.0
     # via lm-eval
+sse-starlette==3.4.5
+    # via mcp
 starlette==1.3.1
     # via
     #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
     #   fastapi
+    #   mcp
+    #   model-hosting-container-standards
+    #   prometheus-fastapi-instrumentator
+    #   sse-starlette
     #   starlette-testclient
 starlette-testclient==0.4.1
     # via schemathesis
@@ -933,6 +1159,8 @@ statsmodels==0.14.4
     # via genai-perf
 structlog==25.4.0
     # via gpt-oss
+supervisor==4.3.0
+    # via model-hosting-container-standards
 sympy==1.13.3
     # via
     #   einx
@@ -962,6 +1190,7 @@ tifffile==2025.3.30
 tiktoken==0.12.0
     # via
     #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
     #   gpt-oss
     #   lm-eval
     #   mistral-common
@@ -973,6 +1202,7 @@ timm==1.0.17
 tokenizers==0.22.2
     # via
     #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
     #   -r requirements/test/cuda.in
     #   transformers
 torch==2.11.0+cu130
@@ -981,6 +1211,7 @@ torch==2.11.0+cu130
     #   -r requirements/test/cuda.in
     #   accelerate
     #   bitsandbytes
+    #   compressed-tensors
     #   encodec
     #   instanttensor
     #   mteb
@@ -994,6 +1225,7 @@ torch==2.11.0+cu130
     #   torchvision
     #   vector-quantize-pytorch
     #   vocos
+    #   xgrammar
 torchaudio==2.11.0+cu130
     # via
     #   -c requirements/cuda.txt
@@ -1009,6 +1241,7 @@ torchvision==0.26.0+cu130
     #   timm
 tqdm==4.67.3
     # via
+    #   -r requirements/test/../common.txt
     #   datasets
     #   evaluate
     #   huggingface-hub
@@ -1016,6 +1249,7 @@ tqdm==4.67.3
     #   mteb
     #   nltk
     #   open-clip-torch
+    #   openai
     #   optuna
     #   peft
     #   pqdm
@@ -1025,15 +1259,20 @@ tqdm==4.67.3
 transformers==5.5.3
     # via
     #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
     #   -r requirements/test/cuda.in
+    #   compressed-tensors
     #   genai-perf
     #   peft
     #   sentence-transformers
     #   transformers-stream-generator
+    #   xgrammar
 transformers-stream-generator==0.0.5
     # via -r requirements/test/cuda.in
 triton==3.6.0
-    # via torch
+    # via
+    #   torch
+    #   xgrammar
 tritonclient==2.64.0
     # via -r requirements/test/cuda.in
 typepy==1.3.2
@@ -1041,8 +1280,10 @@ typepy==1.3.2
     #   dataproperty
     #   pytablewriter
     #   tabledata
-typer==0.15.2
+typer==0.26.8
     # via
+    #   fastapi-cli
+    #   fastapi-cloud-cli
     #   fastsafetensors
     #   huggingface-hub
     #   perceptron
@@ -1050,9 +1291,13 @@ typer==0.15.2
 typing-extensions==4.15.0
     # via
     #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
     #   aiosignal
     #   albumentations
     #   alembic
+    #   anthropic
+    #   anyio
+    #   apache-tvm-ffi
     #   azure-core
     #   azure-identity
     #   azure-storage-blob
@@ -1062,9 +1307,13 @@ typing-extensions==4.15.0
     #   huggingface-hub
     #   librosa
     #   lm-eval
+    #   mcp
     #   mistral-common
     #   mteb
+    #   openai
     #   opentelemetry-api
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
     #   opentelemetry-sdk
     #   opentelemetry-semantic-conventions
     #   pqdm
@@ -1072,17 +1321,20 @@ typing-extensions==4.15.0
     #   pydantic-core
     #   pydantic-extra-types
     #   pytest-asyncio
+    #   rich-toolkit
     #   schemathesis
     #   sentence-transformers
     #   sqlalchemy
     #   starlette
     #   torch
-    #   typer
     #   typing-inspection
+    #   xgrammar
 typing-inspection==0.4.2
     # via
     #   fastapi
+    #   mcp
     #   pydantic
+    #   pydantic-settings
 tzdata==2024.2
     # via pandas
 urllib3==2.2.3
@@ -1092,23 +1344,41 @@ urllib3==2.2.3
     #   docker
     #   requests
     #   responses
+    #   sentry-sdk
     #   tritonclient
 uvicorn==0.35.0
-    # via gpt-oss
+    # via
+    #   fastapi
+    #   fastapi-cli
+    #   fastapi-cloud-cli
+    #   gpt-oss
+    #   mcp
+uvloop==0.22.1
+    # via uvicorn
 vector-quantize-pytorch==1.21.2
     # via -r requirements/test/cuda.in
 virtualenv==20.31.2
     # via ray
 vocos==0.1.0
     # via -r requirements/test/cuda.in
+watchfiles==1.2.0
+    # via
+    #   -r requirements/test/../common.txt
+    #   uvicorn
 wcwidth==0.2.13
     # via ftfy
+websockets==16.0
+    # via uvicorn
 werkzeug==3.1.3
     # via schemathesis
 word2number==1.1
     # via lm-eval
 wrapt==1.17.2
     # via smart-open
+xgrammar==0.2.3
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 xxhash==3.5.0
     # via
     #   datasets

requirements/test/rocm.in

@@ -15,7 +15,6 @@ albumentations # required for Nemotron Parse in test_common.py
 av # required for audio_in_video tests
 backoff # required for phi4mm test
 blobfile # required for kimi-vl test
-einops # required for MPT, qwen-vl
 httpx
 librosa # required for audio tests
 vector_quantize_pytorch # required for minicpmo_26 test
@@ -33,7 +32,6 @@ matplotlib # required for qwen-vl test
 mistral_common[image,audio]>=1.11.5 # required for voxtral test
 num2words # required for smolvlm test
 open_clip_torch==2.32.0 # Required for nemotron_vl test, Nemotron Parse in test_common.py
-opencv-python-headless>=4.13.0 # required for video test
 datamodel_code_generator # required for minicpm3 test
 lm-eval[api]>=0.4.12 # required for model evaluation test
 mteb[bm25s]>=2, <3 # required for mteb test
@@ -54,11 +52,9 @@ grpcio-reflection==1.78.0
 
 arctic-inference==0.1.1 # Required for suffix decoding test
 numba==0.65.0 # Required for N-gram speculative decoding
-numpy
 runai-model-streamer[s3,gcs,azure]==0.15.7
 fastsafetensors>=0.3.2
 instanttensor>=0.1.5
-pydantic>=2.12 # 2.11 leads to error on python 3.13
 decord==0.6.0
 
 # Prithvi tests
@@ -74,6 +70,7 @@ gpt-oss>=0.0.7; python_version > '3.11'
 
 perceptron # required for isaac test
 kaldi-native-fbank>=1.18.7 # required for fireredasr2 test
+cohere_melody>=0.9.0 # required for cohere command reasoning parser test
 
 # Newer versions of datasets require torchcoded, that makes the tests fail in CI because of a missing library.
 # Older versions are in conflict with terratorch requirements.

requirements/test/rocm.txt

@@ -130,6 +130,8 @@ cloudpickle==3.1.2
     # via
     #   -r requirements/test/../common.txt
     #   tilelang
+cohere-melody==0.9.0
+    # via -r requirements/test/rocm.in
 colorama==0.4.6
     # via
     #   perceptron
@@ -205,7 +207,6 @@ docstring-parser==0.17.0
 einops==0.8.2
     # via
     #   -r requirements/test/../common.txt
-    #   -r requirements/test/rocm.in
     #   encodec
     #   vector-quantize-pytorch
     #   vocos
@@ -561,7 +562,6 @@ numba==0.65.0
 numpy==2.2.6
     # via
     #   -r requirements/test/../common.txt
-    #   -r requirements/test/rocm.in
     #   accelerate
     #   albumentations
     #   bitsandbytes
@@ -630,7 +630,6 @@ opencv-python-headless==4.13.0.92
     # via
     #   -c requirements/common.txt
     #   -r requirements/test/../common.txt
-    #   -r requirements/test/rocm.in
     #   albumentations
     #   mistral-common
 openpyxl==3.1.5
@@ -834,7 +833,6 @@ pydantic==2.12.5
     # via
     #   -c requirements/common.txt
     #   -r requirements/test/../common.txt
-    #   -r requirements/test/rocm.in
     #   albumentations
     #   anthropic
     #   compressed-tensors

requirements/test/xpu.in

@@ -1,3 +1,5 @@
+-r ../common.txt
+
 # --- Test Infrastructure ---
 tblib
 pytest

requirements/test/xpu.txt

@@ -11,6 +11,7 @@ aiohappyeyeballs==2.6.1
 aiohttp==3.13.4
     # via
     #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
     #   fsspec
     #   gpt-oss
     #   lm-eval
@@ -24,12 +25,25 @@ annotated-doc==0.0.4
     #   typer
 annotated-types==0.7.0
     # via pydantic
+anthropic==0.112.0
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 anyio==4.13.0
     # via
+    #   anthropic
     #   httpx
+    #   mcp
+    #   openai
+    #   sse-starlette
     #   starlette
+    #   watchfiles
+apache-tvm-ffi==0.1.12
+    # via xgrammar
 arctic-inference==0.1.1
     # via -r requirements/test/xpu.in
+astor==0.8.1
+    # via depyf
 attrs==26.1.0
     # via
     #   aiohttp
@@ -39,6 +53,8 @@ audioread==3.0.1
     # via
     #   -r requirements/test/xpu.in
     #   librosa
+blake3==1.0.9
+    # via -r requirements/test/../common.txt
 blobfile==3.0.0
     # via -r requirements/test/xpu.in
 bm25s==0.2.13
@@ -47,13 +63,20 @@ bm25s==0.2.13
     #   mteb
 bounded-pool-executor==0.0.3
     # via pqdm
+cachetools==7.1.4
+    # via -r requirements/test/../common.txt
+cbor2==6.1.2
+    # via -r requirements/test/../common.txt
 certifi==2026.2.25
     # via
     #   httpcore
     #   httpx
     #   requests
+    #   sentry-sdk
 cffi==2.0.0
-    # via soundfile
+    # via
+    #   cryptography
+    #   soundfile
 chardet==5.2.0
     # via mbstrdecoder
 charset-normalizer==3.4.6
@@ -64,13 +87,22 @@ click==8.3.1
     # via
     #   jiwer
     #   nltk
+    #   rich-toolkit
     #   schemathesis
     #   typer
     #   uvicorn
+cloudpickle==3.1.2
+    # via -r requirements/test/../common.txt
 colorama==0.4.6
     # via sacrebleu
+compressed-tensors==0.17.0
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 coverage==7.13.5
     # via pytest-cov
+cryptography==49.0.0
+    # via pyjwt
 dataproperty==1.1.0
     # via
     #   pytablewriter
@@ -82,16 +114,35 @@ datasets==4.8.4
     #   mteb
 decorator==5.2.1
     # via librosa
+depyf==0.20.0
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
+detect-installer==0.1.0
+    # via fastapi-cloud-cli
 dill==0.4.1
     # via
     #   datasets
+    #   depyf
     #   evaluate
     #   lm-eval
     #   multiprocess
+diskcache==5.6.3
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
+distro==1.9.0
+    # via
+    #   anthropic
+    #   openai
+dnspython==2.8.0
+    # via email-validator
 docker==7.1.0
     # via gpt-oss
 docopt==0.6.2
     # via num2words
+docstring-parser==0.18.0
+    # via anthropic
 dpcpp-cpp-rt==2025.3.2
     # via
     #   onemkl-sycl-blas
@@ -100,15 +151,30 @@ dpcpp-cpp-rt==2025.3.2
     #   onemkl-sycl-rng
     #   onemkl-sycl-sparse
     #   torch
+einops==0.8.2
+    # via -r requirements/test/../common.txt
+email-validator==2.3.0
+    # via
+    #   fastapi
+    #   pydantic
 evaluate==0.4.6
     # via lm-eval
 fastapi==0.135.2
     # via
     #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
     #   gpt-oss
+    #   model-hosting-container-standards
+fastapi-cli==0.0.27
+    # via fastapi
+fastapi-cloud-cli==0.21.0
+    # via fastapi-cli
+fastar==0.11.0
+    # via fastapi-cloud-cli
 filelock==3.25.2
     # via
     #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
     #   blobfile
     #   datasets
     #   huggingface-hub
@@ -124,10 +190,16 @@ fsspec==2026.2.0
     #   evaluate
     #   huggingface-hub
     #   torch
+googleapis-common-protos==1.75.0
+    # via
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
 gpt-oss==0.0.8
     # via -r requirements/test/xpu.in
 graphql-core==3.2.8
     # via hypothesis-graphql
+grpcio==1.81.1
+    # via opentelemetry-exporter-otlp-proto-grpc
 h11==0.16.0
     # via
     #   httpcore
@@ -140,11 +212,21 @@ html2text==2025.4.15
     # via gpt-oss
 httpcore==1.0.9
     # via httpx
+httptools==0.8.0
+    # via uvicorn
 httpx==0.28.1
     # via
+    #   anthropic
     #   datasets
+    #   fastapi
+    #   fastapi-cloud-cli
     #   huggingface-hub
+    #   mcp
+    #   model-hosting-container-standards
+    #   openai
     #   schemathesis
+httpx-sse==0.4.3
+    # via mcp
 huggingface-hub==1.10.2
     # via
     #   accelerate
@@ -166,9 +248,12 @@ hypothesis-jsonschema==0.23.1
 idna==3.11
     # via
     #   anyio
+    #   email-validator
     #   httpx
     #   requests
     #   yarl
+ijson==3.5.0
+    # via -r requirements/test/../common.txt
 imageio==2.37.3
     # via scikit-image
 impi-rt==2021.17.2
@@ -212,13 +297,22 @@ intel-sycl-rt==2025.3.2
     #   dpcpp-cpp-rt
     #   oneccl
     #   torch
+interegular==0.3.3
+    # via lm-format-enforcer
 jinja2==3.1.6
     # via
     #   -c requirements/xpu.txt
+    #   fastapi
     #   lm-eval
     #   torch
+jiter==0.15.0
+    # via
+    #   anthropic
+    #   openai
 jiwer==4.0.0
     # via -r requirements/test/xpu.in
+jmespath==1.1.0
+    # via model-hosting-container-standards
 joblib==1.5.3
     # via
     #   librosa
@@ -227,7 +321,9 @@ joblib==1.5.3
 jsonschema==4.26.0
     # via
     #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
     #   hypothesis-jsonschema
+    #   mcp
     #   mistral-common
     #   schemathesis
 jsonschema-rs==0.45.0
@@ -236,16 +332,30 @@ jsonschema-specifications==2025.9.1
     # via jsonschema
 junit-xml==1.9
     # via schemathesis
+lark==1.2.2
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 lazy-loader==0.5
     # via
     #   librosa
     #   scikit-image
 librosa==0.10.2.post1
     # via -r requirements/test/xpu.in
+llguidance==1.7.6
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 llvmlite==0.47.0
     # via numba
 lm-eval==0.4.12
     # via -r requirements/test/xpu.in
+lm-format-enforcer==0.11.3
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
+loguru==0.7.3
+    # via compressed-tensors
 lxml==6.0.2
     # via
     #   blobfile
@@ -262,11 +372,14 @@ mbstrdecoder==1.1.4
     #   dataproperty
     #   pytablewriter
     #   typepy
+mcp==1.28.1
+    # via -r requirements/test/../common.txt
 mdurl==0.1.2
     # via markdown-it-py
 mistral-common==1.11.5
     # via
     #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
     #   -r requirements/test/xpu.in
 mkl==2025.3.1
     # via
@@ -276,6 +389,10 @@ mkl==2025.3.1
     #   onemkl-sycl-rng
     #   onemkl-sycl-sparse
     #   torch
+model-hosting-container-standards==0.1.16
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 modelscope==1.35.3
     # via -r requirements/test/xpu.in
 more-itertools==10.8.0
@@ -284,6 +401,8 @@ mpmath==1.3.0
     # via sympy
 msgpack==1.1.2
     # via librosa
+msgspec==0.21.1
+    # via -r requirements/test/../common.txt
 mteb==2.12.7
     # via -r requirements/test/xpu.in
 multidict==6.7.1
@@ -298,6 +417,8 @@ networkx==3.6.1
     # via
     #   scikit-image
     #   torch
+ninja==1.13.0
+    # via -r requirements/test/../common.txt
 nltk==3.9.4
     # via rouge-score
 num2words==0.5.14
@@ -308,6 +429,7 @@ numba==0.65.0
     #   librosa
 numpy==2.2.6
     # via
+    #   -r requirements/test/../common.txt
     #   accelerate
     #   albumentations
     #   bm25s
@@ -333,6 +455,7 @@ numpy==2.2.6
     #   tifffile
     #   torchvision
     #   transformers
+    #   xgrammar
 oneccl==2021.17.2
     # via
     #   oneccl-devel
@@ -356,15 +479,65 @@ onemkl-sycl-rng==2025.3.1
     # via torch
 onemkl-sycl-sparse==2025.3.1
     # via torch
+openai==2.44.0
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 openai-harmony==0.0.8
     # via
     #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
     #   gpt-oss
 opencv-python-headless==4.13.0.92
     # via
     #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
     #   albumentations
     #   mistral-common
+opentelemetry-api==1.43.0
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+    #   opentelemetry-sdk
+    #   opentelemetry-semantic-conventions
+opentelemetry-exporter-otlp==1.43.0
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
+opentelemetry-exporter-otlp-proto-common==1.43.0
+    # via
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+opentelemetry-exporter-otlp-proto-grpc==1.43.0
+    # via opentelemetry-exporter-otlp
+opentelemetry-exporter-otlp-proto-http==1.43.0
+    # via opentelemetry-exporter-otlp
+opentelemetry-proto==1.43.0
+    # via
+    #   opentelemetry-exporter-otlp-proto-common
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+opentelemetry-sdk==1.43.0
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+    #   opentelemetry-semantic-conventions-ai
+opentelemetry-semantic-conventions==0.64b0
+    # via
+    #   opentelemetry-sdk
+    #   opentelemetry-semantic-conventions-ai
+opentelemetry-semantic-conventions-ai==0.5.1
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
+outlines-core==0.2.14
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 packaging==26.0
     # via
     #   -c requirements/xpu.txt
@@ -373,6 +546,7 @@ packaging==26.0
     #   evaluate
     #   huggingface-hub
     #   lazy-loader
+    #   lm-format-enforcer
     #   modelscope
     #   pooch
     #   pytest
@@ -384,10 +558,13 @@ pandas==3.0.1
     # via
     #   datasets
     #   evaluate
+partial-json-parser==0.2.1.1.post7
+    # via -r requirements/test/../common.txt
 pathvalidate==3.3.1
     # via pytablewriter
 pillow==12.1.1
     # via
+    #   -r requirements/test/../common.txt
     #   imageio
     #   mistral-common
     #   scikit-image
@@ -410,16 +587,37 @@ portalocker==3.2.0
     # via sacrebleu
 pqdm==0.2.0
     # via -r requirements/test/xpu.in
+prometheus-client==0.25.0
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
+    #   prometheus-fastapi-instrumentator
+prometheus-fastapi-instrumentator==8.0.2
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 propcache==0.4.1
     # via
     #   aiohttp
     #   yarl
+protobuf==7.35.1
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
+    #   googleapis-common-protos
+    #   opentelemetry-proto
 psutil==7.2.2
-    # via accelerate
+    # via
+    #   -r requirements/test/../common.txt
+    #   accelerate
 py==1.11.0
     # via pytest-forked
+py-cpuinfo==9.0.0
+    # via -r requirements/test/../common.txt
 pyarrow==23.0.1
     # via datasets
+pybase64==1.4.3
+    # via -r requirements/test/../common.txt
 pycountry==26.2.16
     # via pydantic-extra-types
 pycparser==3.0
@@ -429,23 +627,41 @@ pycryptodomex==3.23.0
 pydantic==2.12.5
     # via
     #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
     #   albumentations
+    #   anthropic
+    #   compressed-tensors
     #   fastapi
+    #   fastapi-cloud-cli
     #   gpt-oss
+    #   lm-format-enforcer
+    #   mcp
     #   mistral-common
+    #   model-hosting-container-standards
     #   mteb
+    #   openai
     #   openai-harmony
     #   pydantic-extra-types
+    #   pydantic-settings
+    #   xgrammar
 pydantic-core==2.41.5
     # via pydantic
 pydantic-extra-types==2.11.1
-    # via mistral-common
+    # via
+    #   fastapi
+    #   mistral-common
+pydantic-settings==2.14.2
+    # via
+    #   fastapi
+    #   mcp
 pyelftools==0.32
     # via triton-xpu
 pygments==2.20.0
     # via
     #   pytest
     #   rich
+pyjwt==2.13.0
+    # via mcp
 pyrate-limiter==4.1.0
     # via schemathesis
 pystemmer==3.0.0
@@ -480,19 +696,36 @@ python-dateutil==2.9.0.post0
     # via
     #   pandas
     #   typepy
+python-dotenv==1.2.2
+    # via
+    #   pydantic-settings
+    #   uvicorn
+python-json-logger==4.1.0
+    # via -r requirements/test/../common.txt
+python-multipart==0.0.32
+    # via
+    #   fastapi
+    #   mcp
 pytrec-eval-terrier==0.5.10
     # via mteb
 pytz==2026.1.post1
     # via typepy
 pyyaml==6.0.3
     # via
+    #   -r requirements/test/../common.txt
     #   accelerate
     #   albumentations
     #   datasets
     #   huggingface-hub
+    #   lm-format-enforcer
     #   schemathesis
     #   timm
     #   transformers
+    #   uvicorn
+pyzmq==27.1.0
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 rapidfuzz==3.12.1
     # via
     #   -r requirements/test/xpu.in
@@ -503,6 +736,7 @@ referencing==0.37.0
     #   jsonschema-specifications
 regex==2026.3.32
     # via
+    #   -r requirements/test/../common.txt
     #   nltk
     #   sacrebleu
     #   tiktoken
@@ -510,6 +744,7 @@ regex==2026.3.32
 requests==2.33.1
     # via
     #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
     #   datasets
     #   docker
     #   evaluate
@@ -518,6 +753,7 @@ requests==2.33.1
     #   mistral-common
     #   modelscope
     #   mteb
+    #   opentelemetry-exporter-otlp-proto-http
     #   pooch
     #   schemathesis
     #   starlette-testclient
@@ -525,8 +761,15 @@ requests==2.33.1
 rich==14.3.3
     # via
     #   mteb
+    #   rich-toolkit
     #   schemathesis
     #   typer
+rich-toolkit==0.20.1
+    # via
+    #   fastapi-cli
+    #   fastapi-cloud-cli
+rignore==0.7.6
+    # via fastapi-cloud-cli
 rouge-score==0.1.2
     # via lm-eval
 rpds-py==0.30.0
@@ -538,6 +781,7 @@ sacrebleu==2.6.0
 safetensors==0.7.0
     # via
     #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
     #   accelerate
     #   timm
     #   transformers
@@ -564,10 +808,18 @@ scipy==1.17.1
     #   sentence-transformers
 sentence-transformers==5.3.0
     # via mteb
+sentencepiece==0.2.1
+    # via -r requirements/test/../common.txt
+sentry-sdk==2.63.0
+    # via fastapi-cloud-cli
+setproctitle==1.3.7
+    # via -r requirements/test/../common.txt
 setuptools==80.10.2
     # via
     #   -c requirements/common.txt
     #   -c requirements/xpu.txt
+    #   -r requirements/test/../common.txt
+    #   model-hosting-container-standards
     #   modelscope
     #   pytablewriter
     #   torch
@@ -576,9 +828,14 @@ shellingham==1.5.4
 six==1.17.0
     # via
     #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
     #   junit-xml
     #   python-dateutil
     #   rouge-score
+sniffio==1.3.1
+    # via
+    #   anthropic
+    #   openai
 sortedcontainers==2.4.0
     # via hypothesis
 soundfile==0.13.1
@@ -593,15 +850,24 @@ soxr==0.5.0.post1
     #   mistral-common
 sqlitedict==2.1.0
     # via lm-eval
+sse-starlette==3.4.5
+    # via mcp
 starlette==1.3.1
     # via
     #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
     #   fastapi
+    #   mcp
+    #   model-hosting-container-standards
+    #   prometheus-fastapi-instrumentator
+    #   sse-starlette
     #   starlette-testclient
 starlette-testclient==0.4.1
     # via schemathesis
 structlog==25.5.0
     # via gpt-oss
+supervisor==4.3.0
+    # via model-hosting-container-standards
 sympy==1.14.0
     # via torch
 tabledata==1.3.4
@@ -636,6 +902,7 @@ tifffile==2026.3.3
 tiktoken==0.12.0
     # via
     #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
     #   gpt-oss
     #   lm-eval
     #   mistral-common
@@ -644,19 +911,23 @@ timm==1.0.17
 tokenizers==0.22.2
     # via
     #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
     #   transformers
 torch==2.12.0+xpu
     # via
     #   -c requirements/xpu.txt
     #   accelerate
+    #   compressed-tensors
     #   mteb
     #   sentence-transformers
     #   timm
     #   torchvision
+    #   xgrammar
 torchvision==0.27.0+xpu
     # via timm
 tqdm==4.67.3
     # via
+    #   -r requirements/test/../common.txt
     #   datasets
     #   evaluate
     #   huggingface-hub
@@ -664,13 +935,19 @@ tqdm==4.67.3
     #   modelscope
     #   mteb
     #   nltk
+    #   openai
     #   pqdm
     #   sentence-transformers
     #   transformers
 transformers==5.5.3
     # via
     #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
+    #   compressed-tensors
     #   sentence-transformers
+    #   xgrammar
+triton==3.7.1
+    # via xgrammar
 triton-xpu==3.7.1
     # via torch
 typepy==1.3.4
@@ -680,36 +957,53 @@ typepy==1.3.4
     #   tabledata
 typer==0.24.1
     # via
+    #   fastapi-cli
+    #   fastapi-cloud-cli
     #   huggingface-hub
     #   transformers
 typing-extensions==4.15.0
     # via
     #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
     #   aiosignal
     #   albumentations
+    #   anthropic
     #   anyio
+    #   apache-tvm-ffi
     #   chz
     #   fastapi
+    #   grpcio
     #   huggingface-hub
     #   librosa
     #   lm-eval
+    #   mcp
     #   mistral-common
     #   mteb
+    #   openai
+    #   opentelemetry-api
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+    #   opentelemetry-sdk
+    #   opentelemetry-semantic-conventions
     #   pqdm
     #   pydantic
     #   pydantic-core
     #   pydantic-extra-types
     #   pytest-asyncio
     #   referencing
+    #   rich-toolkit
     #   schemathesis
     #   sentence-transformers
     #   starlette
     #   torch
     #   typing-inspection
+    #   xgrammar
 typing-inspection==0.4.2
     # via
     #   fastapi
+    #   mcp
     #   pydantic
+    #   pydantic-settings
 umf==1.0.3
     # via
     #   intel-cmplr-lib-ur
@@ -720,12 +1014,30 @@ urllib3==2.6.3
     #   docker
     #   modelscope
     #   requests
+    #   sentry-sdk
 uvicorn==0.42.0
-    # via gpt-oss
+    # via
+    #   fastapi
+    #   fastapi-cli
+    #   fastapi-cloud-cli
+    #   gpt-oss
+    #   mcp
+uvloop==0.22.1
+    # via uvicorn
+watchfiles==1.2.0
+    # via
+    #   -r requirements/test/../common.txt
+    #   uvicorn
+websockets==16.0
+    # via uvicorn
 werkzeug==3.1.7
     # via schemathesis
 word2number==1.1
     # via lm-eval
+xgrammar==0.2.3
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 xxhash==3.6.0
     # via
     #   datasets

rust/Cargo.lock

@@ -272,6 +272,18 @@ version = "1.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
 
+[[package]]
+name = "auto_enums"
+version = "0.8.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2e4487600931c9a89f8db7ffbdf3fbdd45bb7bd85e26861f659a463cd0dff966"
+dependencies = [
+ "derive_utils",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.117",
+]
+
 [[package]]
 name = "auto_impl"
 version = "1.3.0"
@@ -938,6 +950,17 @@ dependencies = [
  "unicode-xid",
 ]
 
+[[package]]
+name = "derive_utils"
+version = "0.15.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "362f47930db19fe7735f527e6595e4900316b893ebf6d48ad3d31be928d57dd6"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.117",
+]
+
 [[package]]
 name = "digest"
 version = "0.10.7"
@@ -1478,9 +1501,9 @@ dependencies = [
 
 [[package]]
 name = "h2"
-version = "0.4.13"
+version = "0.4.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54"
+checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155"
 dependencies = [
  "atomic-waker",
  "bytes",
@@ -1638,9 +1661,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
 
 [[package]]
 name = "hyper"
-version = "1.8.1"
+version = "1.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11"
+checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498"
 dependencies = [
  "atomic-waker",
  "bytes",
@@ -1653,7 +1676,6 @@ dependencies = [
  "httpdate",
  "itoa",
  "pin-project-lite",
- "pin-utils",
  "smallvec",
  "tokio",
  "want",
@@ -2569,15 +2591,14 @@ dependencies = [
 
 [[package]]
 name = "openssl"
-version = "0.10.76"
+version = "0.10.81"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "951c002c75e16ea2c65b8c7e4d3d51d5530d8dfa7d060b4776828c88cfb18ecf"
+checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45"
 dependencies = [
  "bitflags",
  "cfg-if",
  "foreign-types",
  "libc",
- "once_cell",
  "openssl-macros",
  "openssl-sys",
 ]
@@ -2610,9 +2631,9 @@ dependencies = [
 
 [[package]]
 name = "openssl-sys"
-version = "0.9.112"
+version = "0.9.117"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57d55af3b3e226502be1526dfdba67ab0e9c96fc293004e79576b2b9edb0dbdb"
+checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695"
 dependencies = [
  "cc",
  "libc",
@@ -2783,12 +2804,6 @@ version = "0.2.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
 
-[[package]]
-name = "pin-utils"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
-
 [[package]]
 name = "pkg-config"
 version = "0.3.32"
@@ -2988,7 +3003,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7"
 dependencies = [
  "heck",
- "itertools 0.10.5",
+ "itertools 0.14.0",
  "log",
  "multimap",
  "petgraph",
@@ -3009,7 +3024,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b"
 dependencies = [
  "anyhow",
- "itertools 0.10.5",
+ "itertools 0.14.0",
  "proc-macro2",
  "quote",
  "syn 2.0.117",
@@ -3503,9 +3518,9 @@ dependencies = [
 
 [[package]]
 name = "rustls-pki-types"
-version = "1.14.0"
+version = "1.14.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd"
+checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9"
 dependencies = [
  "zeroize",
 ]
@@ -4385,6 +4400,22 @@ dependencies = [
  "serde_json",
 ]
 
+[[package]]
+name = "tls-listener"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1461056cc1ef47003f7ee16e4cef3741068d4c7f6b627bfce49b7c00c120a530"
+dependencies = [
+ "axum",
+ "futures-util",
+ "openssl",
+ "pin-project-lite",
+ "thiserror 2.0.18",
+ "tokio",
+ "tokio-openssl",
+ "tracing",
+]
+
 [[package]]
 name = "tokenizers"
 version = "0.22.2"
@@ -4457,6 +4488,17 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "tokio-openssl"
+version = "0.6.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "59df6849caa43bb7567f9a36f863c447d95a11d5903c9cc334ba32576a27eadd"
+dependencies = [
+ "openssl",
+ "openssl-sys",
+ "tokio",
+]
+
 [[package]]
 name = "tokio-rustls"
 version = "0.26.4"
@@ -5220,6 +5262,7 @@ dependencies = [
  "anyhow",
  "async-openai",
  "asynk-strim-attr",
+ "auto_enums",
  "axum",
  "bytes",
  "clap",
@@ -5227,10 +5270,13 @@ dependencies = [
  "expect-test",
  "futures",
  "http-body",
+ "hyper",
+ "hyper-util",
  "indexmap 2.13.0",
  "itertools 0.14.0",
  "libc",
  "llm-multimodal",
+ "openssl",
  "prost",
  "prost-types",
  "rmp-serde",
@@ -5242,8 +5288,11 @@ dependencies = [
  "sha2",
  "socket2",
  "subtle",
+ "tempfile",
  "thiserror-ext",
+ "tls-listener",
  "tokio",
+ "tokio-openssl",
  "tokio-stream",
  "tokio-util",
  "tonic",
@@ -5261,6 +5310,7 @@ dependencies = [
  "vllm-llm",
  "vllm-metrics",
  "vllm-text",
+ "vllm-tokenizer",
  "zeromq",
 ]
 

rust/Cargo.toml

@@ -26,6 +26,7 @@ arc-swap = "1.9.0"
 async-openai = { version = "0.33.1", default-features = false, features = ["native-tls"] }
 async-trait = "0.1.89"
 asynk-strim-attr = "0.1.0"
+auto_enums = { version = "0.8.9", features = ["tokio1"] }
 axum = "0.8.8"
 base64 = "0.22.1"
 bytemuck = { version = "1.25.0", features = ["extern_crate_alloc"] }
@@ -43,6 +44,12 @@ half = { version = "2.7.1", features = ["bytemuck"] }
 hex = "0.4.3"
 hf-hub = { version = "0.5.0", default-features = false, features = ["tokio"] }
 http-body = "1.0.1"
+hyper = { version = "1.10.1", features = ["http1", "server"] }
+hyper-util = { version = "0.1.20", features = [
+    "server-graceful",
+    "service",
+    "tokio",
+] }
 indexmap = "2.13.0"
 itertools = "0.14.0"
 libc = "0.2.177"
@@ -54,6 +61,7 @@ native-tls-vendored = { package = "native-tls", version = "0.2.18", features = [
 ndarray = { version = "0.16.1", features = ["serde"] }
 openai-harmony = { package = "oss-harmony", git = "https://github.com/oss-harmony/harmony", tag = "v0.0.11", default-features = false }
 openai-protocol = "1.6.0"
+openssl = "0.10"
 parking_lot = "0.12.5"
 paste = "1.0.15"
 prometheus-client = "0.24.0"
@@ -89,6 +97,7 @@ thiserror = "2.0.16"
 thiserror-ext = "0.3.0"
 tiktoken-rs = "0.9.1"
 time = { version = "0.3.47", features = ["formatting", "local-offset", "macros"] }
+tls-listener = { version = "0.11.2", default-features = false, features = ["openssl", "tokio-net", "axum"] }
 tokenizers = "0.22.0"
 tokio = { version = "1.47.1", features = [
     "macros",
@@ -97,6 +106,7 @@ tokio = { version = "1.47.1", features = [
     "sync",
     "time",
 ] }
+tokio-openssl = "0.6"
 tokio-stream = "0.1"
 tokio-util = { version = "0.7.18", features = ["rt"] }
 tonic = "0.14.5"

rust/src/chat/Cargo.toml

@@ -50,6 +50,7 @@ tokio.workspace = true
 tracing-subscriber.workspace = true
 uuid.workspace = true
 vllm-engine-core-client = { workspace = true, features = ["test-util"] }
+vllm-tokenizer = { workspace = true, features = ["test-utils"] }
 zeromq.workspace = true
 
 [lints]

rust/src/chat/src/backend/hf.rs

@@ -154,7 +154,8 @@ mod tests {
     use thiserror_ext::AsReport as _;
     use vllm_text::Prompt;
     use vllm_text::backend::hf::TokenizerSource;
-    use vllm_text::tokenizer::{DynTokenizer, Tokenizer};
+    use vllm_text::tokenizer::DynTokenizer;
+    use vllm_tokenizer::test_utils::TestTokenizer;
 
     use super::HfChatBackend;
     use crate::backend::{ChatBackend, LoadModelBackendsOptions, NewChatOutputProcessorOptions};
@@ -196,32 +197,8 @@ mod tests {
         }
     }
 
-    struct TestTokenizer;
-
-    impl Tokenizer for TestTokenizer {
-        fn encode(
-            &self,
-            _text: &str,
-            _add_special_tokens: bool,
-        ) -> vllm_text::tokenizer::Result<Vec<u32>> {
-            Ok(Vec::new())
-        }
-
-        fn decode(
-            &self,
-            _token_ids: &[u32],
-            _skip_special_tokens: bool,
-        ) -> vllm_text::tokenizer::Result<String> {
-            Ok(String::new())
-        }
-
-        fn token_to_id(&self, _token: &str) -> Option<u32> {
-            None
-        }
-    }
-
     fn test_tokenizer() -> DynTokenizer {
-        Arc::new(TestTokenizer)
+        Arc::new(TestTokenizer::new())
     }
 
     fn backend_for_selection(

rust/src/chat/src/lib.rs

@@ -50,7 +50,8 @@ mod request;
 mod stream;
 
 use vllm_engine_core_client::EngineCoreClient;
-use vllm_engine_core_client::protocol::{ModelDtype, ReasoningParserKwargs};
+use vllm_engine_core_client::protocol::dtype::ModelDtype;
+use vllm_engine_core_client::protocol::request::ReasoningParserKwargs;
 use vllm_llm::Llm;
 use vllm_text::{Prompt, TextLlm, TextRequest};
 

rust/src/chat/src/multimodal.rs

@@ -22,7 +22,7 @@ use llm_multimodal::{
     TrackedMedia,
 };
 use tracing::warn;
-use vllm_engine_core_client::protocol::ModelDtype;
+use vllm_engine_core_client::protocol::dtype::ModelDtype;
 use vllm_engine_core_client::protocol::multimodal::{
     MmBatchedField, MmFeatureSpec, MmFeatures, MmField, MmFieldElem, MmFlatField, MmKwargsItem,
     MmSharedField, MmSlice, PlaceholderRange, SliceSpec,
@@ -563,7 +563,7 @@ mod tests {
 
     use llm_multimodal::TokenId;
     use vllm_engine_core_client::protocol::tensor::WireArrayData;
-    use vllm_text::tokenizer::{IncrementalDecoder, Tokenizer, TokenizerError};
+    use vllm_tokenizer::test_utils::TestTokenizer;
 
     use super::*;
 
@@ -574,60 +574,14 @@ mod tests {
     const LLAMA4_TILE_X_SEPARATOR_ID: u32 = 200093;
     const LLAMA4_TILE_Y_SEPARATOR_ID: u32 = 200094;
 
-    struct TestTokenizer;
-
-    impl Tokenizer for TestTokenizer {
-        fn encode(
-            &self,
-            text: &str,
-            _add_special_tokens: bool,
-        ) -> std::result::Result<Vec<u32>, TokenizerError> {
-            Ok(match text {
-                "<|image|>" => vec![LLAMA4_IMAGE_ID],
-                text => text.bytes().map(u32::from).collect(),
-            })
-        }
-
-        fn decode(
-            &self,
-            _token_ids: &[u32],
-            _skip_special_tokens: bool,
-        ) -> std::result::Result<String, TokenizerError> {
-            Ok(String::new())
-        }
-
-        fn token_to_id(&self, token: &str) -> Option<u32> {
-            match token {
-                "<|image_start|>" => Some(LLAMA4_IMAGE_START_ID),
-                "<|image_end|>" => Some(LLAMA4_IMAGE_END_ID),
-                "<|image|>" => Some(LLAMA4_IMAGE_ID),
-                "<|patch|>" => Some(LLAMA4_PATCH_ID),
-                "<|tile_x_separator|>" => Some(LLAMA4_TILE_X_SEPARATOR_ID),
-                "<|tile_y_separator|>" => Some(LLAMA4_TILE_Y_SEPARATOR_ID),
-                _ => None,
-            }
-        }
-
-        fn id_to_token(&self, id: u32) -> Option<String> {
-            match id {
-                LLAMA4_IMAGE_START_ID => Some("<|image_start|>".to_string()),
-                LLAMA4_IMAGE_END_ID => Some("<|image_end|>".to_string()),
-                LLAMA4_IMAGE_ID => Some("<|image|>".to_string()),
-                LLAMA4_PATCH_ID => Some("<|patch|>".to_string()),
-                LLAMA4_TILE_X_SEPARATOR_ID => Some("<|tile_x_separator|>".to_string()),
-                LLAMA4_TILE_Y_SEPARATOR_ID => Some("<|tile_y_separator|>".to_string()),
-                _ => None,
-            }
-        }
-
-        fn create_decode_stream(
-            &self,
-            _prompt_token_ids: &[u32],
-            _skip_special_tokens: bool,
-            _min_bytes_to_buffer: usize,
-        ) -> Box<dyn IncrementalDecoder + '_> {
-            unreachable!("not used")
-        }
+    fn llama4_tokenizer() -> TestTokenizer {
+        TestTokenizer::new()
+            .with_regular_token("<|image_start|>", LLAMA4_IMAGE_START_ID)
+            .with_regular_token("<|image_end|>", LLAMA4_IMAGE_END_ID)
+            .with_regular_token("<|image|>", LLAMA4_IMAGE_ID)
+            .with_regular_token("<|patch|>", LLAMA4_PATCH_ID)
+            .with_regular_token("<|tile_x_separator|>", LLAMA4_TILE_X_SEPARATOR_ID)
+            .with_regular_token("<|tile_y_separator|>", LLAMA4_TILE_Y_SEPARATOR_ID)
     }
 
     fn test_info(model_type: &str, config: serde_json::Value) -> MultimodalModelInfo {
@@ -635,7 +589,7 @@ mod tests {
             model_id: format!("{model_type}-test"),
             model_type: Some(model_type.to_string()),
             config,
-            tokenizer: TokenizerResolver(Arc::new(TestTokenizer)),
+            tokenizer: TokenizerResolver(Arc::new(llama4_tokenizer())),
         };
         let spec = context
             .resolve_model_spec()

rust/src/chat/src/multimodal/tensor.rs

@@ -2,7 +2,7 @@ use std::collections::HashMap;
 
 use half::{bf16, f16};
 use llm_multimodal::{ModelSpecificValue, PreprocessedImages};
-use vllm_engine_core_client::protocol::ModelDtype;
+use vllm_engine_core_client::protocol::dtype::ModelDtype;
 use vllm_engine_core_client::protocol::multimodal::MmKwargValue as ProtocolKwargValue;
 use vllm_engine_core_client::protocol::tensor::{ShapeExt as _, WireTensor};
 

rust/src/chat/src/output/default/mod.rs

@@ -189,46 +189,19 @@ impl ChatOutputProcessor for DefaultChatOutputProcessor {
 mod tests {
     use std::sync::Arc;
 
-    use vllm_tokenizer::Tokenizer;
+    use vllm_tokenizer::test_utils::TestTokenizer;
 
     use super::DefaultChatOutputProcessor;
     use crate::Error;
     use crate::parser::ParserSelection;
     use crate::request::ChatRequest;
 
-    struct FakeTokenizer;
-
-    impl Tokenizer for FakeTokenizer {
-        fn encode(
-            &self,
-            text: &str,
-            _add_special_tokens: bool,
-        ) -> vllm_tokenizer::Result<Vec<u32>> {
-            Ok(text.chars().map(u32::from).collect())
-        }
-
-        fn decode(
-            &self,
-            token_ids: &[u32],
-            _skip_special_tokens: bool,
-        ) -> vllm_tokenizer::Result<String> {
-            Ok(token_ids
-                .iter()
-                .map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
-                .collect())
-        }
-
-        fn token_to_id(&self, token: &str) -> Option<u32> {
-            match token {
-                "<|channel>" => Some(1),
-                "<channel|>" => Some(2),
-                _ => None,
-            }
-        }
-    }
-
-    fn tokenizer() -> Arc<FakeTokenizer> {
-        Arc::new(FakeTokenizer)
+    fn tokenizer() -> Arc<TestTokenizer> {
+        Arc::new(
+            TestTokenizer::new()
+                .with_regular_token("<|channel>", 256)
+                .with_regular_token("<channel|>", 257),
+        )
     }
 
     #[test]

rust/src/chat/src/output/default/structural_tag.rs

@@ -1,7 +1,9 @@
 //! Applies xgrammar structural-tag constraints for strict tool calling.
 
 use thiserror_ext::AsReport;
-use vllm_engine_core_client::protocol::{StructuredOutputBackend, StructuredOutputsParams};
+use vllm_engine_core_client::protocol::structured_outputs::{
+    StructuredOutputBackend, StructuredOutputsParams,
+};
 use vllm_parser::tool::StructuralTagModel;
 use xgrammar_structural_tag::{
     FunctionDefinition, FunctionToolParam, ToolChoice as StructuralTagToolChoice, ToolParam,
@@ -76,7 +78,9 @@ fn structural_tag_tool_choice(request: &ChatRequest) -> Option<StructuralTagTool
 #[cfg(test)]
 mod tests {
     use serde_json::{Value, json};
-    use vllm_engine_core_client::protocol::{StructuredOutputBackend, StructuredOutputsParams};
+    use vllm_engine_core_client::protocol::structured_outputs::{
+        StructuredOutputBackend, StructuredOutputsParams,
+    };
     use vllm_parser::tool::{Qwen3CoderToolParser, Tool, ToolParser};
 
     use super::*;

rust/src/chat/src/output/harmony/tests.rs

@@ -1,14 +1,8 @@
-//! Harmony output tests share the upstream `openai-harmony` tiktoken cache.
-//!
-//! Use a file lock for tests that load the encoding so `cargo nextest` cannot
-//! start multiple processes that concurrently populate the same cache file.
-
 use std::sync::Arc;
 
 use futures::executor::block_on;
 use futures::{TryStreamExt as _, stream};
 use openai_harmony::chat::{Message, Role};
-use serial_test::file_serial;
 use vllm_text::output::{DecodedLogprobs, DecodedPositionLogprobs, DecodedTextEvent, Finished};
 
 use super::*;
@@ -91,7 +85,6 @@ fn request_with_tools() -> ChatRequest {
 }
 
 #[test]
-#[file_serial(harmony_tiktoken_cache)]
 fn interrupted_final_message_is_preserved() {
     let tokens = completion_tokens(&[text_message("final", "hello")]);
     let events = block_on(collect_events(
@@ -127,7 +120,6 @@ fn interrupted_final_message_is_preserved() {
 }
 
 #[test]
-#[file_serial(harmony_tiktoken_cache)]
 fn eos_flush_preserves_trailing_replacement_text() {
     let mut tokens = completion_tokens(&[text_message("final", "Hi")]);
     tokens.pop();
@@ -153,7 +145,6 @@ fn eos_flush_preserves_trailing_replacement_text() {
 }
 
 #[test]
-#[file_serial(harmony_tiktoken_cache)]
 fn interrupted_analysis_message_is_preserved() {
     let tokens = completion_tokens(&[text_message("analysis", "think")]);
     let events = block_on(collect_events(
@@ -189,7 +180,6 @@ fn interrupted_analysis_message_is_preserved() {
 }
 
 #[test]
-#[file_serial(harmony_tiktoken_cache)]
 fn commentary_preamble_is_visible_but_commentary_tool_payload_is_not() {
     let tokens = completion_tokens(&[
         text_message("commentary", "Let me check."),
@@ -217,7 +207,6 @@ fn commentary_preamble_is_visible_but_commentary_tool_payload_is_not() {
 }
 
 #[test]
-#[file_serial(harmony_tiktoken_cache)]
 fn multiple_messages_get_newline_separators() {
     let tokens = completion_tokens(&[
         text_message("analysis", "first think"),
@@ -249,7 +238,6 @@ fn multiple_messages_get_newline_separators() {
 }
 
 #[test]
-#[file_serial(harmony_tiktoken_cache)]
 fn tool_calls_stream_arguments_and_finish_with_local_id_shape() {
     let tokens = completion_tokens(&[tool_message(
         "get_weather",
@@ -302,7 +290,6 @@ fn tool_calls_stream_arguments_and_finish_with_local_id_shape() {
 }
 
 #[test]
-#[file_serial(harmony_tiktoken_cache)]
 fn semantic_events_precede_same_update_logprobs() {
     let tokens = completion_tokens(&[text_message("final", "hello")]);
     let events = block_on(collect_events(
@@ -353,7 +340,6 @@ fn rejects_generic_parser_overrides() {
 }
 
 #[test]
-#[file_serial(harmony_tiktoken_cache)]
 fn allows_auto_auto_only() {
     validate_harmony_parser_overrides(&ParserSelection::Auto, &ParserSelection::Auto).unwrap();
     let _ = HarmonyChatOutputProcessor::new(&ChatRequest::for_test()).unwrap();

rust/src/chat/src/parser/reasoning/tests.rs

@@ -1,32 +1,9 @@
 use std::sync::Arc;
 
-use vllm_tokenizer::Tokenizer;
+use vllm_tokenizer::test_utils::TestTokenizer;
 
 use super::{ReasoningParserFactory, names};
 
-struct FakeTokenizer;
-
-impl Tokenizer for FakeTokenizer {
-    fn encode(&self, text: &str, _add_special_tokens: bool) -> vllm_tokenizer::Result<Vec<u32>> {
-        Ok(text.chars().map(u32::from).collect())
-    }
-
-    fn decode(
-        &self,
-        token_ids: &[u32],
-        _skip_special_tokens: bool,
-    ) -> vllm_tokenizer::Result<String> {
-        Ok(token_ids
-            .iter()
-            .map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
-            .collect())
-    }
-
-    fn token_to_id(&self, _token: &str) -> Option<u32> {
-        None
-    }
-}
-
 #[test]
 fn factory_contains_and_lists_registered_parsers() {
     let factory = ReasoningParserFactory::new();
@@ -107,7 +84,7 @@ fn factory_resolves_minimax_m3_before_generic_minimax() {
 
 #[test]
 fn factory_rejects_unknown_parser_names() {
-    let tokenizer = Arc::new(FakeTokenizer);
+    let tokenizer = Arc::new(TestTokenizer::new());
     let factory = ReasoningParserFactory::new();
     let error = match factory.create("missing", tokenizer) {
         Ok(_) => panic!("expected parser lookup to fail"),

rust/src/chat/src/parser/unified.rs

@@ -75,39 +75,14 @@ impl UnifiedParserFactory {
 mod tests {
     use std::sync::Arc;
 
-    use vllm_tokenizer::Tokenizer;
+    use vllm_tokenizer::test_utils::TestTokenizer;
 
     use super::{UnifiedParserFactory, names};
 
-    struct FakeTokenizer;
-
-    impl Tokenizer for FakeTokenizer {
-        fn encode(
-            &self,
-            text: &str,
-            _add_special_tokens: bool,
-        ) -> vllm_tokenizer::Result<Vec<u32>> {
-            Ok(text.chars().map(u32::from).collect())
-        }
-
-        fn decode(
-            &self,
-            token_ids: &[u32],
-            _skip_special_tokens: bool,
-        ) -> vllm_tokenizer::Result<String> {
-            Ok(token_ids
-                .iter()
-                .map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
-                .collect())
-        }
-
-        fn token_to_id(&self, token: &str) -> Option<u32> {
-            match token {
-                "<|channel>" => Some(1),
-                "<channel|>" => Some(2),
-                _ => None,
-            }
-        }
+    fn tokenizer() -> TestTokenizer {
+        TestTokenizer::new()
+            .with_regular_token("<|channel>", 256)
+            .with_regular_token("<channel|>", 257)
     }
 
     #[test]
@@ -119,6 +94,6 @@ mod tests {
             factory.resolve_name_for_model("google/gemma-4-27b-it"),
             Some(names::GEMMA4)
         );
-        factory.create(names::GEMMA4, &[], Arc::new(FakeTokenizer)).unwrap();
+        factory.create(names::GEMMA4, &[], Arc::new(tokenizer())).unwrap();
     }
 }

rust/src/chat/tests/chat.rs

@@ -15,21 +15,24 @@ use vllm_chat::{
 use vllm_engine_core_client::protocol::logprobs::{
     Logprobs, MaybeWireLogprobs, PositionLogprobs, TokenLogprob,
 };
-use vllm_engine_core_client::protocol::{
-    EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, EngineCoreRequest, StopReason,
+use vllm_engine_core_client::protocol::output::{
+    EngineCoreFinishReason, EngineCoreOutput, EngineCoreOutputs, StopReason,
 };
+use vllm_engine_core_client::protocol::request::EngineCoreRequest;
 use vllm_engine_core_client::test_utils::{IpcNamespace, spawn_mock_engine_task};
 use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig};
 use vllm_llm::Llm;
-use vllm_text::tokenizer::{DynTokenizer, Tokenizer};
+use vllm_text::tokenizer::DynTokenizer;
 use vllm_text::{
     DecodedLogprobs, DecodedPositionLogprobs, DecodedPromptLogprobs, DecodedTokenLogprob, Prompt,
     TextBackend,
 };
+use vllm_tokenizer::test_utils::TestTokenizer;
 use zeromq::prelude::{SocketRecv, SocketSend};
 use zeromq::{DealerSocket, PushSocket, ZmqMessage};
 
 const SPECIAL_STOP_TOKEN_ID: u32 = 256;
+const UNKNOWN_DECODE_TOKEN_ID: u32 = 10_000;
 
 fn request_output(
     request_id: &str,
@@ -158,45 +161,18 @@ async fn connect_chat_llm_with_ipc(
 struct FakeChatBackend {
     has_template: bool,
     model_id: String,
+    tokenizer: DynTokenizer,
 }
 
-#[derive(Debug)]
-struct FakeChatTokenizer;
-
-impl Tokenizer for FakeChatTokenizer {
-    fn encode(&self, text: &str, _add_special_tokens: bool) -> vllm_tokenizer::Result<Vec<u32>> {
-        Ok(text.bytes().map(u32::from).collect())
-    }
-
-    fn decode(
-        &self,
-        token_ids: &[u32],
-        skip_special_tokens: bool,
-    ) -> vllm_tokenizer::Result<String> {
-        let bytes = token_ids
-            .iter()
-            .filter_map(|id| {
-                if skip_special_tokens && *id == SPECIAL_STOP_TOKEN_ID {
-                    None
-                } else {
-                    Some(*id as u8)
-                }
-            })
-            .collect::<Vec<_>>();
-        Ok(String::from_utf8_lossy(&bytes).into_owned())
-    }
-
-    fn token_to_id(&self, token: &str) -> Option<u32> {
-        match token {
-            "<think>" => Some(0xF001),
-            "</think>" => Some(0xF002),
-            "<|START_THINKING|>" => Some(0xF003),
-            "<|END_THINKING|>" => Some(0xF004),
-            "◁think▷" => Some(0xF005),
-            "◁/think▷" => Some(0xF006),
-            _ => None,
-        }
-    }
+fn fake_chat_tokenizer() -> TestTokenizer {
+    TestTokenizer::new()
+        .with_special_token("<stop>", SPECIAL_STOP_TOKEN_ID)
+        .with_regular_token("<think>", 0xF001)
+        .with_regular_token("</think>", 0xF002)
+        .with_regular_token("<|START_THINKING|>", 0xF003)
+        .with_regular_token("<|END_THINKING|>", 0xF004)
+        .with_regular_token("◁think▷", 0xF005)
+        .with_regular_token("◁/think▷", 0xF006)
 }
 
 impl fmt::Debug for FakeChatBackend {
@@ -210,6 +186,7 @@ impl FakeChatBackend {
         Self {
             has_template: true,
             model_id: "test-model".to_string(),
+            tokenizer: Arc::new(fake_chat_tokenizer()),
         }
     }
 
@@ -217,6 +194,7 @@ impl FakeChatBackend {
         Self {
             has_template: false,
             model_id: "test-model".to_string(),
+            tokenizer: Arc::new(fake_chat_tokenizer()),
         }
     }
 
@@ -224,13 +202,19 @@ impl FakeChatBackend {
         Self {
             has_template: true,
             model_id: model_id.into(),
+            tokenizer: Arc::new(fake_chat_tokenizer()),
         }
     }
+
+    fn with_tokenizer(mut self, tokenizer: DynTokenizer) -> Self {
+        self.tokenizer = tokenizer;
+        self
+    }
 }
 
 impl TextBackend for FakeChatBackend {
     fn tokenizer(&self) -> DynTokenizer {
-        Arc::new(FakeChatTokenizer)
+        Arc::clone(&self.tokenizer)
     }
 
     fn model_id(&self) -> &str {
@@ -282,65 +266,6 @@ impl ChatRenderer for FakeChatBackend {
     }
 }
 
-#[derive(Clone, Debug)]
-struct FailingDecodeBackend {
-    inner: FakeChatBackend,
-}
-
-#[derive(Debug)]
-struct FailingDecodeTokenizer;
-
-impl Tokenizer for FailingDecodeTokenizer {
-    fn encode(&self, text: &str, add_special_tokens: bool) -> vllm_tokenizer::Result<Vec<u32>> {
-        FakeChatTokenizer.encode(text, add_special_tokens)
-    }
-
-    fn decode(
-        &self,
-        token_ids: &[u32],
-        skip_special_tokens: bool,
-    ) -> vllm_tokenizer::Result<String> {
-        if token_ids.contains(&(b'i' as u32)) {
-            return Err(vllm_tokenizer::TokenizerError("decode failed".to_string()));
-        }
-        FakeChatTokenizer.decode(token_ids, skip_special_tokens)
-    }
-
-    fn token_to_id(&self, token: &str) -> Option<u32> {
-        FakeChatTokenizer.token_to_id(token)
-    }
-}
-
-impl TextBackend for FailingDecodeBackend {
-    fn tokenizer(&self) -> DynTokenizer {
-        Arc::new(FailingDecodeTokenizer)
-    }
-
-    fn model_id(&self) -> &str {
-        self.inner.model_id()
-    }
-}
-
-impl ChatBackend for FailingDecodeBackend {
-    fn chat_renderer(&self) -> DynChatRenderer {
-        Arc::new(self.clone())
-    }
-
-    fn new_chat_output_processor(
-        &self,
-        _request: &mut ChatRequest,
-        _options: NewChatOutputProcessorOptions<'_>,
-    ) -> vllm_chat::Result<DynChatOutputProcessor> {
-        Ok(Box::new(DefaultChatOutputProcessor::plain_text_only()))
-    }
-}
-
-impl ChatRenderer for FailingDecodeBackend {
-    fn render(&self, request: &ChatRequest) -> vllm_chat::Result<RenderedPrompt> {
-        self.inner.render(request)
-    }
-}
-
 /// Skip `LogprobsDelta` events that carry only token_ids (no logprobs),
 /// returning the next semantically interesting event.
 async fn next_semantic<S>(stream: &mut S) -> Option<Result<ChatEvent, vllm_chat::Error>>
@@ -738,7 +663,12 @@ async fn chat_stream_reports_decode_failure_as_error_event() {
                 send_outputs(
                     push,
                     EngineCoreOutputs {
-                        outputs: vec![request_output("chat-4", vec![b'i' as u32], None, None)],
+                        outputs: vec![request_output(
+                            "chat-4",
+                            vec![UNKNOWN_DECODE_TOKEN_ID],
+                            None,
+                            None,
+                        )],
                         ..Default::default()
                     },
                 )
@@ -747,9 +677,8 @@ async fn chat_stream_reports_decode_failure_as_error_event() {
         },
     );
 
-    let backend: Arc<dyn ChatTextBackend> = Arc::new(FailingDecodeBackend {
-        inner: FakeChatBackend::new(),
-    });
+    let backend: Arc<dyn ChatTextBackend> =
+        Arc::new(FakeChatBackend::new().with_tokenizer(Arc::new(TestTokenizer::new())));
     let chat = connect_chat_llm_with_ipc(
         EngineCoreClientConfig::new_single(handshake_address),
         &ipc,
@@ -769,7 +698,10 @@ async fn chat_stream_reports_decode_failure_as_error_event() {
 
     match timeout(Duration::from_secs(2), stream.next()).await.unwrap() {
         Some(Err(vllm_chat::Error::Text(vllm_text::Error::Tokenizer(message)))) => {
-            assert_eq!(message, "decode failed");
+            assert_eq!(
+                message,
+                format!("test tokenizer cannot decode unknown token id {UNKNOWN_DECODE_TOKEN_ID}")
+            );
         }
         other => panic!("unexpected event after close: {other:?}"),
     }

rust/src/chat/tests/roundtrip.rs

@@ -1,8 +1,8 @@
-//! Text-level roundtrip tests for the real chat-template and output-processor pairing.
+//! Roundtrip tests for the real chat-template and output-processor pairing.
 //!
 //! The invariant under test is that a structured assistant message rendered as history can be
 //! parsed from the generated assistant completion and then rendered back to the exact same
-//! assistant-completion text.
+//! assistant completion.
 
 use std::pin::Pin;
 use std::sync::Arc;
@@ -18,6 +18,10 @@ use vllm_chat::{
     RendererSelection, load_model_backends,
 };
 use vllm_text::{DecodedTextEvent, Finished, Prompt};
+use vllm_tokenizer::Tokenizer;
+
+const TEXT_COMPLETION_CHUNK_CHARS: usize = 7;
+const TOKEN_COMPLETION_CHUNK_TOKENS: usize = 1;
 
 /// One model/parser configuration used to run the fixed roundtrip fixtures.
 #[derive(Clone)]
@@ -191,14 +195,28 @@ impl RoundtripCase {
             sort_json_keys: false,
         }
     }
+
+    /// GPT-OSS Harmony token-id renderer and native Harmony output processor.
+    fn gpt_oss() -> Self {
+        Self {
+            model_id: "openai/gpt-oss-20b",
+            assistant_stop_suffix: "", // not applicable for token-id cases
+            tool_call_parser: ParserSelection::Auto,
+            reasoning_parser: ParserSelection::Auto,
+            thinking_behavior: ThinkingBehavior::Always { value: true },
+            json_fmt: compact_json_fmt(),
+            sort_json_keys: false,
+        }
+    }
 }
 
 macro_rules! roundtrip_tests {
-    ($($case:ident => [$($fixture:ident),* $(,)?]),+ $(,)?) => {
+    ($($case:ident => [$($(#[$fixture_attr:meta])* $fixture:ident),* $(,)?]),+ $(,)?) => {
         paste::paste! {
             $(
                 $(
                     #[tokio::test]
+                    $(#[$fixture_attr])*
                     #[file_serial([<hf_ $case>])]
                     async fn [<roundtrip_ $case _ $fixture>]() -> Result<()> {
                         [<run_roundtrip_ $fixture>](RoundtripCase::$case()).await
@@ -217,9 +235,9 @@ roundtrip_tests! {
     glm47 => [reasoning_and_content, tool_call_mix],
     seed_oss => [reasoning_and_content],
     step3p5 => [reasoning_and_content],
-
     gemma4 => [tool_call_mix], // Gemma4 strips reasoning in history if there's no tool call
     kimi_k25 => [tool_call_mix], // Kimi K2.5 strips reasoning in history
+    gpt_oss => [tool_call_mix], // Harmony strips reasoning in history if there's no tool call
 }
 
 /// Run the fixed reasoning+content fixture for one model/parser case.
@@ -421,10 +439,10 @@ struct RoundtripResult {
     parsed_message: AssistantMessage,
     /// Assistant-completion suffix cut from rendering the expected assistant as
     /// history.
-    closed_completion: String,
+    closed_completion: Prompt,
     /// Assistant-completion suffix cut after rendering the parsed assistant
     /// back as history.
-    rerendered_closed_completion: String,
+    rerendered_closed_completion: Prompt,
 }
 
 /// Render, parse, and rerender one assistant turn through the production
@@ -436,60 +454,59 @@ async fn run_roundtrip(
     assistant: AssistantMessage,
 ) -> Result<RoundtripResult> {
     let renderer = backends.chat_backend.chat_renderer();
-    let (prompt, closed_completion_text) =
-        render_closed_completion(renderer.as_ref(), request, &assistant)?;
-    let completion_body = closed_completion_text
-        .strip_suffix(case.assistant_stop_suffix)
-        .with_context(|| {
-            format!(
-                "closed assistant completion did not end with {:?}: {:?}",
-                case.assistant_stop_suffix, closed_completion_text
-            )
-        })?;
+    let rendered = render_closed_completion(renderer.as_ref(), request, &assistant)?;
 
-    let parsed_message =
-        parse_completion(case, backends, request, &prompt, completion_body).await?;
-    let (_, rerendered_closed_completion) =
-        render_closed_completion(renderer.as_ref(), request, &parsed_message)?;
+    let parsed_message = parse_completion(case, backends, request, &rendered).await?;
+    let rerendered = render_closed_completion(renderer.as_ref(), request, &parsed_message)?;
 
     Ok(RoundtripResult {
         parsed_message,
-        closed_completion: closed_completion_text,
-        rerendered_closed_completion,
+        closed_completion: rendered.completion,
+        rerendered_closed_completion: rerendered.completion,
     })
 }
 
+/// Rendered prompt/completion artifacts at the renderer boundary.
+struct RenderedTurn {
+    prompt: Prompt,
+    completion: Prompt,
+}
+
 /// Render `history` as a production prompt and `history + assistant` as closed
 /// history, then return the production prompt and assistant-completion suffix.
 fn render_closed_completion(
     renderer: &dyn vllm_chat::ChatRenderer,
     base_request: &ChatRequest,
     assistant: &AssistantMessage,
-) -> Result<(String, String)> {
+) -> Result<RenderedTurn> {
     let mut prompt_request = base_request.clone();
     prompt_request.chat_options.generation_prompt_mode = GenerationPromptMode::StartNewAssistant;
-    let prompt = render_text(renderer, &prompt_request).context("failed to render prompt")?;
+    let prompt = renderer.render(&prompt_request).context("failed to render prompt")?.prompt;
 
     let mut full_request = base_request.clone();
     full_request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt;
     full_request.messages.push(ChatMessage::from(assistant.clone()));
-    let full = render_text(renderer, &full_request).context("failed to render full prompt")?;
+    let full = renderer.render(&full_request).context("failed to render full prompt")?.prompt;
 
-    ensure!(
-        full.starts_with(&prompt),
-        "full prompt must extend production prompt\nprompt: {prompt:?}\nfull: {full:?}"
-    );
-    let completion = full[prompt.len()..].to_string();
+    let completion = match (&prompt, full) {
+        (Prompt::Text(prompt), Prompt::Text(full)) => {
+            ensure!(
+                full.starts_with(prompt),
+                "full prompt must extend production prompt\nprompt: {prompt:?}\nfull: {full:?}"
+            );
+            Prompt::Text(full[prompt.len()..].to_string())
+        }
+        (Prompt::TokenIds(prompt), Prompt::TokenIds(full)) => {
+            ensure!(
+                full.starts_with(prompt),
+                "full prompt must extend production prompt\nprompt: {prompt:?}\nfull: {full:?}"
+            );
+            Prompt::TokenIds(full[prompt.len()..].to_vec())
+        }
+        (prompt, full) => bail!("prompt kind changed between renders: {prompt:?} vs {full:?}"),
+    };
 
-    Ok((prompt, completion))
-}
-
-/// Render one chat request and require a text prompt.
-fn render_text(renderer: &dyn vllm_chat::ChatRenderer, request: &ChatRequest) -> Result<String> {
-    match renderer.render(request)?.prompt {
-        Prompt::Text(text) => Ok(text),
-        other => bail!("roundtrip tests expect text prompts, got {other:?}"),
-    }
+    Ok(RenderedTurn { prompt, completion })
 }
 
 /// Feed one rendered assistant completion body into the real output processor
@@ -498,13 +515,15 @@ async fn parse_completion(
     case: &RoundtripCase,
     backends: &vllm_chat::LoadedModelBackends,
     base_request: &ChatRequest,
-    prompt: &str,
-    completion_body: &str,
+    rendered: &RenderedTurn,
 ) -> Result<AssistantMessage> {
     let tokenizer = backends.text_backend.tokenizer();
-    let prompt_token_ids = tokenizer
-        .encode(prompt, base_request.add_special_tokens)
-        .context("failed to encode rendered prompt")?;
+    let prompt_token_ids = match &rendered.prompt {
+        Prompt::Text(prompt) => tokenizer
+            .encode(prompt, base_request.add_special_tokens)
+            .context("failed to encode rendered prompt")?,
+        Prompt::TokenIds(token_ids) => token_ids.clone(),
+    };
 
     let mut request = base_request.clone();
     let processor = backends.chat_backend.new_chat_output_processor(
@@ -515,7 +534,12 @@ async fn parse_completion(
         },
     )?;
 
-    let decoded = decoded_completion_stream(prompt_token_ids, completion_body);
+    let decoded = decoded_completion_stream(
+        tokenizer.as_ref(),
+        prompt_token_ids,
+        &rendered.completion,
+        case.assistant_stop_suffix,
+    )?;
     let mut events = processor.process(decoded)?;
 
     while let Some(event) = events.next().await {
@@ -538,16 +562,46 @@ async fn parse_completion(
 /// split into small chunks to exercise streaming parser state across marker
 /// and JSON boundaries.
 fn decoded_completion_stream(
+    tokenizer: &dyn Tokenizer,
     prompt_token_ids: Vec<u32>,
-    completion_body: &str,
-) -> Pin<Box<dyn Stream<Item = vllm_chat::Result<DecodedTextEvent>> + Send>> {
-    let prompt_token_count = prompt_token_ids.len();
+    completion: &Prompt,
+    assistant_stop_suffix: &str,
+) -> Result<Pin<Box<dyn Stream<Item = vllm_chat::Result<DecodedTextEvent>> + Send>>> {
     let mut events = vec![DecodedTextEvent::Start {
-        prompt_token_ids: Arc::from(prompt_token_ids.into_boxed_slice()),
+        prompt_token_ids: Arc::from(prompt_token_ids.clone().into_boxed_slice()),
         prompt_logprobs: None,
     }];
 
-    let chunks = split_by_chars(completion_body, 7);
+    let chunks = match completion {
+        Prompt::Text(text) => {
+            let body = text.strip_suffix(assistant_stop_suffix).with_context(|| {
+                format!(
+                    "closed assistant completion did not end with {:?}: {:?}",
+                    assistant_stop_suffix, text
+                )
+            })?;
+            split_by_chars(body, TEXT_COMPLETION_CHUNK_CHARS)
+                .into_iter()
+                .map(|delta| DecodedCompletionChunk {
+                    delta,
+                    token_ids: Vec::new(), // unused for text-level roundtrip cases
+                })
+                .collect()
+        }
+        Prompt::TokenIds(token_ids) => {
+            ensure!(
+                assistant_stop_suffix.is_empty(),
+                "token-id roundtrip cases do not support text stop suffixes"
+            );
+            incremental_decode_chunks(
+                tokenizer,
+                &prompt_token_ids,
+                token_ids,
+                TOKEN_COMPLETION_CHUNK_TOKENS,
+            )?
+        }
+    };
+
     if chunks.is_empty() {
         events.push({
             DecodedTextEvent::TextDelta {
@@ -555,11 +609,7 @@ fn decoded_completion_stream(
                 token_ids: Vec::new(),
                 logprobs: None,
                 finished: Some(Finished {
-                    usage: vllm_llm::TokenUsage {
-                        prompt_token_count: 0,
-                        output_token_count: 0,
-                        cached_token_count: 0,
-                    },
+                    usage: Default::default(),
                     finish_reason: FinishReason::stop_eos(),
                     kv_transfer_params: None,
                 }),
@@ -569,24 +619,26 @@ fn decoded_completion_stream(
         let last_index = chunks.len() - 1;
         for (index, chunk) in chunks.into_iter().enumerate() {
             let finished = (index == last_index).then(|| Finished {
-                usage: vllm_llm::TokenUsage {
-                    prompt_token_count,
-                    output_token_count: completion_body.chars().count(),
-                    cached_token_count: 0,
-                },
+                usage: Default::default(),
                 finish_reason: FinishReason::stop_eos(),
                 kv_transfer_params: None,
             });
             events.push(DecodedTextEvent::TextDelta {
-                delta: chunk,
-                token_ids: Vec::new(),
+                delta: chunk.delta,
+                token_ids: chunk.token_ids,
                 logprobs: None,
                 finished,
             });
         }
     }
 
-    stream::iter(events).map(Ok).boxed()
+    Ok(stream::iter(events).map(Ok).boxed())
+}
+
+/// One decoded completion chunk fed into the output processor.
+struct DecodedCompletionChunk {
+    delta: String,
+    token_ids: Vec<u32>,
 }
 
 /// Split text into chunks containing at most `chunk_chars` Unicode scalar
@@ -612,6 +664,49 @@ fn split_by_chars(text: &str, chunk_chars: usize) -> Vec<String> {
     chunks
 }
 
+/// Split token ids into chunks containing at most `chunk_size` ids.
+fn split_by_count(token_ids: &[u32], chunk_size: usize) -> Vec<Vec<u32>> {
+    token_ids.chunks(chunk_size).map(<[u32]>::to_vec).collect()
+}
+
+/// Decode token ids incrementally using the production tokenizer stream.
+fn incremental_decode_chunks(
+    tokenizer: &dyn Tokenizer,
+    prompt_token_ids: &[u32],
+    token_ids: &[u32],
+    chunk_size: usize,
+) -> Result<Vec<DecodedCompletionChunk>> {
+    let mut decoder = tokenizer.create_decode_stream(prompt_token_ids, false, 0);
+    let mut chunks = Vec::new();
+    for chunk_token_ids in split_by_count(token_ids, chunk_size) {
+        let mut delta = String::new();
+        for token_id in chunk_token_ids.iter().copied() {
+            decoder.push_token(token_id)?;
+            while let Some(chunk) = decoder.next_chunk() {
+                delta.push_str(&chunk);
+            }
+        }
+        chunks.push(DecodedCompletionChunk {
+            delta,
+            token_ids: chunk_token_ids,
+        });
+    }
+
+    let (last_chunk, _) = decoder.flush(None)?;
+    if let Some(last_chunk) = last_chunk {
+        if let Some(delta) = chunks.last_mut() {
+            delta.delta.push_str(&last_chunk);
+        } else {
+            chunks.push(DecodedCompletionChunk {
+                delta: last_chunk,
+                token_ids: Vec::new(),
+            });
+        }
+    }
+
+    Ok(chunks)
+}
+
 /// Build a chat request fixture with parser-enabling tool-choice semantics.
 fn roundtrip_request(
     request_id: impl Into<String>,

rust/src/cmd/src/cli.rs

@@ -25,7 +25,7 @@ use vllm_managed_engine::ManagedEngineConfig;
 use vllm_managed_engine::cli::{ManagedEngineArgs, repartition_managed_engine_args};
 use vllm_server::{
     ApiServerOptions, ChatTemplateContentFormatOption, Config, CoordinatorMode, CorsConfig,
-    HttpListenerMode, ParserSelection, RendererSelection,
+    DEFAULT_KEEP_ALIVE_TIMEOUT, HttpListenerMode, ParserSelection, RendererSelection, TlsConfig,
 };
 
 use crate::cli::unsupported::UnsupportedArgs;
@@ -154,6 +154,11 @@ pub struct SharedRuntimeArgs {
     #[arg(long, default_value_t = 0)]
     #[serde(default)]
     pub shutdown_timeout: u64,
+    /// Maximum idle time (seconds) on a keep-alive HTTP connection before the
+    /// server closes it (default 5).
+    #[arg(long = "http-timeout-keep-alive", env = "VLLM_HTTP_TIMEOUT_KEEP_ALIVE")]
+    #[serde(default)]
+    pub http_timeout_keep_alive: Option<u64>,
 
     /// The file path to the chat template, or the template in single-line form
     /// for the specified model.
@@ -257,6 +262,34 @@ pub struct SharedRuntimeArgs {
     #[serde(default)]
     pub allow_credentials: bool,
 
+    /// The file path to the SSL key file. When omitted, the key is read from
+    /// `--ssl-certfile` (combined PEM).
+    #[arg(long)]
+    #[serde(default)]
+    pub ssl_keyfile: Option<String>,
+
+    /// The file path to the SSL cert file. Enables TLS when set.
+    #[arg(long)]
+    #[serde(default)]
+    pub ssl_certfile: Option<String>,
+
+    /// The CA certificates file used to verify client certificates (mTLS).
+    #[arg(long)]
+    #[serde(default)]
+    pub ssl_ca_certs: Option<String>,
+
+    /// Whether a client certificate is required: 0 = none, 1 = optional,
+    /// 2 = required (mirrors Python's `ssl.CERT_*`).
+    #[arg(long, default_value_t = 0, value_parser = clap::value_parser!(i32).range(0..=2))]
+    #[serde(default)]
+    pub ssl_cert_reqs: i32,
+
+    /// OpenSSL cipher string for HTTPS (TLS 1.2 and below).
+    /// When unset, the linked OpenSSL's default suites are used.
+    #[arg(long)]
+    #[serde(default)]
+    pub ssl_ciphers: Option<String>,
+
     /// Unsupported Python vLLM frontend arguments recognized but not yet
     /// implemented in Rust.
     #[educe(Debug(ignore))]
@@ -277,6 +310,13 @@ impl SharedRuntimeArgs {
         Duration::from_secs(self.shutdown_timeout)
     }
 
+    /// Maximum idle time on a keep-alive HTTP connection before the server
+    /// closes it.
+    pub fn keep_alive_timeout(&self) -> Duration {
+        self.http_timeout_keep_alive
+            .map_or(DEFAULT_KEEP_ALIVE_TIMEOUT, Duration::from_secs)
+    }
+
     /// Apply fallback logic for API key configuration from env variables.
     fn apply_env_api_key_fallback(&mut self) {
         if self.api_key.is_empty()
@@ -301,8 +341,10 @@ impl SharedRuntimeArgs {
     ) -> Config {
         let ready_timeout = self.ready_timeout();
         let shutdown_timeout = self.shutdown_timeout();
+        let keep_alive_timeout = self.keep_alive_timeout();
         let api_server_options = self.api_server_options();
         let cors = self.cors_config();
+        let tls = self.tls_config();
 
         Config {
             transport_mode: TransportMode::Bootstrapped {
@@ -329,10 +371,12 @@ impl SharedRuntimeArgs {
             max_logprobs: self.max_logprobs,
             api_server_options,
             cors,
+            tls,
             api_keys: self.api_key,
             disable_log_stats: self.disable_log_stats,
             grpc_port: self.grpc_port,
             shutdown_timeout,
+            keep_alive_timeout,
         }
     }
 
@@ -349,8 +393,10 @@ impl SharedRuntimeArgs {
     ) -> Config {
         let ready_timeout = self.ready_timeout();
         let shutdown_timeout = self.shutdown_timeout();
+        let keep_alive_timeout = self.keep_alive_timeout();
         let api_server_options = self.api_server_options();
         let cors = self.cors_config();
+        let tls = self.tls_config();
 
         Config {
             transport_mode: TransportMode::HandshakeOwner {
@@ -375,10 +421,12 @@ impl SharedRuntimeArgs {
             max_logprobs: self.max_logprobs,
             api_server_options,
             cors,
+            tls,
             api_keys: self.api_key,
             disable_log_stats: self.disable_log_stats,
             grpc_port: self.grpc_port,
             shutdown_timeout,
+            keep_alive_timeout,
         }
     }
 
@@ -398,6 +446,23 @@ impl SharedRuntimeArgs {
             allow_credentials: self.allow_credentials,
         }
     }
+
+    /// Build the TLS config: `Some` when any `ssl_*` argument is set, else
+    /// `None` (plaintext). The combination is validated in [`Config::validate`].
+    fn tls_config(&self) -> Option<TlsConfig> {
+        let tls_requested = self.ssl_certfile.is_some()
+            || self.ssl_keyfile.is_some()
+            || self.ssl_ca_certs.is_some()
+            || self.ssl_cert_reqs != 0
+            || self.ssl_ciphers.is_some();
+        tls_requested.then(|| TlsConfig {
+            cert_file: self.ssl_certfile.clone(),
+            key_file: self.ssl_keyfile.clone(),
+            ca_certs: self.ssl_ca_certs.clone(),
+            cert_reqs: self.ssl_cert_reqs,
+            ciphers: self.ssl_ciphers.clone(),
+        })
+    }
 }
 
 fn default_engine_ready_timeout_secs() -> u64 {

rust/src/cmd/src/cli/tests.rs

@@ -41,6 +41,7 @@ fn serve_args_forward_python_flags_with_separator() {
                         max_logprobs: None,
                         grpc_port: None,
                         shutdown_timeout: 0,
+                        http_timeout_keep_alive: None,
                         chat_template: None,
                         default_chat_template_kwargs: None,
                         chat_template_content_format: Auto,
@@ -65,6 +66,11 @@ fn serve_args_forward_python_flags_with_separator() {
                             ],
                         ),
                         allow_credentials: false,
+                        ssl_keyfile: None,
+                        ssl_certfile: None,
+                        ssl_ca_certs: None,
+                        ssl_cert_reqs: 0,
+                        ssl_ciphers: None,
                     },
                     managed_engine: ManagedEngineArgs {
                         python: "../vllm/.venv/bin/python",
@@ -363,6 +369,140 @@ fn serve_passes_enable_prompt_tokens_details_into_config() {
     assert!(config.api_server_options.enable_prompt_tokens_details);
 }
 
+#[test]
+fn serve_passes_tls_into_config() {
+    let cli = Cli::try_parse_from([
+        "vllm-rs",
+        "serve",
+        "Qwen/Qwen3-0.6B",
+        "--ssl-certfile",
+        "/tmp/cert.pem",
+        "--ssl-keyfile",
+        "/tmp/key.pem",
+        "--ssl-ca-certs",
+        "/tmp/ca.pem",
+        "--ssl-cert-reqs",
+        "2",
+    ])
+    .unwrap();
+
+    let Command::Serve(args) = cli.command else {
+        panic!("expected serve args");
+    };
+    let config = args.to_frontend_config("tcp://127.0.0.1:62100".to_string());
+    let tls = config.tls.expect("tls configured");
+    assert_eq!(tls.cert_file.as_deref(), Some("/tmp/cert.pem"));
+    assert_eq!(tls.key_file.as_deref(), Some("/tmp/key.pem"));
+    assert_eq!(tls.ca_certs.as_deref(), Some("/tmp/ca.pem"));
+    assert_eq!(tls.cert_reqs, 2);
+}
+
+#[test]
+fn serve_without_ssl_flags_has_no_tls() {
+    let cli = Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B"]).unwrap();
+
+    let Command::Serve(args) = cli.command else {
+        panic!("expected serve args");
+    };
+    let config = args.to_frontend_config("tcp://127.0.0.1:62100".to_string());
+    assert!(config.tls.is_none());
+}
+
+#[test]
+fn serve_ssl_keyfile_without_certfile_fails_validation() {
+    let cli = Cli::try_parse_from([
+        "vllm-rs",
+        "serve",
+        "Qwen/Qwen3-0.6B",
+        "--ssl-keyfile",
+        "/tmp/key.pem",
+    ])
+    .unwrap();
+
+    let Command::Serve(args) = cli.command else {
+        panic!("expected serve args");
+    };
+    let config = args.to_frontend_config("tcp://127.0.0.1:62100".to_string());
+    // TLS is requested (a key was given) but there is no certificate, so
+    // validation fails loud rather than silently serving plaintext.
+    assert_eq!(config.tls.as_ref().expect("tls requested").cert_file, None);
+    let err = config.validate().unwrap_err().to_string();
+    assert!(err.contains("--ssl-certfile is required"), "{err}");
+}
+
+#[test]
+fn serve_mtls_without_ca_certs_fails_validation() {
+    let cli = Cli::try_parse_from([
+        "vllm-rs",
+        "serve",
+        "Qwen/Qwen3-0.6B",
+        "--ssl-certfile",
+        "/tmp/cert.pem",
+        "--ssl-cert-reqs",
+        "2",
+    ])
+    .unwrap();
+
+    let Command::Serve(args) = cli.command else {
+        panic!("expected serve args");
+    };
+    let config = args.to_frontend_config("tcp://127.0.0.1:62100".to_string());
+    // Client-cert verification without a CA bundle has nothing to verify
+    // against, so it fails loud at startup.
+    let err = config.validate().unwrap_err().to_string();
+    assert!(err.contains("--ssl-ca-certs is required"), "{err}");
+}
+
+#[test]
+fn frontend_args_json_passes_tls_into_config() {
+    let cli = Cli::try_parse_from([
+        "vllm-rs",
+        "frontend",
+        "--listen-fd",
+        "3",
+        "--input-address",
+        "ipc:///tmp/input.sock",
+        "--output-address",
+        "ipc:///tmp/output.sock",
+        "--args-json",
+        r#"{"model_tag":"Qwen/Qwen3-0.6B","ssl_certfile":"/tmp/cert.pem","ssl_keyfile":"/tmp/key.pem"}"#,
+    ])
+    .unwrap();
+
+    let Command::Frontend(args) = cli.command else {
+        panic!("expected frontend args");
+    };
+    let config = args.into_config();
+    let tls = config.tls.expect("tls configured");
+    assert_eq!(tls.cert_file.as_deref(), Some("/tmp/cert.pem"));
+    assert_eq!(tls.key_file.as_deref(), Some("/tmp/key.pem"));
+}
+
+#[test]
+fn frontend_args_json_rejects_out_of_range_cert_reqs() {
+    let cli = Cli::try_parse_from([
+        "vllm-rs",
+        "frontend",
+        "--listen-fd",
+        "3",
+        "--input-address",
+        "ipc:///tmp/input.sock",
+        "--output-address",
+        "ipc:///tmp/output.sock",
+        "--args-json",
+        r#"{"model_tag":"Qwen/Qwen3-0.6B","ssl_certfile":"/tmp/cert.pem","ssl_cert_reqs":5}"#,
+    ])
+    .unwrap();
+
+    let Command::Frontend(args) = cli.command else {
+        panic!("expected frontend args");
+    };
+    // The JSON path bypasses clap's range check, so validate() is the only guard.
+    let config = args.into_config();
+    let err = config.validate().unwrap_err().to_string();
+    assert!(err.contains("--ssl-cert-reqs"), "{err}");
+}
+
 #[test]
 fn frontend_args_json_passes_enable_request_id_headers_into_config() {
     let cli = Cli::try_parse_from([
@@ -481,13 +621,13 @@ fn serve_args_reject_unsupported_flag_arg() {
         "vllm-rs",
         "serve",
         "Qwen/Qwen3-0.6B",
-        "--ssl-keyfile",
-        "/tmp/key.pem",
+        "--root-path",
+        "/prefix",
     ])
     .unwrap_err();
 
     expect![[r#"
-        error: invalid value '/tmp/key.pem' for '--ssl-keyfile <SSL_KEYFILE>': argument is not implemented in Rust frontend yet
+        error: invalid value '/prefix' for '--root-path <ROOT_PATH>': argument is not implemented in Rust frontend yet
 
         Remove this unsupported argument to continue.
 
@@ -562,6 +702,7 @@ fn frontend_args_accept_json() {
                         max_logprobs: None,
                         grpc_port: None,
                         shutdown_timeout: 0,
+                        http_timeout_keep_alive: None,
                         chat_template: None,
                         default_chat_template_kwargs: None,
                         chat_template_content_format: Auto,
@@ -586,6 +727,11 @@ fn frontend_args_accept_json() {
                             ],
                         ),
                         allow_credentials: false,
+                        ssl_keyfile: None,
+                        ssl_certfile: None,
+                        ssl_ca_certs: None,
+                        ssl_cert_reqs: 0,
+                        ssl_ciphers: None,
                     },
                 },
             ),
@@ -798,14 +944,14 @@ fn frontend_args_json_rejects_unsupported_fields() {
         "--output-address",
         "ipc:///tmp/output.sock",
         "--args-json",
-        r#"{"model_tag":"Qwen/Qwen3-0.6B","ssl_keyfile":"/tmp/key.pem"}"#,
+        r#"{"model_tag":"Qwen/Qwen3-0.6B","root_path":"/prefix"}"#,
     ])
     .unwrap_err();
 
     expect![[r#"
-        error: invalid value '{"model_tag":"Qwen/Qwen3-0.6B","ssl_keyfile":"/tmp/key.pem"}' for '--args-json <JSON>': 
+        error: invalid value '{"model_tag":"Qwen/Qwen3-0.6B","root_path":"/prefix"}' for '--args-json <JSON>': 
         The following arguments are not implemented in Rust frontend yet:
-        - ssl_keyfile
+        - root_path
 
         Remove these arguments to continue.
 
@@ -825,16 +971,16 @@ fn frontend_args_json_aggregates_multiple_unsupported_fields() {
         "--output-address",
         "ipc:///tmp/output.sock",
         "--args-json",
-        r#"{"model_tag":"Qwen/Qwen3-0.6B","response_role":"assistant","ssl_keyfile":"/tmp/key.pem"}"#,
+        r#"{"model_tag":"Qwen/Qwen3-0.6B","response_role":"assistant","root_path":"/prefix"}"#,
     ])
     .unwrap_err();
 
     let actual = error.to_string().replace(": \n", ":\n");
     expect![[r#"
-        error: invalid value '{"model_tag":"Qwen/Qwen3-0.6B","response_role":"assistant","ssl_keyfile":"/tmp/key.pem"}' for '--args-json <JSON>':
+        error: invalid value '{"model_tag":"Qwen/Qwen3-0.6B","response_role":"assistant","root_path":"/prefix"}' for '--args-json <JSON>':
         The following arguments are not implemented in Rust frontend yet:
         - response_role
-        - ssl_keyfile
+        - root_path
 
         Remove these arguments to continue.
 
@@ -1077,6 +1223,7 @@ fn serve_args_accept_handshake_aliases() {
                         max_logprobs: None,
                         grpc_port: None,
                         shutdown_timeout: 0,
+                        http_timeout_keep_alive: None,
                         chat_template: None,
                         default_chat_template_kwargs: None,
                         chat_template_content_format: Auto,
@@ -1101,6 +1248,11 @@ fn serve_args_accept_handshake_aliases() {
                             ],
                         ),
                         allow_credentials: false,
+                        ssl_keyfile: None,
+                        ssl_certfile: None,
+                        ssl_ca_certs: None,
+                        ssl_cert_reqs: 0,
+                        ssl_ciphers: None,
                     },
                     managed_engine: ManagedEngineArgs {
                         python: "python3",
@@ -1234,10 +1386,12 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
                 ],
                 allow_credentials: false,
             },
+            tls: None,
             api_keys: [],
             disable_log_stats: false,
             grpc_port: None,
             shutdown_timeout: 0ns,
+            keep_alive_timeout: 5s,
         }
     "#]]
     .assert_debug_eq(&Config {
@@ -1315,10 +1469,12 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
                 ],
                 allow_credentials: false,
             },
+            tls: None,
             api_keys: [],
             disable_log_stats: false,
             grpc_port: None,
             shutdown_timeout: 0ns,
+            keep_alive_timeout: 5s,
         }
     "#]]
     .assert_debug_eq(&config);
@@ -1414,10 +1570,12 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
                 ],
                 allow_credentials: false,
             },
+            tls: None,
             api_keys: [],
             disable_log_stats: false,
             grpc_port: None,
             shutdown_timeout: 0ns,
+            keep_alive_timeout: 5s,
         }
     "#]]
     .assert_debug_eq(&config);

rust/src/cmd/src/cli/unsupported.rs

@@ -526,18 +526,6 @@ pub struct ServerUnsupportedArgs {
     #[arg(long)]
     pub disable_access_log_for_endpoints: Option<Noop>,
 
-    /// The file path to the SSL key file.
-    #[arg(long)]
-    pub ssl_keyfile: Option<Unsupported>,
-
-    /// The file path to the SSL cert file.
-    #[arg(long)]
-    pub ssl_certfile: Option<Unsupported>,
-
-    /// The CA certificates file.
-    #[arg(long)]
-    pub ssl_ca_certs: Option<Unsupported>,
-
     /// Refresh SSL Context when SSL certificate files change
     #[arg(
         long,
@@ -547,15 +535,6 @@ pub struct ServerUnsupportedArgs {
     )]
     pub enable_ssl_refresh: Option<Unsupported>,
 
-    /// Whether client certificate is required (see stdlib ssl module's).
-    #[arg(long)]
-    pub ssl_cert_reqs: Option<Unsupported>,
-
-    /// SSL cipher suites for HTTPS (TLS 1.2 and below only).
-    /// Example: 'ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-CHACHA20-POLY1305'
-    #[arg(long)]
-    pub ssl_ciphers: Option<Unsupported>,
-
     /// FastAPI root_path when app is behind a path based routing proxy.
     #[arg(long)]
     pub root_path: Option<Unsupported>,

rust/src/engine-core-client/examples/external_engine_logprobs.rs

@@ -5,9 +5,9 @@ use clap::Parser;
 use futures::StreamExt as _;
 use tokio::time::timeout;
 use tracing_subscriber::EnvFilter;
-use vllm_engine_core_client::protocol::{
-    EngineCoreFinishReason, EngineCoreRequest, EngineCoreSamplingParams,
-};
+use vllm_engine_core_client::protocol::output::EngineCoreFinishReason;
+use vllm_engine_core_client::protocol::request::EngineCoreRequest;
+use vllm_engine_core_client::protocol::sampling::EngineCoreSamplingParams;
 use vllm_engine_core_client::{
     EngineCoreClient, EngineCoreClientConfig, EngineCoreStreamOutput, TransportMode,
 };

rust/src/engine-core-client/src/client.rs

@@ -11,10 +11,11 @@ use tracing::{debug, info, trace};
 use crate::client::imp::{ClientInner, run_abort_loop, run_output_dispatcher_loop};
 use crate::coordinator::CoordinatorHandle;
 use crate::error::{Error, Result};
+use crate::protocol::dtype::ModelDtype;
 use crate::protocol::handshake::EngineCoreReadyResponse;
 use crate::protocol::lora::LoraRequest;
+use crate::protocol::request::{EngineCoreRequest, EngineCoreRequestType};
 use crate::protocol::utility::{EngineCoreUtilityRequest, PauseMode};
-use crate::protocol::{EngineCoreRequest, EngineCoreRequestType, ModelDtype};
 use crate::runtime::{BackgroundShutdownRuntime, build_zmq_runtime};
 use crate::transport::{self, ConnectedEngine};
 

rust/src/engine-core-client/src/client/imp.rs

@@ -16,12 +16,11 @@ use crate::client::stream::EngineCoreStreamOutput;
 use crate::client::{AbortCause, AbortRequest};
 use crate::error::{client_closed, dispatcher_closed, unexpected_dispatcher_output};
 use crate::metrics::{LoraInfoExporter, record_scheduler_stats};
+use crate::protocol::encode_msgpack;
+use crate::protocol::output::{ClassifiedEngineCoreOutputs, EngineCoreOutput, EngineCoreOutputs};
+use crate::protocol::request::EngineCoreRequestType;
 use crate::protocol::stats::SchedulerStats;
 use crate::protocol::utility::UtilityOutput;
-use crate::protocol::{
-    ClassifiedEngineCoreOutputs, EngineCoreOutput, EngineCoreOutputs, EngineCoreRequestType,
-    encode_msgpack,
-};
 use crate::transport::{ConnectedEngine, EngineId};
 use crate::{Error, Result, transport};
 

rust/src/engine-core-client/src/client/state.rs

@@ -7,9 +7,9 @@ use tracing::trace;
 use crate::EngineId;
 use crate::client::stream::EngineCoreStreamOutput;
 use crate::error::{Error, Result};
+use crate::protocol::output::{EngineCoreEventType, EngineCoreFinishReason, EngineCoreOutput};
 use crate::protocol::stats::SchedulerStats;
 use crate::protocol::utility::UtilityOutput;
-use crate::protocol::{EngineCoreEventType, EngineCoreFinishReason, EngineCoreOutput};
 use crate::transport::ConnectedEngine;
 
 pub type OutputSender = mpsc::UnboundedSender<Result<EngineCoreStreamOutput>>;
@@ -100,6 +100,7 @@ impl EngineRoutingState {
 pub struct RequestRegistry {
     closed: bool,
     requests: HashMap<String, TrackedRequest>,
+    active_lora_requests: usize,
     routing_per_engine: BTreeMap<EngineId, EngineRoutingState>,
 }
 
@@ -108,6 +109,7 @@ impl RequestRegistry {
         Self {
             closed: false,
             requests: HashMap::default(),
+            active_lora_requests: 0,
             routing_per_engine: engines
                 .iter()
                 .map(|engine| (engine.engine_id.clone(), EngineRoutingState::default()))
@@ -133,15 +135,19 @@ impl RequestRegistry {
 
         let engine_id = self.choose_engine_for_request(data_parallel_rank)?;
         let (tx, rx) = mpsc::unbounded_channel();
+        let lora = lora_name.map(|adapter_name| LoraRequestState {
+            adapter_name,
+            phase: LoraPhase::Waiting,
+        });
+        if lora.is_some() {
+            self.active_lora_requests += 1;
+        }
         self.requests.insert(
             request_id,
             TrackedRequest {
                 sender: tx,
                 engine_id: engine_id.clone(),
-                lora: lora_name.map(|adapter_name| LoraRequestState {
-                    adapter_name,
-                    phase: LoraPhase::Waiting,
-                }),
+                lora,
             },
         );
 
@@ -230,6 +236,10 @@ impl RequestRegistry {
     /// Snapshot the adapter names of tracked LoRA requests as
     /// (running, waiting) sets. Feeds the `vllm:lora_requests_info` gauge.
     pub fn lora_adapter_states(&self) -> (BTreeSet<String>, BTreeSet<String>) {
+        if self.active_lora_requests == 0 {
+            return (BTreeSet::new(), BTreeSet::new());
+        }
+
         let mut running = BTreeSet::new();
         let mut waiting = BTreeSet::new();
         for lora in self.requests.values().filter_map(|tracked| tracked.lora.as_ref()) {
@@ -283,6 +293,7 @@ impl RequestRegistry {
         }
 
         self.closed = true;
+        self.active_lora_requests = 0;
         std::mem::take(&mut self.requests)
             .into_values()
             .map(|tracked| tracked.sender)
@@ -322,6 +333,9 @@ impl RequestRegistry {
     #[must_use]
     pub fn remove(&mut self, request_id: &str) -> Option<(OutputSender, EngineId)> {
         let tracked = self.requests.remove(request_id)?;
+        if tracked.lora.is_some() {
+            self.active_lora_requests -= 1;
+        }
         self.routing_per_engine
             .get_mut(&tracked.engine_id)
             .expect("request registry must track all known engines")
@@ -359,6 +373,11 @@ impl RequestRegistry {
     pub fn is_closed(&self) -> bool {
         self.closed
     }
+
+    #[cfg(test)]
+    fn active_lora_requests(&self) -> usize {
+        self.active_lora_requests
+    }
 }
 
 /// Internal registry for tracking active utility calls and their waiting
@@ -433,7 +452,7 @@ mod tests {
         EngineLoadSnapshot, EngineRoutingState, RequestRegistry, UtilityRegistry,
     };
     use crate::mock_engine::default_ready_response;
-    use crate::protocol::{
+    use crate::protocol::output::{
         EngineCoreEvent, EngineCoreEventType, EngineCoreFinishReason, EngineCoreOutput,
     };
     use crate::transport::ConnectedEngine;
@@ -574,6 +593,63 @@ mod tests {
         );
     }
 
+    #[test]
+    fn registry_counts_only_active_lora_requests() {
+        let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
+
+        registry.register("req-plain".to_string(), None, None).unwrap();
+        assert_eq!(registry.active_lora_requests(), 0);
+        assert_eq!(
+            registry.lora_adapter_states(),
+            (adapter_names(&[]), adapter_names(&[]))
+        );
+
+        registry
+            .register(
+                "req-lora-a".to_string(),
+                Some("adapter-a".to_string()),
+                None,
+            )
+            .unwrap();
+        registry
+            .register(
+                "req-lora-b".to_string(),
+                Some("adapter-b".to_string()),
+                None,
+            )
+            .unwrap();
+        assert_eq!(registry.active_lora_requests(), 2);
+
+        drop(registry.remove("req-plain"));
+        assert_eq!(registry.active_lora_requests(), 2);
+
+        drop(registry.finish_many(&["req-lora-a".to_string()]));
+        assert_eq!(registry.active_lora_requests(), 1);
+
+        drop(registry.abort_many(&["req-lora-b".to_string()], 0.0));
+        assert_eq!(registry.active_lora_requests(), 0);
+        assert_eq!(
+            registry.lora_adapter_states(),
+            (adapter_names(&[]), adapter_names(&[]))
+        );
+    }
+
+    #[test]
+    fn registry_clears_lora_count_on_close() {
+        let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
+        registry
+            .register("req-lora".to_string(), Some("adapter-a".to_string()), None)
+            .unwrap();
+
+        assert_eq!(registry.active_lora_requests(), 1);
+        drop(registry.close());
+        assert_eq!(registry.active_lora_requests(), 0);
+        assert_eq!(
+            registry.lora_adapter_states(),
+            (adapter_names(&[]), adapter_names(&[]))
+        );
+    }
+
     #[test]
     fn registry_drops_lora_tracking_on_abort() {
         let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);

rust/src/engine-core-client/src/client/stream.rs

@@ -10,7 +10,7 @@ use tracing::{debug, error, warn};
 
 use crate::client::AbortRequest;
 use crate::client::state::OutputReceiver;
-use crate::protocol::{EngineCoreFinishReason, EngineCoreOutput};
+use crate::protocol::output::{EngineCoreFinishReason, EngineCoreOutput};
 use crate::{AbortCause, Error, Result};
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]

rust/src/engine-core-client/src/coordinator/handle.rs

@@ -20,6 +20,27 @@ pub(crate) struct CoordinatorStateSnapshot {
     pub engines_running: bool,
 }
 
+impl CoordinatorStateSnapshot {
+    /// Resume the engines for a `FirstRequest` and return the wave to broadcast
+    /// and the engine to exclude from the wakeup.
+    ///
+    /// The request may have been stamped with a `request_wave` older than
+    /// `current_wave` if a `WaveComplete` advanced it after the command was
+    /// enqueued. Such a request still needs serving, so the current wave is
+    /// broadcast to every engine (`exclude = None`); the wave is never rewound.
+    /// A non-stale request excludes the engine that already received it. Mirrors
+    /// the Python coordinator's front-end path.
+    pub(crate) fn start_wave_for_first_request(
+        &mut self,
+        request_wave: u32,
+        target_engine_index: u32,
+    ) -> (u32, Option<u32>) {
+        self.engines_running = true;
+        let exclude = (request_wave >= self.current_wave).then_some(target_engine_index);
+        (self.current_wave, exclude)
+    }
+}
+
 /// Shared in-process coordinator state.
 pub(crate) type CoordinatorState = Mutex<CoordinatorStateSnapshot>;
 

rust/src/engine-core-client/src/coordinator/inproc.rs

@@ -10,10 +10,9 @@ use zeromq::{XPubSocket, ZmqMessage};
 use crate::client::imp::ClientInner;
 use crate::coordinator::handle::{CoordinatorCommand, CoordinatorState};
 use crate::error::{Error, Result, bail_unexpected_coordinator_output};
-use crate::protocol::{
-    ClassifiedEngineCoreOutputs, DpControlMessage, EngineCoreOutputs, EngineCoreRequestType,
-    encode_msgpack,
-};
+use crate::protocol::encode_msgpack;
+use crate::protocol::output::{ClassifiedEngineCoreOutputs, DpControlMessage, EngineCoreOutputs};
+use crate::protocol::request::EngineCoreRequestType;
 
 /// Coordinator-to-engine `START_DP_WAVE` control payload encoded on the
 /// engine-facing coordinator socket.
@@ -27,9 +26,10 @@ use crate::protocol::{
 struct StartDpWaveMessage {
     /// DP wave number that all engines should start processing.
     wave: u32,
-    /// Engine index that already received the triggering request and should not
-    /// receive an extra wakeup notification.
-    exclude_engine_index: u32,
+    /// Engine index that already received the triggering request and so does not
+    /// need an extra wakeup. `None` wakes every engine (used when the triggering
+    /// request was for a stale wave).
+    exclude_engine_index: Option<u32>,
 }
 
 /// Background half of the in-process coordinator.
@@ -57,7 +57,11 @@ impl InProcCoordinatorRunner {
     }
 
     /// Broadcast Python-compatible `START_DP_WAVE` to all connected engines.
-    async fn broadcast_start_wave(&mut self, wave: u32, exclude_engine_index: u32) -> Result<()> {
+    async fn broadcast_start_wave(
+        &mut self,
+        wave: u32,
+        exclude_engine_index: Option<u32>,
+    ) -> Result<()> {
         let payload = encode_msgpack(&StartDpWaveMessage {
             wave,
             exclude_engine_index,
@@ -86,13 +90,17 @@ impl InProcCoordinatorRunner {
                         engine_id: target_engine_id.to_vec(),
                     }
                 })?;
-                self.state.lock().current_wave = wave;
+                let (current_wave, exclude) = {
+                    let mut state = self.state.lock();
+                    state.start_wave_for_first_request(wave, target_engine_index)
+                };
                 debug!(
-                    wave,
-                    exclude_engine_index = target_engine_index,
+                    current_wave,
+                    request_wave = wave,
+                    ?exclude,
                     "starting DP wave after first request while engines were paused"
                 );
-                self.broadcast_start_wave(wave, target_engine_index).await?;
+                self.broadcast_start_wave(current_wave, exclude).await?;
             }
         }
         Ok(())
@@ -150,7 +158,7 @@ impl InProcCoordinatorRunner {
                             exclude_engine_index = engine_index,
                             "starting DP wave after stale-wave notification from engine"
                         );
-                        self.broadcast_start_wave(wave, engine_index).await?;
+                        self.broadcast_start_wave(wave, Some(engine_index)).await?;
                     }
                 }
             },
@@ -202,3 +210,48 @@ impl InProcCoordinatorRunner {
         inner.close_registries(Arc::new(error));
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use crate::coordinator::handle::CoordinatorStateSnapshot;
+
+    /// A `FirstRequest` for the current wave starts that wave and excludes the
+    /// engine that already received the triggering request.
+    #[test]
+    fn first_request_for_current_wave_excludes_target() {
+        let mut state = CoordinatorStateSnapshot {
+            current_wave: 3,
+            engines_running: false,
+        };
+
+        let (wave, exclude) = state.start_wave_for_first_request(3, 2);
+
+        assert_eq!(wave, 3);
+        assert_eq!(exclude, Some(2));
+        assert!(state.engines_running);
+        assert_eq!(state.current_wave, 3);
+    }
+
+    /// A `FirstRequest` whose wave was superseded by a racing `WaveComplete`
+    /// (`request_wave < current_wave`) must still start the request's wave: it
+    /// broadcasts the current wave and wakes every engine (`exclude = None`)
+    /// rather than rewinding the wave or dropping the request.
+    #[test]
+    fn stale_first_request_starts_current_wave_for_all_engines() {
+        let mut state = CoordinatorStateSnapshot {
+            current_wave: 4,
+            engines_running: false,
+        };
+
+        // Request stamped with wave 3 while the coordinator already advanced to 4.
+        let (wave, exclude) = state.start_wave_for_first_request(3, 2);
+
+        assert_eq!(
+            wave, 4,
+            "must broadcast the current wave, not the stale one"
+        );
+        assert_eq!(exclude, None, "a stale request must wake every engine");
+        assert!(state.engines_running);
+        assert_eq!(state.current_wave, 4, "wave must not be rewound");
+    }
+}

rust/src/engine-core-client/src/mock_engine.rs

@@ -8,8 +8,9 @@ use zeromq::{DealerSocket, PushSocket, SocketOptions, SubSocket, ZmqMessage};
 
 use crate::EngineId;
 use crate::error::{Error, Result, bail_unexpected_handshake_message};
+use crate::protocol::dtype::ModelDtype;
 use crate::protocol::handshake::{EngineCoreReadyResponse, HandshakeInitMessage, ReadyMessage};
-use crate::protocol::{ModelDtype, decode_msgpack, encode_msgpack};
+use crate::protocol::{decode_msgpack, encode_msgpack};
 
 /// Default model length advertised by reusable mock engine helpers.
 pub const DEFAULT_MOCK_MAX_MODEL_LEN: u64 = 1024 * 1024;

rust/src/engine-core-client/src/protocol/handshake.rs

@@ -2,7 +2,8 @@ use std::collections::BTreeMap;
 
 use serde::{Deserialize, Serialize};
 
-use crate::protocol::{ModelDtype, OpaqueValue};
+use crate::protocol::OpaqueValue;
+use crate::protocol::dtype::ModelDtype;
 
 /// Decoded engine startup-handshake payload sent on the handshake socket.
 ///

rust/src/engine-core-client/src/protocol/logprobs.rs

@@ -9,8 +9,7 @@ use enum_as_inner::EnumAsInner;
 use serde::{Deserialize, Deserializer, Serialize};
 
 use self::wire::*;
-use super::{EngineCoreOutput, EngineCoreOutputs, decode_msgpack};
-use crate::error::{Error, Result, bail_ext_value_decode, ext_value_decode};
+use crate::error::{Error, Result, bail_ext_value_decode};
 use crate::protocol::tensor::{WireArrayData, WireNdArray};
 
 /// One token candidate and its logprob metadata for a single sequence position.
@@ -160,7 +159,7 @@ impl Serialize for MaybeWireLogprobs {
 impl MaybeWireLogprobs {
     /// Resolve the wire representation into decoded logprobs by looking up aux
     /// frames and decoding raw views as needed.
-    fn resolve<Frame>(self, frames: &[Frame], field_prefix: &str) -> Result<Self>
+    pub(super) fn resolve<Frame>(self, frames: &[Frame], field_prefix: &str) -> Result<Self>
     where
         Frame: AsRef<[u8]>,
     {
@@ -171,37 +170,6 @@ impl MaybeWireLogprobs {
     }
 }
 
-impl EngineCoreOutputs {
-    /// Resolve all wire-format fields in-place by looking up aux frames and
-    /// decoding raw-view payloads as needed.
-    fn resolve_in_place<Frame>(&mut self, frames: &[Frame]) -> Result<()>
-    where
-        Frame: AsRef<[u8]>,
-    {
-        for output in &mut self.outputs {
-            output.resolve_in_place(frames)?;
-        }
-        Ok(())
-    }
-}
-
-impl EngineCoreOutput {
-    /// Resolve all wire-format fields in-place by looking up aux frames and
-    /// decoding raw-view payloads as needed.
-    fn resolve_in_place<Frame>(&mut self, frames: &[Frame]) -> Result<()>
-    where
-        Frame: AsRef<[u8]>,
-    {
-        self.new_logprobs = (self.new_logprobs.take())
-            .map(|value| value.resolve(frames, "new_logprobs"))
-            .transpose()?;
-        self.new_prompt_logprobs_tensors = (self.new_prompt_logprobs_tensors.take())
-            .map(|value| value.resolve(frames, "new_prompt_logprobs_tensors"))
-            .transpose()?;
-        Ok(())
-    }
-}
-
 impl WireLogprobs {
     /// Convert semantic per-position logprobs into the Python wire tuple shape.
     ///
@@ -315,16 +283,3 @@ impl WireLogprobs {
         Ok(Logprobs { positions })
     }
 }
-
-/// Decode one ordinary or multipart engine-core output message into the strong
-/// typed public protocol shape.
-pub fn decode_engine_core_outputs<Frame>(frames: &[Frame]) -> Result<EngineCoreOutputs>
-where
-    Frame: AsRef<[u8]>,
-{
-    let first_frame = frames.first().ok_or_else(|| ext_value_decode!("missing output frame"))?;
-
-    let mut outputs: EngineCoreOutputs = decode_msgpack(first_frame.as_ref())?;
-    outputs.resolve_in_place(frames)?;
-    Ok(outputs)
-}