Apply pre-commit formatting and use torch.accelerator API

- Apply ruff, clang-format formatting fixes
- Replace torch.cuda.set_device/device_count/synchronize with
  torch.accelerator equivalents per project convention

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Signed-off-by: Alexander Matveev <alexm-redhat@dgx-b200-02.mgmt.accl-001.lab.rdu2.dc.redhat.com>

Signed-off-by:  <>

.gitignore

@@ -15,6 +15,9 @@ vllm/third_party/flashmla/flash_mla_interface.py
 # DeepGEMM vendored package built from source
 vllm/third_party/deep_gemm/
 
+# fmha_sm100 vendored package built from source
+vllm/third_party/fmha_sm100/
+
 # triton jit
 .triton
 

CMakeLists.txt

@@ -324,6 +324,7 @@ endif()
 
 set(VLLM_EXT_SRC
   "csrc/quantization/activation_kernels.cu"
+  "csrc/push_all_reduce.cu"
   "csrc/torch_bindings.cpp")
 
 if(VLLM_GPU_LANG STREQUAL "CUDA")
@@ -440,6 +441,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
     "csrc/libtorch_stable/quantization/gptq/q_gemm.cu"
     "csrc/libtorch_stable/pos_encoding_kernels.cu"
     "csrc/libtorch_stable/fused_qknorm_rope_kernel.cu"
+    "csrc/libtorch_stable/fused_minimax_m3_qknorm_rope_kv_insert_kernel.cu"
     "csrc/libtorch_stable/layernorm_kernels.cu"
     "csrc/libtorch_stable/layernorm_quant_kernels.cu"
     "csrc/libtorch_stable/quantization/fused_kernels/fused_layernorm_dynamic_per_token_quant.cu"
@@ -1398,6 +1400,7 @@ endif()
 # For CUDA we also build and ship some external projects.
 if (VLLM_GPU_LANG STREQUAL "CUDA")
     include(cmake/external_projects/deepgemm.cmake)
+    include(cmake/external_projects/fmha_sm100.cmake)
     include(cmake/external_projects/flashmla.cmake)
     include(cmake/external_projects/qutlass.cmake)
 

benchmarks/kernels/benchmark_device_communicators.py

@@ -33,6 +33,7 @@ from vllm.distributed.device_communicators.custom_all_reduce import CustomAllred
 from vllm.distributed.device_communicators.flashinfer_all_reduce import (
     FlashInferAllReduce,
 )
+from vllm.distributed.device_communicators.push_all_reduce import PushAllReduce
 from vllm.distributed.device_communicators.pynccl import (
     PyNcclCommunicator,
     register_nccl_symmetric_ops,
@@ -80,6 +81,7 @@ class CommunicatorBenchmark:
 
         # Initialize communicators
         self.custom_allreduce = None
+        self.push_ar_comm = None
         self.pynccl_comm = None
         self.symm_mem_comm = None
         self.symm_mem_comm_multimem = None
@@ -106,6 +108,23 @@ class CommunicatorBenchmark:
             )
             self.custom_allreduce = None
 
+        try:
+            self.push_ar_comm = PushAllReduce(
+                group=self.cpu_group,
+                device=self.device,
+                max_size=self.max_size_override,
+            )
+            if not self.push_ar_comm.disabled:
+                logger.info("Rank %s: PushAllReduce initialized", self.rank)
+            else:
+                logger.info("Rank %s: PushAllReduce disabled", self.rank)
+                self.push_ar_comm = None
+        except Exception as e:
+            logger.warning(
+                "Rank %s: Failed to initialize PushAllReduce: %s", self.rank, e
+            )
+            self.push_ar_comm = None
+
         try:
             self.pynccl_comm = PyNcclCommunicator(
                 group=self.cpu_group, device=self.device
@@ -216,6 +235,19 @@ class CommunicatorBenchmark:
                 )
             )
 
+        if self.push_ar_comm is not None:
+            comm = self.push_ar_comm
+            communicators.append(
+                (
+                    "push_ar",
+                    lambda t, c=comm: c.all_reduce(t),
+                    lambda t, c=comm: c.should_use(t),
+                    comm.capture(),
+                    {},
+                    None,
+                )
+            )
+
         if self.pynccl_comm is not None:
             comm = self.pynccl_comm
             communicators.append(

cmake/external_projects/fmha_sm100.cmake

@@ -0,0 +1,50 @@
+include(FetchContent)
+
+# If FMHA_SM100_SRC_DIR is set, fmha_sm100 is installed from that directory
+# instead of downloading. This is useful for local MSA development.
+if(DEFINED ENV{FMHA_SM100_SRC_DIR})
+  set(FMHA_SM100_SRC_DIR $ENV{FMHA_SM100_SRC_DIR})
+endif()
+
+if(FMHA_SM100_SRC_DIR)
+  FetchContent_Declare(
+    fmha_sm100
+    SOURCE_DIR ${FMHA_SM100_SRC_DIR}
+    CONFIGURE_COMMAND ""
+    BUILD_COMMAND ""
+  )
+else()
+  FetchContent_Declare(
+    fmha_sm100
+    GIT_REPOSITORY https://github.com/vllm-project/MSA.git
+    GIT_TAG 544eee5e09ae2dfa774d5b06739013f9b7402c57
+    GIT_PROGRESS TRUE
+    CONFIGURE_COMMAND ""
+    BUILD_COMMAND ""
+  )
+endif()
+
+FetchContent_GetProperties(fmha_sm100)
+if(NOT fmha_sm100_POPULATED)
+  FetchContent_Populate(fmha_sm100)
+endif()
+message(STATUS "fmha_sm100 is available at ${fmha_sm100_SOURCE_DIR}")
+
+add_custom_target(fmha_sm100)
+
+install(FILES
+  "${fmha_sm100_SOURCE_DIR}/python/fmha_sm100/__init__.py"
+  "${fmha_sm100_SOURCE_DIR}/python/fmha_sm100/sparse.py"
+  DESTINATION vllm/third_party/fmha_sm100
+  COMPONENT fmha_sm100)
+
+install(DIRECTORY "${fmha_sm100_SOURCE_DIR}/python/fmha_sm100/cute/"
+  DESTINATION vllm/third_party/fmha_sm100/cute
+  COMPONENT fmha_sm100
+  FILES_MATCHING
+    REGEX "/__pycache__(/.*)?$" EXCLUDE
+    REGEX ".*\\.pyc$" EXCLUDE
+    PATTERN "example.py" EXCLUDE
+    PATTERN "test_*.py" EXCLUDE
+    PATTERN "*.py"
+    PATTERN "build_k2q_csr.cu")

csrc/libtorch_stable/activation_kernels.cu

@@ -10,11 +10,20 @@
 
 namespace vllm {
 
-template <typename scalar_t, scalar_t (*ACT_FN)(const scalar_t&),
+// `alpha` and `beta` are applied to opposite operands:
+//   - alpha lives INSIDE the activation (the activated half): the gated
+//     activation computes act_half * sigmoid(alpha * act_half).
+//   - beta is added to the OTHER (non-activated) half before the multiply.
+// So the result is always ACT(act_half, alpha) * (other_half + beta).
+// Which half is which depends on `act_first` (see below). Defaults
+// alpha=1.0, beta=0.0 reproduce the plain SwiGLU/GeGLU behavior.
+template <typename scalar_t, scalar_t (*ACT_FN)(const scalar_t&, const float),
           bool act_first, bool HAS_CLAMP>
 __device__ __forceinline__ scalar_t compute(const scalar_t& x,
                                             const scalar_t& y,
-                                            const float limit) {
+                                            const float limit,
+                                            const float alpha,
+                                            const float beta) {
   if constexpr (act_first) {
     scalar_t gate = x;
     scalar_t up = y;
@@ -22,7 +31,9 @@ __device__ __forceinline__ scalar_t compute(const scalar_t& x,
       gate = (scalar_t)fminf((float)gate, limit);
       up = (scalar_t)fmaxf(fminf((float)up, limit), -limit);
     }
-    return ACT_FN(gate) * up;
+    // act_first: gate is the activated half -> alpha applies to gate;
+    // beta is added to up (the non-activated half).
+    return (scalar_t)(ACT_FN(gate, alpha) * ((float)up + beta));
   } else {
     scalar_t gate = x;
     scalar_t up = y;
@@ -30,55 +41,68 @@ __device__ __forceinline__ scalar_t compute(const scalar_t& x,
       gate = (scalar_t)fmaxf(fminf((float)gate, limit), -limit);
       up = (scalar_t)fminf((float)up, limit);
     }
-    return gate * ACT_FN(up);
+    // !act_first: up is the activated half -> alpha applies to up;
+    // beta is added to gate (the non-activated half).
+    return (scalar_t)(((float)gate + beta) * ACT_FN(up, alpha));
   }
 }
 
-template <typename packed_t, packed_t (*PACKED_ACT_FN)(const packed_t&),
+template <typename packed_t,
+          packed_t (*PACKED_ACT_FN)(const packed_t&, const float),
           bool act_first, bool HAS_CLAMP>
 __device__ __forceinline__ packed_t packed_compute(const packed_t& x,
                                                    const packed_t& y,
-                                                   const float limit) {
+                                                   const float limit,
+                                                   const float alpha,
+                                                   const float beta) {
   if constexpr (act_first) {
     packed_t gate = x;
     packed_t up = y;
+    float2 u = cast_to_float2(up);
     if constexpr (HAS_CLAMP) {
       float2 g = cast_to_float2(gate);
-      float2 u = cast_to_float2(up);
       g.x = fminf(g.x, limit);
       g.y = fminf(g.y, limit);
       u.x = fmaxf(fminf(u.x, limit), -limit);
       u.y = fmaxf(fminf(u.y, limit), -limit);
       gate = cast_to_packed<packed_t>(g);
-      up = cast_to_packed<packed_t>(u);
     }
-    return packed_mul(PACKED_ACT_FN(gate), up);
+    // act_first: gate is the activated half -> alpha applies to gate;
+    // beta is added to up (the non-activated half).
+    float2 activated = cast_to_float2(PACKED_ACT_FN(gate, alpha));
+    activated.x *= u.x + beta;
+    activated.y *= u.y + beta;
+    return cast_to_packed<packed_t>(activated);
   } else {
     packed_t gate = x;
     packed_t up = y;
+    float2 g = cast_to_float2(gate);
     if constexpr (HAS_CLAMP) {
-      float2 g = cast_to_float2(gate);
       float2 u = cast_to_float2(up);
       g.x = fmaxf(fminf(g.x, limit), -limit);
       g.y = fmaxf(fminf(g.y, limit), -limit);
       u.x = fminf(u.x, limit);
       u.y = fminf(u.y, limit);
-      gate = cast_to_packed<packed_t>(g);
       up = cast_to_packed<packed_t>(u);
     }
-    return packed_mul(gate, PACKED_ACT_FN(up));
+    // !act_first: up is the activated half -> alpha applies to up;
+    // beta is added to gate (the non-activated half).
+    float2 activated = cast_to_float2(PACKED_ACT_FN(up, alpha));
+    activated.x *= g.x + beta;
+    activated.y *= g.y + beta;
+    return cast_to_packed<packed_t>(activated);
   }
 }
 
 // Activation and gating kernel template.
 template <typename scalar_t, typename packed_t,
-          scalar_t (*ACT_FN)(const scalar_t&),
-          packed_t (*PACKED_ACT_FN)(const packed_t&), bool act_first,
-          bool use_vec, bool HAS_CLAMP, bool use_256b = false>
+          scalar_t (*ACT_FN)(const scalar_t&, const float),
+          packed_t (*PACKED_ACT_FN)(const packed_t&, const float),
+          bool act_first, bool use_vec, bool HAS_CLAMP, bool use_256b = false>
 __global__ void act_and_mul_kernel(
     scalar_t* __restrict__ out,          // [..., d]
     const scalar_t* __restrict__ input,  // [..., 2, d]
-    const int d, const float limit) {
+    const int d, const float limit, const float alpha, const float beta) {
   const scalar_t* x_ptr = input + blockIdx.x * 2 * d;
   const scalar_t* y_ptr = x_ptr + d;
   scalar_t* out_ptr = out + blockIdx.x * d;
@@ -105,7 +129,7 @@ __global__ void act_and_mul_kernel(
       for (int j = 0; j < pvec_t::NUM_ELTS; j++) {
         x.elts[j] =
             packed_compute<packed_t, PACKED_ACT_FN, act_first, HAS_CLAMP>(
-                x.elts[j], y.elts[j], limit);
+                x.elts[j], y.elts[j], limit, alpha, beta);
       }
       if constexpr (use_256b) {
         st256(x, &out_vec[i]);
@@ -118,29 +142,34 @@ __global__ void act_and_mul_kernel(
     for (int64_t idx = threadIdx.x; idx < d; idx += blockDim.x) {
       const scalar_t x = VLLM_LDG(&x_ptr[idx]);
       const scalar_t y = VLLM_LDG(&y_ptr[idx]);
-      out_ptr[idx] =
-          compute<scalar_t, ACT_FN, act_first, HAS_CLAMP>(x, y, limit);
+      out_ptr[idx] = compute<scalar_t, ACT_FN, act_first, HAS_CLAMP>(
+          x, y, limit, alpha, beta);
     }
   }
 }
 
+// Gated activations take an `alpha` argument that scales the sigmoid input
+// (`x * sigmoid(alpha * x)`). alpha defaults to 1.0 at all call sites, which
+// is exactly SiLU; only the clamp path (silu_and_mul_with_clamp) passes a
+// non-default alpha. Activations that do not use alpha simply ignore it.
 template <typename T>
-__device__ __forceinline__ T silu_kernel(const T& x) {
-  // x * sigmoid(x)
-  return (T)(((float)x) / (1.0f + expf((float)-x)));
+__device__ __forceinline__ T silu_kernel(const T& x, const float alpha) {
+  // x * sigmoid(alpha * x)
+  return (T)(((float)x) / (1.0f + expf((float)-x * alpha)));
 }
 
 template <typename packed_t>
-__device__ __forceinline__ packed_t packed_silu_kernel(const packed_t& val) {
-  // x * sigmoid(x)
+__device__ __forceinline__ packed_t packed_silu_kernel(const packed_t& val,
+                                                       const float alpha) {
+  // x * sigmoid(alpha * x)
   float2 fval = cast_to_float2(val);
-  fval.x = fval.x / (1.0f + expf(-fval.x));
-  fval.y = fval.y / (1.0f + expf(-fval.y));
+  fval.x = fval.x / (1.0f + expf(-fval.x * alpha));
+  fval.y = fval.y / (1.0f + expf(-fval.y * alpha));
   return cast_to_packed<packed_t>(fval);
 }
 
 template <typename T>
-__device__ __forceinline__ T gelu_kernel(const T& x) {
+__device__ __forceinline__ T gelu_kernel(const T& x, const float /*alpha*/) {
   // Equivalent to PyTorch GELU with 'none' approximation.
   // Refer to:
   // https://github.com/pytorch/pytorch/blob/8ac9b20d4b090c213799e81acf48a55ea8d437d6/aten/src/ATen/native/cuda/ActivationGeluKernel.cu#L36-L38
@@ -150,7 +179,8 @@ __device__ __forceinline__ T gelu_kernel(const T& x) {
 }
 
 template <typename packed_t>
-__device__ __forceinline__ packed_t packed_gelu_kernel(const packed_t& val) {
+__device__ __forceinline__ packed_t packed_gelu_kernel(const packed_t& val,
+                                                       const float /*alpha*/) {
   // Equivalent to PyTorch GELU with 'none' approximation.
   // Refer to:
   // https://github.com/pytorch/pytorch/blob/8ac9b20d4b090c213799e81acf48a55ea8d437d6/aten/src/ATen/native/cuda/ActivationGeluKernel.cu#L36-L38
@@ -162,7 +192,8 @@ __device__ __forceinline__ packed_t packed_gelu_kernel(const packed_t& val) {
 }
 
 template <typename T>
-__device__ __forceinline__ T gelu_tanh_kernel(const T& x) {
+__device__ __forceinline__ T gelu_tanh_kernel(const T& x,
+                                              const float /*alpha*/) {
   // Equivalent to PyTorch GELU with 'tanh' approximation.
   // Refer to:
   // https://github.com/pytorch/pytorch/blob/8ac9b20d4b090c213799e81acf48a55ea8d437d6/aten/src/ATen/native/cuda/ActivationGeluKernel.cu#L25-L30
@@ -176,7 +207,7 @@ __device__ __forceinline__ T gelu_tanh_kernel(const T& x) {
 
 template <typename packed_t>
 __device__ __forceinline__ packed_t
-packed_gelu_tanh_kernel(const packed_t& val) {
+packed_gelu_tanh_kernel(const packed_t& val, const float /*alpha*/) {
   // Equivalent to PyTorch GELU with 'tanh' approximation.
   // Refer to:
   // https://github.com/pytorch/pytorch/blob/8ac9b20d4b090c213799e81acf48a55ea8d437d6/aten/src/ATen/native/cuda/ActivationGeluKernel.cu#L25-L30
@@ -202,7 +233,7 @@ packed_gelu_tanh_kernel(const packed_t& val) {
 // clamped (max only) and up input is clamped (both sides) before the
 // activation function is applied.
 #define LAUNCH_ACTIVATION_GATE_KERNEL(KERNEL, PACKED_KERNEL, ACT_FIRST,        \
-                                      HAS_CLAMP, LIMIT)                        \
+                                      HAS_CLAMP, LIMIT, ALPHA, BETA)           \
   auto dtype = input.scalar_type();                                            \
   int d = input.size(-1) / 2;                                                  \
   int64_t num_tokens = input.numel() / input.size(-1);                         \
@@ -230,7 +261,7 @@ packed_gelu_tanh_kernel(const packed_t& val) {
             PACKED_KERNEL<typename vllm::PackedTypeConverter<scalar_t>::Type>, \
             ACT_FIRST, true, HAS_CLAMP, true><<<grid, block, 0, stream>>>(     \
             out.mutable_data_ptr<scalar_t>(),                                  \
-            input.const_data_ptr<scalar_t>(), d, LIMIT);                       \
+            input.const_data_ptr<scalar_t>(), d, LIMIT, ALPHA, BETA);          \
       });                                                                      \
     } else {                                                                   \
       VLLM_STABLE_DISPATCH_FLOATING_TYPES(dtype, "act_and_mul_kernel", [&] {   \
@@ -240,7 +271,7 @@ packed_gelu_tanh_kernel(const packed_t& val) {
             PACKED_KERNEL<typename vllm::PackedTypeConverter<scalar_t>::Type>, \
             ACT_FIRST, true, HAS_CLAMP, false><<<grid, block, 0, stream>>>(    \
             out.mutable_data_ptr<scalar_t>(),                                  \
-            input.const_data_ptr<scalar_t>(), d, LIMIT);                       \
+            input.const_data_ptr<scalar_t>(), d, LIMIT, ALPHA, BETA);          \
       });                                                                      \
     }                                                                          \
   } else {                                                                     \
@@ -252,7 +283,7 @@ packed_gelu_tanh_kernel(const packed_t& val) {
           PACKED_KERNEL<typename vllm::PackedTypeConverter<scalar_t>::Type>,   \
           ACT_FIRST, false, HAS_CLAMP><<<grid, block, 0, stream>>>(            \
           out.mutable_data_ptr<scalar_t>(), input.const_data_ptr<scalar_t>(),  \
-          d, LIMIT);                                                           \
+          d, LIMIT, ALPHA, BETA);                                              \
     });                                                                        \
   }
 
@@ -260,14 +291,18 @@ void silu_and_mul(torch::stable::Tensor& out,    // [..., d]
                   torch::stable::Tensor& input)  // [..., 2 * d]
 {
   LAUNCH_ACTIVATION_GATE_KERNEL(vllm::silu_kernel, vllm::packed_silu_kernel,
-                                true, false, 0.0f);
+                                true, false, 0.0f, 1.0f, 0.0f);
 }
 
 void silu_and_mul_clamp(torch::stable::Tensor& out,    // [..., d]
                         torch::stable::Tensor& input,  // [..., 2 * d]
-                        double limit) {
+                        double limit, double alpha, double beta) {
+  // out = (gate.clamp(max=limit) * sigmoid(alpha * gate.clamp(max=limit)))
+  //       * (up.clamp(+-limit) + beta)
+  // alpha=1.0, beta=0.0 reduce this to silu(gate) * up.
   LAUNCH_ACTIVATION_GATE_KERNEL(vllm::silu_kernel, vllm::packed_silu_kernel,
-                                true, true, (float)limit);
+                                true, true, (float)limit, (float)alpha,
+                                (float)beta);
 }
 
 void mul_and_silu(torch::stable::Tensor& out,    // [..., d]
@@ -276,21 +311,22 @@ void mul_and_silu(torch::stable::Tensor& out,    // [..., d]
   // The difference between mul_and_silu and silu_and_mul is that mul_and_silu
   // applies the silu to the latter half of the input.
   LAUNCH_ACTIVATION_GATE_KERNEL(vllm::silu_kernel, vllm::packed_silu_kernel,
-                                false, false, 0.0f);
+                                false, false, 0.0f, 1.0f, 0.0f);
 }
 
 void gelu_and_mul(torch::stable::Tensor& out,    // [..., d]
                   torch::stable::Tensor& input)  // [..., 2 * d]
 {
   LAUNCH_ACTIVATION_GATE_KERNEL(vllm::gelu_kernel, vllm::packed_gelu_kernel,
-                                true, false, 0.0f);
+                                true, false, 0.0f, 1.0f, 0.0f);
 }
 
 void gelu_tanh_and_mul(torch::stable::Tensor& out,    // [..., d]
                        torch::stable::Tensor& input)  // [..., 2 * d]
 {
-  LAUNCH_ACTIVATION_GATE_KERNEL(
-      vllm::gelu_tanh_kernel, vllm::packed_gelu_tanh_kernel, true, false, 0.0f);
+  LAUNCH_ACTIVATION_GATE_KERNEL(vllm::gelu_tanh_kernel,
+                                vllm::packed_gelu_tanh_kernel, true, false,
+                                0.0f, 1.0f, 0.0f);
 }
 
 namespace vllm {

csrc/libtorch_stable/fp32_router_gemm.cu

@@ -175,49 +175,52 @@ void invokeFp32RouterGemm(float* output, InputT const* mat_a,
 }
 
 // ---------------------------------------------------------------------------
-// Explicit instantiations: M=1..32, E=256, H=3072, for both input types
+// Explicit instantiations: M=1..32, for both input types, for the supported
+// (E, H) pairs:  (256, 3072) [MiniMax-M2/M2.5]  and  (128, 6144) [MiniMax-M3].
 // ---------------------------------------------------------------------------
 
-#define INSTANTIATE(T, M)                              \
-  template void invokeFp32RouterGemm<T, M, 256, 3072>( \
-      float*, T const*, float const*, cudaStream_t);
+#define INSTANTIATE(T, M, E, H)                                    \
+  template void invokeFp32RouterGemm<T, M, E, H>(float*, T const*, \
+                                                 float const*, cudaStream_t);
 
-#define INSTANTIATE_ALL(T) \
-  INSTANTIATE(T, 1)        \
-  INSTANTIATE(T, 2)        \
-  INSTANTIATE(T, 3)        \
-  INSTANTIATE(T, 4)        \
-  INSTANTIATE(T, 5)        \
-  INSTANTIATE(T, 6)        \
-  INSTANTIATE(T, 7)        \
-  INSTANTIATE(T, 8)        \
-  INSTANTIATE(T, 9)        \
-  INSTANTIATE(T, 10)       \
-  INSTANTIATE(T, 11)       \
-  INSTANTIATE(T, 12)       \
-  INSTANTIATE(T, 13)       \
-  INSTANTIATE(T, 14)       \
-  INSTANTIATE(T, 15)       \
-  INSTANTIATE(T, 16)       \
-  INSTANTIATE(T, 17)       \
-  INSTANTIATE(T, 18)       \
-  INSTANTIATE(T, 19)       \
-  INSTANTIATE(T, 20)       \
-  INSTANTIATE(T, 21)       \
-  INSTANTIATE(T, 22)       \
-  INSTANTIATE(T, 23)       \
-  INSTANTIATE(T, 24)       \
-  INSTANTIATE(T, 25)       \
-  INSTANTIATE(T, 26)       \
-  INSTANTIATE(T, 27)       \
-  INSTANTIATE(T, 28)       \
-  INSTANTIATE(T, 29)       \
-  INSTANTIATE(T, 30)       \
-  INSTANTIATE(T, 31)       \
-  INSTANTIATE(T, 32)
+#define INSTANTIATE_ALL(T, E, H) \
+  INSTANTIATE(T, 1, E, H)        \
+  INSTANTIATE(T, 2, E, H)        \
+  INSTANTIATE(T, 3, E, H)        \
+  INSTANTIATE(T, 4, E, H)        \
+  INSTANTIATE(T, 5, E, H)        \
+  INSTANTIATE(T, 6, E, H)        \
+  INSTANTIATE(T, 7, E, H)        \
+  INSTANTIATE(T, 8, E, H)        \
+  INSTANTIATE(T, 9, E, H)        \
+  INSTANTIATE(T, 10, E, H)       \
+  INSTANTIATE(T, 11, E, H)       \
+  INSTANTIATE(T, 12, E, H)       \
+  INSTANTIATE(T, 13, E, H)       \
+  INSTANTIATE(T, 14, E, H)       \
+  INSTANTIATE(T, 15, E, H)       \
+  INSTANTIATE(T, 16, E, H)       \
+  INSTANTIATE(T, 17, E, H)       \
+  INSTANTIATE(T, 18, E, H)       \
+  INSTANTIATE(T, 19, E, H)       \
+  INSTANTIATE(T, 20, E, H)       \
+  INSTANTIATE(T, 21, E, H)       \
+  INSTANTIATE(T, 22, E, H)       \
+  INSTANTIATE(T, 23, E, H)       \
+  INSTANTIATE(T, 24, E, H)       \
+  INSTANTIATE(T, 25, E, H)       \
+  INSTANTIATE(T, 26, E, H)       \
+  INSTANTIATE(T, 27, E, H)       \
+  INSTANTIATE(T, 28, E, H)       \
+  INSTANTIATE(T, 29, E, H)       \
+  INSTANTIATE(T, 30, E, H)       \
+  INSTANTIATE(T, 31, E, H)       \
+  INSTANTIATE(T, 32, E, H)
 
-INSTANTIATE_ALL(float)
-INSTANTIATE_ALL(__nv_bfloat16)
+INSTANTIATE_ALL(float, 256, 3072)
+INSTANTIATE_ALL(__nv_bfloat16, 256, 3072)
+INSTANTIATE_ALL(float, 128, 6144)
+INSTANTIATE_ALL(__nv_bfloat16, 128, 6144)
 
 #undef INSTANTIATE_ALL
 #undef INSTANTIATE

csrc/libtorch_stable/fp32_router_gemm_entry.cu

@@ -22,36 +22,42 @@ inline int getSMVersion() {
 
 }  // namespace
 
-static constexpr int FP32_NUM_EXPERTS = 256;
-static constexpr int FP32_HIDDEN_DIM = 3072;
 static constexpr int FP32_MAX_TOKENS = 32;
 
+// Supported (hidden_dim, num_experts) pairs (must match the instantiations in
+// fp32_router_gemm.cu): (3072, 256) for MiniMax-M2/M2.5, (6144, 128) for M3.
+static inline bool fp32_router_gemm_supported(int hidden_dim, int num_experts) {
+  return (hidden_dim == 3072 && num_experts == 256) ||
+         (hidden_dim == 6144 && num_experts == 128);
+}
+
 // Forward declarations — 4 template params must match fp32_router_gemm.cu
 template <typename InputT, int kNumTokens, int kNumExperts, int kHiddenDim>
 void invokeFp32RouterGemm(float* output, InputT const* mat_a,
                           float const* mat_b, cudaStream_t stream);
 
-// LoopUnroller templated on InputT
-template <typename InputT, int kBegin, int kEnd>
+// LoopUnroller templated on InputT, kNumExperts and kHiddenDim
+template <typename InputT, int kNumExperts, int kHiddenDim, int kBegin,
+          int kEnd>
 struct Fp32LoopUnroller {
   static void unroll(int num_tokens, float* output, InputT const* mat_a,
                      float const* mat_b, cudaStream_t stream) {
     if (num_tokens == kBegin) {
-      invokeFp32RouterGemm<InputT, kBegin, FP32_NUM_EXPERTS, FP32_HIDDEN_DIM>(
+      invokeFp32RouterGemm<InputT, kBegin, kNumExperts, kHiddenDim>(
           output, mat_a, mat_b, stream);
     } else {
-      Fp32LoopUnroller<InputT, kBegin + 1, kEnd>::unroll(num_tokens, output,
-                                                         mat_a, mat_b, stream);
+      Fp32LoopUnroller<InputT, kNumExperts, kHiddenDim, kBegin + 1,
+                       kEnd>::unroll(num_tokens, output, mat_a, mat_b, stream);
     }
   }
 };
 
-template <typename InputT, int kEnd>
-struct Fp32LoopUnroller<InputT, kEnd, kEnd> {
+template <typename InputT, int kNumExperts, int kHiddenDim, int kEnd>
+struct Fp32LoopUnroller<InputT, kNumExperts, kHiddenDim, kEnd, kEnd> {
   static void unroll(int num_tokens, float* output, InputT const* mat_a,
                      float const* mat_b, cudaStream_t stream) {
     if (num_tokens == kEnd) {
-      invokeFp32RouterGemm<InputT, kEnd, FP32_NUM_EXPERTS, FP32_HIDDEN_DIM>(
+      invokeFp32RouterGemm<InputT, kEnd, kNumExperts, kHiddenDim>(
           output, mat_a, mat_b, stream);
     } else {
       throw std::invalid_argument(
@@ -60,6 +66,23 @@ struct Fp32LoopUnroller<InputT, kEnd, kEnd> {
   }
 };
 
+// Dispatch over the supported (num_experts, hidden_dim) pairs.
+template <typename InputT>
+void dispatchFp32RouterGemm(int num_experts, int hidden_dim, int num_tokens,
+                            float* output, InputT const* mat_a,
+                            float const* mat_b, cudaStream_t stream) {
+  if (num_experts == 256 && hidden_dim == 3072) {
+    Fp32LoopUnroller<InputT, 256, 3072, 1, FP32_MAX_TOKENS>::unroll(
+        num_tokens, output, mat_a, mat_b, stream);
+  } else if (num_experts == 128 && hidden_dim == 6144) {
+    Fp32LoopUnroller<InputT, 128, 6144, 1, FP32_MAX_TOKENS>::unroll(
+        num_tokens, output, mat_a, mat_b, stream);
+  } else {
+    throw std::invalid_argument(
+        "fp32_router_gemm: unsupported (hidden_dim, num_experts) pair");
+  }
+}
+
 void fp32_router_gemm(
     torch::stable::Tensor& output,       // [num_tokens, num_experts]
     torch::stable::Tensor const& mat_a,  // [num_tokens, hidden_dim]
@@ -85,10 +108,10 @@ void fp32_router_gemm(
   STD_TORCH_CHECK(
       mat_a.size(1) == mat_b.size(1),
       "fp32_router_gemm: mat_a and mat_b must have the same hidden_dim");
-  STD_TORCH_CHECK(hidden_dim == FP32_HIDDEN_DIM,
-                  "fp32_router_gemm: expected hidden_dim=3072");
-  STD_TORCH_CHECK(num_experts == FP32_NUM_EXPERTS,
-                  "fp32_router_gemm: expected num_experts=256");
+  STD_TORCH_CHECK(
+      fp32_router_gemm_supported(hidden_dim, num_experts),
+      "fp32_router_gemm: supported (hidden_dim, num_experts) pairs are "
+      "(3072, 256) and (6144, 128)");
   STD_TORCH_CHECK(num_tokens <= FP32_MAX_TOKENS,
                   "fp32_router_gemm: num_tokens must be in [0, 32]");
   STD_TORCH_CHECK(
@@ -113,12 +136,13 @@ void fp32_router_gemm(
   if (mat_a.scalar_type() == torch::headeronly::ScalarType::BFloat16) {
     auto const* mat_a_ptr =
         reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr());
-    Fp32LoopUnroller<__nv_bfloat16, 1, FP32_MAX_TOKENS>::unroll(
-        num_tokens, out_ptr, mat_a_ptr, mat_b_ptr, stream);
+    dispatchFp32RouterGemm<__nv_bfloat16>(num_experts, hidden_dim, num_tokens,
+                                          out_ptr, mat_a_ptr, mat_b_ptr,
+                                          stream);
   } else {
     auto const* mat_a_ptr = reinterpret_cast<float const*>(mat_a.data_ptr());
-    Fp32LoopUnroller<float, 1, FP32_MAX_TOKENS>::unroll(
-        num_tokens, out_ptr, mat_a_ptr, mat_b_ptr, stream);
+    dispatchFp32RouterGemm<float>(num_experts, hidden_dim, num_tokens, out_ptr,
+                                  mat_a_ptr, mat_b_ptr, stream);
   }
 }
 

csrc/libtorch_stable/fused_minimax_m3_qknorm_rope_kv_insert_kernel.cu

@@ -0,0 +1,635 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ * SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+ *
+ * Horizontally-fused MiniMax-M3 attention pre-processing kernel.
+ *
+ * Replaces the per-token Python sequence in
+ * ``MiniMaxM3SparseAttention.forward`` / ``MiniMaxM3Attention.forward``:
+ *
+ *     q  = q_norm(q);  k = k_norm(k);  q, k = rotary_emb(pos, q, k)
+ *     index_q = index_q_norm(index_q);  index_k = index_k_norm(index_k)
+ *     index_q, index_k = rotary_emb(pos, index_q, index_k)
+ *     _insert_kv(k, v, index_k)
+ *
+ * All branches share head_dim=128 and the *same* partial-NeoX RoPE table
+ * (``rotary_dim`` rotated, the trailing dims pass through).  The four norms
+ * are Gemma-style RMSNorm (``x * rsqrt(mean(x^2)+eps) * (1 + weight)``) with
+ * independent weights.
+ *
+ * Everything lives in a single fused ``qkv`` tensor.  The sparse layer's
+ * fused projection (MinimaxM3QKVParallelLinearWithIndexer) emits, per token::
+ *
+ *     [ q | k | v | index_q | index_k ]   (the "5 results")
+ *
+ * while the dense layer emits just ``[ q | k | v ]``.  The kernel reads the
+ * index branch straight out of that packed row -- no separate index tensors.
+ *
+ * One kernel, one grid; each warp owns one (token, head-slot) pair.  Slot
+ * enumeration per token:
+ *     [0, nq)                         Q  heads   -> norm(q_w)  + RoPE, write
+ * qkv [nq, nq+nkv)                    K  heads   -> norm(k_w)  + RoPE, write
+ * qkv
+ *                                                  (+ insert into key cache)
+ *     [nq+nkv, nq+2*nkv)              V  heads   -> insert into value cache
+ *     IQ heads (niq)                             -> norm(iq_w) + RoPE, write iq
+ *     IK       (1)                               -> norm(ik_w) + RoPE
+ *                                                  (+ insert into index cache)
+ *
+ * The IQ/IK warps address the index_q/index_k sub-blocks *inside* qkv at the
+ * fixed physical offsets (nq+2*nkv)*128 and (nq+2*nkv+niq)*128.
+ *
+ * Dense vs sparse is a compile-time choice via the ``kIsSparse``/``kInsertKV``
+ * template bools (3 instantiations: dense <false,false>, sparse-profiling
+ * <true,false>, sparse-serving <true,true>), so the index slots, the V slots
+ * and the cache inserts fold away entirely on paths that don't use them. The
+ * dense layer passes no caches/index: norm+RoPE happens in place and the
+ * generic ``Attention`` layer owns the cache write.
+ *
+ * Q/K and (sparse) index_q/index_k are all rewritten in place inside the fused
+ * ``qkv`` tensor.  Caches (bf16) are scatter-written by slot.
+ */
+
+#include <cmath>
+#include <cuda_runtime.h>
+#include <type_traits>
+
+#include "torch_utils.h"
+
+#include "../cuda_compat.h"
+#include "../type_convert.cuh"
+#include "dispatch_utils.h"
+
+#ifndef FINAL_MASK
+  #ifdef USE_ROCM
+    #define FINAL_MASK 0xffffffffffffffffULL
+  #else
+    #define FINAL_MASK 0xffffffffu
+  #endif
+#endif
+
+namespace vllm {
+namespace minimax_m3_fused_ops {
+
+namespace {
+inline int getSMVersion() {
+  auto* props = get_device_prop();
+  return props->major * 10 + props->minor;
+}
+}  // namespace
+
+// ────────────────────────────────────────────────────────────────────────────
+// Constants (hard-coded for MiniMax-M3-preview).
+// ────────────────────────────────────────────────────────────────────────────
+constexpr int kHeadDim = 128;
+constexpr int kNumLanes = 32;
+constexpr int kElemsPerLane = kHeadDim / kNumLanes;  // 4
+
+// ────────────────────────────────────────────────────────────────────────────
+// Helpers
+// ────────────────────────────────────────────────────────────────────────────
+__device__ __forceinline__ float warpReduceSum(float val) {
+#pragma unroll
+  for (int mask = 16; mask > 0; mask >>= 1) {
+    val += __shfl_xor_sync(FINAL_MASK, val, mask, 32);
+  }
+  return val;
+}
+
+// Gemma RMSNorm over the full head (no-op when ``weight == nullptr``), rounded
+// back to scalar_t like the materialized unfused norm output, followed by
+// partial NeoX RoPE on the leading ``rotary_dim`` dims. Each lane owns
+// ``kElemsPerLane`` contiguous dims [laneId*4, laneId*4+4).
+template <typename scalar_t>
+__device__ __forceinline__ void normAndRope(
+    float (&elems)[kElemsPerLane], int const laneId, float const eps,
+    scalar_t const* __restrict__ weight,  // [kHeadDim] or nullptr (no norm)
+    bool const do_rope, int const rotary_dim,
+    scalar_t const* __restrict__ cos_ptr,  // cos_sin_cache + pos*rotary_dim
+    bool const apply_norm) {
+  // ── Gemma RMSNorm: x * rsqrt(mean(x^2)+eps) * (1 + w) ──────────────────
+  if (apply_norm) {
+    float sumsq = 0.0f;
+#pragma unroll
+    for (int i = 0; i < kElemsPerLane; i++) sumsq += elems[i] * elems[i];
+    sumsq = warpReduceSum(sumsq);
+    float const rms_rcp = rsqrtf(sumsq / static_cast<float>(kHeadDim) + eps);
+#pragma unroll
+    for (int i = 0; i < kElemsPerLane; i++) {
+      int const dim = laneId * kElemsPerLane + i;
+      float const w = 1.0f + static_cast<float>(weight[dim]);
+      elems[i] = elems[i] * rms_rcp * w;
+    }
+  }
+
+  // ── Partial NeoX RoPE on dims [0, rotary_dim) ──────────────────────────
+  // half = rotary_dim/2.  Pair (i, i+half) for i in [0, half).  Lane L owns
+  // dims [4L, 4L+4); since half is a multiple of 4, a lane lies wholly in the
+  // first half (own=x[i]) or second half (own=x[i+half]); its partner lives
+  // ``half/4`` lanes away (XOR with that distance).
+  if (do_rope) {
+    int const half = rotary_dim / 2;
+    int const dim0 = laneId * kElemsPerLane;
+    bool const in_rope = dim0 < rotary_dim;
+    int const lane_xor = half / kElemsPerLane;  // partner-lane distance
+
+    float partner[kElemsPerLane];
+#pragma unroll
+    for (int i = 0; i < kElemsPerLane; i++) {
+      partner[i] = __shfl_xor_sync(FINAL_MASK, elems[i], lane_xor, 32);
+    }
+    if (in_rope) {
+      bool const first_half = dim0 < half;
+      int const i_base = first_half ? dim0 : (dim0 - half);  // cos/sin index
+      scalar_t const* sin_ptr = cos_ptr + half;
+#pragma unroll
+      for (int i = 0; i < kElemsPerLane; i++) {
+        float const c = static_cast<float>(cos_ptr[i_base + i]);
+        float const s = static_cast<float>(sin_ptr[i_base + i]);
+        if (first_half) {
+          elems[i] = elems[i] * c - partner[i] * s;
+        } else {
+          elems[i] = elems[i] * c + partner[i] * s;
+        }
+      }
+    }
+  }
+}
+
+// Load 4 contiguous bf16 -> 4 fp32 registers.
+template <typename scalar_t>
+__device__ __forceinline__ void loadElems(scalar_t const* __restrict__ src,
+                                          float (&elems)[kElemsPerLane]) {
+  using Converter = vllm::_typeConvert<scalar_t>;
+  uint2 v = *reinterpret_cast<uint2 const*>(src);
+  auto const* p =
+      reinterpret_cast<typename Converter::packed_hip_type const*>(&v);
+#pragma unroll
+  for (int i = 0; i < kElemsPerLane / 2; i++) {
+    float2 f2 = Converter::convert(p[i]);
+    elems[2 * i] = f2.x;
+    elems[2 * i + 1] = f2.y;
+  }
+}
+
+// Store 4 fp32 registers -> 4 contiguous bf16.
+template <typename scalar_t>
+__device__ __forceinline__ void storeElems(
+    scalar_t* __restrict__ dst, float const (&elems)[kElemsPerLane]) {
+  using Converter = vllm::_typeConvert<scalar_t>;
+  uint2 v;
+  auto* p = reinterpret_cast<typename Converter::packed_hip_type*>(&v);
+#pragma unroll
+  for (int i = 0; i < kElemsPerLane / 2; i++) {
+    p[i] = Converter::convert(make_float2(elems[2 * i], elems[2 * i + 1]));
+  }
+  *reinterpret_cast<uint2*>(dst) = v;
+}
+
+// ────────────────────────────────────────────────────────────────────────────
+// Kernel
+// ────────────────────────────────────────────────────────────────────────────
+// Grid: 1D, ceil(num_tokens * slots_per_token / warps_per_block).
+// Each warp = one (token, slot).
+//
+// `kIsSparse` and `kInsertKV` are compile-time template bools, so all the
+// branch decisions that distinguish the dense layer from the sparse layer
+// (index slots, KV/index inserts, V slots) fold away per instantiation.
+// Three instantiations are built: dense <false,false>, sparse-profiling
+// <true,false> and sparse-serving <true,true>.  Slots per token:
+//     Q : nq                            (always — norm+RoPE)
+//     K : nkv                           (always — norm+RoPE; +K-cache insert)
+//     V : nkv  only if kInsertKV        (V-cache insert; no warps in dense)
+//     IQ: niq  only if kIsSparse        (norm+RoPE)
+//     IK: 1    only if kIsSparse        (norm+RoPE; +index-cache insert)
+template <typename scalar_t, bool kIsSparse, bool kInsertKV>
+__global__ void fusedMiniMaxM3QNormRopeKVInsertKernel(
+    scalar_t* __restrict__ qkv,  // [N, qkv_row] in/out (packs index if sparse)
+    scalar_t* __restrict__ q_out,        // [N, nq*128] contiguous, or nullptr
+    scalar_t* __restrict__ index_q_out,  // [N, niq*128] contiguous, or nullptr
+    scalar_t const* __restrict__ q_norm_w,
+    scalar_t const* __restrict__ k_norm_w,
+    scalar_t const* __restrict__ iq_norm_w,
+    scalar_t const* __restrict__ ik_norm_w,
+    scalar_t const* __restrict__ cos_sin_cache,  // [max_pos, rotary_dim]
+    int64_t const* __restrict__ positions,       // [N] i64
+    int64_t const* __restrict__ slot_mapping,    // main K/V slots or nullptr
+    int64_t const* __restrict__ index_slot_mapping,  // index K slots/nullptr
+    scalar_t* __restrict__ kv_cache,     // [nb,2,bs,nkv,128] or nullptr
+    scalar_t* __restrict__ index_cache,  // [nb*bs, 128] or nullptr
+    float const eps, int const rotary_dim, int const num_tokens, int const nq,
+    int const nkv, int const niq, int const block_size,
+    // kv_cache strides (in elements) for logical shape [nb, 2, bs, nkv, 128].
+    // The head_dim (last) dim is always innermost-contiguous (stride 1), so the
+    // NHD/HND layout choice is fully captured by these four strides: NHD keeps
+    // s_token < s_head, HND swaps them. dim_base addresses head_dim directly.
+    int64_t const kv_s_block, int64_t const kv_s_kv, int64_t const kv_s_token,
+    int64_t const kv_s_head) {
+#if (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 800) && !defined(USE_ROCM)
+  // _typeConvert<BFloat16> is unavailable on pre-Ampere; the M3 kernel only
+  // runs with bf16/fp16 inputs in practice.  Discard the bf16 body there.
+  if constexpr (std::is_same_v<scalar_t, c10::BFloat16>) {
+    return;
+  } else {
+#endif
+    int const warpsPerBlock = blockDim.x / 32;
+    int const laneId = threadIdx.x % 32;
+    int const globalWarpIdx = blockIdx.x * warpsPerBlock + (threadIdx.x / 32);
+
+    // Slot layout (compile-time gated: dense has neither V nor index slots).
+    int const v_slots = kInsertKV ? nkv : 0;
+    int const idx_slots = kIsSparse ? niq + 1 : 0;
+    int const slots_per_token = nq + nkv + v_slots + idx_slots;
+
+    int const tokenIdx = globalWarpIdx / slots_per_token;
+    int const slot = globalWarpIdx % slots_per_token;
+    if (tokenIdx >= num_tokens) return;
+
+    // Slot boundaries.
+    int const k_begin = nq;
+    int const v_begin = nq + nkv;             // valid only when kInsertKV
+    int const iq_begin = nq + nkv + v_slots;  // index block start
+    int const ik_slot = iq_begin + niq;       // valid only when kIsSparse
+
+    bool const isQ = slot < k_begin;
+    bool const isK = slot >= k_begin && slot < v_begin;
+    bool isV = false;
+    if constexpr (kInsertKV) isV = slot >= v_begin && slot < v_begin + nkv;
+    bool isIQ = false, isIK = false;
+    if constexpr (kIsSparse) {
+      isIQ = slot >= iq_begin && slot < ik_slot;
+      isIK = slot == ik_slot;
+    }
+
+    int const dim_base = laneId * kElemsPerLane;
+    // Physical row width of qkv: the dense layer packs [q|k|v]; the sparse
+    // layer additionally packs [index_q (niq heads) | index_k (1 head)].
+    int const qkv_row = (nq + 2 * nkv + (kIsSparse ? (niq + 1) : 0)) * kHeadDim;
+
+    // ── Resolve source pointer + per-branch parameters. ────────────────────
+    scalar_t* row_ptr = nullptr;       // in-place output location
+    scalar_t const* norm_w = nullptr;  // nullptr -> skip norm (V)
+    bool do_rope = true;
+    int head = 0;  // kv head index for inserts
+
+    if (isQ) {
+      row_ptr =
+          qkv + static_cast<int64_t>(tokenIdx) * qkv_row + slot * kHeadDim;
+      norm_w = q_norm_w;
+    } else if (isK) {
+      head = slot - k_begin;
+      row_ptr =
+          qkv + static_cast<int64_t>(tokenIdx) * qkv_row + slot * kHeadDim;
+      norm_w = k_norm_w;
+    } else if (isV) {
+      // qkv V section starts at slot index (nq + nkv): slot * kHeadDim is the
+      // correct in-tensor offset.
+      head = slot - v_begin;
+      row_ptr =
+          qkv + static_cast<int64_t>(tokenIdx) * qkv_row + slot * kHeadDim;
+      norm_w = nullptr;  // V: no norm, no rope
+      do_rope = false;
+    } else if (isIQ) {
+      // index_q sub-block lives at physical offset (nq+2*nkv)*128 in qkv.
+      int const ih = slot - iq_begin;
+      row_ptr = qkv + static_cast<int64_t>(tokenIdx) * qkv_row +
+                (nq + 2 * nkv + ih) * kHeadDim;
+      norm_w = iq_norm_w;
+    } else {  // isIK -- single shared index key at (nq+2*nkv+niq)*128.
+      row_ptr = qkv + static_cast<int64_t>(tokenIdx) * qkv_row +
+                (nq + 2 * nkv + niq) * kHeadDim;
+      norm_w = ik_norm_w;
+    }
+
+    // Store destination.  Q and index_q are gathered into dedicated contiguous
+    // output buffers (when provided) so the downstream SM100 sparse kernel's
+    // flat TMA descriptor can address them as [tokens*heads, head_dim]; this
+    // folds the de-interleaving into the store the kernel already does, instead
+    // of a separate q.contiguous() copy.  Everything else stays in place.
+    scalar_t* store_ptr = row_ptr;
+    if (isQ && q_out != nullptr) {
+      store_ptr = q_out + static_cast<int64_t>(tokenIdx) * nq * kHeadDim +
+                  slot * kHeadDim;
+    } else if (isIQ && index_q_out != nullptr) {
+      store_ptr = index_q_out +
+                  static_cast<int64_t>(tokenIdx) * niq * kHeadDim +
+                  (slot - iq_begin) * kHeadDim;
+    }
+
+    // PDL: wait for the predecessor kernel (the qkv-projection GEMM that
+    // produces ``qkv``) to finish before touching any global memory.  No-op
+    // when PDL is not enabled on the launch.  The CUDA runtime wrapper emits
+    // the griddepcontrol.wait PTX with the required memory clobber internally.
+#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
+    cudaGridDependencySynchronize();
+#endif
+
+    // ── Load -> norm+rope (fp32) -> store back in place. ───────────────────
+    float elems[kElemsPerLane];
+    loadElems<scalar_t>(row_ptr + dim_base, elems);
+
+    if (!isV) {
+      int64_t const pos = positions[tokenIdx];
+      scalar_t const* cos_ptr = cos_sin_cache + pos * rotary_dim;
+      normAndRope<scalar_t>(elems, laneId, eps, norm_w, do_rope, rotary_dim,
+                            cos_ptr, /*apply_norm=*/norm_w != nullptr);
+      storeElems<scalar_t>(store_ptr + dim_base, elems);
+    }
+
+    // ── Cache inserts (sparse serving only). ───────────────────────────────
+    if constexpr (kInsertKV) {
+      // Guard (not early-return) so every thread reaches the PDL trigger below.
+      int64_t const sm = (isK || isV)
+                             ? slot_mapping[tokenIdx]
+                             : (isIK ? index_slot_mapping[tokenIdx] : -1);
+      if (sm >= 0) {  // skip padded / unscheduled tokens
+        if (isIK) {
+          scalar_t* dst = index_cache + sm * kHeadDim + dim_base;
+          storeElems<scalar_t>(dst, elems);
+        } else if (isK || isV) {
+          // kv_cache logical shape [num_blocks, 2, block_size, nkv, head_dim].
+          // Paging is logical (block = sm/block_size, token = sm%block_size);
+          // the physical NHD/HND layout is honoured via the passed strides.
+          int64_t const b = sm / block_size;
+          int64_t const t = sm % block_size;
+          int const kv = isK ? 0 : 1;
+          int64_t const off =
+              b * kv_s_block + kv * kv_s_kv + t * kv_s_token + head * kv_s_head;
+          storeElems<scalar_t>(kv_cache + off + dim_base, elems);
+        }
+      }
+    }
+
+    // PDL: signal that this kernel is done so a dependent successor may launch
+    // early.  No-op when PDL is not enabled on the launch.
+#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
+    cudaTriggerProgrammaticLaunchCompletion();
+#endif
+#if (!defined(__CUDA_ARCH__) || __CUDA_ARCH__ < 800) && !defined(USE_ROCM)
+  }
+#endif
+}
+
+// ────────────────────────────────────────────────────────────────────────────
+// Launch wrapper
+// ────────────────────────────────────────────────────────────────────────────
+template <typename scalar_t>
+void launchFusedMiniMaxM3(scalar_t* qkv, scalar_t* q_out, scalar_t* index_q_out,
+                          scalar_t const* q_norm_w, scalar_t const* k_norm_w,
+                          scalar_t const* iq_norm_w, scalar_t const* ik_norm_w,
+                          scalar_t const* cos_sin_cache,
+                          int64_t const* positions, int64_t const* slot_mapping,
+                          int64_t const* index_slot_mapping, scalar_t* kv_cache,
+                          scalar_t* index_cache, float const eps,
+                          int const rotary_dim, int const num_tokens,
+                          int const nq, int const nkv, int const niq,
+                          int const block_size, int64_t const kv_s_block,
+                          int64_t const kv_s_kv, int64_t const kv_s_token,
+                          int64_t const kv_s_head, bool const has_index,
+                          bool const insert_kv, cudaStream_t stream) {
+  // Slot count must match the kernel's compile-time gating.
+  int const v_slots = insert_kv ? nkv : 0;
+  int const idx_slots = has_index ? niq + 1 : 0;
+  int const slots_per_token = nq + nkv + v_slots + idx_slots;
+
+  constexpr int kBlockSize = 256;
+  constexpr int kWarpsPerBlock = kBlockSize / 32;
+  int64_t const total_warps =
+      static_cast<int64_t>(num_tokens) * slots_per_token;
+  int const grid =
+      static_cast<int>((total_warps + kWarpsPerBlock - 1) / kWarpsPerBlock);
+  if (grid == 0) return;
+
+#ifndef USE_ROCM
+  // PDL: enable programmatic stream serialization whenever the hardware
+  // supports it (SM90+).  On pre-Hopper GPUs the attribute is unavailable, so
+  // leave numAttrs = 0 and launch as a regular kernel via cudaLaunchKernelEx.
+  static int const sm_version = getSMVersion();
+  cudaLaunchConfig_t config;
+  config.gridDim = dim3(grid);
+  config.blockDim = dim3(kBlockSize);
+  config.dynamicSmemBytes = 0;
+  config.stream = stream;
+  cudaLaunchAttribute attrs[1];
+  attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization;
+  attrs[0].val.programmaticStreamSerializationAllowed = 1;
+  config.attrs = attrs;
+  config.numAttrs = (sm_version >= 90) ? 1 : 0;
+
+  #define LAUNCH(IS_SPARSE, INSERT)                                           \
+    cudaLaunchKernelEx(                                                       \
+        &config,                                                              \
+        fusedMiniMaxM3QNormRopeKVInsertKernel<scalar_t, IS_SPARSE, INSERT>,   \
+        qkv, q_out, index_q_out, q_norm_w, k_norm_w, iq_norm_w, ik_norm_w,    \
+        cos_sin_cache, positions, slot_mapping, index_slot_mapping, kv_cache, \
+        index_cache, eps, rotary_dim, num_tokens, nq, nkv, niq, block_size,   \
+        kv_s_block, kv_s_kv, kv_s_token, kv_s_head)
+#else
+  // ROCm: standard kernel launch syntax (no PDL/stream serialization).
+  // clang-format off
+  #define LAUNCH(IS_SPARSE, INSERT)                                          \
+    fusedMiniMaxM3QNormRopeKVInsertKernel<scalar_t, IS_SPARSE, INSERT>       \
+        <<<grid, kBlockSize, 0, stream>>>(                                   \
+            qkv, q_out, index_q_out, q_norm_w, k_norm_w, iq_norm_w,          \
+            ik_norm_w, cos_sin_cache, positions, slot_mapping,               \
+            index_slot_mapping, kv_cache, index_cache, eps, rotary_dim,      \
+            num_tokens, nq, nkv, niq, block_size, kv_s_block, kv_s_kv,       \
+            kv_s_token, kv_s_head)
+  // clang-format on
+#endif
+
+  if (has_index) {
+    if (insert_kv) {
+      LAUNCH(true, true);  // sparse serving
+    } else {
+      LAUNCH(true, false);  // sparse profiling
+    }
+  } else {
+    // Dense layer: never has an index branch and never inserts here (the
+    // generic Attention layer owns the KV insert).
+    LAUNCH(false, false);
+  }
+#undef LAUNCH
+}
+
+}  // namespace minimax_m3_fused_ops
+}  // namespace vllm
+
+// ────────────────────────────────────────────────────────────────────────────
+// Torch op wrapper
+// ────────────────────────────────────────────────────────────────────────────
+void fused_minimax_m3_qknorm_rope_kv_insert(
+    torch::stable::Tensor& qkv,  // [N, qkv_row] (packs index if sparse)
+    torch::stable::Tensor const& q_norm_weight,  // [128]
+    torch::stable::Tensor const& k_norm_weight,  // [128]
+    torch::stable::Tensor const& cos_sin_cache,  // [max_pos, rotary_dim]
+    torch::stable::Tensor const& positions,      // [N] i64
+    int64_t num_heads, int64_t num_kv_heads, int64_t rotary_dim, double eps,
+    std::optional<torch::stable::Tensor> index_q_norm_weight,  // [128]
+    std::optional<torch::stable::Tensor> index_k_norm_weight,  // [128]
+    int64_t num_index_heads,                                  // niq; 0 => dense
+    std::optional<torch::stable::Tensor> slot_mapping,        // [N] i64
+    std::optional<torch::stable::Tensor> index_slot_mapping,  // [N] i64
+    std::optional<torch::stable::Tensor> kv_cache,     // [nb,2,bs,nkv,128]
+    std::optional<torch::stable::Tensor> index_cache,  // [nb,bs,128]
+    int64_t block_size,
+    std::optional<torch::stable::Tensor> q_out,  // [N, nq*128] contiguous
+    std::optional<torch::stable::Tensor>
+        index_q_out) {  // [N, niq*128] contiguous
+  STD_TORCH_CHECK(qkv.is_cuda() && qkv.is_contiguous(),
+                  "qkv must be contiguous CUDA");
+  STD_TORCH_CHECK(
+      positions.is_cuda() &&
+          positions.scalar_type() == torch::headeronly::ScalarType::Long,
+      "positions must be int64 CUDA");
+  STD_TORCH_CHECK(cos_sin_cache.is_cuda() && cos_sin_cache.is_contiguous(),
+                  "cos_sin_cache must be contiguous CUDA");
+  STD_TORCH_CHECK(cos_sin_cache.scalar_type() == qkv.scalar_type(),
+                  "cos_sin_cache dtype must match qkv");
+  STD_TORCH_CHECK(
+      cos_sin_cache.dim() == 2 && cos_sin_cache.size(1) == rotary_dim,
+      "cos_sin_cache shape [max_pos, rotary_dim]");
+
+  STD_TORCH_CHECK(q_norm_weight.scalar_type() == qkv.scalar_type() &&
+                      k_norm_weight.scalar_type() == qkv.scalar_type(),
+                  "q/k norm weight dtype must match qkv");
+  STD_TORCH_CHECK(
+      q_norm_weight.numel() == vllm::minimax_m3_fused_ops::kHeadDim &&
+          k_norm_weight.numel() == vllm::minimax_m3_fused_ops::kHeadDim,
+      "q/k norm weight must have 128 elements");
+  STD_TORCH_CHECK(rotary_dim > 0 && rotary_dim % 8 == 0 &&
+                      rotary_dim <= vllm::minimax_m3_fused_ops::kHeadDim,
+                  "rotary_dim must be a positive multiple of 8 and <= 128");
+
+  int const num_tokens = static_cast<int>(qkv.size(0));
+  int const nq = static_cast<int>(num_heads);
+  int const nkv = static_cast<int>(num_kv_heads);
+  int const niq = static_cast<int>(num_index_heads);
+
+  // The sparse layer packs the index branch ([index_q (niq heads) | index_k
+  // (1 head)]) right after [q|k|v] in the same row; the dense layer does not.
+  bool const has_index = niq > 0;
+  bool const insert_kv = kv_cache.has_value();
+  int const kHeadDim = vllm::minimax_m3_fused_ops::kHeadDim;
+  int const expected_row =
+      (nq + 2 * nkv + (has_index ? niq + 1 : 0)) * kHeadDim;
+  STD_TORCH_CHECK(qkv.size(1) == expected_row,
+                  "qkv last dim must be (num_heads + 2*num_kv_heads"
+                  " + num_index_heads + 1) * 128 for sparse, "
+                  "(num_heads + 2*num_kv_heads) * 128 for dense");
+
+  // Only the sparse layer inserts here (dense lets the generic Attention layer
+  // own the KV write); there is no dense+insert kernel instantiation.
+  STD_TORCH_CHECK(
+      !insert_kv || has_index,
+      "insert mode (kv_cache) requires the index branch (sparse layer)");
+  if (has_index) {
+    STD_TORCH_CHECK(
+        index_q_norm_weight.has_value() && index_k_norm_weight.has_value(),
+        "index branch requires both index norm weights");
+    STD_TORCH_CHECK(index_q_norm_weight->scalar_type() == qkv.scalar_type() &&
+                        index_k_norm_weight->scalar_type() == qkv.scalar_type(),
+                    "index norm weights dtype must match qkv");
+    STD_TORCH_CHECK(index_q_norm_weight->numel() == kHeadDim &&
+                        index_k_norm_weight->numel() == kHeadDim,
+                    "index norm weights must have 128 elements");
+  }
+  // kv_cache strides (logical shape [nb, 2, bs, nkv, head_dim]). Read straight
+  // off the tensor so the kernel honours whatever physical layout the attention
+  // backend allocated (NHD: stride order (0,1,2,3,4); HND: (0,1,3,2,4)). No new
+  // op argument is needed -- the strides ride along with the tensor itself.
+  int64_t kv_s_block = 0, kv_s_kv = 0, kv_s_token = 0, kv_s_head = 0;
+  torch::stable::Tensor const* effective_index_slot_mapping = nullptr;
+  if (insert_kv) {
+    STD_TORCH_CHECK(
+        slot_mapping.has_value() && slot_mapping->is_cuda() &&
+            slot_mapping->scalar_type() == torch::headeronly::ScalarType::Long,
+        "insert mode requires int64 CUDA slot_mapping");
+    STD_TORCH_CHECK(
+        !index_slot_mapping.has_value() ||
+            (index_slot_mapping->is_cuda() &&
+             index_slot_mapping->scalar_type() ==
+                 torch::headeronly::ScalarType::Long &&
+             index_slot_mapping->numel() == slot_mapping->numel()),
+        "index_slot_mapping must be int64 CUDA with slot_mapping length");
+    STD_TORCH_CHECK(kv_cache->scalar_type() == qkv.scalar_type(),
+                    "kv_cache dtype must match qkv (bf16 cache only)");
+    STD_TORCH_CHECK(index_cache.has_value() &&
+                        index_cache->scalar_type() == qkv.scalar_type(),
+                    "insert mode requires matching index_cache");
+    STD_TORCH_CHECK(kv_cache->dim() == 5 && kv_cache->stride(4) == 1,
+                    "kv_cache must be [nb,2,bs,nkv,head_dim] with contiguous "
+                    "head_dim (stride(4)==1)");
+    kv_s_block = kv_cache->stride(0);
+    kv_s_kv = kv_cache->stride(1);
+    kv_s_token = kv_cache->stride(2);
+    kv_s_head = kv_cache->stride(3);
+    effective_index_slot_mapping = index_slot_mapping.has_value()
+                                       ? &index_slot_mapping.value()
+                                       : &slot_mapping.value();
+  }
+  // Optional contiguous gather targets: when given, the normed/roped q (and
+  // index_q) are written here instead of in place, so callers avoid a separate
+  // .contiguous() copy.  index_q_out only makes sense on the sparse path.
+  if (q_out.has_value()) {
+    STD_TORCH_CHECK(
+        q_out->is_cuda() && q_out->is_contiguous() &&
+            q_out->scalar_type() == qkv.scalar_type(),
+        "q_out must be a contiguous CUDA tensor matching qkv dtype");
+    STD_TORCH_CHECK(
+        q_out->numel() == static_cast<int64_t>(num_tokens) * nq * kHeadDim,
+        "q_out must have num_tokens * num_heads * 128 elements");
+  }
+  if (index_q_out.has_value()) {
+    STD_TORCH_CHECK(
+        has_index,
+        "index_q_out requires the index branch (num_index_heads > 0)");
+    STD_TORCH_CHECK(
+        index_q_out->is_cuda() && index_q_out->is_contiguous() &&
+            index_q_out->scalar_type() == qkv.scalar_type(),
+        "index_q_out must be a contiguous CUDA tensor matching qkv dtype");
+    STD_TORCH_CHECK(index_q_out->numel() ==
+                        static_cast<int64_t>(num_tokens) * niq * kHeadDim,
+                    "index_q_out must have num_tokens * num_index_heads * 128 "
+                    "elements");
+  }
+
+  const torch::stable::accelerator::DeviceGuard device_guard(
+      qkv.get_device_index());
+  auto stream = get_current_cuda_stream(qkv.get_device_index());
+
+  VLLM_STABLE_DISPATCH_HALF_TYPES(
+      qkv.scalar_type(), "fused_minimax_m3_qknorm_rope_kv_insert", [&] {
+        using st = scalar_t;
+        vllm::minimax_m3_fused_ops::launchFusedMiniMaxM3<st>(
+            reinterpret_cast<st*>(qkv.data_ptr()),
+            q_out.has_value() ? reinterpret_cast<st*>(q_out->data_ptr())
+                              : nullptr,
+            index_q_out.has_value()
+                ? reinterpret_cast<st*>(index_q_out->data_ptr())
+                : nullptr,
+            reinterpret_cast<st const*>(q_norm_weight.data_ptr()),
+            reinterpret_cast<st const*>(k_norm_weight.data_ptr()),
+            has_index
+                ? reinterpret_cast<st const*>(index_q_norm_weight->data_ptr())
+                : nullptr,
+            has_index
+                ? reinterpret_cast<st const*>(index_k_norm_weight->data_ptr())
+                : nullptr,
+            reinterpret_cast<st const*>(cos_sin_cache.data_ptr()),
+            reinterpret_cast<int64_t const*>(positions.data_ptr()),
+            insert_kv
+                ? reinterpret_cast<int64_t const*>(slot_mapping->data_ptr())
+                : nullptr,
+            insert_kv ? reinterpret_cast<int64_t const*>(
+                            effective_index_slot_mapping->data_ptr())
+                      : nullptr,
+            insert_kv ? reinterpret_cast<st*>(kv_cache->data_ptr()) : nullptr,
+            (insert_kv && has_index)
+                ? reinterpret_cast<st*>(index_cache->data_ptr())
+                : nullptr,
+            static_cast<float>(eps), static_cast<int>(rotary_dim), num_tokens,
+            nq, nkv, niq, static_cast<int>(block_size), kv_s_block, kv_s_kv,
+            kv_s_token, kv_s_head, has_index, insert_kv, stream);
+      });
+}

csrc/libtorch_stable/ops.h

@@ -281,6 +281,24 @@ minimax_allreduce_rms_qk(torch::stable::Tensor qkv,
                          int64_t const nranks, double const eps);
 #endif
 
+// Horizontally-fused MiniMax-M3 QK-norm + partial NeoX RoPE (+ optional KV /
+// index-cache insert). Dense layer: norm+RoPE only; sparse layer: also packs
+// the index branch and scatters k/v/index_k into their paged caches.
+void fused_minimax_m3_qknorm_rope_kv_insert(
+    torch::stable::Tensor& qkv, torch::stable::Tensor const& q_norm_weight,
+    torch::stable::Tensor const& k_norm_weight,
+    torch::stable::Tensor const& cos_sin_cache,
+    torch::stable::Tensor const& positions, int64_t num_heads,
+    int64_t num_kv_heads, int64_t rotary_dim, double eps,
+    std::optional<torch::stable::Tensor> index_q_norm_weight,
+    std::optional<torch::stable::Tensor> index_k_norm_weight,
+    int64_t num_index_heads, std::optional<torch::stable::Tensor> slot_mapping,
+    std::optional<torch::stable::Tensor> index_slot_mapping,
+    std::optional<torch::stable::Tensor> kv_cache,
+    std::optional<torch::stable::Tensor> index_cache, int64_t block_size,
+    std::optional<torch::stable::Tensor> q_out,
+    std::optional<torch::stable::Tensor> index_q_out);
+
 // Sampler kernels (shared CUDA/ROCm)
 void apply_repetition_penalties_(
     torch::stable::Tensor& logits, const torch::stable::Tensor& prompt_mask,
@@ -346,7 +364,8 @@ void free_shared_buffer(int64_t buffer);
 // Activation kernels (shared CUDA/ROCm)
 void silu_and_mul(torch::stable::Tensor& out, torch::stable::Tensor& input);
 void silu_and_mul_clamp(torch::stable::Tensor& out,
-                        torch::stable::Tensor& input, double limit);
+                        torch::stable::Tensor& input, double limit,
+                        double alpha = 1.0, double beta = 0.0);
 void mul_and_silu(torch::stable::Tensor& out, torch::stable::Tensor& input);
 void gelu_and_mul(torch::stable::Tensor& out, torch::stable::Tensor& input);
 void gelu_tanh_and_mul(torch::stable::Tensor& out,

csrc/libtorch_stable/quantization/fp4/nvfp4_utils.cuh

@@ -237,21 +237,30 @@ __device__ __forceinline__ fp4_packed_t cvt_warp_fp16_to_fp4(
   // Get the final absolute maximum values.
   float vecMax = float(__hmax(localMax.x, localMax.y));
 
-  // Get the SF (max value of the vector / max value of e2m1).
-  // maximum value of e2m1 = 6.0.
-  // TODO: use half as compute data type.
-  float SFValue = SFScaleVal * (vecMax * reciprocal_approximate_ftz(6.0f));
   // 8 bits representation of the SF.
+  float SFValue;
   uint8_t fp8SFVal;
-  // Write the SF to global memory (STG.8).
+
   if constexpr (UE8M0_SF) {
-    // Extract the 8 exponent bits from float32.
-    // float 32bits = 1 sign bit + 8 exponent bits + 23 mantissa bits.
-    uint32_t tmp = reinterpret_cast<uint32_t&>(SFValue) >> 23;
-    fp8SFVal = tmp & 0xff;
-    // Convert back to fp32.
-    reinterpret_cast<uint32_t&>(SFValue) = tmp << 23;
+    // OCP MX spec E8M0 scale computation (MXFP4 path):
+    // scale_exp = biased_exponent(round_up(vecMax)) - 2
+    //   -2 because max E2M1 value is 6.0 ≈ 2^2.58; we use 2^2=4 as the
+    //   safe divisor so that max_val / scale <= 6.0 for values near 2^n.
+    uint32_t max_bits = __float_as_uint(vecMax);
+    // Add rounding bias at mantissa bit 21 (equivalent to bf16 val_to_add=32
+    // at bit 5). Threshold: values with mantissa >= 0.75 (i.e. >= 1.75*2^n)
+    // round up to the next power of 2.
+    uint32_t rounded_bits = (max_bits + (1u << 21)) & 0xFF800000u;
+    uint32_t biased_exp = (rounded_bits >> 23) & 0xFFu;
+    uint32_t scale_exp = (biased_exp > 2u) ? (biased_exp - 2u) : 0u;
+    scale_exp = min(scale_exp, 254u);
+    fp8SFVal = static_cast<uint8_t>(scale_exp);
+    // Reconstruct scale as float32: scale = 2^(scale_exp - 127)
+    uint32_t sf_bits = scale_exp << 23;
+    SFValue = __uint_as_float(sf_bits);
   } else {
+    // NVFP4 path: scale = max / 6.0, stored as E4M3.
+    SFValue = SFScaleVal * (vecMax * reciprocal_approximate_ftz(6.0f));
     // Here SFValue is always positive, so E4M3 is the same as UE4M3.
     __nv_fp8_e4m3 tmp = __nv_fp8_e4m3(SFValue);
     reinterpret_cast<__nv_fp8_e4m3&>(fp8SFVal) = tmp;
@@ -262,13 +271,21 @@ __device__ __forceinline__ fp4_packed_t cvt_warp_fp16_to_fp4(
   // Write the SF to global memory (STG.8).
   if (SFout) *SFout = fp8SFVal;
 
-  // Get the output scale.
-  // Recipe: final_scale = reciprocal(fp32(fp8(SFValue * SFScaleVal))) *
-  //                       reciprocal(SFScaleVal))
-  float outputScale =
-      SFValue != 0.0f ? reciprocal_approximate_ftz(
+  // Get the output scale (= 1 / SFValue for the MXFP4/UE8M0 path where
+  // SFScaleVal=1).  Use exact division for UE8M0 to ensure bit-exact scaling
+  // that matches the reference QDQ implementation (dividing by a power-of-2
+  // scale is exact in IEEE 754).
+  float outputScale;
+  if constexpr (UE8M0_SF) {
+    // SFValue is always a power of 2 for UE8M0, so 1/SFValue is exact.
+    outputScale = SFValue != 0.0f ? (1.0f / SFValue) : 0.0f;
+  } else {
+    // NVFP4 path: use fast approximate reciprocal (original behavior).
+    outputScale = SFValue != 0.0f
+                      ? reciprocal_approximate_ftz(
                             SFValue * reciprocal_approximate_ftz(SFScaleVal))
                       : 0.0f;
+  }
 
   // Convert the input to float.
   float2 fp2Vals[CVT_FP4_ELTS_PER_THREAD / 2];

csrc/libtorch_stable/torch_bindings.cpp

@@ -461,6 +461,18 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
       "float eps) -> (Tensor, Tensor)");
 #endif
 
+  // Horizontally-fused MiniMax-M3 QK-norm + partial NeoX RoPE + KV-insert.
+  ops.def(
+      "fused_minimax_m3_qknorm_rope_kv_insert("
+      "Tensor! qkv, Tensor q_norm_weight, Tensor k_norm_weight, "
+      "Tensor cos_sin_cache, Tensor positions, int num_heads, "
+      "int num_kv_heads, int rotary_dim, float eps, "
+      "Tensor? index_q_norm_weight, Tensor? index_k_norm_weight, "
+      "int num_index_heads, "
+      "Tensor? slot_mapping, Tensor? index_slot_mapping, "
+      "Tensor!? kv_cache, Tensor!? index_cache, "
+      "int block_size, Tensor!? q_out, Tensor!? index_q_out) -> ()");
+
   // Apply repetition penalties to logits in-place.
   ops.def(
       "apply_repetition_penalties_(Tensor! logits, Tensor prompt_mask, "
@@ -488,9 +500,11 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
   ops.def("mul_and_silu(Tensor! out, Tensor input) -> ()");
 
   // SwiGLU activation with input clamping.
+  // alpha scales the sigmoid (gate * sigmoid(alpha * gate)); beta is added to
+  // the up half (up + beta). Defaults alpha=1.0, beta=0.0 give silu(gate)*up.
   ops.def(
-      "silu_and_mul_with_clamp(Tensor! result, Tensor input, float limit) "
-      "-> ()");
+      "silu_and_mul_with_clamp(Tensor! result, Tensor input, float limit, "
+      "float alpha=1.0, float beta=0.0) -> ()");
 
   // Activation function used in GeGLU with `none` approximation.
   ops.def("gelu_and_mul(Tensor! out, Tensor input) -> ()");
@@ -679,6 +693,8 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
   ops.impl("minimax_allreduce_rms", TORCH_BOX(&minimax_allreduce_rms));
   ops.impl("minimax_allreduce_rms_qk", TORCH_BOX(&minimax_allreduce_rms_qk));
 #endif
+  ops.impl("fused_minimax_m3_qknorm_rope_kv_insert",
+           TORCH_BOX(&fused_minimax_m3_qknorm_rope_kv_insert));
 
   // Sampler kernels (shared CUDA/ROCm)
   ops.impl("apply_repetition_penalties_",

csrc/ops.h

@@ -50,7 +50,8 @@ void rotary_embedding(torch::Tensor& positions, torch::Tensor& query,
 
 void silu_and_mul(torch::Tensor& out, torch::Tensor& input);
 
-void silu_and_mul_clamp(torch::Tensor& out, torch::Tensor& input, double limit);
+void silu_and_mul_clamp(torch::Tensor& out, torch::Tensor& input, double limit,
+                        double alpha = 1.0, double beta = 0.0);
 
 void silu_and_mul_quant(torch::Tensor& out, torch::Tensor& input,
                         torch::Tensor& scale);

csrc/push_all_reduce.cu

@@ -0,0 +1,90 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+//
+// C++ bridge functions for push-based allreduce.
+// Exposes PushAllReduceManager to Python via torch custom ops.
+
+#include "push_all_reduce.cuh"
+
+#include <c10/cuda/CUDAGuard.h>
+#include <c10/cuda/CUDAStream.h>
+#include <torch/all.h>
+
+using fptr_t = int64_t;
+using namespace vllm::push_ar;
+
+// Initialize the manager; returns opaque pointer as int64_t
+fptr_t init_push_ar(int64_t rank, int64_t world_size, int64_t push_buffer_bytes,
+                    int64_t max_num_cta) {
+  auto* mgr = new PushAllReduceManager(
+      static_cast<int>(rank), static_cast<int>(world_size), push_buffer_bytes,
+      static_cast<int>(max_num_cta));
+  return reinterpret_cast<fptr_t>(mgr);
+}
+
+// Get IPC handle as a byte tensor
+torch::Tensor get_push_ar_ipc_handle(fptr_t _mgr) {
+  auto* mgr = reinterpret_cast<PushAllReduceManager*>(_mgr);
+  cudaIpcMemHandle_t handle = mgr->get_ipc_handle();
+  auto t = torch::from_blob(&handle, {static_cast<int64_t>(sizeof(handle))},
+                            torch::kUInt8)
+               .clone();
+  return t;
+}
+
+// Post-init with peer IPC handles
+void post_init_push_ar(fptr_t _mgr, torch::Tensor all_handles) {
+  auto* mgr = reinterpret_cast<PushAllReduceManager*>(_mgr);
+  int world_size = all_handles.size(0);
+  std::vector<cudaIpcMemHandle_t> handles(world_size);
+  for (int i = 0; i < world_size; i++) {
+    memcpy(&handles[i], all_handles[i].data_ptr(), sizeof(cudaIpcMemHandle_t));
+  }
+  mgr->post_init(handles);
+}
+
+// Check weak contiguity (same logic as vLLM custom_all_reduce.cu)
+static bool _is_weak_contiguous(const torch::Tensor& t) {
+  return t.is_contiguous() ||
+         (t.storage().nbytes() - t.storage_offset() * t.element_size() ==
+          static_cast<size_t>(t.numel()) * t.element_size());
+}
+
+// Perform allreduce
+void push_ar_all_reduce(fptr_t _mgr, torch::Tensor& inp, torch::Tensor& out) {
+  auto* mgr = reinterpret_cast<PushAllReduceManager*>(_mgr);
+  const at::cuda::OptionalCUDAGuard device_guard(device_of(inp));
+  auto stream = c10::cuda::getCurrentCUDAStream().stream();
+
+  TORCH_CHECK_EQ(inp.scalar_type(), out.scalar_type());
+  TORCH_CHECK_EQ(inp.numel(), out.numel());
+  TORCH_CHECK(_is_weak_contiguous(inp), "Input must be contiguous");
+  TORCH_CHECK(_is_weak_contiguous(out), "Output must be contiguous");
+
+  switch (out.scalar_type()) {
+    case at::ScalarType::BFloat16:
+      mgr->allreduce<nv_bfloat16>(
+          stream, reinterpret_cast<nv_bfloat16*>(inp.data_ptr()),
+          reinterpret_cast<nv_bfloat16*>(out.data_ptr()), out.numel());
+      break;
+    case at::ScalarType::Half:
+      mgr->allreduce<half>(stream, reinterpret_cast<half*>(inp.data_ptr()),
+                           reinterpret_cast<half*>(out.data_ptr()),
+                           out.numel());
+      break;
+    case at::ScalarType::Float:
+      mgr->allreduce<float>(stream, reinterpret_cast<float*>(inp.data_ptr()),
+                            reinterpret_cast<float*>(out.data_ptr()),
+                            out.numel());
+      break;
+    default:
+      TORCH_CHECK(false,
+                  "push allreduce: unsupported dtype (need bf16/fp16/fp32)");
+  }
+}
+
+// Dispose the manager
+void dispose_push_ar(fptr_t _mgr) {
+  auto* mgr = reinterpret_cast<PushAllReduceManager*>(_mgr);
+  delete mgr;
+}

csrc/push_all_reduce.cuh

@@ -0,0 +1,244 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+//
+// Host-side manager for push-based allreduce.
+// Manages IPC storage, PushController initialization, and kernel launch.
+// Replaces SGLang's CustomAllReduceBase + CustomAllReducePush.
+
+#pragma once
+#include "push_all_reduce_kernel.cuh"
+
+#include <cuda_runtime.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cstring>
+#include <numeric>
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+namespace vllm {
+namespace push_ar {
+
+#define PUSH_AR_CUDACHECK(cmd)                                                 \
+  do {                                                                         \
+    cudaError_t e = cmd;                                                       \
+    if (e != cudaSuccess) {                                                    \
+      throw std::runtime_error(std::string("push_all_reduce CUDA error at ") + \
+                               __FILE__ + ":" + std::to_string(__LINE__) +     \
+                               " '" + cudaGetErrorString(e) + "'");            \
+    }                                                                          \
+  } while (0)
+
+class PushAllReduceManager {
+ public:
+  PushAllReduceManager(int rank, int world_size, int64_t push_buffer_bytes,
+                       int max_num_cta)
+      : rank_(rank),
+        world_size_(world_size),
+        push_buffer_bytes_(push_buffer_bytes),
+        max_num_cta_(max_num_cta),
+        storage_(nullptr) {
+    assert(world_size_ >= 2 && world_size_ <= 8);
+    assert(max_num_cta_ > 0 && max_num_cta_ <= 512);
+    assert(push_buffer_bytes_ > 0);
+
+    // Determine PDL support from device capability
+    int device_id;
+    PUSH_AR_CUDACHECK(cudaGetDevice(&device_id));
+    int major;
+    PUSH_AR_CUDACHECK(cudaDeviceGetAttribute(
+        &major, cudaDevAttrComputeCapabilityMajor, device_id));
+    use_pdl_ = (major >= 9);  // Hopper (sm90) or newer
+
+    // Allocate storage
+    storage_bytes_ = push_signal_bytes() + push_buffer_total_bytes();
+    PUSH_AR_CUDACHECK(cudaMalloc(&storage_, storage_bytes_));
+    // Zeros signals (epoch=0 for all CTAs) AND push buffer
+    // (0x0000 = IEEE 754 positive-zero = "empty" sentinel)
+    PUSH_AR_CUDACHECK(cudaMemset(storage_, 0, storage_bytes_));
+
+    peer_storage_.resize(world_size_, nullptr);
+  }
+
+  ~PushAllReduceManager() {
+    for (int i = 0; i < world_size_; i++) {
+      if (i != rank_ && peer_storage_[i] != nullptr) {
+        cudaIpcCloseMemHandle(peer_storage_[i]);
+      }
+    }
+    if (storage_) {
+      cudaFree(storage_);
+    }
+  }
+
+  // Phase 1: Return IPC handle for local storage
+  cudaIpcMemHandle_t get_ipc_handle() {
+    cudaIpcMemHandle_t handle;
+    PUSH_AR_CUDACHECK(cudaIpcGetMemHandle(&handle, storage_));
+    return handle;
+  }
+
+  // Phase 2: Open peer IPC handles and init PushController
+  void post_init(const std::vector<cudaIpcMemHandle_t>& peer_handles) {
+    assert(peer_handles.size() == (size_t)world_size_);
+    for (int i = 0; i < world_size_; i++) {
+      if (i == rank_) {
+        peer_storage_[i] = storage_;
+      } else {
+        PUSH_AR_CUDACHECK(cudaIpcOpenMemHandle(&peer_storage_[i],
+                                               peer_handles[i],
+                                               cudaIpcMemLazyEnablePeerAccess));
+      }
+    }
+    // Create PushController pointing to local signal region
+    push_ctrl_ = PushController(get_push_signal(storage_));
+    ctrl_initialized_ = true;
+  }
+
+  // Main allreduce dispatch
+  template <typename T>
+  void allreduce(cudaStream_t stream, T* input, T* output, int num_elements) {
+    assert(ctrl_initialized_);
+    assert(num_elements > 0);
+
+    const uint32_t num_items = static_cast<uint32_t>(num_elements);
+    const int num_threads = select_num_threads<T>(num_items);
+
+    // Verify input fits in push buffer (runtime check, not compiled out)
+    const int64_t input_bytes = static_cast<int64_t>(sizeof(T)) * num_elements;
+    if (input_bytes > push_buffer_bytes_) {
+      throw std::runtime_error("push_all_reduce: input (" +
+                               std::to_string(input_bytes) +
+                               " bytes) exceeds push buffer capacity (" +
+                               std::to_string(push_buffer_bytes_) + " bytes)");
+    }
+
+    // Build kernel params
+    AllReducePushData params;
+    for (int i = 0; i < world_size_; i++) {
+      params.buffer[i] = get_push_buffer(peer_storage_[i]);
+    }
+    // Fill remaining buffer slots with nullptr (safety for kMaxNumGPU=8)
+    for (int i = world_size_; i < (int)kMaxNumGPU; i++) {
+      params.buffer[i] = nullptr;
+    }
+    params.input = input;
+    params.output = output;
+    params.rank = rank_;
+    params.num_items = num_items;
+    params.buffer_bytes = static_cast<uint32_t>(push_buffer_bytes_);
+    params.epoch_bytes = world_size_ * params.buffer_bytes;
+
+    // Build launch config for cudaLaunchKernelEx
+    cudaLaunchConfig_t config = {};
+    config.gridDim = dim3(max_num_cta_);
+    config.blockDim = dim3(num_threads);
+    config.dynamicSmemBytes = 0;
+    config.stream = stream;
+
+    cudaLaunchAttribute attrs[1];
+    config.numAttrs = 0;
+    config.attrs = attrs;
+
+    if (use_pdl_) {
+      attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization;
+      attrs[0].val.programmaticStreamSerializationAllowed = 1;
+      config.numAttrs = 1;
+    }
+
+    // Template dispatch: world_size x pdl
+    launch_kernel<T>(config, params);
+  }
+
+ private:
+  // Template dispatch helper
+  template <typename T>
+  void launch_kernel(const cudaLaunchConfig_t& config,
+                     const AllReducePushData& params) {
+    // Dispatch on world_size and use_pdl
+    if (world_size_ == 8) {
+      if (use_pdl_) {
+        auto kernel = all_reduce_one_shot_push_kernel<T, 8, true>;
+        cudaLaunchKernelEx(&config, kernel, params, push_ctrl_);
+      } else {
+        auto kernel = all_reduce_one_shot_push_kernel<T, 8, false>;
+        cudaLaunchKernelEx(&config, kernel, params, push_ctrl_);
+      }
+    } else if (world_size_ == 4) {
+      if (use_pdl_) {
+        auto kernel = all_reduce_one_shot_push_kernel<T, 4, true>;
+        cudaLaunchKernelEx(&config, kernel, params, push_ctrl_);
+      } else {
+        auto kernel = all_reduce_one_shot_push_kernel<T, 4, false>;
+        cudaLaunchKernelEx(&config, kernel, params, push_ctrl_);
+      }
+    } else if (world_size_ == 2) {
+      if (use_pdl_) {
+        auto kernel = all_reduce_one_shot_push_kernel<T, 2, true>;
+        cudaLaunchKernelEx(&config, kernel, params, push_ctrl_);
+      } else {
+        auto kernel = all_reduce_one_shot_push_kernel<T, 2, false>;
+        cudaLaunchKernelEx(&config, kernel, params, push_ctrl_);
+      }
+    } else if (world_size_ == 6) {
+      if (use_pdl_) {
+        auto kernel = all_reduce_one_shot_push_kernel<T, 6, true>;
+        cudaLaunchKernelEx(&config, kernel, params, push_ctrl_);
+      } else {
+        auto kernel = all_reduce_one_shot_push_kernel<T, 6, false>;
+        cudaLaunchKernelEx(&config, kernel, params, push_ctrl_);
+      }
+    }
+  }
+
+  // Thread count selection (from SGLang CustomAllReducePush::all_reduce)
+  template <typename T>
+  int select_num_threads(uint32_t num_items) const {
+    constexpr uint32_t kVecSize = 16 / (sizeof(T) * 2);
+    for (const auto t : {128u, 256u, 512u}) {
+      if (t * max_num_cta_ * 2 * kVecSize >= num_items) {
+        return static_cast<int>(t);
+      }
+    }
+    return 1024;
+  }
+
+  // Storage layout helpers
+  int64_t push_signal_bytes() const {
+    return align128(sizeof(uint32_t) * max_num_cta_);
+  }
+
+  int64_t push_buffer_total_bytes() const {
+    return align128(PushController::kNumStages * world_size_ *
+                    push_buffer_bytes_);
+  }
+
+  void* get_push_signal(void* base) const {
+    return base;  // signals start at offset 0
+  }
+
+  void* get_push_buffer(void* base) const {
+    return static_cast<char*>(base) + push_signal_bytes();
+  }
+
+  static int64_t align128(int64_t size) { return ((size + 127) / 128) * 128; }
+
+  // Members
+  int rank_;
+  int world_size_;
+  int64_t push_buffer_bytes_;
+  int max_num_cta_;
+  bool use_pdl_;
+
+  void* storage_;
+  int64_t storage_bytes_;
+  std::vector<void*> peer_storage_;
+
+  PushController push_ctrl_;
+  bool ctrl_initialized_ = false;
+};
+
+}  // namespace push_ar
+}  // namespace vllm

csrc/push_all_reduce_kernel.cuh

@@ -0,0 +1,538 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+//
+// Push-based 2-buffer allreduce kernel, ported from SGLang's
+// all_reduce_one_shot_push_kernel. Uses epoch-based double-buffered
+// protocol with positive-zero sentinel for data arrival detection.
+//
+// Original source:
+//   sglang/jit_kernel/csrc/distributed/custom_all_reduce_push.cuh
+// Protocol reference: SGLang commit edb1b3f
+//
+// Changes from SGLang:
+//   - All code placed in namespace vllm::push_ar
+//   - SGL_DEVICE macros replaced with __device__ __forceinline__
+//   - SGL_CUDA_ARCH replaced with __CUDA_ARCH__
+//   - std::integral (C++20) replaced with explicit overloads (C++17)
+//   - kMaxVecBytes hardcoded to 16 (push kernel uses 16-byte vectors)
+//   - TVM/FFI dependencies removed; file is self-contained
+
+#pragma once
+#include <cuda_bf16.h>
+#include <cuda_fp16.h>
+#include <cuda_runtime.h>
+
+#include <cstddef>
+#include <cstdint>
+
+namespace vllm {
+namespace push_ar {
+
+// ============================================================
+// Section A: Type aliases (from SGLang utils.cuh lines 50-75)
+// ============================================================
+using fp32_t = float;
+using fp16_t = __half;
+using bf16_t = __nv_bfloat16;
+using fp32x2_t = float2;
+using fp16x2_t = __half2;
+using bf16x2_t = __nv_bfloat162;
+
+static constexpr uint32_t kWarpThreads = 32u;
+
+// ============================================================
+// Section B: kMaxVecBytes (from SGLang utils.cuh line 112)
+// ============================================================
+// Hardcoded to 16 since the push kernel uses 16-byte vectors.
+// The kernel's kVecSize = 16 / (sizeof(DType) * 2) yields
+// AlignedVector<packed_t<DType>, kVecSize> = 16 bytes always.
+inline constexpr std::size_t kMaxVecBytes = 16;
+
+// ============================================================
+// Section C: PDL helpers (from SGLang utils.cuh lines 119-148)
+// ============================================================
+// CHANGED: SGL_ARCH_HOPPER_OR_GREATER -> __CUDA_ARCH__ >= 900
+template <bool kUsePDL>
+__device__ __forceinline__ void PDLWaitPrimary() {
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900
+  if constexpr (kUsePDL) {
+    asm volatile("griddepcontrol.wait;" ::: "memory");
+  }
+#endif
+}
+
+template <bool kUsePDL>
+__device__ __forceinline__ void PDLTriggerSecondary() {
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 900
+  if constexpr (kUsePDL) {
+    asm volatile("griddepcontrol.launch_dependents;" :::);
+  }
+#endif
+}
+
+// ============================================================
+// Section D: Pointer offset helpers (from SGLang utils.cuh 181-195)
+// ============================================================
+// CHANGED: Removed std::integral (C++20) constraint.
+// Use explicit overloads for 1 and 2 offsets (C++17 compatible).
+
+// Byte-level offset (replaces pointer::offset<char>)
+__device__ __forceinline__ void* ptr_byte_offset(void* ptr, int64_t off1) {
+  return static_cast<char*>(ptr) + off1;
+}
+
+__device__ __forceinline__ void* ptr_byte_offset(void* ptr, int64_t off1,
+                                                 int64_t off2) {
+  return static_cast<char*>(ptr) + off1 + off2;
+}
+
+// Typed offset for AlignedVector load/store addressing
+// (replaces pointer::offset<T>)
+template <typename T>
+__device__ __forceinline__ void* ptr_typed_offset(void* ptr, int64_t offset) {
+  return static_cast<T*>(ptr) + offset;
+}
+
+template <typename T>
+__device__ __forceinline__ const void* ptr_typed_offset(const void* ptr,
+                                                        int64_t offset) {
+  return static_cast<const T*>(ptr) + offset;
+}
+
+// Host-side pointer offset (for storage layout calculations)
+inline void* host_ptr_offset(void* ptr, int64_t off) {
+  return static_cast<char*>(ptr) + off;
+}
+
+// ============================================================
+// Section E: dtype_trait system (from SGLang type.cuh)
+// ============================================================
+// COPY AS IS with namespace adjustment.
+
+template <typename T>
+struct dtype_trait {};
+
+template <>
+struct dtype_trait<fp32_t> {
+  using self_t = fp32_t;
+  using packed_t = fp32x2_t;
+  template <typename S>
+  __device__ __forceinline__ static self_t from(const S& value) {
+    return static_cast<fp32_t>(value);
+  }
+  __device__ __forceinline__ static self_t from(const fp16_t& x) {
+    return __half2float(x);
+  }
+  __device__ __forceinline__ static self_t from(const bf16_t& x) {
+    return __bfloat162float(x);
+  }
+};
+
+template <>
+struct dtype_trait<fp16_t> {
+  using self_t = fp16_t;
+  using packed_t = fp16x2_t;
+  template <typename S>
+  __device__ __forceinline__ static self_t from(const S& value) {
+    return static_cast<fp16_t>(value);
+  }
+};
+
+template <>
+struct dtype_trait<bf16_t> {
+  using self_t = bf16_t;
+  using packed_t = bf16x2_t;
+  template <typename S>
+  __device__ __forceinline__ static self_t from(const S& value) {
+    return static_cast<bf16_t>(value);
+  }
+};
+
+template <>
+struct dtype_trait<fp32x2_t> {
+  using self_t = fp32x2_t;
+  template <typename S>
+  __device__ __forceinline__ static self_t from(const S& value) {
+    return static_cast<fp32x2_t>(value);
+  }
+  __device__ __forceinline__ static self_t from(const fp16x2_t& x) {
+    return __half22float2(x);
+  }
+  __device__ __forceinline__ static self_t from(const bf16x2_t& x) {
+    return __bfloat1622float2(x);
+  }
+};
+
+template <>
+struct dtype_trait<fp16x2_t> {
+  using self_t = fp16x2_t;
+  template <typename S>
+  __device__ __forceinline__ static self_t from(const S& value) {
+    return static_cast<fp16x2_t>(value);
+  }
+  __device__ __forceinline__ static self_t from(const fp32x2_t& x) {
+    return __float22half2_rn(x);
+  }
+};
+
+template <>
+struct dtype_trait<bf16x2_t> {
+  using self_t = bf16x2_t;
+  template <typename S>
+  __device__ __forceinline__ static self_t from(const S& value) {
+    return static_cast<bf16x2_t>(value);
+  }
+  __device__ __forceinline__ static self_t from(const fp32x2_t& x) {
+    return __float22bfloat162_rn(x);
+  }
+};
+
+template <typename T>
+using packed_t = typename dtype_trait<T>::packed_t;
+
+template <typename To, typename From>
+__device__ __forceinline__ To cast(const From& value) {
+  return dtype_trait<To>::from(value);
+}
+
+// ============================================================
+// Section F: AlignedVector (from SGLang vec.cuh lines 73-116)
+// ============================================================
+// COPY AS IS with SGL_DEVICE -> __device__ __forceinline__
+// and kMaxVecBytes = 16.
+
+namespace detail {
+
+template <std::size_t N>
+struct uint_trait {};
+template <>
+struct uint_trait<1> {
+  using type = uint8_t;
+};
+template <>
+struct uint_trait<2> {
+  using type = uint16_t;
+};
+template <>
+struct uint_trait<4> {
+  using type = uint32_t;
+};
+template <>
+struct uint_trait<8> {
+  using type = uint64_t;
+};
+
+template <typename T>
+using sized_int = typename uint_trait<sizeof(T)>::type;
+
+}  // namespace detail
+
+template <typename T, std::size_t N>
+struct alignas(sizeof(T) * N) AlignedStorage {
+  T data[N];
+};
+
+template <typename T, std::size_t N>
+struct AlignedVector {
+ private:
+  static_assert((N > 0 && (N & (N - 1)) == 0) && sizeof(T) * N <= kMaxVecBytes,
+                "CUDA vector size exceeds arch limit (max 16 bytes)");
+  using element_t = typename detail::sized_int<T>;
+  using storage_t = AlignedStorage<element_t, N>;
+
+ public:
+  __device__ __forceinline__ void load(const void* ptr, int64_t offset = 0) {
+    m_storage = reinterpret_cast<const storage_t*>(ptr)[offset];
+  }
+  __device__ __forceinline__ void store(void* ptr, int64_t offset = 0) const {
+    reinterpret_cast<storage_t*>(ptr)[offset] = m_storage;
+  }
+  __device__ __forceinline__ void fill(T value) {
+    const auto store_value = *reinterpret_cast<element_t*>(&value);
+#pragma unroll
+    for (std::size_t i = 0; i < N; ++i) {
+      m_storage.data[i] = store_value;
+    }
+  }
+  __device__ __forceinline__ auto operator[](std::size_t idx) -> T& {
+    return reinterpret_cast<T*>(&m_storage)[idx];
+  }
+  __device__ __forceinline__ auto operator[](std::size_t idx) const -> T {
+    return reinterpret_cast<const T*>(&m_storage)[idx];
+  }
+
+ private:
+  storage_t m_storage;
+};
+
+// ============================================================
+// Section G: PushController (from SGLang common.cuh lines 93-118)
+// ============================================================
+// COPY AS IS with SGL_DEVICE -> __device__ __forceinline__
+
+static constexpr uint32_t kMaxNumGPU = 8;
+
+struct PushController {
+  using SignalType = uint32_t;
+  static constexpr int64_t kNumStages = 2;  // double-buffered epochs
+
+  PushController() : m_local_signal(nullptr) {}
+
+  PushController(void* ptr) : m_local_signal(static_cast<SignalType*>(ptr)) {}
+
+  __device__ __forceinline__ SignalType epoch() const {
+    return m_local_signal[blockIdx.x];
+  }
+
+  __device__ __forceinline__ void exit() const {
+    __syncthreads();
+    if (threadIdx.x == 0) {
+      exit_unsafe(blockIdx.x);
+    }
+  }
+
+  __device__ __forceinline__ void exit_unsafe(uint32_t which) const {
+    auto& signal = m_local_signal[which];
+    signal = (signal + 1) % kNumStages;
+  }
+
+  SignalType* m_local_signal;
+};
+
+// ============================================================
+// Section H: AllReducePushData (from SGLang
+//   custom_all_reduce_push.cuh 23-31)
+// ============================================================
+// COPY AS IS.
+
+struct AllReducePushData {
+  void* __restrict__ buffer[kMaxNumGPU];
+  const void* input;
+  void* output;
+  uint32_t rank;
+  uint32_t num_items;
+  uint32_t buffer_bytes;
+  uint32_t epoch_bytes;
+};
+
+// ============================================================
+// Section I: fp_trait sentinel types (from SGLang push.cuh 35-64)
+// ============================================================
+// COPY AS IS.
+
+template <typename T>
+struct fp_trait {};
+
+template <>
+struct fp_trait<bf16_t> {
+  using type = uint16_t;
+  [[maybe_unused]] static constexpr uint16_t pos_zero = 0x0000u;
+  [[maybe_unused]] static constexpr uint16_t neg_zero = 0x8000u;
+};
+
+template <>
+struct fp_trait<fp16_t> {
+  using type = uint16_t;
+  [[maybe_unused]] static constexpr uint16_t pos_zero = 0x0000u;
+  [[maybe_unused]] static constexpr uint16_t neg_zero = 0x8000u;
+};
+
+template <>
+struct fp_trait<float> {
+  using type = uint32_t;
+  [[maybe_unused]] static constexpr uint32_t pos_zero = 0x00000000u;
+  [[maybe_unused]] static constexpr uint32_t neg_zero = 0x80000000u;
+};
+
+// ============================================================
+// Section J: Sentinel helpers (from SGLang push.cuh 66-84)
+// ============================================================
+// COPY AS IS.
+
+template <typename DType>
+__device__ __forceinline__ void clear_pos_zero(DType& val) {
+  using Trait = fp_trait<DType>;
+  const auto ptr = reinterpret_cast<typename Trait::type*>(&val);
+  if (*ptr == Trait::pos_zero) *ptr = Trait::neg_zero;
+}
+
+template <typename DType>
+__device__ __forceinline__ bool is_pos_zero(const DType& val) {
+  using Trait = fp_trait<DType>;
+  const auto ptr = reinterpret_cast<const typename Trait::type*>(&val);
+  return *ptr == Trait::pos_zero;
+}
+
+template <typename DType>
+__device__ __forceinline__ DType get_pos_zero() {
+  using Trait = fp_trait<DType>;
+  const auto value = Trait::pos_zero;
+  return *reinterpret_cast<const DType*>(&value);
+}
+
+// ============================================================
+// Section K: Volatile 16-byte load/store (from SGLang push.cuh 87-105)
+// ============================================================
+// CHANGED: pointer::offset<T> -> ptr_typed_offset<T>
+
+template <typename T>
+__device__ __forceinline__ void ld_global_volatile_16B(T& x, const void* addr,
+                                                       int64_t offset) {
+  static_assert(alignof(T) == 16 && sizeof(T) == 16);
+  addr = ptr_typed_offset<T>(addr, offset);
+  uint4 val;
+  asm volatile("ld.volatile.global.v4.b32 {%0, %1, %2, %3}, [%4];"
+               : "=r"(val.x), "=r"(val.y), "=r"(val.z), "=r"(val.w)
+               : "l"(addr));
+  x = *reinterpret_cast<const T*>(&val);
+}
+
+template <typename T>
+__device__ __forceinline__ void st_global_volatile_16B(const T& x, void* addr,
+                                                       int64_t offset) {
+  static_assert(alignof(T) == 16 && sizeof(T) == 16);
+  const uint4 val = *reinterpret_cast<const uint4*>(&x);
+  addr = ptr_typed_offset<T>(addr, offset);
+  asm volatile("st.volatile.global.v4.b32 [%4], {%0, %1, %2, %3};" ::"r"(val.x),
+               "r"(val.y), "r"(val.z), "r"(val.w), "l"(addr));
+}
+
+// ============================================================
+// Section L: reduce_impl (from SGLang custom_all_reduce.cuh 331-354)
+// ============================================================
+// COPY AS IS.
+
+template <typename DType2, size_t N, uint32_t M>
+__device__ __forceinline__ auto reduce_impl(
+    AlignedVector<DType2, N> (&storage)[M]) -> AlignedVector<DType2, N> {
+  fp32x2_t acc[N] = {};
+#pragma unroll
+  for (uint32_t i = 0; i < M; ++i) {
+#pragma unroll
+    for (uint32_t j = 0; j < N; ++j) {
+      const auto [x, y] = cast<fp32x2_t>(storage[i][j]);
+      auto& [x_acc, y_acc] = acc[j];
+      x_acc += x;
+      y_acc += y;
+    }
+  }
+  AlignedVector<DType2, N> result;
+#pragma unroll
+  for (uint32_t j = 0; j < N; ++j) {
+    result[j] = cast<DType2>(acc[j]);
+  }
+  return result;
+}
+
+// ============================================================
+// Section M: push_impl (from SGLang push.cuh 107-128)
+// ============================================================
+// CHANGED: pointer::offset -> ptr_byte_offset
+
+template <typename DType, uint32_t kNumGPU>
+__device__ __forceinline__ void push_impl(DType* (&push_buf)[kNumGPU],
+                                          const void* data,
+                                          uint32_t num_items) {
+  constexpr uint32_t kVecSize = 16 / (sizeof(DType) * 2);
+  using Storage = AlignedVector<packed_t<DType>, kVecSize>;
+
+  for (auto i = blockIdx.x;; i += gridDim.x) {
+    const auto offset = i * blockDim.x + threadIdx.x;
+    if (offset * kVecSize * 2 >= num_items) break;
+    Storage vec;
+    vec.load(data, offset);
+#pragma unroll
+    for (uint32_t j = 0; j < kVecSize; ++j) {
+      clear_pos_zero(vec[j].x);
+      clear_pos_zero(vec[j].y);
+    }
+#pragma unroll
+    for (uint32_t i = 0; i < kNumGPU; ++i) {
+      st_global_volatile_16B(vec, push_buf[i], offset);
+    }
+  }
+}
+
+// ============================================================
+// Section N: poll_impl (from SGLang push.cuh 130-165)
+// ============================================================
+// COPY AS IS.
+
+template <typename DType, uint32_t kNumGPU>
+__device__ __forceinline__ void poll_impl(DType* (&poll_buf)[kNumGPU],
+                                          void* data, uint32_t num_items) {
+  constexpr uint32_t kVecSize = 16 / (sizeof(DType) * 2);
+  using Storage = AlignedVector<packed_t<DType>, kVecSize>;
+
+  for (auto i = blockIdx.x;; i += gridDim.x) {
+    const auto offset = i * blockDim.x + threadIdx.x;
+    if (offset * kVecSize * 2 >= num_items) break;
+    Storage storage[kNumGPU];
+
+    while (true) {
+      bool has_pos_zero = false;
+#pragma unroll
+      for (uint32_t i = 0; i < kNumGPU; ++i) {
+        ld_global_volatile_16B(storage[i], poll_buf[i], offset);
+#pragma unroll
+        for (auto j = 0; j < kVecSize; ++j) {
+          has_pos_zero |= is_pos_zero(storage[i][j].x);
+          has_pos_zero |= is_pos_zero(storage[i][j].y);
+        }
+      }
+      if (!has_pos_zero) break;
+    }
+
+    const Storage result = reduce_impl(storage);
+    result.store(data, offset);
+
+    Storage pos_zeros;
+    pos_zeros.fill({get_pos_zero<DType>(), get_pos_zero<DType>()});
+#pragma unroll
+    for (uint32_t i = 0; i < kNumGPU; ++i) {
+      pos_zeros.store(poll_buf[i], offset);
+    }
+  }
+}
+
+// ============================================================
+// Section O: THE KERNEL (from SGLang push.cuh 167-196)
+// ============================================================
+// COPY AS IS. CHANGED: CUSTOM_AR_KERNEL macro expanded.
+// The kernel uses __grid_constant__ for params passed by value.
+// cudaLaunchKernelEx copies params to constant memory before launch.
+
+template <typename DType, uint32_t kNumGPU, bool kUsePDL>
+__global__ __launch_bounds__(1024, 1) void all_reduce_one_shot_push_kernel(
+    const AllReducePushData __grid_constant__ params,
+    const PushController __grid_constant__ ctrl) {
+  const auto [buffer, input, output, rank, num_items, buffer_bytes,
+              epoch_bytes] = params;
+
+  PDLWaitPrimary<kUsePDL>();
+
+  // Phase 1: Push data from input to all ranks' push buffers
+  const auto epoch_offset = ctrl.epoch() * epoch_bytes;
+  DType* push_buf[kNumGPU];
+#pragma unroll
+  for (uint32_t i = 0; i < kNumGPU; ++i) {
+    push_buf[i] = static_cast<DType*>(
+        ptr_byte_offset(buffer[i], rank * buffer_bytes, epoch_offset));
+  }
+  push_impl(push_buf, input, num_items);
+
+  PDLTriggerSecondary<kUsePDL>();
+
+  // Phase 2: Poll local buffer, reduce, write output, reset
+  DType* poll_buf[kNumGPU];
+#pragma unroll
+  for (uint32_t i = 0; i < kNumGPU; ++i) {
+    poll_buf[i] = static_cast<DType*>(
+        ptr_byte_offset(buffer[rank], i * buffer_bytes, epoch_offset));
+  }
+  poll_impl(poll_buf, output, num_items);
+  ctrl.exit();
+}
+
+}  // namespace push_ar
+}  // namespace vllm

csrc/torch_bindings.cpp

@@ -104,4 +104,25 @@ TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _cuda_utils), cuda_utils) {
 }
 #endif
 
+// Push-based allreduce (ported from SGLang)
+fptr_t init_push_ar(int64_t rank, int64_t world_size, int64_t push_buffer_bytes,
+                    int64_t max_num_cta);
+torch::Tensor get_push_ar_ipc_handle(fptr_t _mgr);
+void post_init_push_ar(fptr_t _mgr, torch::Tensor all_handles);
+void push_ar_all_reduce(fptr_t _mgr, torch::Tensor& inp, torch::Tensor& out);
+void dispose_push_ar(fptr_t _mgr);
+
+TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _push_ar), push_ar) {
+  push_ar.def("init_push_ar", &init_push_ar);
+
+  push_ar.def("get_push_ar_ipc_handle", &get_push_ar_ipc_handle);
+
+  push_ar.def("post_init_push_ar", &post_init_push_ar);
+
+  push_ar.def("push_ar_all_reduce(int mgr, Tensor inp, Tensor! out) -> ()");
+  push_ar.impl("push_ar_all_reduce", torch::kCUDA, &push_ar_all_reduce);
+
+  push_ar.def("dispose_push_ar", &dispose_push_ar);
+}
+
 REGISTER_EXTENSION(TORCH_EXTENSION_NAME)

docs/design/attention_backends.md

@@ -170,8 +170,8 @@ Priority is **1 = highest** (tried first).
 | Backend | Version | Dtypes | KV Dtypes | Block Sizes | Head Sizes | Sink | Non-Causal | MM Prefix | DCP | Attention Types | Compute Cap. |
 | ------- | ------- | ------ | --------- | ----------- | ---------- | ---- | ---------- | --------- | --- | --------------- | ------------ |
 | `CPU_ATTN` | | fp16, bf16, fp32 | `auto`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | %16 | 32, 64, 80, 96, 112, 128, 160, 192, 224, 256, 512 | ❌ | ❌ | ❌ | ❌ | All | N/A |
-| `FLASHINFER` | Native† | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | 16, 32, 64 | 64, 128, 256, 512 | ❌ | ❌ | ❌ | ✅ | Decoder | 7.x-9.x |
-| `FLASHINFER` | TRTLLM† | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2`, `nvfp4` | 16, 32, 64 | 64, 128, 256, 512 | ✅ | ❌ | ❌ | ✅ | Decoder | 10.x |
+| `FLASHINFER` | Native† | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | 16, 32, 64, 128, 256, 512, 1024 | 64, 128, 256, 512 | ❌ | ❌ | ❌ | ✅ | Decoder | 7.x-9.x |
+| `FLASHINFER` | TRTLLM† | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2`, `nvfp4` | 16, 32, 64, 128, 256, 512, 1024 | 64, 128, 256, 512 | ✅ | ❌ | ❌ | ✅ | Decoder | 10.x |
 | `FLASH_ATTN` | FA2* | fp16, bf16 | `auto`, `float16`, `bfloat16` | %16 | Any | ❌ | ✅ | ❌ | ✅ | All | ≥8.0 |
 | `FLASH_ATTN` | FA3* | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | %16 | Any | ✅ | ✅ | ❌ | ✅ | All | 9.x |
 | `FLASH_ATTN` | FA4* | fp16, bf16 | `auto`, `float16`, `bfloat16` | %16 | Any | ✅ | ✅ | ❌ | ✅ | All | ≥10.0 |
@@ -188,6 +188,18 @@ Priority is **1 = highest** (tried first).
 >
 > **\*** Specify the FlashAttention version via `--attention-config.flash_attn_version=2`, `3`, or `4`. Default is FA4 on SM100+ (Blackwell), FA3 on SM90 (Hopper), FA2 otherwise.
 
+## MiniMax M3 Sparse Attention Backends
+
+Block-sparse GQA backend used by MiniMax M3 sparse ("lightning indexer")
+layers. It is wired in directly by the model and is not part of the
+automatic priority lists above. A lightning indexer scores KV blocks, the
+top-k blocks (plus fixed init/local blocks) are selected, and attention
+attends only to those blocks; index keys live in a separate side cache.
+
+| Backend | Dtypes | KV Dtypes | Block Sizes | Head Sizes | Sink | Non-Causal | MM Prefix | DCP | Attention Types | Compute Cap. |
+| ------- | ------ | --------- | ----------- | ---------- | ---- | ---------- | --------- | --- | --------------- | ------------ |
+| `MINIMAX_M3_SPARSE` | bf16, fp16 | `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | 128 | 128 | ❌ | ❌ | ❌ | ❌ | Decoder | Any |
+
 ## MLA (Multi-head Latent Attention) Backends
 
 MLA uses separate backends for prefill and decode phases.

docs/features/tool_calling.md

@@ -109,18 +109,18 @@ vLLM supports the `tool_choice='none'` option in the chat completion API. When t
 
 ## Constrained Decoding Behavior
 
-Whether vLLM enforces the tool parameter schema during generation depends on the `tool_choice` mode:
+Whether vLLM enforces the tool parameter schema during generation depends on the `tool_choice` mode and the per-tool `strict` field:
 
 | `tool_choice` value | Schema-constrained decoding | Behavior |
 | --- | --- | --- |
 | Named function | Yes (via structured outputs backend) | Arguments are guaranteed to be valid JSON conforming to the function's parameter schema. |
 | `"required"` | Yes (via structured outputs backend) | Same as named function. The model must produce at least one tool call. |
-| `"auto"` | Depends on the parser | Model-specific structural-tag parsers can constrain tool-call arguments with structured outputs. Other parsers generate freely and extract tool calls from raw text. |
+| `"auto"` | Only when `strict: true` is set on at least one tool | Structural-tag parsers constrain tool-call arguments when a tool opts in with `strict: true`. Without it, the model generates freely and tool calls are extracted from raw text. |
 | `"none"` | N/A | No tool calls are produced. |
 
 ### Strict Mode
 
-Strict tool calling makes function-call arguments adhere to the function schema instead of relying only on best-effort parsing. vLLM implements strict tool calling for structural-tag based tool parsers by using the structured outputs backend under the hood.
+For `tool_choice="required"` or named function calling, structural-tag constraints are always applied regardless of the `strict` field. For `tool_choice="auto"`, setting `strict: true` on at least one tool opts in to structural-tag constraints; without it, the model generates freely and tool calls are extracted from raw text. The `strict` field is supported across all three API surfaces: Chat Completion, Responses, and Anthropic Messages.
 
 For best compatibility with strict schema enforcement, define tool parameter schemas in the OpenAI strict-schema style:
 
@@ -128,16 +128,12 @@ For best compatibility with strict schema enforcement, define tool parameter sch
 * Mark all fields in `properties` as required.
 * Represent optional fields by allowing `null`, for example `{"type": ["string", "null"]}`.
 
-vLLM controls structural-tag strict tool calling with the `VLLM_ENFORCE_STRICT_TOOL_CALLING` environment variable. It defaults to `true`.
+vLLM also provides a global toggle via the `VLLM_ENFORCE_STRICT_TOOL_CALLING` environment variable (defaults to `true`). When set to `false`, vLLM does not attach structural tags for tool calling regardless of the per-tool `strict` field. This environment variable only affects structural-tag based tool calling; it does not change schema-derived structured outputs used by named function calling or `tool_choice="required"`.
 
 ```bash
 VLLM_ENFORCE_STRICT_TOOL_CALLING=false vllm serve ...
 ```
 
-When this variable is `true`, structural-tag based tool parsers attach a structural tag to the request, so the structured outputs backend can constrain the model-specific tool-call format and function-call arguments. When it is `false`, vLLM does not attach structural tags for tool calling. In that case, `tool_choice="auto"` falls back to best-effort parser extraction from the raw model output, and no structural-tag constraint is applied.
-
-This environment variable only affects structural-tag based tool calling. It does not change schema-derived structured outputs used by named function calling or `tool_choice="required"`.
-
 ## Automatic Function Calling
 
 To enable this feature, you should set the following flags:
@@ -156,7 +152,7 @@ from HuggingFace; and you can find an example of this in a `tokenizer_config.jso
 If your favorite tool-calling model is not supported, please feel free to contribute a parser & tool use chat template!
 
 !!! note
-    With `tool_choice="auto"`, schema-level constraint depends on the selected parser and `VLLM_ENFORCE_STRICT_TOOL_CALLING`. Structural-tag parsers can enforce tool-call constraints when it is `true`; when it is `false`, or when the selected parser has no structural-tag support, vLLM extracts tool calls from raw text, so arguments may occasionally be malformed or violate the function's parameter schema.
+    With `tool_choice="auto"`, schema-level constraint requires both `VLLM_ENFORCE_STRICT_TOOL_CALLING=true` (the default) and at least one tool with `strict: true`. When these conditions are met and the selected parser supports structural tags, vLLM constrains tool-call arguments. Otherwise, vLLM extracts tool calls from raw text, so arguments may occasionally be malformed or violate the function's parameter schema.
 
 ### Hermes Models (`hermes`)
 

docs/getting_started/installation/gpu.rocm.inc.md

@@ -27,6 +27,19 @@ If you need a different ROCm version or want to use an existing PyTorch installa
 --8<-- [end:set-up-using-python]
 --8<-- [start:pre-built-wheels]
 
+!!! warning "Python 3.12 required for ROCm wheels"
+
+    ROCm pre-built wheels are only available for **Python 3.12**. If you are using a different Python version (e.g. 3.11 or 3.13), the installer **will silently fall back** to the CUDA wheel from PyPI, which will fail on AMD GPUs with errors like `libcudart.so: cannot open shared object file`.
+
+    To check your Python version: `python3 --version`
+
+    If you need Python 3.12, you can create an isolated environment with `uv`:
+
+    ```bash
+    uv venv --python 3.12 --seed --managed-python
+    source .venv/bin/activate
+    ```
+
 To install the latest version of vLLM for Python 3.12, ROCm 7.0 and `glibc >= 2.35`.
 
 ```bash

docs/models/pooling_models/README.md

@@ -184,7 +184,7 @@ Our online Server provides endpoints that correspond to the offline APIs:
 - Corresponding to `LLM.classify`:
     - [Classification API](classify.md#online-serving)(`/classify`)
 - Corresponding to `LLM.score`:
-    - [Score API](scoring.md#score-api)(`/score`)
+    - [Score API](scoring.md#score-api) (`/score`, `/v1/score`)
     - [Cohere Rerank API](scoring.md#rerank-api) (`/rerank`, `/v1/rerank`, `/v2/rerank`)
 - Pooling API (`/pooling`) is similar to `LLM.encode`, being applicable to all types of pooling models.
 

docs/models/pooling_models/scoring.md

@@ -19,7 +19,7 @@ The score models is designed to compute similarity scores between two input prom
 - Offline APIs:
     - `LLM.score`
 - Online APIs:
-    - [Score API](scoring.md#score-api) (`/score`)
+    - [Score API](scoring.md#score-api) (`/score`, `/v1/score`)
     - [Cohere Rerank API](scoring.md#rerank-api) (`/rerank`, `/v1/rerank`, `/v2/rerank`)
 
 !!! note
@@ -157,7 +157,7 @@ A code example can be found here: [examples/basic/offline_inference/score.py](..
 
 ### Score API
 
-Our Score API (`/score`) is similar to `LLM.score`, compute similarity scores between two input prompts.
+Our Score API (`/score`, `/v1/score`) is similar to `LLM.score`, compute similarity scores between two input prompts.
 
 #### Parameters
 

docs/models/supported_models.md

@@ -488,7 +488,6 @@ th {
 | `TeleChat2ForCausalLM` | TeleChat2 | `Tele-AI/TeleChat2-3B`, `Tele-AI/TeleChat2-7B`, `Tele-AI/TeleChat2-35B`, etc. | ✅︎ | ✅︎ |
 | `TeleChat3ForCausalLM` | TeleChat3 | `Tele-AI/TeleChat3-36B-Thinking`, `Tele-AI/TeleChat3-Coder-36B-Thinking`, etc. | ✅︎ | ✅︎ |
 | `TeleFLMForCausalLM` | TeleFLM | `CofeAI/FLM-2-52B-Instruct-2407`, `CofeAI/Tele-FLM`, etc. | ✅︎ | ✅︎ |
-| `XverseForCausalLM` | XVERSE | `xverse/XVERSE-7B-Chat`, `xverse/XVERSE-13B-Chat`, `xverse/XVERSE-65B-Chat`, etc. | ✅︎ | ✅︎ |
 | `MiniMaxM1ForCausalLM` | MiniMax-Text | `MiniMaxAI/MiniMax-M1-40k`, `MiniMaxAI/MiniMax-M1-80k`, etc. | | |
 | `MiniMaxText01ForCausalLM` | MiniMax-Text | `MiniMaxAI/MiniMax-Text-01`, etc. | | |
 | `Zamba2ForCausalLM` | Zamba2 | `Zyphra/Zamba2-7B-instruct`, `Zyphra/Zamba2-2.7B-instruct`, `Zyphra/Zamba2-1.2B-instruct`, etc. | | |

docs/serving/online_serving/README.md

@@ -9,12 +9,13 @@ We currently support the following OpenAI APIs:
 - [Completions API](./openai_compatible_server.md#completions-api) (`/v1/completions`)
     - Only applicable to [text generation models](../../models/generative_models.md).
     - *Note: `suffix` parameter is not supported.*
-- [Responses API](./openai_compatible_server.md#responses-api) (`/v1/responses`)
-    - Only applicable to [text generation models](../../models/generative_models.md).
 - [Chat Completions API](./openai_compatible_server.md#chat-api) (`/v1/chat/completions`)
     - Only applicable to [text generation models](../../models/generative_models.md) with a [chat template](./openai_compatible_server.md#chat-template).
     - *Note: `user` parameter is ignored.*
     - *Note:* Setting the `parallel_tool_calls` parameter to `false` ensures vLLM only returns zero or one tool call per request. Setting it to `true` (the default) allows returning more than one tool call per request. There is no guarantee more than one tool call will be returned if this is set to `true`, as that behavior is model dependent and not all models are designed to support parallel tool calls.
+- [Chat Completions batch API](./openai_compatible_server.md#chat-api) (`/v1/chat/completions/batch`)
+- [Responses API](./openai_compatible_server.md#responses-api) (`/v1/responses`, `/v1/responses/{response_id}`, `/v1/responses/{response_id}/cancel`)
+    - Only applicable to [text generation models](../../models/generative_models.md).
 - [Embeddings API](../../models/pooling_models/embed.md#openai-compatible-embeddings-api) (`/v1/embeddings`)
     - Only applicable to [embedding models](../../models/pooling_models/embed.md).
 - [Transcriptions API](./speech_to_text.md#transcriptions-api) (`/v1/audio/transcriptions`)
@@ -24,7 +25,7 @@ We currently support the following OpenAI APIs:
 
 ## Anthropic APIs
 
-- Anthropic messages API (`/v1/messages`)
+- Anthropic messages API (`/v1/messages`, `/v1/messages/count_tokens`)
 
 ## Cohere APIs
 
@@ -35,10 +36,6 @@ We currently support the following OpenAI APIs:
     - Implements [Jina AI's v1 rerank API](https://jina.ai/reranker/)
     - compatible with [Cohere's v1 & v2 rerank APIs](https://docs.cohere.com/v2/reference/rerank)
 
-## SageMaker APIs
-
-- `/invocations` - SageMaker-compatible endpoint (routes to the same inference functions as `/v1` endpoints)
-
 ## Pooling APIs
 
 For further details on pooling models, please refer to [this page](../../models/pooling_models/README.md).
@@ -51,7 +48,7 @@ For further details on pooling models, please refer to [this page](../../models/
     - [OpenAI-compatible Embeddings API](../../models/pooling_models/embed.md#openai-compatible-embeddings-api) (`/v1/embeddings`)
     - Only applicable to [embedding models](../../models/pooling_models/embed.md).
 - [Scoring Usages](../../models/pooling_models/scoring.md)
-    - [Score API](../../models/pooling_models/scoring.md#score-api) (`/score`)
+    - [Score API](../../models/pooling_models/scoring.md#score-api) (`/score`, `/v1/score`)
     - [Cohere Rerank API](../../models/pooling_models/scoring.md#rerank-api) (`/rerank`, `/v1/rerank`, `/v2/rerank`)
     - Applicable to [score models](../../models/pooling_models/scoring.md) (cross-encoder, bi-encoder, late-interaction).
 - [Pooling API](../../models/pooling_models/README.md#pooling-api) (`/pooling`)
@@ -68,17 +65,6 @@ For further details on speech to text, please refer to [this page](speech_to_tex
 - [Realtime API](./speech_to_text.md#realtime-api) (`/v1/realtime`)
     - Only applicable to [Automatic Speech Recognition (ASR) models](../../models/supported_models.md#realtime-transcription).
 
-## Disaggregated APIs
-
-### Renderer APIs
-
-For further details on renderer APIs, please refer to [this page](renderer.md).
-
-- [Completions Render API](renderer.md) (`/v1/completions/render`)
-    - Render completion requests
-- [Chat Completions Render API](renderer.md) (`/v1/chat/completions/render`)
-    - Render chat completions
-
 ## Custom APIs
 
 - [Classification API](../../models/pooling_models/classify.md#classification-api) (`/classify`)
@@ -91,14 +77,79 @@ For further details on renderer APIs, please refer to [this page](renderer.md).
     - Applicable to [CausalLM models](../../models/generative_models.md) (task `"generate"`).
     - Computes next-token probabilities for specified `label_token_ids`.
 
-## Utility APIs
+## Instrumentator APIs
+
+### Basic APIs
+
+- `/version` - Version information
+- `/load` - Server load metrics
+- `/v1/models` - List available models
+- `/health` - Health check
+
+### Metrics APIs
+
+For further details on metrics, please refer to [this page](../../design/metrics.md).
+
+- `/metrics` - Prometheus-compatible metrics HTTP endpoint
+
+### Offline API Documentation
+
+The FastAPI `/docs` endpoint requires an internet connection by default. To enable offline access in air-gapped environments, use the `--enable-offline-docs` flag:
+
+```bash
+vllm serve NousResearch/Meta-Llama-3-8B-Instruct --enable-offline-docs
+```
+
+### LoRA dynamic loading
+
+LoRA dynamic loading & unloading is enabled in the API server. This should ONLY be used for local development!
+
+- `/v1/load_lora_adapter` - LoRA dynamic loading
+- `/v1/unload_lora_adapter` - LoRA dynamic unloading
+
+### Profiling APIs
+
+For further details on profiling vLLM, please refer to [this page](../../contributing/profiling.md).
+
+- `/start_profile` - Start PyTorch profiler
+- `/stop_profile` - Stop PyTorch profiler
+
+### SageMaker APIs
+
+- `/ping` - SageMaker health check
+- `/invocations` - SageMaker-compatible endpoint (routes to the same inference functions as `/v1` endpoints)
+
+## Disaggregated Everything
+
+### Tokens IN <> Tokens OUT
+
+- `/inference/v1/generate` - Generate completions
+- `/abort_requests` - Abort in-flight requests (only when `--tokens-only` is also set)
+
+### Renderer APIs
+
+For further details on renderer APIs, please refer to [this page](renderer.md).
+
+- [Completions Render API](renderer.md) (`/v1/completions/render`)
+    - Render completion requests
+- [Chat Completions Render API](renderer.md) (`/v1/chat/completions/render`)
+    - Render chat completions
+
+### Derenderer APIs
+
+- `/v1/completions/derender` - Derenderer completion requests
+- `/v1/chat/completions/derender` - Derenderer chat completion requests
+
+## Tokenize APIs
 
 - `/tokenize` - Tokenize text
 - `/detokenize` - Detokenize tokens
-- `/health` - Health check
-- `/ping` - SageMaker health check
-- `/version` - Version information
-- `/load` - Server load metrics
+- `/tokenizer_info` - Get comprehensive tokenizer information including chat templates and configuration
+
+## Elastic Expert Parallelism (EEP)
+
+- `/scale_elastic_ep` - Trigger scaling operations
+- `/is_scaling_elastic_ep` - Check if scaling is in progress
 
 ## Server in development mode
 
@@ -120,7 +171,9 @@ For further details on Weight Transfer, please refer to [this page](../../traini
 - `/resume` - Resume generation
 - `/is_paused` - Check if generation is paused
 - `/init_weight_transfer_engine` - Initialize weight transfer engine for RLHF
+- `/start_weight_update` - Prepares the inference engine for a weight update.
 - `/update_weights` - Update model weights (can alter model behavior)
+- `/finish_weight_update` - Finalizes the weight update
 - `/get_world_size` - Get distributed world size
 
 ### Collective RPC
@@ -189,14 +242,6 @@ the detected format, which can be one of:
 If the result is not what you expect, you can set the `--chat-template-content-format` CLI argument
 to override which format to use.
 
-## Offline API Documentation
-
-The FastAPI `/docs` endpoint requires an internet connection by default. To enable offline access in air-gapped environments, use the `--enable-offline-docs` flag:
-
-```bash
-vllm serve NousResearch/Meta-Llama-3-8B-Instruct --enable-offline-docs
-```
-
 ## Ray Serve LLM
 
 Ray Serve LLM enables scalable, production-grade serving of the vLLM engine. It integrates tightly with vLLM and extends it with features such as auto-scaling, load balancing, and back-pressure.

docs/serving/online_serving/openai_compatible_server.md

@@ -9,12 +9,13 @@ We currently support the following OpenAI APIs:
 - [Completions API](#completions-api) (`/v1/completions`)
     - Only applicable to [text generation models](../../models/generative_models.md).
     - *Note: `suffix` parameter is not supported.*
-- [Responses API](#responses-api) (`/v1/responses`)
-    - Only applicable to [text generation models](../../models/generative_models.md).
 - [Chat Completions API](#chat-api) (`/v1/chat/completions`)
     - Only applicable to [text generation models](../../models/generative_models.md) with a [chat template](../online_serving/README.md#chat-template).
     - *Note: `user` parameter is ignored.*
     - *Note:* Setting the `parallel_tool_calls` parameter to `false` ensures vLLM only returns zero or one tool call per request. Setting it to `true` (the default) allows returning more than one tool call per request. There is no guarantee more than one tool call will be returned if this is set to `true`, as that behavior is model dependent and not all models are designed to support parallel tool calls.
+- [Chat Completions batch API](#chat-api) (`/v1/chat/completions/batch`)
+- [Responses API](#responses-api) (`/v1/responses`, `/v1/responses/{response_id}`, `/v1/responses/{response_id}/cancel`)
+    - Only applicable to [text generation models](../../models/generative_models.md).
 - [Embeddings API](../../models/pooling_models/embed.md#openai-compatible-embeddings-api) (`/v1/embeddings`)
     - Only applicable to [embedding models](../../models/pooling_models/embed.md).
 - [Transcriptions API](./speech_to_text.md#transcriptions-api) (`/v1/audio/transcriptions`)

pyproject.toml

@@ -162,6 +162,8 @@ dout = "dout"
 Pn = "Pn"
 arange = "arange"
 thw = "thw"
+# temporal position ids (parallels hpos/wpos in vision RoPE)
+tpos = "tpos"
 subtile = "subtile"
 HSA = "HSA"
 setp = "setp"

requirements/common.txt

@@ -29,6 +29,7 @@ xgrammar >= 0.2.1, < 1.0.0; platform_machine == "x86_64" or platform_machine ==
 typing_extensions >= 4.10
 filelock >= 3.16.1 # need to contain https://github.com/tox-dev/filelock/pull/317
 partial-json-parser # used for parsing partial JSON outputs
+jsonschema >= 4.23.0 # required for MiniMax M3 tool schema validation
 pyzmq >= 25.0.0
 msgspec
 mistral_common[image] >= 1.11.3

requirements/test/cuda.txt

@@ -360,6 +360,7 @@ jsonpointer==3.0.0
     # via jsonschema
 jsonschema==4.23.0
     # via
+    #   -c requirements/common.txt
     #   hypothesis-jsonschema
     #   mistral-common
     #   ray

requirements/test/rocm.txt

@@ -439,6 +439,8 @@ jsonpointer==3.1.0
     # via jsonschema
 jsonschema==4.26.0
     # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
     #   hypothesis-jsonschema
     #   mcp
     #   mistral-common

requirements/test/xpu.txt

@@ -229,6 +229,7 @@ jsonlines==4.0.0
     # via lm-eval
 jsonschema==4.26.0
     # via
+    #   -c requirements/common.txt
     #   hypothesis-jsonschema
     #   mistral-common
     #   schemathesis

rust/Cargo.lock

@@ -5804,6 +5804,7 @@ dependencies = [
  "enum-as-inner",
  "expect-test",
  "futures",
+ "parking_lot",
  "rmp-serde",
  "serde",
  "serde_json",

rust/src/chat/src/lib.rs

@@ -235,6 +235,12 @@ impl ChatLlm {
         Ok(token_ids)
     }
 
+    /// Abort in-flight requests by their external (user-supplied) request ids.
+    pub async fn abort(&self, external_ids: &[String]) -> Result<()> {
+        self.text.abort(external_ids).await?;
+        Ok(())
+    }
+
     /// Shut down the underlying LLM client and its background tasks.
     pub async fn shutdown(self) -> Result<()> {
         self.text.shutdown().await?;
@@ -271,7 +277,7 @@ mod tests {
         )
         .unwrap_err();
 
-        expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, gemma4, glm45, glm47, granite4, hermes, hy_v3, internlm, kimi_k2, llama3_json, llama4_json, minimax_m2, mistral, phi4_mini_json, qwen3_coder, qwen3_xml)"].assert_eq(&error.to_report_string());
+        expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, gemma4, glm45, glm47, granite4, hermes, hy_v3, internlm, kimi_k2, llama3_json, llama4_json, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml)"].assert_eq(&error.to_report_string());
     }
 
     #[test]
@@ -282,6 +288,6 @@ mod tests {
         )
         .unwrap_err();
 
-        expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, kimi, kimi_k2, minimax_m2, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
+        expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, kimi, kimi_k2, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
     }
 }

rust/src/chat/src/output/default/mod.rs

@@ -37,6 +37,7 @@ trait_set! {
 pub struct DefaultChatOutputProcessor {
     reasoning_parser: Option<Box<dyn ReasoningParser>>,
     tool_parser: Option<Box<dyn ToolParser>>,
+    parallel_tool_calls: bool,
 }
 
 impl DefaultChatOutputProcessor {
@@ -74,6 +75,7 @@ impl DefaultChatOutputProcessor {
         Ok(Self {
             reasoning_parser,
             tool_parser,
+            parallel_tool_calls: request.parallel_tool_calls,
         })
     }
 
@@ -86,6 +88,7 @@ impl DefaultChatOutputProcessor {
         Self {
             reasoning_parser: None,
             tool_parser: None,
+            parallel_tool_calls: true,
         }
     }
 
@@ -159,7 +162,7 @@ impl ChatOutputProcessor for DefaultChatOutputProcessor {
     fn process(self: Box<Self>, decoded: DynDecodedTextEventStream) -> Result<DynChatEventStream> {
         let reasoning = reasoning_event_stream(decoded, self.reasoning_parser);
         let tool = tool_event_stream(reasoning, self.tool_parser);
-        let structured = structured_chat_event_stream(tool);
+        let structured = structured_chat_event_stream(tool, self.parallel_tool_calls);
 
         Ok(structured.boxed())
     }

rust/src/chat/src/output/default/tool.rs

@@ -473,7 +473,7 @@ mod tests {
             })));
         let parser = DeepSeekV4ToolParser::create(&deepseek_v4_test_tools()).unwrap();
         let assistant_events = tool_event_stream(stream::iter(events), Some(parser));
-        let chat_events = structured_chat_event_stream(assistant_events);
+        let chat_events = structured_chat_event_stream(assistant_events, true);
 
         ChatEventStream::new("req_deepseek_v4".to_string(), Box::pin(chat_events))
             .collect_message()
@@ -717,9 +717,10 @@ mod tests {
 
         let message = ChatEventStream::new(
             "req_fallback".to_string(),
-            Box::pin(structured_chat_event_stream(stream::iter(
-                events.into_iter().map(Ok),
-            ))),
+            Box::pin(structured_chat_event_stream(
+                stream::iter(events.into_iter().map(Ok)),
+                true,
+            )),
         )
         .collect_message()
         .await
@@ -968,9 +969,10 @@ mod tests {
         ));
         let collected = ChatEventStream::new(
             "req_final_only".to_string(),
-            Box::pin(structured_chat_event_stream(stream::iter(
-                events.into_iter().map(Ok),
-            ))),
+            Box::pin(structured_chat_event_stream(
+                stream::iter(events.into_iter().map(Ok)),
+                true,
+            )),
         )
         .collect_message()
         .await

rust/src/chat/src/output/harmony/mod.rs

@@ -35,6 +35,7 @@ use crate::request::ChatRequest;
 pub struct HarmonyChatOutputProcessor {
     encoding: &'static HarmonyEncoding,
     tool_calls_enabled: bool,
+    parallel_tool_calls: bool,
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
@@ -76,6 +77,7 @@ impl HarmonyChatOutputProcessor {
         Ok(Self {
             encoding: harmony_encoding()?,
             tool_calls_enabled: request.tool_parsing_enabled(),
+            parallel_tool_calls: request.parallel_tool_calls,
         })
     }
 }
@@ -110,7 +112,11 @@ impl ChatOutputProcessor for HarmonyChatOutputProcessor {
     fn process(self: Box<Self>, decoded: DynDecodedTextEventStream) -> Result<DynChatEventStream> {
         let assistant =
             harmony_assistant_event_stream(decoded, self.encoding, self.tool_calls_enabled);
-        Ok(crate::output::structured::structured_chat_event_stream(assistant).boxed())
+        Ok(crate::output::structured::structured_chat_event_stream(
+            assistant,
+            self.parallel_tool_calls,
+        )
+        .boxed())
     }
 }
 

rust/src/chat/src/output/structured.rs

@@ -53,16 +53,22 @@ struct StructuredEventState {
     open_tool_call: Option<OpenToolCall>,
     /// Next OpenAI-compatible tool-call ordinal.
     next_tool_call_index: usize,
+    /// Whether more than one tool call may be surfaced northbound.
+    parallel_tool_calls: bool,
+    /// Whether the current tool-call parse is being suppressed.
+    suppressing_tool_call: bool,
 }
 
 impl StructuredEventState {
     /// Create one fresh assembly state for a new streamed response.
-    fn new() -> Self {
+    fn new(parallel_tool_calls: bool) -> Self {
         Self {
             message: AssistantMessage::default(),
             open_text_block: None,
             open_tool_call: None,
             next_tool_call_index: 0,
+            parallel_tool_calls,
+            suppressing_tool_call: false,
         }
     }
 
@@ -98,6 +104,12 @@ impl StructuredEventState {
 
         let index = self.next_tool_call_index;
         self.next_tool_call_index += 1;
+        if !self.parallel_tool_calls && index >= 1 {
+            self.suppressing_tool_call = true;
+            return Ok(events);
+        }
+
+        self.suppressing_tool_call = false;
         self.open_tool_call = Some(OpenToolCall {
             index,
             id: id.clone(),
@@ -110,6 +122,10 @@ impl StructuredEventState {
 
     /// Append one incremental tool-call arguments delta.
     fn push_tool_call_arguments(&mut self, delta: String) -> Result<Vec<ChatEvent>> {
+        if self.suppressing_tool_call {
+            return Ok(Vec::new());
+        }
+
         let mut events = Vec::new();
         let Some(open_tool_call) = self.open_tool_call.as_mut() else {
             return Err(Error::ToolCallStreamInvariant {
@@ -207,6 +223,11 @@ impl StructuredEventState {
 
     /// Finalize the currently open tool call, if present.
     fn close_open_tool_call(&mut self, events: &mut Vec<ChatEvent>) {
+        if self.suppressing_tool_call {
+            self.suppressing_tool_call = false;
+            return;
+        }
+
         let Some(open_tool_call) = self.open_tool_call.take() else {
             return;
         };
@@ -229,11 +250,12 @@ impl StructuredEventState {
 #[try_stream]
 pub(crate) async fn structured_chat_event_stream(
     stream: impl AssistantEventStream,
+    parallel_tool_calls: bool,
     mut y: TryYielder<ChatEvent, Error>,
 ) -> Result<()> {
     pin_mut!(stream);
 
-    let mut state = StructuredEventState::new();
+    let mut state = StructuredEventState::new(parallel_tool_calls);
 
     while let Some(event) = stream.next().await.transpose()? {
         match event {
@@ -315,7 +337,7 @@ mod tests {
             }),
         ]);
 
-        let events = structured_chat_event_stream(events)
+        let events = structured_chat_event_stream(events, true)
             .collect::<Vec<_>>()
             .await
             .into_iter()
@@ -369,7 +391,7 @@ mod tests {
             }),
         ]);
 
-        let events = structured_chat_event_stream(events)
+        let events = structured_chat_event_stream(events, true)
             .collect::<Vec<_>>()
             .await
             .into_iter()
@@ -420,7 +442,7 @@ mod tests {
             }),
         ]);
 
-        let events = structured_chat_event_stream(events)
+        let events = structured_chat_event_stream(events, true)
             .collect::<Vec<_>>()
             .await
             .into_iter()
@@ -471,7 +493,7 @@ mod tests {
             }),
         ]);
 
-        let events = structured_chat_event_stream(events)
+        let events = structured_chat_event_stream(events, true)
             .collect::<Vec<_>>()
             .await
             .into_iter()
@@ -499,7 +521,7 @@ mod tests {
             delta: "{}".to_string(),
         })]);
 
-        let err = structured_chat_event_stream(events)
+        let err = structured_chat_event_stream(events, true)
             .collect::<Vec<_>>()
             .await
             .into_iter()
@@ -509,4 +531,56 @@ mod tests {
 
         assert!(matches!(err, Error::ToolCallStreamInvariant { .. }));
     }
+
+    #[tokio::test]
+    async fn structured_stream_suppresses_later_tool_calls_when_parallel_disabled() {
+        let events = stream::iter(vec![
+            Ok(AssistantEvent::ToolCallStart {
+                id: "call_1".to_string(),
+                name: "first".to_string(),
+            }),
+            Ok(AssistantEvent::ToolCallArgumentsDelta {
+                delta: r#"{"a":1}"#.to_string(),
+            }),
+            Ok(AssistantEvent::ToolCallStart {
+                id: "call_2".to_string(),
+                name: "second".to_string(),
+            }),
+            Ok(AssistantEvent::ToolCallArgumentsDelta {
+                delta: r#"{"b":2}"#.to_string(),
+            }),
+            Ok(AssistantEvent::Done {
+                usage: vllm_llm::TokenUsage {
+                    prompt_token_count: 1,
+                    output_token_count: 1,
+                    cached_token_count: 0,
+                },
+                finish_reason: FinishReason::stop_eos(),
+                kv_transfer_params: None,
+            }),
+        ]);
+
+        let events = structured_chat_event_stream(events, false)
+            .collect::<Vec<_>>()
+            .await
+            .into_iter()
+            .collect::<crate::Result<Vec<_>>>()
+            .unwrap();
+
+        assert!(matches!(
+            events[0],
+            ChatEvent::ToolCallStart { index: 0, .. }
+        ));
+        assert!(matches!(
+            events[1],
+            ChatEvent::ToolCallArgumentsDelta { index: 0, .. }
+        ));
+        assert!(matches!(events[2], ChatEvent::ToolCallEnd { index: 0, .. }));
+        let ChatEvent::Done { message, .. } = &events[3] else {
+            panic!("expected done");
+        };
+        let tool_calls = message.tool_calls().collect::<Vec<_>>();
+        assert_eq!(tool_calls.len(), 1);
+        assert_eq!(tool_calls[0].name, "first");
+    }
 }

rust/src/chat/src/parser/reasoning/mod.rs

@@ -5,9 +5,9 @@ use std::sync::LazyLock;
 pub use vllm_reasoning_parser::{
     CohereCmdReasoningParser, DeepSeekR1ReasoningParser, DeepSeekV3ReasoningParser,
     DeepSeekV4ReasoningParser, Gemma4ReasoningParser, Glm45ReasoningParser, KimiK2ReasoningParser,
-    KimiReasoningParser, MiniMaxM2ReasoningParser, NemotronV3ReasoningParser, Qwen3ReasoningParser,
-    ReasoningDelta, ReasoningError, ReasoningParser, SeedOssReasoningParser, Step3ReasoningParser,
-    Step3p5ReasoningParser,
+    KimiReasoningParser, MiniMaxM2ReasoningParser, MiniMaxM3ReasoningParser,
+    NemotronV3ReasoningParser, Qwen3ReasoningParser, ReasoningDelta, ReasoningError,
+    ReasoningParser, SeedOssReasoningParser, Step3ReasoningParser, Step3p5ReasoningParser,
 };
 use vllm_tokenizer::DynTokenizer;
 
@@ -24,6 +24,7 @@ pub mod names {
     pub const KIMI: &str = "kimi";
     pub const KIMI_K2: &str = "kimi_k2";
     pub const MINIMAX_M2: &str = "minimax_m2";
+    pub const MINIMAX_M3: &str = "minimax_m3";
     pub const NEMOTRON_V3: &str = "nemotron_v3";
     pub const QWEN3: &str = "qwen3";
     pub const SEED_OSS: &str = "seed_oss";
@@ -62,6 +63,7 @@ impl ReasoningParserFactory {
             .register_parser::<KimiReasoningParser>(names::KIMI)
             .register_parser::<KimiK2ReasoningParser>(names::KIMI_K2)
             .register_parser::<MiniMaxM2ReasoningParser>(names::MINIMAX_M2)
+            .register_parser::<MiniMaxM3ReasoningParser>(names::MINIMAX_M3)
             .register_parser::<NemotronV3ReasoningParser>(names::NEMOTRON_V3)
             .register_parser::<Qwen3ReasoningParser>(names::QWEN3)
             .register_parser::<SeedOssReasoningParser>(names::SEED_OSS)
@@ -90,6 +92,8 @@ impl ReasoningParserFactory {
             .register_pattern("step3", names::STEP3)
             .register_pattern("seed-oss", names::SEED_OSS)
             .register_pattern("seedoss", names::SEED_OSS)
+            .register_pattern("minimax-m3", names::MINIMAX_M3)
+            .register_pattern("mm-m3", names::MINIMAX_M3)
             .register_pattern("minimax", names::MINIMAX_M2)
             .register_pattern("mm-m2", names::MINIMAX_M2)
             .register_pattern("cohere", names::COHERE_CMD)

rust/src/chat/src/parser/reasoning/tests.rs

@@ -34,10 +34,12 @@ fn factory_contains_and_lists_registered_parsers() {
     assert!(factory.contains(names::DEEPSEEK_V4));
     assert!(factory.contains(names::SEED_OSS));
     assert!(factory.contains(names::STEP3P5));
+    assert!(factory.contains(names::MINIMAX_M3));
     assert!(factory.list().contains(&names::QWEN3.to_string()));
     assert!(factory.list().contains(&names::DEEPSEEK_V4.to_string()));
     assert!(factory.list().contains(&names::SEED_OSS.to_string()));
     assert!(factory.list().contains(&names::STEP3P5.to_string()));
+    assert!(factory.list().contains(&names::MINIMAX_M3.to_string()));
 }
 
 #[test]
@@ -88,6 +90,19 @@ fn factory_routes_seed_oss_models() {
     );
 }
 
+#[test]
+fn factory_resolves_minimax_m3_before_generic_minimax() {
+    let factory = ReasoningParserFactory::new();
+    assert_eq!(
+        factory.resolve_name_for_model("MiniMaxAI/Minimax-M3-preview"),
+        Some(names::MINIMAX_M3)
+    );
+    assert_eq!(
+        factory.resolve_name_for_model("mm-m3"),
+        Some(names::MINIMAX_M3)
+    );
+}
+
 #[test]
 fn factory_rejects_unknown_parser_names() {
     let tokenizer = Arc::new(FakeTokenizer);

rust/src/chat/src/parser/tool/mod.rs

@@ -6,8 +6,9 @@ pub use vllm_tool_parser::{
     DeepSeekV3ToolParser, DeepSeekV4ToolParser, DeepSeekV31ToolParser, DeepSeekV32ToolParser,
     Gemma4ToolParser, Glm45MoeToolParser, Glm47MoeToolParser, Granite4ToolParser, HermesToolParser,
     HyV3ToolParser, Internlm2ToolParser, KimiK2ToolParser, Llama3JsonToolParser,
-    MinimaxM2ToolParser, MistralToolParser, Phi4MiniJsonToolParser, Qwen3CoderToolParser,
-    Qwen3XmlToolParser, ToolCallDelta, ToolParser, ToolParserError, ToolParserOutput,
+    MinimaxM2ToolParser, MinimaxM3ToolParser, MistralToolParser, Phi4MiniJsonToolParser,
+    Qwen3CoderToolParser, Qwen3XmlToolParser, ToolCallDelta, ToolParser, ToolParserError,
+    ToolParserOutput,
 };
 
 use crate::parser::ParserFactory;
@@ -32,6 +33,7 @@ pub mod names {
     pub const LLAMA3_JSON: &str = "llama3_json";
     pub const LLAMA4_JSON: &str = "llama4_json";
     pub const MINIMAX_M2: &str = "minimax_m2";
+    pub const MINIMAX_M3: &str = "minimax_m3";
     pub const MISTRAL: &str = "mistral";
     pub const PHI4_MINI_JSON: &str = "phi4_mini_json";
     pub const QWEN3_CODER: &str = "qwen3_coder";
@@ -73,6 +75,7 @@ impl ToolParserFactory {
             .register_parser::<Llama3JsonToolParser>(names::LLAMA3_JSON)
             .register_parser::<Llama3JsonToolParser>(names::LLAMA4_JSON)
             .register_parser::<MinimaxM2ToolParser>(names::MINIMAX_M2)
+            .register_parser::<MinimaxM3ToolParser>(names::MINIMAX_M3)
             .register_parser::<MistralToolParser>(names::MISTRAL)
             .register_parser::<Phi4MiniJsonToolParser>(names::PHI4_MINI_JSON)
             .register_parser::<Qwen3XmlToolParser>(names::QWEN3_XML)
@@ -111,6 +114,8 @@ impl ToolParserFactory {
             .register_pattern("gemma-4", names::GEMMA4)
             .register_pattern("granite-4", names::GRANITE4)
             .register_pattern("kimi-k2", names::KIMI_K2)
+            .register_pattern("minimax-m3", names::MINIMAX_M3)
+            .register_pattern("mm-m3", names::MINIMAX_M3)
             .register_pattern("minimax", names::MINIMAX_M2)
             .register_pattern("mm-m2", names::MINIMAX_M2);
 

rust/src/chat/src/parser/tool/tests.rs

@@ -157,6 +157,14 @@ fn factory_new_resolves_default_patterns() {
         factory.resolve_name_for_model("tencent/Hy3-preview"),
         Some(names::HY_V3)
     );
+    assert_eq!(
+        factory.resolve_name_for_model("MiniMax/MiniMax-M3-Text"),
+        Some(names::MINIMAX_M3)
+    );
+    assert_eq!(
+        factory.resolve_name_for_model("org/mm-m3-base"),
+        Some(names::MINIMAX_M3)
+    );
     assert_eq!(
         factory.resolve_name_for_model("MiniMax/MiniMax-M2-01"),
         Some(names::MINIMAX_M2)

rust/src/chat/src/request.rs

@@ -406,6 +406,10 @@ pub struct ChatRequest {
     pub tools: Vec<ChatTool>,
     /// Tool-choice behavior for this request.
     pub tool_choice: ChatToolChoice,
+    /// Whether the model may return more than one tool call per response.
+    ///
+    /// When `false`, only the first parsed tool call is surfaced northbound.
+    pub parallel_tool_calls: bool,
     /// Text decode options for incremental detokenization.
     pub decode_options: TextDecodeOptions,
     /// Whether to emit intermediate northbound content deltas before the
@@ -442,6 +446,7 @@ impl ChatRequest {
             chat_options: ChatOptions::default(),
             tools: Vec::new(),
             tool_choice: ChatToolChoice::None,
+            parallel_tool_calls: true,
             decode_options: TextDecodeOptions::default(),
             intermediate: true,
             priority: 0,

rust/src/engine-core-client/src/client.rs

@@ -490,6 +490,10 @@ impl EngineCoreClient {
             return Ok(());
         }
 
+        // Finalize the consumer streams first, before the engine round-trip.
+        let all_request_ids: Vec<String> = abortable.values().flatten().cloned().collect();
+        self.inner.abort_requests_locally(&all_request_ids);
+
         for (engine_id, request_ids) in abortable {
             self.inner.do_abort_requests(&engine_id, &request_ids).await?;
         }

rust/src/engine-core-client/src/client/imp.rs

@@ -1,5 +1,6 @@
 use std::collections::{BTreeMap, BTreeSet};
 use std::sync::Arc;
+use std::time::{SystemTime, UNIX_EPOCH};
 
 use arc_swap::ArcSwapOption;
 use parking_lot::Mutex;
@@ -126,6 +127,20 @@ impl ClientInner {
         self.request_reg.lock().finish_many(request_ids)
     }
 
+    /// Finalize client-initiated aborts by pushing a terminal `Abort` output
+    /// down each request's stream and removing it from the registry. Returns
+    /// the request ids that were still active. See [`RequestRegistry::abort_many`].
+    pub fn abort_requests_locally<'a>(
+        &self,
+        request_ids: impl IntoIterator<Item = &'a String>,
+    ) -> Vec<String> {
+        let timestamp = SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .map(|d| d.as_secs_f64())
+            .unwrap_or(0.0);
+        self.request_reg.lock().abort_many(request_ids, timestamp)
+    }
+
     /// Apply one scheduler stats update for the given engine to the local
     /// routing state. Returns `false` if the engine is unknown to the
     /// client.

rust/src/engine-core-client/src/client/state.rs

@@ -9,7 +9,7 @@ use crate::client::stream::EngineCoreStreamOutput;
 use crate::error::{Error, Result};
 use crate::protocol::stats::SchedulerStats;
 use crate::protocol::utility::UtilityOutput;
-use crate::protocol::{EngineCoreEventType, EngineCoreOutput};
+use crate::protocol::{EngineCoreEventType, EngineCoreFinishReason, EngineCoreOutput};
 use crate::transport::ConnectedEngine;
 
 pub type OutputSender = mpsc::UnboundedSender<Result<EngineCoreStreamOutput>>;
@@ -289,6 +289,34 @@ impl RequestRegistry {
             .collect()
     }
 
+    /// Finalize client-initiated aborts: remove each request and push a
+    /// terminal output with `finish_reason = Abort` down its stream before the
+    /// sender drops. Returns the request ids that were still active.
+    pub fn abort_many<'a>(
+        &mut self,
+        request_ids: impl IntoIterator<Item = &'a String>,
+        timestamp: f64,
+    ) -> Vec<String> {
+        let mut aborted = Vec::new();
+        for request_id in request_ids {
+            let Some((sender, engine_id)) = self.remove(request_id) else {
+                continue;
+            };
+            let output = EngineCoreStreamOutput {
+                engine_index: engine_id.engine_index().unwrap_or(0),
+                timestamp,
+                output: EngineCoreOutput {
+                    request_id: request_id.clone(),
+                    finish_reason: Some(EngineCoreFinishReason::Abort),
+                    ..EngineCoreOutput::default()
+                },
+            };
+            let _ = sender.send(Ok(output));
+            aborted.push(request_id.clone());
+        }
+        aborted
+    }
+
     /// Remove one request from the local registry. Returns the tracked entry if
     /// it exists.
     #[must_use]

rust/src/engine-core-client/src/mock_engine.rs

@@ -15,6 +15,8 @@ use crate::protocol::{ModelDtype, decode_msgpack, encode_msgpack};
 pub const DEFAULT_MOCK_MAX_MODEL_LEN: u64 = 1024 * 1024;
 /// Default KV block count advertised by reusable mock engine helpers.
 pub const DEFAULT_MOCK_NUM_GPU_BLOCKS: u64 = 0;
+/// Default KV block size (tokens per block)
+pub const DEFAULT_MOCK_BLOCK_SIZE: u64 = 16;
 
 /// Startup behavior for one mock engine joining a frontend.
 #[derive(Debug, Clone)]
@@ -46,9 +48,12 @@ pub fn default_ready_response() -> EngineCoreReadyResponse {
     EngineCoreReadyResponse {
         max_model_len: DEFAULT_MOCK_MAX_MODEL_LEN,
         num_gpu_blocks: DEFAULT_MOCK_NUM_GPU_BLOCKS,
+        block_size: DEFAULT_MOCK_BLOCK_SIZE,
         dp_stats_address: None,
         dtype: ModelDtype::Float32,
         vllm_version: "test-vllm-version".to_string(),
+        kv_cache_size_tokens: None,
+        kv_cache_max_concurrency: None,
     }
 }
 

rust/src/engine-core-client/src/protocol/handshake.rs

@@ -28,7 +28,7 @@ pub struct ReadyMessage {
 /// profiling).
 ///
 /// Original Python definition:
-/// <https://github.com/vllm-project/vllm/blob/c8d98f81f6/vllm/v1/engine/__init__.py#L67-L77>
+/// <https://github.com/vllm-project/vllm/blob/c9340e6f35/vllm/v1/engine/__init__.py#L68-L80>
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct EngineCoreReadyResponse {
     /// Engine-reported maximum model context length (auto-fitted after
@@ -36,12 +36,18 @@ pub struct EngineCoreReadyResponse {
     pub max_model_len: u64,
     /// Number of GPU blocks available for KV cache on this engine.
     pub num_gpu_blocks: u64,
+    /// KV cache block size (tokens per block).
+    pub block_size: u64,
     /// DP coordinator stats publish address, if applicable.
     pub dp_stats_address: Option<String>,
     /// Effective model dtype after Python vLLM resolves `--dtype`.
     pub dtype: ModelDtype,
     /// Python vLLM version reported by the engine process.
     pub vllm_version: String,
+    /// Total KV cache capacity in tokens, if reported.
+    pub kv_cache_size_tokens: Option<u64>,
+    /// Maximum achievable request concurrency given the KV cache, if reported.
+    pub kv_cache_max_concurrency: Option<f64>,
 }
 
 /// Frontend-owned ZMQ addresses that are sent to the engine during startup

rust/src/engine-core-client/src/tests/client.rs

@@ -1939,7 +1939,7 @@ async fn multi_engine_abort_is_grouped_and_utility_fans_out_to_all_engines() {
 
     let (shutdown_tx_0, engine_task_0) = spawn_mock_engine_task(
         handshake_address.clone(),
-        b"engine-0".to_vec(),
+        EngineId::from_engine_index(0).into_frame().to_vec(),
         |dealer, push| {
             Box::pin(async move {
                 let utility = recv_engine_message(dealer).await;
@@ -1993,7 +1993,7 @@ async fn multi_engine_abort_is_grouped_and_utility_fans_out_to_all_engines() {
     tokio::time::sleep(Duration::from_millis(50)).await;
     let (shutdown_tx_1, engine_task_1) = spawn_mock_engine_task(
         handshake_address.clone(),
-        b"engine-1".to_vec(),
+        EngineId::from_engine_index(1).into_frame().to_vec(),
         |dealer, push| {
             Box::pin(async move {
                 let utility = recv_engine_message(dealer).await;
@@ -2445,6 +2445,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
     let inline_prompt_frames = lines.next().expect("missing inline prompt logprobs fixture line");
     let multipart_prompt_frames =
         lines.next().expect("missing multipart prompt logprobs fixture line");
+    let ready_response_hex = lines.next().expect("missing ready response fixture line");
 
     let request_bytes = hex::decode(request_hex).unwrap();
     let multimodal_request_bytes = hex::decode(multimodal_request_hex).unwrap();
@@ -2554,6 +2555,23 @@ fn python_msgpack_fixtures_match_rust_encoding() {
             .as_ref()
             .expect("multipart prompt logprobs decoded"),
     );
+
+    let map_keys = |bytes: &[u8]| -> BTreeSet<String> {
+        match decode_value(bytes) {
+            Value::Map(entries) => entries
+                .into_iter()
+                .filter_map(|(key, _)| key.as_str().map(str::to_owned))
+                .collect(),
+            other => panic!("ready response should encode as a map, got {other:?}"),
+        }
+    };
+    let python_ready_keys = map_keys(&hex::decode(ready_response_hex).unwrap());
+    let rust_ready_keys =
+        map_keys(&rmp_serde::to_vec_named(&crate::mock_engine::default_ready_response()).unwrap());
+    assert_eq!(
+        rust_ready_keys, python_ready_keys,
+        "EngineCoreReadyResponse drifted from the Python dataclass",
+    );
 }
 
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]

rust/src/engine-core-client/src/tests/python_compat.py

@@ -10,6 +10,7 @@
 # ]
 # ///
 
+from dataclasses import dataclass
 from enum import Enum, IntEnum
 
 import msgpack
@@ -337,6 +338,28 @@ multipart_prompt_logprobs = engine_outputs_wire(
     )
 )
 
+
+@dataclass
+class EngineCoreReadyResponse:
+    max_model_len: int
+    num_gpu_blocks: int
+    block_size: int
+    dp_stats_address: str | None
+    dtype: str
+    vllm_version: str
+    kv_cache_size_tokens: int | None = None
+    kv_cache_max_concurrency: float | None = None
+
+
+ready_response = EngineCoreReadyResponse(
+    max_model_len=32768,
+    num_gpu_blocks=1000,
+    block_size=16,
+    dp_stats_address=None,
+    dtype="float32",
+    vllm_version="0.0.0",
+)
+
 print(msgspec.msgpack.encode(request).hex())
 print(msgpack.packb(multimodal_request_wire, use_bin_type=True).hex())
 print(msgspec.msgpack.encode(outputs).hex())
@@ -354,3 +377,4 @@ print(
         for frame in encode_output_frames(multipart_prompt_logprobs, size_threshold=1)
     )
 )
+print(msgspec.msgpack.encode(ready_response).hex())

rust/src/llm/Cargo.toml

@@ -11,6 +11,7 @@ test-util = []
 easy-ext.workspace = true
 enum-as-inner.workspace = true
 futures.workspace = true
+parking_lot.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 thiserror.workspace = true

rust/src/llm/src/inflight.rs

@@ -0,0 +1,179 @@
+//! Tracking of the external→internal request-id mapping for in-flight requests.
+//!
+//! When request-id randomization is enabled (the default), [`crate::Llm`]
+//! rewrites the external (user-supplied) request id into a unique internal
+//! engine id before reaching engine-core. Engine-core only ever knows the
+//! internal id, so aborting a request by its external id requires resolving it
+//! back to the internal id(s) first.
+
+use std::collections::HashMap;
+use std::sync::{Arc, Weak};
+
+use parking_lot::Mutex;
+
+/// external id → internal id → number of live guards holding that edge.
+type InflightMap = HashMap<String, HashMap<String, usize>>;
+
+/// Maps external (user-supplied) request ids to the set of live internal engine
+/// request ids they currently expand into.
+///
+/// One external id may map to multiple internal ids: duplicate external ids
+/// submitted concurrently each get their own randomized internal id, and an
+/// abort by the shared external id must reach all of them. Edges are
+/// refcounted: with randomization disabled the same (external, internal) pair
+/// can be tracked by several guards in sequence (e.g. a finished request whose
+/// stream is still held alongside a fresh submission reusing the id), and the
+/// edge must survive until the last guard drops.
+#[derive(Default)]
+pub(crate) struct InflightRequests {
+    map: Arc<Mutex<InflightMap>>,
+}
+
+impl InflightRequests {
+    pub(crate) fn new() -> Self {
+        Self::default()
+    }
+
+    /// Record that `internal` is now an in-flight engine request for the
+    /// `external` request id, returning a guard that removes the edge when the
+    /// request's output stream is dropped (on clean finish or cancellation).
+    pub(crate) fn track(&self, external: String, internal: String) -> RequestGuard {
+        *self
+            .map
+            .lock()
+            .entry(external.clone())
+            .or_default()
+            .entry(internal.clone())
+            .or_insert(0) += 1;
+        RequestGuard {
+            map: Arc::downgrade(&self.map),
+            external,
+            internal,
+        }
+    }
+
+    /// Resolve external request ids to the internal engine ids currently
+    /// in-flight for them. Unknown or already-finished ids contribute nothing.
+    pub(crate) fn resolve(&self, external_ids: &[String]) -> Vec<String> {
+        let map = self.map.lock();
+        external_ids
+            .iter()
+            .filter_map(|external| map.get(external))
+            .flat_map(|internal_ids| internal_ids.keys())
+            .cloned()
+            .collect()
+    }
+
+    #[cfg(test)]
+    fn is_empty(&self) -> bool {
+        self.map.lock().is_empty()
+    }
+}
+
+/// RAII guard that releases one refcount on a single external→internal edge
+/// when dropped, removing the edge once no live guard holds it.
+///
+/// Held by the per-request output stream, so cleanup runs whether the stream
+/// terminates cleanly or is cancelled. A [`Weak`] handle is used so a stream
+/// outliving its owning [`InflightRequests`] does not keep the map alive.
+pub(crate) struct RequestGuard {
+    map: Weak<Mutex<InflightMap>>,
+    external: String,
+    internal: String,
+}
+
+impl Drop for RequestGuard {
+    fn drop(&mut self) {
+        let Some(map) = self.map.upgrade() else {
+            return;
+        };
+        let mut map = map.lock();
+        if let Some(internal_ids) = map.get_mut(&self.external) {
+            if let Some(count) = internal_ids.get_mut(&self.internal) {
+                *count -= 1;
+                if *count == 0 {
+                    internal_ids.remove(&self.internal);
+                }
+            }
+            if internal_ids.is_empty() {
+                map.remove(&self.external);
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn resolves_external_to_internal() {
+        let inflight = InflightRequests::new();
+        let _guard = inflight.track("ext".to_string(), "ext-abc".to_string());
+
+        assert_eq!(
+            inflight.resolve(&["ext".to_string()]),
+            vec!["ext-abc".to_string()]
+        );
+        assert!(inflight.resolve(&["unknown".to_string()]).is_empty());
+    }
+
+    #[test]
+    fn one_external_maps_to_many_internal() {
+        let inflight = InflightRequests::new();
+        let _g1 = inflight.track("dup".to_string(), "dup-1".to_string());
+        let _g2 = inflight.track("dup".to_string(), "dup-2".to_string());
+
+        let mut resolved = inflight.resolve(&["dup".to_string()]);
+        resolved.sort();
+        assert_eq!(resolved, vec!["dup-1".to_string(), "dup-2".to_string()]);
+    }
+
+    #[test]
+    fn dropping_guard_removes_only_its_own_edge_then_cleans_empty_key() {
+        let inflight = InflightRequests::new();
+        let g1 = inflight.track("dup".to_string(), "dup-1".to_string());
+        let g2 = inflight.track("dup".to_string(), "dup-2".to_string());
+
+        drop(g1);
+        assert_eq!(
+            inflight.resolve(&["dup".to_string()]),
+            vec!["dup-2".to_string()]
+        );
+
+        drop(g2);
+        assert!(inflight.resolve(&["dup".to_string()]).is_empty());
+        assert!(
+            inflight.is_empty(),
+            "empty external key must be removed, not left dangling"
+        );
+    }
+
+    #[test]
+    fn identical_edges_are_refcounted_across_guards() {
+        // With request-id randomization disabled, internal == external, so two
+        // tracked requests can share the exact same edge. Dropping one guard
+        // (e.g. a stale stream, or the error path of a rejected duplicate
+        // submission) must not untrack the other still-live request.
+        let inflight = InflightRequests::new();
+        let g1 = inflight.track("x".to_string(), "x".to_string());
+        let g2 = inflight.track("x".to_string(), "x".to_string());
+
+        drop(g1);
+        assert_eq!(inflight.resolve(&["x".to_string()]), vec!["x".to_string()]);
+
+        drop(g2);
+        assert!(inflight.resolve(&["x".to_string()]).is_empty());
+        assert!(inflight.is_empty());
+    }
+
+    #[test]
+    fn guard_drop_is_a_noop_after_inflight_is_gone() {
+        let guard = {
+            let inflight = InflightRequests::new();
+            inflight.track("ext".to_string(), "ext-abc".to_string())
+        };
+        // Dropping the guard after the owning map is gone must not panic.
+        drop(guard);
+    }
+}

rust/src/llm/src/lib.rs

@@ -2,6 +2,7 @@ use tracing::Span;
 use vllm_engine_core_client::EngineCoreClient;
 
 mod error;
+mod inflight;
 mod log_stats;
 mod output;
 mod request;
@@ -15,18 +16,22 @@ pub use output::{
 pub use request::GenerateRequest;
 pub use vllm_engine_core_client::protocol::logprobs::{Logprobs, PositionLogprobs, TokenLogprob};
 
+use crate::inflight::InflightRequests;
 use crate::log_stats::StatsLogger;
 use crate::request_metrics::RequestMetricsTracker;
 
-/// Thin generate-only facade over [`EngineCoreClient`].
+/// Thin generate-and-abort facade over [`EngineCoreClient`].
 ///
 /// This mirrors the narrow public shape of Python `AsyncLLM.generate()` and
 /// `abort()`, but keeps the boundary close to raw engine-core requests and
-/// outputs.
+/// outputs. It tracks an in-flight external→internal request-id index (see
+/// [`InflightRequests`]) so that aborts issued against external (user-supplied)
+/// ids can be resolved to the internal engine ids that engine-core understands.
 pub struct Llm {
     client: EngineCoreClient,
     randomize_request_id: bool,
     stats_logger: Option<StatsLogger>,
+    inflight: InflightRequests,
 }
 
 impl Llm {
@@ -37,6 +42,7 @@ impl Llm {
             client,
             randomize_request_id: true,
             stats_logger: None,
+            inflight: InflightRequests::new(),
         }
     }
 
@@ -72,9 +78,15 @@ impl Llm {
     pub async fn generate(&self, req: GenerateRequest) -> Result<GenerateOutputStream> {
         let prepared = req.prepare(self.randomize_request_id)?;
         let prompt_token_ids = prepared.prompt_token_ids().into();
+        let external_request_id = prepared
+            .engine_request
+            .external_req_id
+            .clone()
+            .expect("prepare always sets external_req_id");
+        let internal_request_id = prepared.engine_request.request_id.clone();
 
         // Record internal engine-core request ID in the current tracing span.
-        Span::current().record("engine_request_id", &prepared.engine_request.request_id);
+        Span::current().record("engine_request_id", &internal_request_id);
 
         let request_metrics = RequestMetricsTracker::new(
             self.client.model_name().to_string(),
@@ -84,14 +96,32 @@ impl Llm {
             1,
         );
         let stream = self.client.call(prepared.engine_request).await?;
+        let guard = self.inflight.track(external_request_id, internal_request_id);
 
         Ok(GenerateOutputStream::new(
             prompt_token_ids,
             stream,
             request_metrics,
+            guard,
         ))
     }
 
+    /// Abort in-flight requests by their external (user-supplied) request ids.
+    ///
+    /// External ids are resolved to the internal engine ids actually known to
+    /// engine-core (one external id may map to several internal ids). Unknown
+    /// or already-finished ids resolve to nothing and are a safe no-op. The
+    /// tracking entries themselves are removed when the corresponding output
+    /// streams are dropped, not here.
+    pub async fn abort(&self, external_ids: &[String]) -> Result<()> {
+        let internal_ids = self.inflight.resolve(external_ids);
+        if internal_ids.is_empty() {
+            return Ok(());
+        }
+        self.client.abort(&internal_ids).await?;
+        Ok(())
+    }
+
     /// Shut down the underlying engine-core client and its background tasks.
     pub async fn shutdown(self) -> Result<()> {
         self.client.shutdown().await?;

rust/src/llm/src/output.rs

@@ -12,6 +12,7 @@ use vllm_engine_core_client::protocol::{EngineCoreFinishReason, StopReason};
 use vllm_engine_core_client::{AbortCause, EngineCoreOutputStream};
 
 use crate::error::Result;
+use crate::inflight::RequestGuard;
 use crate::request_metrics::{RequestMetricsTracker, current_unix_timestamp_secs};
 
 /// Token usage metadata for one request.
@@ -195,12 +196,17 @@ impl GenerateOutput {
 
 /// Stream of per-request generate outputs for one request.
 ///
-/// - A normal termination of the stream represents a clean completion of the request.
-/// - For errors, unexpected closes, or explicit aborts, the stream terminates with an error.
+/// - A normal termination of the stream represents a clean completion of the
+///   request, including a client-initiated abort, which yields a final output
+///   with `finish_reason = Abort` before the stream ends.
+/// - For errors or unexpected engine-side closes, the stream terminates with an error.
 pub struct GenerateOutputStream {
     pending_prompt_info: Option<GeneratePromptInfo>,
     raw_stream: EngineCoreOutputStream,
     request_metrics: RequestMetricsTracker,
+    /// Removes this request's external→internal tracking edge on drop. Held for
+    /// its `Drop` side effect only; never read directly.
+    _request_guard: RequestGuard,
 }
 
 impl GenerateOutputStream {
@@ -210,6 +216,7 @@ impl GenerateOutputStream {
         prompt_token_ids: Arc<[u32]>,
         raw_stream: EngineCoreOutputStream,
         request_metrics: RequestMetricsTracker,
+        request_guard: RequestGuard,
     ) -> Self {
         Self {
             pending_prompt_info: Some(GeneratePromptInfo {
@@ -218,6 +225,7 @@ impl GenerateOutputStream {
             }),
             raw_stream,
             request_metrics,
+            _request_guard: request_guard,
         }
     }
 

rust/src/llm/src/request_metrics.rs

@@ -98,27 +98,33 @@ impl RequestMetricsTracker {
             self.observe_events(engine_index, events);
         }
 
-        if self.is_prefilling {
-            if let Some(prefill_stats) = &output.prefill_stats {
-                record_prompt_tokens(&self.model_name, engine_index, prefill_stats);
+        // Only outputs that actually carry tokens drive token-timing metrics.
+        // A terminal output with no new tokens (e.g. the synthesized abort
+        // output) must not log a stray time-to-first-token or inter-token
+        // sample.
+        if !output.new_token_ids.is_empty() {
+            if self.is_prefilling {
+                if let Some(prefill_stats) = &output.prefill_stats {
+                    record_prompt_tokens(&self.model_name, engine_index, prefill_stats);
+                }
+                self.first_token_latency = received_at - self.arrival_time;
+                observe_time_to_first_token_seconds(
+                    &self.model_name,
+                    engine_index,
+                    self.first_token_latency,
+                );
+                self.first_token_ts = batch_timestamp;
+                self.is_prefilling = false;
+            } else if self.last_token_ts > 0.0 {
+                observe_inter_token_latency_seconds(
+                    &self.model_name,
+                    engine_index,
+                    batch_timestamp - self.last_token_ts,
+                );
             }
-            self.first_token_latency = received_at - self.arrival_time;
-            observe_time_to_first_token_seconds(
-                &self.model_name,
-                engine_index,
-                self.first_token_latency,
-            );
-            self.first_token_ts = batch_timestamp;
-            self.is_prefilling = false;
-        } else if self.last_token_ts > 0.0 {
-            observe_inter_token_latency_seconds(
-                &self.model_name,
-                engine_index,
-                batch_timestamp - self.last_token_ts,
-            );
-        }
 
-        self.last_token_ts = batch_timestamp;
+            self.last_token_ts = batch_timestamp;
+        }
     }
 
     /// Emit the terminal request metrics once a finished output has been

rust/src/llm/tests/generate.rs

@@ -554,6 +554,142 @@ async fn duplicate_external_request_ids_are_randomized_before_reaching_engine_co
     llm.shutdown().await.unwrap();
 }
 
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn abort_resolves_external_request_id_to_internal_before_reaching_engine() {
+    let ipc = IpcNamespace::new().unwrap();
+    let handshake_address = ipc.handshake_endpoint();
+    let engine_id = b"engine-abort".to_vec();
+
+    let (shutdown_tx, engine_task) = spawn_mock_engine_task(
+        handshake_address.clone(),
+        engine_id.clone(),
+        |dealer, push| {
+            Box::pin(async move {
+                let add = recv_engine_message(dealer).await;
+                assert_eq!(add[0].as_ref(), &[0x00]);
+                let request: EngineCoreRequest = rmp_serde::from_slice(&add[1]).unwrap();
+                assert_eq!(request.external_req_id.as_deref(), Some("req-abort"));
+                assert!(request.request_id.starts_with("req-abort-"));
+                assert_ne!(request.request_id, "req-abort");
+
+                send_outputs(
+                    push,
+                    EngineCoreOutputs {
+                        outputs: vec![request_output(&request.request_id, vec![7], None)],
+                        ..Default::default()
+                    },
+                )
+                .await;
+
+                // The abort frame must carry the internal engine id, not the
+                // external "req-abort" id the caller aborted by.
+                let abort =
+                    timeout(Duration::from_secs(1), recv_engine_message(dealer)).await.unwrap();
+                assert_eq!(abort[0].as_ref(), &[0x01]);
+                let aborted_ids: Vec<String> = rmp_serde::from_slice(&abort[1]).unwrap();
+                assert_eq!(aborted_ids, vec![request.request_id]);
+            })
+        },
+    );
+
+    let llm = connect_async_llm_with_ipc(handshake_address, 0, "test-model", &ipc).await;
+    let mut stream = llm.generate(sample_generate_request("req-abort", 4)).await.unwrap();
+    let internal_id = stream.request_id().to_string();
+    assert_ne!(internal_id, "req-abort");
+
+    assert_eq!(stream.next().await.unwrap().unwrap().token_ids, vec![7]);
+
+    // Abort by the external id; engine-core only knows the internal id.
+    llm.abort(&["req-abort".to_string()]).await.unwrap();
+
+    // The consumer stream is finalized locally with a clean abort terminal
+    // rather than hanging or surfacing as RequestStreamClosed. The engine sends
+    // no final output for a client abort, so this output is synthesized.
+    let terminal = stream.next().await.unwrap().unwrap();
+    assert_eq!(terminal.finish_reason, Some(FinishReason::Abort));
+    assert!(terminal.token_ids.is_empty());
+    assert!(stream.next().await.is_none());
+
+    let _ = shutdown_tx.send(());
+    engine_task.await.unwrap();
+    drop(stream);
+    llm.shutdown().await.unwrap();
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+async fn abort_by_external_id_aborts_all_internal_requests() {
+    let ipc = IpcNamespace::new().unwrap();
+    let handshake_address = ipc.handshake_endpoint();
+    let engine_id = b"engine-abort-many".to_vec();
+
+    let (shutdown_tx, engine_task) = spawn_mock_engine_task(
+        handshake_address.clone(),
+        engine_id.clone(),
+        |dealer, push| {
+            Box::pin(async move {
+                let add_1 = recv_engine_message(dealer).await;
+                assert_eq!(add_1[0].as_ref(), &[0x00]);
+                let request_1: EngineCoreRequest = rmp_serde::from_slice(&add_1[1]).unwrap();
+
+                let add_2 = recv_engine_message(dealer).await;
+                assert_eq!(add_2[0].as_ref(), &[0x00]);
+                let request_2: EngineCoreRequest = rmp_serde::from_slice(&add_2[1]).unwrap();
+
+                assert_eq!(request_1.external_req_id.as_deref(), Some("req-dup-abort"));
+                assert_eq!(request_2.external_req_id.as_deref(), Some("req-dup-abort"));
+                assert_ne!(request_1.request_id, request_2.request_id);
+
+                send_outputs(
+                    push,
+                    EngineCoreOutputs {
+                        outputs: vec![
+                            request_output(&request_1.request_id, vec![7], None),
+                            request_output(&request_2.request_id, vec![8], None),
+                        ],
+                        ..Default::default()
+                    },
+                )
+                .await;
+
+                // A single abort by the shared external id must abort both
+                // internal engine ids it expanded into.
+                let abort =
+                    timeout(Duration::from_secs(1), recv_engine_message(dealer)).await.unwrap();
+                assert_eq!(abort[0].as_ref(), &[0x01]);
+                let mut aborted_ids: Vec<String> = rmp_serde::from_slice(&abort[1]).unwrap();
+                aborted_ids.sort();
+                let mut expected = vec![request_1.request_id, request_2.request_id];
+                expected.sort();
+                assert_eq!(aborted_ids, expected);
+            })
+        },
+    );
+
+    let llm = connect_async_llm_with_ipc(handshake_address, 0, "test-model", &ipc).await;
+    let mut stream_1 = llm.generate(sample_generate_request("req-dup-abort", 4)).await.unwrap();
+    let mut stream_2 = llm.generate(sample_generate_request("req-dup-abort", 4)).await.unwrap();
+    assert_ne!(stream_1.request_id(), stream_2.request_id());
+
+    assert_eq!(stream_1.next().await.unwrap().unwrap().token_ids, vec![7]);
+    assert_eq!(stream_2.next().await.unwrap().unwrap().token_ids, vec![8]);
+
+    llm.abort(&["req-dup-abort".to_string()]).await.unwrap();
+
+    // Both internal requests the external id expanded into are finalized with a
+    // clean abort terminal.
+    for stream in [&mut stream_1, &mut stream_2] {
+        let terminal = stream.next().await.unwrap().unwrap();
+        assert_eq!(terminal.finish_reason, Some(FinishReason::Abort));
+        assert!(stream.next().await.is_none());
+    }
+
+    let _ = shutdown_tx.send(());
+    engine_task.await.unwrap();
+    drop(stream_1);
+    drop(stream_2);
+    llm.shutdown().await.unwrap();
+}
+
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 async fn generate_records_request_metrics_in_prometheus_output() {
     let ipc = IpcNamespace::new().unwrap();

rust/src/reasoning-parser/src/lib.rs

@@ -19,6 +19,7 @@ mod deepseek_r1;
 mod delimited;
 mod gemma4;
 mod kimi;
+mod minimax_m3;
 mod qwen3;
 mod seed_oss;
 mod step3p5;
@@ -31,6 +32,7 @@ pub use self::deepseek_r1::DeepSeekR1ReasoningParser;
 pub(crate) use self::delimited::DelimitedReasoningParser;
 pub use self::gemma4::Gemma4ReasoningParser;
 pub use self::kimi::KimiReasoningParser;
+pub use self::minimax_m3::MiniMaxM3ReasoningParser;
 pub use self::qwen3::Qwen3ReasoningParser;
 pub use self::seed_oss::SeedOssReasoningParser;
 pub use self::step3p5::Step3p5ReasoningParser;

rust/src/reasoning-parser/src/minimax_m3.rs

@@ -0,0 +1,98 @@
+use vllm_tokenizer::DynTokenizer;
+
+use super::{DelimitedReasoningParser, ReasoningDelta, ReasoningParser, Result};
+
+const M3_THINK_START: &str = "<mm:think>";
+const M3_THINK_END: &str = "</mm:think>";
+
+/// Reasoning parser for MiniMax M3 style outputs.
+///
+/// MiniMax M3 uses `<mm:think>...</mm:think>` delimiters. Its chat template may
+/// prefill either delimiter depending on the requested thinking mode, so the
+/// shared delimited parser derives the starting state from the rendered prompt.
+pub struct MiniMaxM3ReasoningParser {
+    inner: DelimitedReasoningParser,
+    /// True until the first response text is classified. Only this position may
+    /// drop a stray `</mm:think>` emitted at the start of a response.
+    at_response_start: bool,
+    /// Holds an initial suffix like `</mm` while it may still complete into the
+    /// leading closer on a later chunk.
+    leading_end_buffer: String,
+}
+
+impl MiniMaxM3ReasoningParser {
+    /// Create a MiniMax M3 parser backed by the shared delimited state machine.
+    pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
+        Ok(Self {
+            inner: DelimitedReasoningParser::new(tokenizer, M3_THINK_START, M3_THINK_END, false)?,
+            at_response_start: true,
+            leading_end_buffer: String::new(),
+        })
+    }
+
+    /// Drop a response-leading `</mm:think>` while preserving later unmatched
+    /// closers as ordinary content.
+    fn push_inner(&mut self, delta: &str) -> ReasoningDelta {
+        if self.at_response_start && !self.inner.in_reasoning() {
+            self.leading_end_buffer.push_str(delta);
+            let buffered = std::mem::take(&mut self.leading_end_buffer);
+
+            if buffered.is_empty() {
+                return ReasoningDelta::default();
+            }
+            if let Some(rest) = buffered.strip_prefix(M3_THINK_END) {
+                self.at_response_start = false;
+                return self.inner.push(rest);
+            }
+            if M3_THINK_END.starts_with(buffered.as_str()) {
+                self.leading_end_buffer = buffered;
+                return ReasoningDelta::default();
+            }
+
+            self.at_response_start = false;
+            return self.inner.push(&buffered);
+        }
+
+        self.inner.push(delta)
+    }
+}
+
+fn append_delta(target: &mut ReasoningDelta, delta: ReasoningDelta) {
+    if let Some(reasoning) = delta.reasoning {
+        target.push_reasoning(&reasoning);
+    }
+    if let Some(content) = delta.content {
+        target.push_content(&content);
+    }
+}
+
+impl ReasoningParser for MiniMaxM3ReasoningParser {
+    fn create(tokenizer: DynTokenizer) -> Result<Box<dyn ReasoningParser>>
+    where
+        Self: Sized + 'static,
+    {
+        Ok(Box::new(Self::new(tokenizer)?))
+    }
+
+    fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
+        self.inner.initialize(prompt_token_ids);
+        self.at_response_start = true;
+        self.leading_end_buffer.clear();
+        Ok(())
+    }
+
+    fn push(&mut self, delta: &str) -> Result<ReasoningDelta> {
+        Ok(self.push_inner(delta))
+    }
+
+    fn finish(&mut self) -> Result<ReasoningDelta> {
+        let mut delta = ReasoningDelta::default();
+        if !self.leading_end_buffer.is_empty() {
+            let pending = std::mem::take(&mut self.leading_end_buffer);
+            self.at_response_start = false;
+            append_delta(&mut delta, self.inner.push(&pending));
+        }
+        append_delta(&mut delta, self.inner.finish());
+        Ok(delta)
+    }
+}

rust/src/reasoning-parser/src/tests.rs

@@ -3,7 +3,8 @@ use std::sync::Arc;
 use vllm_tokenizer::Tokenizer;
 
 use super::{
-    DeepSeekR1ReasoningParser, DelimitedReasoningParser, Qwen3ReasoningParser, ReasoningParser,
+    DeepSeekR1ReasoningParser, DelimitedReasoningParser, MiniMaxM3ReasoningParser,
+    Qwen3ReasoningParser, ReasoningParser,
 };
 
 pub(crate) struct FakeTokenizer;
@@ -32,6 +33,8 @@ impl Tokenizer for FakeTokenizer {
             "<|END_THINKING|>" => Some(4),
             "◁think▷" => Some(5),
             "◁/think▷" => Some(6),
+            "<mm:think>" => Some(8),
+            "</mm:think>" => Some(9),
             "<seed:think>" => Some(10),
             "</seed:think>" => Some(11),
             _ => None,
@@ -161,3 +164,66 @@ fn deepseek_r1_stops_scanning_at_last_special_token() {
     assert_eq!(delta.reasoning.as_deref(), Some("reason"));
     assert_eq!(delta.content.as_deref(), Some("answer"));
 }
+
+#[test]
+fn minimax_m3_handles_explicit_think_delimiters() {
+    let tokenizer = Arc::new(FakeTokenizer);
+    let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
+
+    let delta = parser.push("<mm:think>reason</mm:think>answer").unwrap();
+    assert_eq!(delta.reasoning.as_deref(), Some("reason"));
+    assert_eq!(delta.content.as_deref(), Some("answer"));
+}
+
+#[test]
+fn minimax_m3_drops_leading_end_marker() {
+    let tokenizer = Arc::new(FakeTokenizer);
+    let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
+
+    let delta = parser.push("</mm:think>answer").unwrap();
+    assert_eq!(delta.reasoning, None);
+    assert_eq!(delta.content.as_deref(), Some("answer"));
+}
+
+#[test]
+fn minimax_m3_preserves_non_leading_end_marker() {
+    let tokenizer = Arc::new(FakeTokenizer);
+    let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
+
+    let delta = parser.push("XXX</mm:think>YYY").unwrap();
+    assert_eq!(delta.reasoning, None);
+    assert_eq!(delta.content.as_deref(), Some("XXX</mm:think>YYY"));
+}
+
+#[test]
+fn minimax_m3_drops_split_leading_end_marker() {
+    let tokenizer = Arc::new(FakeTokenizer);
+    let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
+
+    assert!(parser.push("</mm").unwrap().is_empty());
+    let delta = parser.push(":think>answer").unwrap();
+    assert_eq!(delta.reasoning, None);
+    assert_eq!(delta.content.as_deref(), Some("answer"));
+}
+
+#[test]
+fn minimax_m3_uses_prompt_prefilled_start_marker() {
+    let tokenizer = Arc::new(FakeTokenizer);
+    let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
+    parser.initialize(&[8]).unwrap();
+
+    let delta = parser.push("reason</mm:think>answer").unwrap();
+    assert_eq!(delta.reasoning.as_deref(), Some("reason"));
+    assert_eq!(delta.content.as_deref(), Some("answer"));
+}
+
+#[test]
+fn minimax_m3_uses_prompt_prefilled_end_marker() {
+    let tokenizer = Arc::new(FakeTokenizer);
+    let mut parser = MiniMaxM3ReasoningParser::new(tokenizer).unwrap();
+    parser.initialize(&[9]).unwrap();
+
+    let delta = parser.push("answer").unwrap();
+    assert_eq!(delta.reasoning, None);
+    assert_eq!(delta.content.as_deref(), Some("answer"));
+}

rust/src/server/src/lib.rs

@@ -35,9 +35,24 @@ use crate::routes::build_router;
 use crate::server_info::ServerInfoSnapshot;
 use crate::state::AppState;
 
+/// Resolve the public model names accepted by the frontend.
+fn effective_served_model_names(model: &str, served_model_name: &[String]) -> Vec<String> {
+    if served_model_name.is_empty() {
+        vec![model.to_string()]
+    } else {
+        served_model_name.to_vec()
+    }
+}
+
 /// Build the shared application state for one configured model and one engine
 /// client.
 async fn build_state(config: &Config) -> Result<Arc<AppState>> {
+    // If no served names are specified, fall back to the backend model path so
+    // that the API always has at least one valid model ID. Use the same primary
+    // public name for frontend-side metrics labels.
+    let served_model_names = effective_served_model_names(&config.model, &config.served_model_name);
+    let metrics_model_name = served_model_names[0].clone();
+
     // Load both backends from the same model metadata so they stay in sync.
     let loaded = load_model_backends(
         &config.model,
@@ -68,7 +83,7 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
     let client = EngineCoreClient::connect(EngineCoreClientConfig {
         transport_mode: config.transport_mode.clone(),
         coordinator_mode,
-        model_name: config.model.clone(),
+        model_name: metrics_model_name,
         client_index: 0,
     })
     .await
@@ -81,14 +96,6 @@ async fn build_state(config: &Config) -> Result<Arc<AppState>> {
         .with_tool_call_parser(config.tool_call_parser.clone())
         .with_reasoning_parser(config.reasoning_parser.clone());
 
-    // If no served names are specified, fall back to the backend model path so
-    // that the API always has at least one valid model ID.
-    let served_model_names = if config.served_model_name.is_empty() {
-        vec![config.model.clone()]
-    } else {
-        config.served_model_name.clone()
-    };
-
     Ok(Arc::new(
         AppState::new(served_model_names, chat)
             .with_api_server_options(config.api_server_options)
@@ -258,3 +265,26 @@ where
         .unwrap_or_else(|| Instant::now() + config.shutdown_timeout);
     state.shutdown(shutdown_deadline).await
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn effective_served_model_names_falls_back_to_backend_model() {
+        assert_eq!(
+            effective_served_model_names("backend-model", &[]),
+            vec!["backend-model"]
+        );
+    }
+
+    #[test]
+    fn effective_served_model_names_preserves_public_names() {
+        let served_names = vec!["public-model".to_string(), "public-alias".to_string()];
+
+        assert_eq!(
+            effective_served_model_names("backend-model", &served_names),
+            served_names
+        );
+    }
+}

rust/src/server/src/routes/openai/chat_completions/convert.rs

@@ -142,6 +142,7 @@ pub(super) fn prepare_chat_request(
         },
         tools: convert_tools(request.tools)?,
         tool_choice: convert_tool_choice(request.tool_choice.as_ref())?,
+        parallel_tool_calls: request.parallel_tool_calls.unwrap_or(true),
         decode_options: vllm_text::output::TextDecodeOptions {
             skip_special_tokens: request.skip_special_tokens,
             include_stop_str_in_output: request.include_stop_str_in_output,
@@ -412,6 +413,33 @@ mod tests {
         }
     }
 
+    #[test]
+    fn prepare_chat_request_maps_parallel_tool_calls() {
+        let mut request = base_request();
+        request.parallel_tool_calls = Some(false);
+
+        let prepared = prepare_chat_request(
+            request,
+            &served(&["Qwen/Qwen1.5-0.5B-Chat"]),
+            ResolvedRequestContext::default(),
+        )
+        .expect("request is valid");
+
+        assert!(!prepared.chat_request.parallel_tool_calls);
+    }
+
+    #[test]
+    fn prepare_chat_request_defaults_parallel_tool_calls_to_true() {
+        let prepared = prepare_chat_request(
+            base_request(),
+            &served(&["Qwen/Qwen1.5-0.5B-Chat"]),
+            ResolvedRequestContext::default(),
+        )
+        .expect("request is valid");
+
+        assert!(prepared.chat_request.parallel_tool_calls);
+    }
+
     #[test]
     fn prepare_chat_request_maps_text_parts() {
         let mut request = base_request();

rust/src/server/src/routes/openai/chat_completions/validate.rs

@@ -93,13 +93,6 @@ pub(super) fn validate_request_compat(
     // ---- Reject parameters that are accepted for deserialization but not yet
     // implemented ----
 
-    if request.parallel_tool_calls.is_some() {
-        bail_invalid_request!(
-            param = "parallel_tool_calls",
-            "parallel_tool_calls is not supported."
-        );
-    }
-
     reject_non_default(
         request.length_penalty.as_ref(),
         "length_penalty",

rust/src/server/src/routes/tests.rs

@@ -1677,6 +1677,85 @@ async fn http_metrics_record_list_models_requests() {
     );
 }
 
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+#[serial]
+async fn request_metrics_use_served_model_name_label() {
+    let ipc = IpcNamespace::new().expect("create ipc namespace");
+    let handshake_address = ipc.handshake_endpoint();
+    let engine_id = b"engine-openai-served-model-metrics".to_vec();
+
+    let engine_task = MockEngineTask::new(spawn_mock_engine_task(
+        handshake_address.clone(),
+        engine_id.clone(),
+        |dealer, push| {
+            boxed_test_future(async move {
+                let add = recv_engine_message(dealer).await;
+                let request: EngineCoreRequest =
+                    rmp_serde::from_slice(&add[1]).expect("decode request");
+                send_outputs(
+                    push,
+                    engine_outputs_for_request(&request.request_id, default_stream_output_specs()),
+                )
+                .await;
+            })
+        },
+    ));
+
+    let client = EngineCoreClient::connect(
+        EngineCoreClientConfig::new_single(handshake_address)
+            .with_model_name("served-model-metrics")
+            .with_local_input_output_addresses(
+                Some(ipc.input_endpoint()),
+                Some(ipc.output_endpoint()),
+            ),
+    )
+    .await
+    .expect("connect client");
+    let chat = ChatLlm::from_shared_backend(test_llm(client), Arc::new(FakeChatBackend::new()));
+    let mut app = build_router(Arc::new(AppState::new(
+        vec![
+            "served-model-metrics".to_string(),
+            "served-model-alias".to_string(),
+        ],
+        chat,
+    )));
+    let before = METRICS.render().unwrap();
+
+    let response = app
+        .call(
+            Request::builder()
+                .method("POST")
+                .uri("/v1/chat/completions")
+                .header("content-type", "application/json")
+                .body(Body::from(
+                    json!({
+                        "model": "served-model-alias",
+                        "stream": false,
+                        "messages": [{"role": "user", "content": "hello"}]
+                    })
+                    .to_string(),
+                ))
+                .expect("build request"),
+        )
+        .await
+        .expect("call app");
+
+    assert_eq!(response.status(), StatusCode::OK);
+    let _ = to_bytes(response.into_body(), usize::MAX).await.unwrap();
+
+    let after = METRICS.render().unwrap();
+    assert_eq!(
+        metric_delta(
+            &before,
+            &after,
+            "vllm:request_success_total",
+            Some("model_name=\"served-model-metrics\",engine=\"0\",finished_reason=\"stop\""),
+        ),
+        1.0
+    );
+    engine_task.await.expect("mock engine task");
+}
+
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 #[serial]
 async fn wrong_model_returns_not_found() {

rust/src/server/src/routes/tokenize/types.rs

@@ -83,6 +83,7 @@ impl TokenizeChatRequest {
             },
             tools: convert_tools(self.tools)?,
             tool_choice: ChatToolChoice::Auto,
+            parallel_tool_calls: true,
             decode_options: TextDecodeOptions::default(),
             intermediate: false,
             priority: 0,

rust/src/text/src/lib.rs

@@ -151,6 +151,12 @@ impl TextLlm {
         Ok((text_request, raw_stream))
     }
 
+    /// Abort in-flight requests by their external (user-supplied) request ids.
+    pub async fn abort(&self, external_ids: &[String]) -> Result<()> {
+        self.llm.abort(external_ids).await?;
+        Ok(())
+    }
+
     /// Shut down the underlying LLM client and its background tasks.
     pub async fn shutdown(self) -> Result<()> {
         self.llm.shutdown().await?;

rust/src/tool-parser/python/src/lib.rs

@@ -38,6 +38,8 @@ macro_rules! tool_parser_factory {
 
 // Export a tool parser to Python by registering it here.
 tool_parser_factory! {
+    MinimaxM3ToolParser,
+
     // Below are the parsers just for testing purposes on Python side.
     DeepSeekV4ToolParser,
     KimiK2ToolParser,

rust/src/tool-parser/src/lib.rs

@@ -10,6 +10,7 @@ mod hy_v3;
 mod json;
 mod kimi_k2;
 mod minimax_m2;
+mod minimax_m3;
 mod parameters;
 mod qwen_coder;
 #[cfg(any(test, feature = "test-util"))]
@@ -30,6 +31,7 @@ pub use json::{
 };
 pub use kimi_k2::KimiK2ToolParser;
 pub use minimax_m2::MinimaxM2ToolParser;
+pub use minimax_m3::MinimaxM3ToolParser;
 pub use qwen_coder::Qwen3CoderToolParser;
 use serde::{Deserialize, Serialize};
 use serde_json::Value;

rust/src/tool-parser/src/minimax_m3.rs

@@ -0,0 +1,885 @@
+use winnow::ascii::{multispace0 as ws0, multispace1 as ws1};
+use winnow::combinator::{alt, delimited, seq};
+use winnow::error::{ContextError, ErrMode};
+use winnow::prelude::*;
+use winnow::stream::Partial;
+use winnow::token::{literal, rest, take_until};
+
+use super::parameters::{ParamElement, ParamInput, ToolSchemas};
+use super::utils::{parse_buffered_event, safe_text_len};
+use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput};
+use crate::Tool;
+
+const NAMESPACE: &str = "]<]minimax[>[";
+const TOOL_CALL_START: &str = "]<]minimax[>[<tool_call>";
+const TOOL_CALL_END: &str = "]<]minimax[>[</tool_call>";
+const INVOKE_START: &str = "]<]minimax[>[<invoke";
+const INVOKE_END: &str = "]<]minimax[>[</invoke>";
+const ELEMENT_START: &str = "]<]minimax[>[<";
+const ELEMENT_END_START: &str = "]<]minimax[>[</";
+const MIXED_TEXT_FIELD: &str = "$text";
+
+type MinimaxM3Input<'i> = Partial<&'i str>;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum MinimaxM3Mode {
+    Text,
+    ToolBlock,
+    Done,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+enum MinimaxM3Event {
+    Text {
+        len: usize,
+    },
+    ToolBlockStart,
+    Invoke {
+        name: String,
+        params: Vec<(String, ParamInput)>,
+    },
+    ToolBlockEnd,
+    IgnoredRest,
+}
+
+/// Tool parser for MiniMax M3 namespace-delimited XML-style tool calls.
+///
+/// Example tool call content with recursive parameters:
+///
+/// ```text
+/// ]<]minimax[>[<tool_call>
+/// ]<]minimax[>[<invoke name="create_order">
+/// ]<]minimax[>[<user_id>42]<]minimax[>[</user_id>
+/// ]<]minimax[>[<shipping>
+/// ]<]minimax[>[<city>Singapore]<]minimax[>[</city>
+/// ]<]minimax[>[<zip>018956]<]minimax[>[</zip>
+/// ]<]minimax[>[</shipping>
+/// ]<]minimax[>[<items>
+/// ]<]minimax[>[<item>
+/// ]<]minimax[>[<sku>book-001]<]minimax[>[</sku>
+/// ]<]minimax[>[<qty>2]<]minimax[>[</qty>
+/// ]<]minimax[>[</item>
+/// ]<]minimax[>[</items>
+/// ]<]minimax[>[</invoke>
+/// ]<]minimax[>[</tool_call>
+/// ```
+///
+/// With a schema where `shipping` is an object and `items` is an array of
+/// objects, recursive parameter conversion produces:
+///
+/// ```json
+/// {
+///   "user_id": 42,
+///   "shipping": {
+///     "city": "Singapore",
+///     "zip": 18956
+///   },
+///   "items": [
+///     {
+///       "sku": "book-001",
+///       "qty": 2
+///     }
+///   ]
+/// }
+/// ```
+///
+/// MiniMax M3 emits the namespace marker `]<]minimax[>[` before each structural
+/// tag. Arguments are emitted only after a full `<invoke>` block is parsed.
+pub struct MinimaxM3ToolParser {
+    buffer: String,
+    mode: MinimaxM3Mode,
+    emitted_tool_count: usize,
+    tool_parameters: ToolSchemas,
+}
+
+impl MinimaxM3ToolParser {
+    /// Create a MiniMax M3 tool parser.
+    pub fn new(tools: &[Tool]) -> Self {
+        Self {
+            buffer: String::new(),
+            mode: MinimaxM3Mode::Text,
+            emitted_tool_count: 0,
+            tool_parameters: ToolSchemas::from_tools(tools),
+        }
+    }
+
+    /// Apply one parsed MiniMax M3 event to parser state and output.
+    fn apply_event(&mut self, event: MinimaxM3Event, output: &mut ToolParserOutput) -> Result<()> {
+        match event {
+            MinimaxM3Event::Text { len: consumed_len } => {
+                output.normal_text.push_str(&self.buffer[..consumed_len]);
+            }
+            MinimaxM3Event::ToolBlockStart => self.mode = MinimaxM3Mode::ToolBlock,
+            MinimaxM3Event::Invoke { name, params } => {
+                let arguments = self.tool_parameters.convert_params_with_schema(&name, params);
+                let arguments = serde_json::to_string(&arguments)
+                    .map_err(|error| parsing_failed!("failed to serialize arguments: {}", error))?;
+
+                output.calls.push(ToolCallDelta {
+                    tool_index: self.emitted_tool_count,
+                    name: Some(name),
+                    arguments,
+                });
+                self.emitted_tool_count += 1;
+            }
+            MinimaxM3Event::ToolBlockEnd => self.mode = MinimaxM3Mode::Done,
+            MinimaxM3Event::IgnoredRest => {}
+        }
+        Ok(())
+    }
+}
+
+impl ToolParser for MinimaxM3ToolParser {
+    fn create(tools: &[Tool]) -> Result<Box<dyn ToolParser>>
+    where
+        Self: Sized + 'static,
+    {
+        Ok(Box::new(Self::new(tools)))
+    }
+
+    fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
+        self.buffer.push_str(chunk);
+
+        while let Some((event, consumed_len)) = parse_buffered_event(&self.buffer, |input| {
+            parse_next_minimax_m3_event(input, self.mode)
+        })? {
+            self.apply_event(event, output)?;
+            self.buffer.drain(..consumed_len);
+        }
+
+        Ok(())
+    }
+
+    fn finish(&mut self) -> Result<ToolParserOutput> {
+        let mut output = ToolParserOutput::default();
+        match self.mode {
+            MinimaxM3Mode::Text => {
+                output.normal_text.push_str(&self.buffer);
+            }
+            MinimaxM3Mode::ToolBlock => {
+                if !self.buffer.trim_start().is_empty() {
+                    return Err(parsing_failed!("incomplete MiniMax M3 tool call"));
+                }
+            }
+            MinimaxM3Mode::Done => {}
+        }
+        let _ = self.reset();
+        Ok(output)
+    }
+
+    fn reset(&mut self) -> String {
+        self.mode = MinimaxM3Mode::Text;
+        self.emitted_tool_count = 0;
+        std::mem::take(&mut self.buffer)
+    }
+}
+
+/// Parse a MiniMax M3 event for the current parser mode.
+fn parse_next_minimax_m3_event(
+    input: &mut MinimaxM3Input<'_>,
+    mode: MinimaxM3Mode,
+) -> ModalResult<MinimaxM3Event> {
+    match mode {
+        MinimaxM3Mode::Text => parse_text_event(input),
+        MinimaxM3Mode::ToolBlock => parse_tool_block_event(input),
+        MinimaxM3Mode::Done => ignored_rest_event(input),
+    }
+}
+
+/// Parse a text-mode MiniMax M3 event.
+fn parse_text_event(input: &mut MinimaxM3Input<'_>) -> ModalResult<MinimaxM3Event> {
+    alt((tool_block_start_event, safe_text_event)).parse_next(input)
+}
+
+/// Parse a MiniMax M3 tool-block start marker.
+fn tool_block_start_event(input: &mut MinimaxM3Input<'_>) -> ModalResult<MinimaxM3Event> {
+    literal(TOOL_CALL_START).value(MinimaxM3Event::ToolBlockStart).parse_next(input)
+}
+
+/// Parse a safe text run before the next MiniMax M3 marker.
+fn safe_text_event(input: &mut MinimaxM3Input<'_>) -> ModalResult<MinimaxM3Event> {
+    safe_text_len(input, TOOL_CALL_START).map(|len| MinimaxM3Event::Text { len })
+}
+
+/// Parse one event inside a MiniMax M3 tool block.
+fn parse_tool_block_event(input: &mut MinimaxM3Input<'_>) -> ModalResult<MinimaxM3Event> {
+    alt((tool_block_end_event, invoke_event)).parse_next(input)
+}
+
+/// Parse a MiniMax M3 tool-block end marker.
+fn tool_block_end_event(input: &mut MinimaxM3Input<'_>) -> ModalResult<MinimaxM3Event> {
+    (ws0, literal(TOOL_CALL_END))
+        .value(MinimaxM3Event::ToolBlockEnd)
+        .parse_next(input)
+}
+
+/// Parse a complete MiniMax M3 invoke block.
+fn invoke_event(input: &mut MinimaxM3Input<'_>) -> ModalResult<MinimaxM3Event> {
+    let (name, body) = seq!(
+        _: ws0,
+        _: literal(INVOKE_START),
+        _: (ws1, literal("name=")),
+        partial_attr_value,
+        _: literal(">"),
+        take_until(0.., INVOKE_END),
+        _: literal(INVOKE_END),
+    )
+    .parse_next(input)?;
+    let params = parse_invoke_params(body)?;
+
+    Ok(MinimaxM3Event::Invoke {
+        name: name.trim().to_string(),
+        params,
+    })
+}
+
+/// Parse all parameter elements inside a complete MiniMax M3 invoke body.
+fn parse_invoke_params(invoke_body: &str) -> ModalResult<Vec<(String, ParamInput)>> {
+    let mut input = invoke_body;
+    let mut elements = Vec::new();
+
+    loop {
+        let _ = ws0.parse_next(&mut input)?;
+        if input.is_empty() {
+            break;
+        }
+        if input.starts_with(ELEMENT_START) {
+            elements.push(parameter_element(&mut input)?);
+            continue;
+        }
+        if input.starts_with(NAMESPACE) {
+            return malformed();
+        }
+        // Be tolerant: ordinary text at an invokeparameter boundary ends this invoke.
+        // Keep parsed parameters and drop the remaining invoke body.
+        break;
+    }
+
+    Ok(elements.into_iter().map(|element| (element.name, element.value)).collect())
+}
+
+/// Parse a MiniMax M3 parameter element.
+fn parameter_element(input: &mut &str) -> ModalResult<ParamElement> {
+    let name = open_element_tag(input)?.to_string();
+    let value = element_body(input, &name)?;
+    close_element_tag(input, &name)?;
+    Ok(ParamElement { name, value })
+}
+
+/// Parse a MiniMax M3 opening element tag.
+fn open_element_tag<'i>(input: &mut &'i str) -> ModalResult<&'i str> {
+    let name = seq!(
+        _: literal(ELEMENT_START),
+        take_until(1.., ">"),
+        _: literal(">"),
+    )
+    .parse_next(input)?;
+
+    let name = name.0;
+    if name.starts_with('/') || name.trim().is_empty() {
+        return malformed();
+    }
+
+    Ok(name)
+}
+
+/// Parse a MiniMax M3 closing element tag.
+fn close_element_tag(input: &mut &str, name: &str) -> ModalResult<()> {
+    literal(ELEMENT_END_START).void().parse_next(input)?;
+    literal(name).void().parse_next(input)?;
+    literal(">").void().parse_next(input)
+}
+
+/// Parse the body of one MiniMax M3 element.
+fn element_body(input: &mut &str, closing_name: &str) -> ModalResult<ParamInput> {
+    let close_tag = format!("{ELEMENT_END_START}{closing_name}>");
+    let mut text = String::new();
+    let mut elements = Vec::new();
+
+    loop {
+        text.push_str(text_until_namespace(input)?);
+
+        if input.starts_with(&close_tag) {
+            // Close tag reached, end of element body.
+            break;
+        }
+        if input.starts_with(ELEMENT_START) {
+            // Child element start reached, parse child element recursively.
+            elements.push(parameter_element(input)?);
+            continue;
+        }
+        if input.starts_with(NAMESPACE) {
+            // Unexpected namespace marker.
+            return malformed();
+        }
+    }
+
+    if elements.is_empty() {
+        Ok(ParamInput::Text(text))
+    } else {
+        if !text.trim().is_empty() {
+            push_mixed_text_element(&mut elements, text);
+        }
+        Ok(ParamInput::Elements(elements))
+    }
+}
+
+/// Parse text until the next MiniMax M3 namespace marker.
+fn text_until_namespace<'i>(input: &mut &'i str) -> ModalResult<&'i str> {
+    take_until(0.., NAMESPACE).parse_next(input)
+}
+
+/// Preserve mixed text content under a reserved object field.
+///
+/// By default, the field name is `$text`, but if that collides with an existing
+/// child element name, prepend `$` until there is no collision.
+fn push_mixed_text_element(elements: &mut Vec<ParamElement>, text: String) {
+    let mut name = MIXED_TEXT_FIELD.to_string();
+    while elements.iter().any(|element| element.name == name) {
+        name.insert(0, '$');
+    }
+    elements.push(ParamElement {
+        name,
+        value: ParamInput::Text(text),
+    });
+}
+
+/// Parse a quoted or unquoted XML attribute value from partial streaming input.
+fn partial_attr_value<'i>(input: &mut MinimaxM3Input<'i>) -> ModalResult<&'i str> {
+    alt((
+        delimited(literal("\""), take_until(1.., "\""), literal("\"")),
+        delimited(literal("'"), take_until(1.., "'"), literal("'")),
+        take_until(1.., ">"),
+    ))
+    .parse_next(input)
+}
+
+/// Parse ignored rest after the MiniMax M3 tool block ends.
+fn ignored_rest_event(input: &mut MinimaxM3Input<'_>) -> ModalResult<MinimaxM3Event> {
+    rest.value(MinimaxM3Event::IgnoredRest).parse_next(input)
+}
+
+fn malformed<T>() -> ModalResult<T> {
+    Err(ErrMode::Cut(ContextError::new()))
+}
+
+#[cfg(test)]
+mod tests {
+    use expect_test::expect;
+    use serde_json::{Value, json};
+    use thiserror_ext::AsReport;
+
+    use super::{
+        ELEMENT_END_START, ELEMENT_START, INVOKE_END, INVOKE_START, MinimaxM3ToolParser,
+        TOOL_CALL_END, TOOL_CALL_START, ToolParser,
+    };
+    use crate::test_utils::{collect_stream, split_by_chars, test_tools};
+    use crate::{Tool, ToolParserTestExt as _};
+
+    fn element(name: &str, body: &str) -> String {
+        format!("{ELEMENT_START}{name}>{body}{ELEMENT_END_START}{name}>")
+    }
+
+    fn invoke(function_name: &str, body: &str) -> String {
+        format!("{INVOKE_START} name=\"{function_name}\">{body}{INVOKE_END}")
+    }
+
+    fn build_tool_block(invokes: &[(&str, String)]) -> String {
+        let invokes = invokes
+            .iter()
+            .map(|(function_name, body)| invoke(function_name, body))
+            .collect::<Vec<_>>()
+            .join("\n");
+        format!("{TOOL_CALL_START}\n{invokes}\n{TOOL_CALL_END}")
+    }
+
+    fn m3_test_tools() -> Vec<Tool> {
+        let mut tools = test_tools();
+        tools.push(Tool {
+            name: "create_order".to_string(),
+            description: None,
+            parameters: json!({
+                "type": "object",
+                "properties": {
+                    "user_id": { "type": "integer" },
+                    "urgent": { "type": "boolean" },
+                    "note": { "type": "string" },
+                    "shipping": {
+                        "type": "object",
+                        "properties": {
+                            "city": { "type": "string" },
+                            "zip": { "type": "integer" }
+                        }
+                    },
+                    "items": {
+                        "type": "array",
+                        "items": {
+                            "type": "object",
+                            "properties": {
+                                "sku": { "type": "string" },
+                                "qty": { "type": "integer" }
+                            }
+                        }
+                    },
+                    "metadata": {
+                        "type": "object",
+                        "additionalProperties": { "type": "integer" }
+                    },
+                    "duplicate_demo": {
+                        "type": "object",
+                        "properties": {
+                            "tag": { "type": "string" }
+                        }
+                    },
+                    "schema_mismatch_array": {
+                        "type": "array",
+                        "items": { "type": "integer" }
+                    }
+                }
+            }),
+            strict: None,
+        });
+        tools
+    }
+
+    fn order_arguments() -> String {
+        let shipping = element(
+            "shipping",
+            &format!(
+                "{}{}",
+                element("city", "Singapore"),
+                element("zip", "018956")
+            ),
+        );
+        let first_item = element(
+            "item",
+            &format!("{}{}", element("sku", "book-001"), element("qty", "2")),
+        );
+        let second_item = element(
+            "item",
+            &format!("{}{}", element("sku", "pen-007"), element("qty", "5")),
+        );
+        let items = element("items", &format!("{first_item}{second_item}"));
+        let metadata = element(
+            "metadata",
+            &format!("{}{}", element("score", "42"), element("rank", "7")),
+        );
+        let duplicate_demo = element(
+            "duplicate_demo",
+            &format!("{}{}", element("tag", "a"), element("tag", "b")),
+        );
+        let schema_mismatch_array = element(
+            "schema_mismatch_array",
+            &format!("{}{}", element("x", "1"), element("x", "2")),
+        );
+
+        [
+            element("user_id", "42"),
+            element("urgent", "true"),
+            element("note", "Please leave at front desk."),
+            shipping,
+            items,
+            metadata,
+            duplicate_demo,
+            schema_mismatch_array,
+            element(
+                "unknown_struct",
+                &format!("{}{}", element("a", "1"), element("a", "2")),
+            ),
+        ]
+        .join("")
+    }
+
+    #[test]
+    fn minimax_m3_parse_complete_without_tool_call_keeps_text() {
+        let mut parser = MinimaxM3ToolParser::new(&m3_test_tools());
+        let output = parser.parse_complete("Hello, world!").unwrap();
+
+        assert_eq!(output.normal_text, "Hello, world!");
+        assert!(output.calls.is_empty());
+    }
+
+    #[test]
+    fn minimax_m3_parse_complete_extracts_single_tool_call() {
+        let mut parser = MinimaxM3ToolParser::new(&m3_test_tools());
+        let output = parser
+            .parse_complete(&build_tool_block(&[(
+                "get_weather",
+                format!("{}{}", element("city", "Seattle"), element("days", "5")),
+            )]))
+            .unwrap();
+
+        assert!(output.normal_text.is_empty());
+        assert_eq!(output.calls.len(), 1);
+        assert_eq!(output.calls[0].name.as_deref(), Some("get_weather"));
+        assert_eq!(
+            serde_json::from_str::<Value>(&output.calls[0].arguments).unwrap(),
+            json!({ "city": "Seattle", "days": 5 })
+        );
+    }
+
+    #[test]
+    fn minimax_m3_parse_complete_preserves_prefix_and_ignores_trailing_text() {
+        let mut parser = MinimaxM3ToolParser::new(&m3_test_tools());
+        let output = format!(
+            "Let me check. {} This trailing text is ignored.",
+            build_tool_block(&[("get_weather", element("city", "Seattle"))])
+        );
+        let output = parser.parse_complete(&output).unwrap();
+
+        assert_eq!(output.normal_text, "Let me check. ");
+        assert_eq!(output.calls.len(), 1);
+    }
+
+    #[test]
+    fn minimax_m3_parse_complete_extracts_multiple_invokes() {
+        let mut parser = MinimaxM3ToolParser::new(&m3_test_tools());
+        let output = parser
+            .parse_complete(&build_tool_block(&[
+                ("get_weather", element("city", "Seattle")),
+                ("get_weather", element("city", "NYC")),
+            ]))
+            .unwrap();
+
+        assert_eq!(output.calls.len(), 2);
+        assert_eq!(output.calls[0].tool_index, 0);
+        assert_eq!(output.calls[1].tool_index, 1);
+        assert_eq!(
+            serde_json::from_str::<Value>(&output.calls[0].arguments).unwrap(),
+            json!({ "city": "Seattle" })
+        );
+        assert_eq!(
+            serde_json::from_str::<Value>(&output.calls[1].arguments).unwrap(),
+            json!({ "city": "NYC" })
+        );
+    }
+
+    #[test]
+    fn minimax_m3_invoke_body_junk_drops_rest_of_invoke() {
+        let mut parser = MinimaxM3ToolParser::new(&m3_test_tools());
+        let output = parser
+            .parse_complete(&build_tool_block(&[(
+                "get_weather",
+                [
+                    element("city", "Seattle"),
+                    "I need to use the city above.".to_string(),
+                    element("days", "5"),
+                ]
+                .join(""),
+            )]))
+            .unwrap();
+
+        assert_eq!(
+            serde_json::from_str::<Value>(&output.calls[0].arguments).unwrap(),
+            json!({ "city": "Seattle" })
+        );
+    }
+
+    #[test]
+    fn minimax_m3_parse_complete_converts_schema_types() {
+        let mut parser = MinimaxM3ToolParser::new(&m3_test_tools());
+        let output = parser
+            .parse_complete(&build_tool_block(&[(
+                "convert",
+                [
+                    element("whole", "5.0"),
+                    element("flag", "true"),
+                    element("payload", r#"{"nested":true}"#),
+                    element("items", "[1,2]"),
+                    element("empty", "42"),
+                ]
+                .join(""),
+            )]))
+            .unwrap();
+
+        assert_eq!(
+            serde_json::from_str::<Value>(&output.calls[0].arguments).unwrap(),
+            json!({
+                "whole": 5.0,
+                "flag": true,
+                "payload": { "nested": true },
+                "items": [1, 2],
+                "empty": "42",
+            })
+        );
+    }
+
+    #[test]
+    fn minimax_m3_parse_complete_converts_nested_arguments() {
+        let mut parser = MinimaxM3ToolParser::new(&m3_test_tools());
+        let output = parser
+            .parse_complete(&build_tool_block(&[("create_order", order_arguments())]))
+            .unwrap();
+
+        assert_eq!(
+            serde_json::from_str::<Value>(&output.calls[0].arguments).unwrap(),
+            json!({
+                "user_id": 42,
+                "urgent": true,
+                "note": "Please leave at front desk.",
+                "shipping": {
+                    "city": "Singapore",
+                    "zip": 18956
+                },
+                "items": [
+                    {
+                        "sku": "book-001",
+                        "qty": 2
+                    },
+                    {
+                        "sku": "pen-007",
+                        "qty": 5
+                    }
+                ],
+                "metadata": {
+                    "score": 42,
+                    "rank": 7
+                },
+                "duplicate_demo": {
+                    "tag": ["a", "b"]
+                },
+                "schema_mismatch_array": [1, 2],
+                "unknown_struct": {
+                    "a": ["1", "2"]
+                }
+            })
+        );
+    }
+
+    #[test]
+    fn minimax_m3_parse_complete_handles_multiline_leaf_parameters() {
+        let mut parser = MinimaxM3ToolParser::new(&m3_test_tools());
+        let output = parser
+            .parse_complete(&build_tool_block(&[(
+                "calculate_area",
+                [
+                    element("shape", "\nrectangle\n"),
+                    element("dimensions", r#"{"width":10,"height":20}"#),
+                    element("precision", "2"),
+                ]
+                .join(""),
+            )]))
+            .unwrap();
+
+        assert_eq!(
+            serde_json::from_str::<Value>(&output.calls[0].arguments).unwrap(),
+            json!({
+                "shape": "\nrectangle\n",
+                "dimensions": { "width": 10, "height": 20 },
+                "precision": 2,
+            })
+        );
+    }
+
+    #[test]
+    fn minimax_m3_streaming_extracts_single_tool_call() {
+        let mut parser = MinimaxM3ToolParser::new(&m3_test_tools());
+        let output = collect_stream(
+            &mut parser,
+            &[
+                TOOL_CALL_START,
+                &invoke("get_weather", &element("city", "Seattle")),
+                TOOL_CALL_END,
+            ],
+        );
+
+        assert!(output.normal_text.is_empty());
+        assert_eq!(output.calls.len(), 1);
+        assert_eq!(output.calls[0].name.as_deref(), Some("get_weather"));
+        assert_eq!(
+            serde_json::from_str::<Value>(&output.calls[0].arguments).unwrap(),
+            json!({ "city": "Seattle" })
+        );
+    }
+
+    #[test]
+    fn minimax_m3_streaming_preserves_prefix_text() {
+        let mut parser = MinimaxM3ToolParser::new(&m3_test_tools());
+        let output = collect_stream(
+            &mut parser,
+            &[
+                "Let me check. ",
+                TOOL_CALL_START,
+                &invoke("get_weather", &element("city", "Seattle")),
+                TOOL_CALL_END,
+            ],
+        );
+
+        assert_eq!(output.normal_text, "Let me check. ");
+        assert_eq!(output.calls.len(), 1);
+    }
+
+    #[test]
+    fn minimax_m3_streaming_without_tool_call_emits_text_incrementally() {
+        let mut parser = MinimaxM3ToolParser::new(&m3_test_tools());
+        let output = collect_stream(&mut parser, &["Hello, ", "world!"]);
+
+        assert_eq!(output.normal_text, "Hello, world!");
+        assert!(output.calls.is_empty());
+    }
+
+    #[test]
+    fn minimax_m3_streaming_handles_marker_split_across_chunks() {
+        let text = build_tool_block(&[("get_weather", element("city", "Seattle"))]);
+        let chunks = split_by_chars(&text, 3);
+        let mut parser = MinimaxM3ToolParser::new(&m3_test_tools());
+        let output = collect_stream(&mut parser, &chunks);
+
+        assert_eq!(output.calls.len(), 1);
+        assert!(output.normal_text.is_empty());
+    }
+
+    #[test]
+    fn minimax_m3_streaming_extracts_multiple_invokes_in_order() {
+        let text = build_tool_block(&[
+            ("get_weather", element("city", "Seattle")),
+            ("get_weather", element("city", "NYC")),
+        ]);
+        let chunks = split_by_chars(&text, 7);
+        let mut parser = MinimaxM3ToolParser::new(&m3_test_tools());
+        let output = collect_stream(&mut parser, &chunks);
+
+        assert_eq!(output.calls.len(), 2);
+        assert_eq!(output.calls[0].tool_index, 0);
+        assert_eq!(output.calls[1].tool_index, 1);
+    }
+
+    #[test]
+    fn minimax_m3_streaming_does_not_emit_incomplete_tool_call() {
+        let mut parser = MinimaxM3ToolParser::new(&m3_test_tools());
+        let output = parser
+            .parse_chunk(&format!(
+                "{TOOL_CALL_START}{INVOKE_START} name=\"get_weather\">"
+            ))
+            .unwrap();
+
+        assert!(output.normal_text.is_empty());
+        assert!(output.calls.is_empty());
+    }
+
+    #[test]
+    fn minimax_m3_streaming_ignores_text_after_tool_block() {
+        let text = format!(
+            "{} ignored",
+            build_tool_block(&[("get_weather", element("city", "Seattle"))])
+        );
+        let chunks = split_by_chars(&text, 5);
+        let mut parser = MinimaxM3ToolParser::new(&m3_test_tools());
+        let output = collect_stream(&mut parser, &chunks);
+
+        assert!(output.normal_text.is_empty());
+        assert_eq!(output.calls.len(), 1);
+    }
+
+    #[test]
+    fn minimax_m3_finish_fails_incomplete_tool_call() {
+        let mut parser = MinimaxM3ToolParser::new(&m3_test_tools());
+        parser
+            .parse_chunk(&format!(
+                "{TOOL_CALL_START}{INVOKE_START} name=\"get_weather\">"
+            ))
+            .unwrap();
+
+        assert!(parser.finish().is_err());
+    }
+
+    #[test]
+    fn minimax_m3_finish_recovers_after_bare_tool_block_start() {
+        let mut parser = MinimaxM3ToolParser::new(&m3_test_tools());
+        parser.parse_chunk(TOOL_CALL_START).unwrap();
+
+        let output = parser.finish().unwrap();
+        assert!(output.normal_text.is_empty());
+        assert!(output.calls.is_empty());
+    }
+
+    #[test]
+    fn minimax_m3_finish_recovers_completed_invoke_with_whitespace_tail() {
+        let mut parser = MinimaxM3ToolParser::new(&m3_test_tools());
+        let output = parser
+            .parse_complete(&format!(
+                "{}\n{}\n  \n",
+                TOOL_CALL_START,
+                invoke("get_weather", &element("city", "Seattle"))
+            ))
+            .unwrap();
+
+        assert_eq!(output.calls.len(), 1);
+        assert_eq!(
+            serde_json::from_str::<Value>(&output.calls[0].arguments).unwrap(),
+            json!({ "city": "Seattle" })
+        );
+    }
+
+    #[test]
+    fn minimax_m3_finish_fails_partial_outer_end_marker() {
+        let mut parser = MinimaxM3ToolParser::new(&m3_test_tools());
+        parser
+            .parse_chunk(&format!(
+                "{}\n{}\n{}",
+                TOOL_CALL_START,
+                invoke("get_weather", &element("city", "Seattle")),
+                &TOOL_CALL_END[..3]
+            ))
+            .unwrap();
+
+        assert!(parser.finish().is_err());
+    }
+
+    #[test]
+    fn minimax_m3_malformed_tool_call_fails_fast() {
+        let mut parser = MinimaxM3ToolParser::new(&m3_test_tools());
+        let error = parser
+            .parse_chunk(&format!(
+                "{TOOL_CALL_START}{ELEMENT_START}bad>{TOOL_CALL_END}"
+            ))
+            .unwrap_err();
+
+        expect!["tool parser parsing failed: "].assert_eq(&error.to_report_string());
+    }
+
+    #[test]
+    fn minimax_m3_mixed_content_is_preserved_as_text_field() {
+        let mut parser = MinimaxM3ToolParser::new(&m3_test_tools());
+        let body = element(
+            "payload",
+            &format!("text before {} text after", element("child", "value")),
+        );
+        let output = parser.parse_complete(&build_tool_block(&[("convert", body)])).unwrap();
+
+        assert_eq!(
+            serde_json::from_str::<Value>(&output.calls[0].arguments).unwrap(),
+            json!({
+                "payload": {
+                    "child": "value",
+                    "$text": "text before  text after"
+                }
+            })
+        );
+    }
+
+    #[test]
+    fn minimax_m3_mixed_text_field_avoids_child_name_collision() {
+        let mut parser = MinimaxM3ToolParser::new(&m3_test_tools());
+        let body = element(
+            "payload",
+            &format!(
+                "text{}{}",
+                element("$text", "child text"),
+                element("child", "value")
+            ),
+        );
+        let output = parser.parse_complete(&build_tool_block(&[("convert", body)])).unwrap();
+
+        assert_eq!(
+            serde_json::from_str::<Value>(&output.calls[0].arguments).unwrap(),
+            json!({
+                "payload": {
+                    "$text": "child text",
+                    "$$text": "text",
+                    "child": "value"
+                }
+            })
+        );
+    }
+}

setup.py

@@ -432,6 +432,19 @@ class cmake_build_ext(build_ext):
                     dirs_exist_ok=True,
                 )
 
+            # copy vendored fmha_sm100 package from build_lib to source tree
+            # for editable installs
+            fmha_sm100_build = os.path.join(
+                self.build_lib, "vllm", "third_party", "fmha_sm100"
+            )
+            if os.path.exists(fmha_sm100_build):
+                print(f"Copying {fmha_sm100_build} to vllm/third_party/fmha_sm100")
+                shutil.copytree(
+                    fmha_sm100_build,
+                    "vllm/third_party/fmha_sm100",
+                    dirs_exist_ok=True,
+                )
+
 
 class precompiled_build_ext(build_ext):
     """Disables extension building when using precompiled binaries."""
@@ -787,6 +800,7 @@ class precompiled_wheel_utils:
                 )
                 # DeepGEMM: extract all files (.py, .so, .cuh, .h, .hpp, etc.)
                 deep_gemm_regex = re.compile(r"vllm/third_party/deep_gemm/.*")
+                fmha_sm100_regex = re.compile(r"vllm/third_party/fmha_sm100/.*")
                 file_members = []
                 for member in wheel.filelist:
                     if member.filename in exact_members:
@@ -812,6 +826,7 @@ class precompiled_wheel_utils:
                         or triton_kernels_regex.match(member.filename)
                         or flashmla_regex.match(member.filename)
                         or deep_gemm_regex.match(member.filename)
+                        or fmha_sm100_regex.match(member.filename)
                     ):
                         file_members.append(member)
 
@@ -1120,6 +1135,8 @@ if _is_cuda():
         # DeepGEMM requires CUDA 12.3+ (SM90/SM100)
         # Optional since it won't build on unsupported architectures
         ext_modules.append(CMakeExtension(name="vllm._deep_gemm_C", optional=True))
+    # fmha_sm100 is a Python/CuTe-DSL package installed into vllm.third_party.
+    ext_modules.append(CMakeExtension(name="vllm.fmha_sm100", optional=True))
 
 if _is_cpu():
     import platform
@@ -1150,6 +1167,8 @@ package_data = {
         "third_party/deep_gemm/include/**/*.cuh",
         "third_party/deep_gemm/include/**/*.h",
         "third_party/deep_gemm/include/**/*.hpp",
+        # fmha_sm100 sparse CuTe-DSL helper kernels (vendored via cmake)
+        "third_party/fmha_sm100/cute/src/sm100/build_k2q_csr/build_k2q_csr.cu",
     ]
 }
 

tests/distributed/_test_push_ar_worker.py

@@ -0,0 +1,61 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""
+Worker helper for push allreduce unit tests.
+Run via torch.multiprocessing.spawn from test_push_all_reduce.py.
+
+Provides init/teardown helpers that create separate gloo (CPU) and
+nccl (device) process groups for PushAllReduce (which needs gloo for
+IPC handle exchange) and NCCL reference reduction (which needs nccl).
+"""
+
+import os
+import socket
+
+import torch
+import torch.distributed as dist
+
+
+def find_free_port() -> int:
+    """Find a free TCP port for distributed init."""
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        s.bind(("localhost", 0))
+        return s.getsockname()[1]
+
+
+# Global references to process groups created by init_groups
+_cpu_group = None
+_nccl_group = None
+
+
+def init_groups(rank: int, world_size: int, port: int):
+    """Initialize gloo (CPU) and nccl process groups.
+
+    PushAllReduce uses the gloo group for IPC handle exchange.
+    NCCL group is used for reference allreduce.
+    """
+    global _cpu_group, _nccl_group
+
+    os.environ["MASTER_ADDR"] = "localhost"
+    os.environ["MASTER_PORT"] = str(port)
+
+    torch.accelerator.set_device_index(rank)
+
+    dist.init_process_group(backend="gloo", rank=rank, world_size=world_size)
+    _cpu_group = dist.group.WORLD
+
+    # Create a separate NCCL group for reference allreduce
+    _nccl_group = dist.new_group(backend="nccl")
+
+
+def get_cpu_group():
+    return _cpu_group
+
+
+def get_nccl_group():
+    return _nccl_group
+
+
+def teardown():
+    """Clean up distributed groups."""
+    dist.destroy_process_group()

tests/distributed/test_pipeline_parallel.py

@@ -155,9 +155,6 @@ TEXT_GENERATION_MODELS = {
     "stabilityai/stablelm-3b-4e1t": PPTestSettings.fast(),
     "bigcode/starcoder2-3b": PPTestSettings.fast(),
     "upstage/solar-pro-preview-instruct": PPTestSettings.fast(load_format="dummy"),
-    # FIXME: Cannot load tokenizer in latest transformers version.
-    # Need to use tokenizer from `meta-llama/Llama-2-7b-chat-hf`
-    # "xverse/XVERSE-7B-Chat": PPTestSettings.fast(),
     # [Encoder-only]
     # TODO: Implement PP
     # "facebook/bart-base": PPTestSettings.fast(),

tests/entrypoints/openai/chat_completion/test_serving_chat.py

@@ -23,7 +23,11 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
     ChatCompletionRequest,
     ChatCompletionResponse,
 )
-from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
+from vllm.entrypoints.openai.chat_completion.serving import (
+    OpenAIServingChat,
+    _get_mm_token_counts,
+    _make_prompt_tokens_details,
+)
 from vllm.entrypoints.openai.engine.protocol import (
     ErrorResponse,
     RequestResponseMetadata,
@@ -37,6 +41,7 @@ from vllm.entrypoints.openai.parser.harmony_utils import get_encoding
 from vllm.entrypoints.serve.render.serving import OpenAIServingRender
 from vllm.exceptions import VLLMValidationError
 from vllm.inputs import TokensPrompt
+from vllm.multimodal.inputs import PlaceholderRange
 from vllm.outputs import CompletionOutput, RequestOutput
 from vllm.parser import HarmonyParser
 from vllm.renderers.hf import HfRenderer
@@ -635,6 +640,37 @@ def test_async_serving_chat_init():
     assert serving_completion.chat_template == CHAT_TEMPLATE
 
 
+def test_mm_prompt_tokens_details():
+    # Text-only input has no multimodal placeholders.
+    assert _get_mm_token_counts({"type": "tokens"}) == {}
+
+    # Per-modality counts sum each modality's placeholder ranges.
+    counts = _get_mm_token_counts(
+        {
+            "mm_placeholders": {
+                "image": [
+                    PlaceholderRange(offset=0, length=576),
+                    PlaceholderRange(offset=600, length=24),
+                ],
+                "video": [PlaceholderRange(offset=700, length=1200)],
+            }
+        }
+    )
+    assert counts == {"image": 600, "video": 1200}
+
+    # Gated off, or nothing to report -> no details.
+    assert _make_prompt_tokens_details(False, 5, counts) is None
+    assert _make_prompt_tokens_details(True, None, None) is None
+
+    # Zero cached_tokens is still reported (not None), matching the cached-only
+    # behavior; multimodal counts ride alongside even when cached_tokens is None.
+    assert _make_prompt_tokens_details(True, 0, None).cached_tokens == 0
+    details = _make_prompt_tokens_details(True, None, counts)
+    assert details.cached_tokens is None
+    assert details.multimodal_tokens == {"image": 600, "video": 1200}
+    assert _make_prompt_tokens_details(True, 3, counts).cached_tokens == 3
+
+
 @pytest.mark.asyncio
 async def test_serving_chat_returns_correct_model_name():
     mock_engine = MagicMock(spec=AsyncLLM)
@@ -1314,6 +1350,21 @@ class TestServingChatWithHarmony:
             else serving_chat.chat_completion_full_generator
         )
 
+        chat_template_kwargs = serving_chat._effective_chat_template_kwargs(req)
+        if stream:
+            extra_kwargs: dict[str, Any] = {
+                "chat_template_kwargs": chat_template_kwargs,
+            }
+        else:
+            parser = None
+            if serving_chat.parser_cls is not None:
+                parser = serving_chat.parser_cls(
+                    tokenizer,
+                    req.tools,
+                    chat_template_kwargs=chat_template_kwargs,
+                )
+            extra_kwargs = {"parser": parser}
+
         result = generator_func(
             request=req,
             result_generator=result_generator(),
@@ -1325,7 +1376,7 @@ class TestServingChatWithHarmony:
                 request_id=req.request_id,
                 model_name=req.model,
             ),
-            chat_template_kwargs=serving_chat._effective_chat_template_kwargs(req),
+            **extra_kwargs,
         )
 
         if stream:

tests/entrypoints/speech_to_text/test_upload_size_limit.py

@@ -0,0 +1,142 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Regression tests for the speech-to-text upload size pre-check.
+
+These tests verify that over-limit audio uploads are rejected *before*
+the full file is materialized into memory, closing the vulnerability
+where vLLM would allocate memory proportional to an oversized upload
+before enforcing the VLLM_MAX_AUDIO_CLIP_FILESIZE_MB limit.
+"""
+
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from vllm.entrypoints.speech_to_text.base.utils import read_upload_with_limit
+from vllm.exceptions import VLLMValidationError
+
+
+def _make_upload_file(data: bytes, *, size: int | None = None) -> AsyncMock:
+    """Create a mock UploadFile that yields data in chunks."""
+    mock = AsyncMock()
+    mock.size = size
+
+    offset = 0
+
+    async def _read(n: int = -1):
+        nonlocal offset
+        if n <= 0:
+            chunk = data[offset:]
+            offset = len(data)
+            return chunk
+        chunk = data[offset : offset + n]
+        offset += len(chunk)
+        return chunk
+
+    mock.read = AsyncMock(side_effect=_read)
+    return mock
+
+
+@pytest.mark.asyncio
+async def test_rejects_oversized_upload_via_content_length():
+    """File is rejected early when file.size exceeds the limit."""
+    max_mb = 1
+    oversized_bytes = max_mb * 1024 * 1024 + 1
+
+    upload = _make_upload_file(b"", size=oversized_bytes)
+
+    with pytest.raises(VLLMValidationError, match="Maximum file size exceeded"):
+        await read_upload_with_limit(upload, max_size_mb=max_mb)
+
+    upload.read.assert_not_called()
+
+
+@pytest.mark.asyncio
+async def test_rejects_oversized_upload_via_chunked_read():
+    """File is rejected mid-read without materializing the full content."""
+    max_mb = 1
+    max_bytes = max_mb * 1024 * 1024
+    oversized_data = b"\x00" * (max_bytes + 1024)
+
+    upload = _make_upload_file(oversized_data, size=None)
+
+    with pytest.raises(VLLMValidationError, match="Maximum file size exceeded"):
+        await read_upload_with_limit(upload, max_size_mb=max_mb)
+
+
+@pytest.mark.asyncio
+async def test_accepts_file_within_limit():
+    """File within the limit is read successfully."""
+    max_mb = 1
+    data = b"\x00" * (512 * 1024)  # 512 KiB, well under 1 MB
+
+    upload = _make_upload_file(data, size=len(data))
+    result = await read_upload_with_limit(upload, max_size_mb=max_mb)
+
+    assert result == data
+
+
+@pytest.mark.asyncio
+async def test_accepts_file_at_exact_limit():
+    """File exactly at the limit boundary is accepted."""
+    max_mb = 1
+    max_bytes = max_mb * 1024 * 1024
+    data = b"\x00" * max_bytes
+
+    upload = _make_upload_file(data, size=len(data))
+    result = await read_upload_with_limit(upload, max_size_mb=max_mb)
+
+    assert result == data
+
+
+@pytest.mark.asyncio
+async def test_rejects_at_one_byte_over_limit():
+    """File one byte over the limit is rejected."""
+    max_mb = 1
+    max_bytes = max_mb * 1024 * 1024
+    data = b"\x00" * (max_bytes + 1)
+
+    upload = _make_upload_file(data, size=None)
+
+    with pytest.raises(VLLMValidationError, match="Maximum file size exceeded"):
+        await read_upload_with_limit(upload, max_size_mb=max_mb)
+
+
+@pytest.mark.asyncio
+async def test_uses_env_default_when_no_limit_specified():
+    """Uses VLLM_MAX_AUDIO_CLIP_FILESIZE_MB when max_size_mb is not given."""
+    with patch("vllm.entrypoints.speech_to_text.base.utils.envs") as mock_envs:
+        mock_envs.VLLM_MAX_AUDIO_CLIP_FILESIZE_MB = 2
+        max_bytes = 2 * 1024 * 1024
+        oversized_data = b"\x00" * (max_bytes + 1)
+
+        upload = _make_upload_file(oversized_data, size=None)
+
+        with pytest.raises(VLLMValidationError, match="Maximum file size exceeded"):
+            await read_upload_with_limit(upload)
+
+
+@pytest.mark.asyncio
+async def test_chunked_read_does_not_fully_materialize():
+    """Verify that for large oversized files, we stop reading early.
+
+    The function reads in 64 KiB chunks and aborts once the accumulated
+    size exceeds the limit. We confirm that far fewer read calls were made
+    than would be required to fully materialize the file.
+    """
+    max_mb = 1
+    max_bytes = max_mb * 1024 * 1024
+    large_size = max_bytes * 10  # 10x the limit
+    data = b"\x00" * large_size
+
+    upload = _make_upload_file(data, size=None)
+
+    with pytest.raises(VLLMValidationError):
+        await read_upload_with_limit(upload, max_size_mb=max_mb)
+
+    chunk_size = 64 * 1024
+    calls_for_full_read = large_size // chunk_size + 1
+    calls_to_exceed_limit = max_bytes // chunk_size + 1
+    actual_calls = upload.read.call_count
+    assert actual_calls <= calls_to_exceed_limit + 1
+    assert actual_calls < calls_for_full_read

tests/kernels/attention/test_flashmla_sparse.py

@@ -122,3 +122,24 @@ def test_sparse_flashmla_prefill_smoke():
     assert out.shape == (s_q, h_q, d_v)
     assert max_logits.shape == (s_q, h_q)
     assert lse.shape == (s_q, h_q)
+
+
+def test_deepseek_v4_prefill_chunk_planning_expands_for_short_sequences():
+    from vllm.v1.attention.backends.mla.sparse_swa import DeepseekSparseSWAMetadata
+
+    metadata = DeepseekSparseSWAMetadata(
+        block_table=torch.empty(0, dtype=torch.int32),
+        slot_mapping=torch.empty(0, dtype=torch.int32),
+        block_size=64,
+        num_prefills=5,
+        prefill_seq_lens_cpu=torch.tensor([80, 96, 112, 128, 144], dtype=torch.int32),
+        prefill_query_lens_cpu=torch.tensor([4, 4, 4, 4, 4], dtype=torch.int32),
+        prefill_window_size=64,
+        prefill_max_model_len=1024,
+        prefill_max_num_batched_tokens=128,
+    )
+
+    chunk_plan = metadata.get_prefill_chunk_plan(compress_ratio=4, prefill_chunk_size=4)
+
+    # the adaptive plan keeps all 5 in one chunk
+    assert chunk_plan == [(0, 5, 36, 103)]

tests/kernels/attention/test_minimax_m3.py

@@ -0,0 +1,854 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Correctness tests for MiniMax M3 sparse prefill attention kernels."""
+
+import pytest
+import torch
+
+from vllm import _custom_ops as ops
+from vllm.models.minimax_m3.common.indexer import (
+    MiniMaxM3IndexerBackend,
+)
+from vllm.models.minimax_m3.common.ops.index_topk import (
+    minimax_m3_index_decode,
+    minimax_m3_index_score,
+    minimax_m3_index_topk,
+)
+from vllm.models.minimax_m3.common.ops.sparse_attn import (
+    minimax_m3_sparse_attn,
+    minimax_m3_sparse_attn_decode,
+)
+from vllm.models.minimax_m3.common.sparse_attention import (
+    MiniMaxM3SparseBackend,
+)
+from vllm.platforms import current_platform
+from vllm.v1.attention.backends.utils import set_kv_cache_layout
+from vllm.v1.kv_cache_interface import FullAttentionSpec, MLAAttentionSpec
+from vllm.v1.worker.gpu.attn_utils import _reshape_kv_cache
+from vllm.v1.worker.utils import AttentionGroup
+
+if not (current_platform.is_cuda() or current_platform.is_rocm()):
+    pytest.skip(
+        "MiniMax M3 attention kernels require CUDA or ROCm.",
+        allow_module_level=True,
+    )
+
+
+@pytest.fixture
+def kv_layout(request):
+    """Set the global KV cache layout for one test and restore it after."""
+    set_kv_cache_layout(request.param)
+    try:
+        yield request.param
+    finally:
+        set_kv_cache_layout(None)
+
+
+def _stride_order_for(backend: type[MiniMaxM3SparseBackend], ndim: int) -> tuple:
+    """Mirror the allocator's stride-order resolution (identity fallback)."""
+    try:
+        stride_order = backend.get_kv_cache_stride_order()
+        assert len(stride_order) == ndim
+    except (AttributeError, NotImplementedError):
+        stride_order = tuple(range(ndim))
+    return stride_order
+
+
+def _allocate_main_kv_via_contract(
+    num_pages: int, device: torch.device | str = "cuda"
+) -> torch.Tensor:
+    """Build the main KV cache exactly as the production allocator does for the
+    currently active layout: allocate the physical (permuted) tensor, then
+    expose the inverse-permuted logical-NHD view the backend sees."""
+    logical_shape = MiniMaxM3SparseBackend.get_kv_cache_shape(
+        num_pages, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM
+    )
+    stride_order = _stride_order_for(MiniMaxM3SparseBackend, len(logical_shape))
+    physical_shape = tuple(logical_shape[i] for i in stride_order)
+    inv_order = [stride_order.index(i) for i in range(len(stride_order))]
+    raw = torch.randn(physical_shape, device=device, dtype=DTYPE)
+    return raw.permute(*inv_order)
+
+
+NUM_Q_HEADS = 32
+NUM_KV_HEADS = 2
+HEAD_DIM = 128
+BLOCK_SIZE = 128
+DTYPE = torch.bfloat16
+SM_SCALE = HEAD_DIM**-0.5
+TOPK = 16
+
+
+# Index top-k kernels.
+def _reference_index_topk(
+    idx_q: torch.Tensor,
+    index_kv_cache: torch.Tensor,
+    block_table: torch.Tensor,
+    q_lens: torch.Tensor,
+    seq_lens: torch.Tensor,
+    prefix_lens: torch.Tensor,
+    topk: int,
+    init_blocks: int,
+    local_blocks: int,
+    sm_scale: float,
+) -> torch.Tensor:
+    total_q, num_idx_heads, _ = idx_q.shape
+    out = torch.full(
+        (num_idx_heads, total_q, topk), -1, device=idx_q.device, dtype=torch.int32
+    )
+
+    q_start = 0
+    for req_id, (q_len, seq_len, prefix_len) in enumerate(
+        zip(q_lens.tolist(), seq_lens.tolist(), prefix_lens.tolist())
+    ):
+        q_end = q_start + q_len
+        q = idx_q[q_start:q_end]
+        num_blocks = (seq_len + BLOCK_SIZE - 1) // BLOCK_SIZE
+        pages = block_table[req_id, :num_blocks]
+        k = index_kv_cache[pages].reshape(num_blocks * BLOCK_SIZE, -1)
+        score = torch.einsum("qhd,kd->hqk", q.float(), k.float()) * sm_scale
+
+        q_pos = prefix_len + torch.arange(q_len, device=idx_q.device)
+        k_pos = torch.arange(k.shape[0], device=idx_q.device)
+        score.masked_fill_(k_pos[None, :] > q_pos[:, None], -float("inf"))
+        score = score.reshape(num_idx_heads, q_len, num_blocks, BLOCK_SIZE)
+        score_tensor = score.max(dim=3).values
+
+        valid_blocks = (q_pos + BLOCK_SIZE) // BLOCK_SIZE
+        for local_q, num_valid_blocks in enumerate(valid_blocks.tolist()):
+            end = min(init_blocks, num_valid_blocks)
+            score_tensor[:, local_q, :end] = 1e30
+            start = max(0, num_valid_blocks - local_blocks)
+            score_tensor[:, local_q, start:num_valid_blocks] = 1e29
+
+            k = min(topk, num_valid_blocks)
+            topk_idx = score_tensor[:, local_q].topk(k, dim=1).indices
+            out[:, q_start + local_q, :k] = topk_idx
+        q_start = q_end
+
+    return out
+
+
+def _assert_topk_indices_equal_unordered(
+    actual: torch.Tensor,
+    expected: torch.Tensor,
+) -> None:
+    """Compare selected sparse blocks without requiring a deterministic order."""
+    assert actual.shape == expected.shape
+    actual_flat = actual.cpu().reshape(-1, actual.shape[-1]).tolist()
+    expected_flat = expected.cpu().reshape(-1, expected.shape[-1]).tolist()
+    for actual_row, expected_row in zip(actual_flat, expected_flat):
+        assert set(actual_row) == set(expected_row)
+
+
+def test_prefill_index_topk_correctness():
+    topk = 6
+    init_blocks = 0
+    local_blocks = 1
+    num_idx_heads = 2
+    head_dim = 16
+    q_lens = torch.tensor((4, 3), device="cuda", dtype=torch.int32)
+    prefix_lens = torch.tensor((0, 1024), device="cuda", dtype=torch.int32)
+    seq_lens = prefix_lens + q_lens
+    batch = q_lens.numel()
+    max_seq_len = seq_lens.max().item()
+    max_blocks = (max_seq_len + BLOCK_SIZE - 1) // BLOCK_SIZE
+    num_pages = batch * max_blocks
+
+    cu_seqlens = torch.zeros(batch + 1, device="cuda", dtype=torch.int32)
+    cu_seqlens[1:] = q_lens.cumsum(0)
+    block_table = torch.randperm(num_pages, device="cuda", dtype=torch.int32).reshape(
+        batch, max_blocks
+    )
+    idx_q = torch.ones(q_lens.sum().item(), num_idx_heads, head_dim, device="cuda")
+    index_kv_cache = torch.empty(num_pages, BLOCK_SIZE, head_dim, device="cuda")
+    for req_id in range(batch):
+        for block_id in range(max_blocks):
+            page = block_table[req_id, block_id]
+            index_kv_cache[page].fill_(block_id + 1)
+
+    score = minimax_m3_index_score(
+        idx_q,
+        index_kv_cache,
+        block_table,
+        cu_seqlens,
+        seq_lens,
+        prefix_lens,
+        max_query_len=q_lens.max().item(),
+        max_seq_len=max_seq_len,
+        num_kv_heads=num_idx_heads,
+        sm_scale=head_dim**-0.5,
+    )
+    actual = minimax_m3_index_topk(
+        score,
+        cu_seqlens,
+        prefix_lens,
+        max_query_len=q_lens.max().item(),
+        topk=topk,
+        init_blocks=init_blocks,
+        local_blocks=local_blocks,
+    )
+    expected = _reference_index_topk(
+        idx_q,
+        index_kv_cache,
+        block_table,
+        q_lens,
+        seq_lens,
+        prefix_lens,
+        topk,
+        init_blocks,
+        local_blocks,
+        head_dim**-0.5,
+    )
+    _assert_topk_indices_equal_unordered(actual, expected)
+
+
+@pytest.mark.parametrize("decode_query_len", [1, 4])
+@pytest.mark.parametrize("num_padded_reqs", [0, 2])
+def test_decode_index_topk_correctness(
+    decode_query_len: int,
+    num_padded_reqs: int,
+):
+    topk = 6
+    init_blocks = 0
+    local_blocks = 1
+    num_idx_heads = 2
+    head_dim = 16
+    active_seq_lens = torch.tensor((7, 129, 1025), device="cuda", dtype=torch.int32)
+    q_lens = torch.full_like(active_seq_lens, decode_query_len)
+    prefix_lens = active_seq_lens - decode_query_len
+    active_batch = active_seq_lens.numel()
+    batch = active_batch + num_padded_reqs
+    seq_lens = torch.cat(
+        [
+            active_seq_lens,
+            torch.zeros(num_padded_reqs, device="cuda", dtype=torch.int32),
+        ]
+    )
+    max_seq_len = active_seq_lens.max().item()
+    max_blocks = (max_seq_len + BLOCK_SIZE - 1) // BLOCK_SIZE
+    num_pages = active_batch * max_blocks
+
+    active_block_table = torch.randperm(
+        num_pages, device="cuda", dtype=torch.int32
+    ).reshape(active_batch, max_blocks)
+    block_table = torch.zeros(batch, max_blocks, device="cuda", dtype=torch.int32)
+    block_table[:active_batch] = active_block_table
+    idx_q = torch.randn(
+        batch * decode_query_len, num_idx_heads, head_dim, device="cuda"
+    )
+    index_kv_cache = torch.randn(num_pages, BLOCK_SIZE, head_dim, device="cuda")
+
+    actual = minimax_m3_index_decode(
+        idx_q,
+        index_kv_cache,
+        block_table,
+        seq_lens,
+        max_seq_len=max_seq_len,
+        topk=topk,
+        init_blocks=init_blocks,
+        local_blocks=local_blocks,
+        num_kv_heads=num_idx_heads,
+        sm_scale=head_dim**-0.5,
+        decode_query_len=decode_query_len,
+    )
+    expected = torch.full_like(actual, -1)
+    active_tokens = active_batch * decode_query_len
+    expected[:, :active_tokens] = _reference_index_topk(
+        idx_q[:active_tokens],
+        index_kv_cache,
+        block_table[:active_batch],
+        q_lens,
+        active_seq_lens,
+        prefix_lens,
+        topk,
+        init_blocks,
+        local_blocks,
+        head_dim**-0.5,
+    )
+    _assert_topk_indices_equal_unordered(actual, expected)
+
+
+# Sparse attention kernels.
+def _reference_sparse_attn(
+    q: torch.Tensor,
+    kv_cache: torch.Tensor,
+    topk_idx: torch.Tensor,
+    block_table: torch.Tensor,
+    q_lens: torch.Tensor,
+    seq_lens: torch.Tensor,
+    prefix_lens: torch.Tensor,
+) -> torch.Tensor:
+    out = torch.empty_like(q, dtype=torch.float32)
+    gqa_group_size = NUM_Q_HEADS // NUM_KV_HEADS
+    q_start = 0
+    for req_id, (q_len, seq_len, prefix_len) in enumerate(
+        zip(q_lens.tolist(), seq_lens.tolist(), prefix_lens.tolist())
+    ):
+        q_end = q_start + q_len
+        q_req = q[q_start:q_end]
+        positions = torch.arange(seq_len, device="cuda")
+        pages = block_table[req_id, positions // BLOCK_SIZE]
+        rows = positions % BLOCK_SIZE
+        k_req = kv_cache[pages, 0, rows]
+        v_req = kv_cache[pages, 1, rows].float()
+
+        q_pos = prefix_len + torch.arange(q_len, device="cuda")
+        key_blocks = positions // BLOCK_SIZE
+        causal_mask = positions.unsqueeze(0) <= q_pos.unsqueeze(1)
+
+        for kv_head in range(NUM_KV_HEADS):
+            selected = topk_idx[kv_head, q_start:q_end]
+            selected_mask = (key_blocks[None, :, None] == selected[:, None, :]).any(-1)
+            mask = causal_mask & selected_mask
+            head_start = kv_head * gqa_group_size
+            head_end = head_start + gqa_group_size
+
+            q_heads = q_req[:, head_start:head_end].transpose(0, 1)
+            k_head = k_req[:, kv_head].T.expand(gqa_group_size, -1, -1)
+            scores = torch.bmm(q_heads, k_head, out_dtype=torch.float32)
+            scores = scores.transpose(0, 1) * SM_SCALE
+            probs = torch.softmax(
+                scores.masked_fill(~mask[:, None, :], -float("inf")), -1
+            )
+            out[q_start:q_end, head_start:head_end] = torch.einsum(
+                "qhk,kd->qhd", probs, v_req[:, kv_head]
+            )
+        q_start += q_len
+    return out.to(q.dtype)
+
+
+@pytest.mark.parametrize("kv_layout", ["NHD", "HND"], indirect=True)
+@pytest.mark.parametrize(
+    ("q_lens", "kv_lens"),
+    [
+        ((129, 257), (129, 257)),
+        ((65, 129, 257), (129, 257, 385)),
+    ],
+)
+def test_prefill_sparse_attention_correctness(
+    kv_layout: str,
+    q_lens: tuple[int, ...],
+    kv_lens: tuple[int, ...],
+):
+    assert len(q_lens) == len(kv_lens)
+    assert all(kv_len >= q_len for q_len, kv_len in zip(q_lens, kv_lens))
+
+    # Build paged-KV metadata, including a non-identity page order.
+    batch = len(q_lens)
+    pages_per_req = [(kv_len + BLOCK_SIZE - 1) // BLOCK_SIZE for kv_len in kv_lens]
+    max_blocks = max(pages_per_req)
+    num_pages = sum(pages_per_req)
+    physical_pages = torch.randperm(num_pages, device="cuda", dtype=torch.int32)
+    block_table = torch.zeros(batch, max_blocks, device="cuda", dtype=torch.int32)
+    base_page = 0
+    for req_id, num_req_pages in enumerate(pages_per_req):
+        block_table[req_id, :num_req_pages] = physical_pages[
+            base_page : base_page + num_req_pages
+        ]
+        base_page += num_req_pages
+
+    q_lens_t = torch.tensor(q_lens, device="cuda", dtype=torch.int32)
+    seq_lens = torch.tensor(kv_lens, device="cuda", dtype=torch.int32)
+    prefix_lens = seq_lens - q_lens_t
+    cu_seqlens = torch.zeros(batch + 1, device="cuda", dtype=torch.int32)
+    cu_seqlens[1:] = q_lens_t.cumsum(0)
+    total_q = sum(q_lens)
+    max_seqlen_q = max(q_lens)
+
+    q_shape = (total_q, NUM_Q_HEADS, HEAD_DIM)
+    q = torch.randn(q_shape, device="cuda", dtype=DTYPE)
+    # Allocate the main KV cache through the backend layout contract so the
+    # physical storage matches the active layout (contiguous NHD or strided
+    # HND), while the kernels and reference see the logical-NHD view.
+    kv_cache = _allocate_main_kv_via_contract(num_pages)
+
+    # Build sparse block indices with the same contract as the real M3 indexer:
+    # one forced local block, then score-selected older causal blocks.
+    topk_shape = (NUM_KV_HEADS, total_q, TOPK)
+    topk_idx = torch.full(topk_shape, -1, device="cuda", dtype=torch.int32)
+    q_start = 0
+    for q_len, prefix_len in zip(q_lens_t.tolist(), prefix_lens.tolist()):
+        for local_q in range(q_len):
+            current_block = (prefix_len + local_q) // BLOCK_SIZE
+            older_blocks = torch.randperm(
+                current_block, device="cuda", dtype=torch.int32
+            )
+            selected = torch.cat(
+                [
+                    torch.tensor([current_block], device="cuda", dtype=torch.int32),
+                    older_blocks[: TOPK - 1],
+                ]
+            )
+            topk_idx[:, q_start + local_q, : selected.numel()] = selected
+        q_start += q_len
+
+    actual = torch.empty_like(q)
+    minimax_m3_sparse_attn(
+        q,
+        kv_cache,
+        topk_idx,
+        block_table,
+        cu_seqlens,
+        seq_lens,
+        prefix_lens,
+        max_seqlen_q,
+        NUM_KV_HEADS,
+        SM_SCALE,
+        actual,
+    )
+
+    expected = _reference_sparse_attn(
+        q,
+        kv_cache,
+        topk_idx,
+        block_table,
+        q_lens_t,
+        seq_lens,
+        prefix_lens,
+    )
+    torch.accelerator.synchronize()
+
+    error = (actual.float() - expected.float()).abs()
+    assert error.mean().item() < 2.5e-4
+    assert error.max().item() < 1.7e-2
+
+
+def test_main_backend_layout_contract():
+    """The main sparse backend exposes the logical-NHD shape and the
+    flash_attn-style stride order for each layout."""
+    nb, bs, h, d = 7, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM
+    logical = MiniMaxM3SparseBackend.get_kv_cache_shape(nb, bs, h, d)
+    assert logical == (nb, 2, bs, h, d)
+    # The old HND-ordered shape is no longer the logical shape.
+    assert logical != (nb, 2, h, bs, d)
+
+    try:
+        set_kv_cache_layout("HND")
+        assert MiniMaxM3SparseBackend.get_kv_cache_stride_order() == (0, 1, 3, 2, 4)
+        set_kv_cache_layout("NHD")
+        assert MiniMaxM3SparseBackend.get_kv_cache_stride_order() == (0, 1, 2, 3, 4)
+    finally:
+        set_kv_cache_layout(None)
+
+    for layout in ("NHD", "HND"):
+        try:
+            set_kv_cache_layout(layout)
+            order = MiniMaxM3SparseBackend.get_kv_cache_stride_order()
+        finally:
+            set_kv_cache_layout(None)
+        # Valid permutation: no duplicates, covers every axis.
+        assert set(order) == set(range(len(order)))
+
+    # M3 has no cross-layer KV blocks.
+    with pytest.raises(NotImplementedError):
+        MiniMaxM3SparseBackend.get_kv_cache_stride_order(
+            include_num_layers_dimension=True
+        )
+
+
+def test_main_backend_unknown_layout_raises(monkeypatch):
+    """An unrecognized layout (injected past env-var validation) is rejected."""
+    import vllm.models.minimax_m3.common.sparse_attention as sparse_attn_mod
+
+    monkeypatch.setattr(sparse_attn_mod, "get_kv_cache_layout", lambda: "BOGUS")
+    with pytest.raises(ValueError, match="Unknown cache layout format"):
+        MiniMaxM3SparseBackend.get_kv_cache_stride_order()
+
+
+def test_indexer_backend_stride_order_is_identity():
+    """The 3-dim indexer cache must not inherit the parent's 5-element stride
+    order; it overrides to the 3-element identity so the allocator keeps the
+    contiguous layout."""
+    assert MiniMaxM3IndexerBackend.get_kv_cache_stride_order() == (0, 1, 2)
+
+    # Cross-layer (per-layer-stacked) KV blocks are not supported.
+    with pytest.raises(NotImplementedError):
+        MiniMaxM3IndexerBackend.get_kv_cache_stride_order(
+            include_num_layers_dimension=True
+        )
+
+    # The stride order matches the 3-dim indexer shape rank.
+    indexer_shape = MiniMaxM3IndexerBackend.get_kv_cache_shape(
+        5, BLOCK_SIZE, 1, HEAD_DIM
+    )
+    assert len(indexer_shape) == 3
+    assert _stride_order_for(MiniMaxM3IndexerBackend, len(indexer_shape)) == (0, 1, 2)
+
+
+def test_hnd_allocation_is_byte_identical_to_transpose():
+    """Under HND the backend-visible logical view is byte-identical to the
+    pre-change allocate-HND-then-transpose(2, 3) workaround."""
+    nb, bs, h, d = 4, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM
+    logical = MiniMaxM3SparseBackend.get_kv_cache_shape(nb, bs, h, d)
+    try:
+        set_kv_cache_layout("HND")
+        stride_order = MiniMaxM3SparseBackend.get_kv_cache_stride_order()
+    finally:
+        set_kv_cache_layout(None)
+
+    physical_shape = tuple(logical[i] for i in stride_order)
+    # The physical (permuted) shape equals the old hardcoded HND shape.
+    assert physical_shape == (nb, 2, h, bs, d)
+
+    inv_order = [stride_order.index(i) for i in range(len(stride_order))]
+    raw = torch.empty(physical_shape, device="cuda", dtype=DTYPE)
+    view = raw.permute(*inv_order)
+    expected = raw.view((nb, 2, h, bs, d)).transpose(2, 3)
+
+    assert view.shape == expected.shape
+    assert view.stride() == expected.stride()
+    assert view.storage_offset() == expected.storage_offset()
+
+    # Negative: the identity (wrong) stride order under HND does not reproduce
+    # the transpose view.
+    wrong_view = raw.view(logical)
+    assert wrong_view.stride() != expected.stride()
+
+
+def test_main_cache_is_block_first_and_unpadded():
+    """The allocator's contiguous-view branch (not the padded-strided branch)
+    is used for the main GQA cache: its spec is unpadded and the physical
+    layout keeps num_blocks as the first dimension under both layouts."""
+    from vllm.v1.kv_cache_interface import FullAttentionSpec
+
+    spec = FullAttentionSpec(
+        block_size=BLOCK_SIZE,
+        num_kv_heads=NUM_KV_HEADS,
+        head_size=HEAD_DIM,
+        head_size_v=HEAD_DIM,
+        dtype=DTYPE,
+    )
+    # Unpadded -> allocator uses kv_tensor.view(...) rather than as_strided().
+    assert spec.page_size_padded is None
+
+    logical = MiniMaxM3SparseBackend.get_kv_cache_shape(
+        4, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM
+    )
+    for layout in ("NHD", "HND"):
+        try:
+            set_kv_cache_layout(layout)
+            order = MiniMaxM3SparseBackend.get_kv_cache_stride_order()
+        finally:
+            set_kv_cache_layout(None)
+        inv_order = [order.index(i) for i in range(len(order))]
+        # Physical first dim is num_blocks (block-first); required by the
+        # padded-strided branch's block-first assumption if it were ever taken.
+        assert inv_order[0] == 0
+        assert logical[order[0]] == logical[0]
+
+
+def _build_decode_inputs(
+    seq_lens_list: tuple[int, ...],
+    decode_query_len: int = 1,
+    num_padded_reqs: int = 0,
+):
+    """Shared decode setup: uniform query tokens per request, a non-identity
+    block table, and topk indices selecting the current block plus older causal
+    blocks for each query token."""
+    active_batch = len(seq_lens_list)
+    batch = active_batch + num_padded_reqs
+    pages_per_req = [(s + BLOCK_SIZE - 1) // BLOCK_SIZE for s in seq_lens_list]
+    max_blocks = max(pages_per_req)
+    num_pages = sum(pages_per_req)
+    physical_pages = torch.randperm(num_pages, device="cuda", dtype=torch.int32)
+    block_table = torch.zeros(batch, max_blocks, device="cuda", dtype=torch.int32)
+    base_page = 0
+    for req_id, num_req_pages in enumerate(pages_per_req):
+        block_table[req_id, :num_req_pages] = physical_pages[
+            base_page : base_page + num_req_pages
+        ]
+        base_page += num_req_pages
+
+    seq_lens = torch.tensor(
+        (*seq_lens_list, *([0] * num_padded_reqs)),
+        device="cuda",
+        dtype=torch.int32,
+    )
+    q = torch.randn(
+        batch * decode_query_len, NUM_Q_HEADS, HEAD_DIM, device="cuda", dtype=DTYPE
+    )
+
+    topk_idx = torch.full(
+        (NUM_KV_HEADS, batch * decode_query_len, TOPK),
+        -1,
+        device="cuda",
+        dtype=torch.int32,
+    )
+    token_id = 0
+    for req_id, seq_len in enumerate(seq_lens_list):
+        for local_q in range(decode_query_len):
+            query_pos = seq_len - decode_query_len + local_q
+            current_block = query_pos // BLOCK_SIZE
+            older_blocks = torch.randperm(
+                current_block, device="cuda", dtype=torch.int32
+            )
+            selected = torch.cat(
+                [
+                    torch.tensor([current_block], device="cuda", dtype=torch.int32),
+                    older_blocks[: TOPK - 1],
+                ]
+            )
+            topk_idx[:, token_id, : selected.numel()] = selected
+            token_id += 1
+
+    return q, block_table, seq_lens, topk_idx, num_pages
+
+
+@pytest.mark.parametrize("kv_layout", ["NHD", "HND"], indirect=True)
+@pytest.mark.parametrize(
+    "seq_lens_list",
+    [(130, 257), (129, 200, 384)],
+)
+@pytest.mark.parametrize("decode_query_len", [1, 4])
+@pytest.mark.parametrize("num_padded_reqs", [0, 2])
+def test_decode_sparse_attention_correctness(
+    kv_layout: str,
+    seq_lens_list: tuple[int, ...],
+    decode_query_len: int,
+    num_padded_reqs: int,
+):
+    """Decode (split-K) parity under both layouts: this is the only coverage of
+    the decode-site cache feed, and the strided HND case fails if the kernel
+    ignores the cache strides."""
+    torch.manual_seed(0)
+    q, block_table, seq_lens, topk_idx, num_pages = _build_decode_inputs(
+        seq_lens_list, decode_query_len, num_padded_reqs
+    )
+    kv_cache = _allocate_main_kv_via_contract(num_pages)
+
+    actual = torch.empty_like(q)
+    minimax_m3_sparse_attn_decode(
+        q,
+        kv_cache,
+        topk_idx,
+        block_table,
+        seq_lens,
+        NUM_KV_HEADS,
+        SM_SCALE,
+        actual,
+        decode_query_len,
+    )
+
+    # Reuse the prefill reference: decode is a uniform query chunk ending at
+    # seq_len - 1 for each request.
+    active_batch = len(seq_lens_list)
+    active_tokens = active_batch * decode_query_len
+    q_lens_t = torch.full(
+        (len(seq_lens_list),), decode_query_len, device="cuda", dtype=torch.int32
+    )
+    active_seq_lens = seq_lens[:active_batch]
+    prefix_lens = active_seq_lens - q_lens_t
+    expected = _reference_sparse_attn(
+        q[:active_tokens],
+        kv_cache,
+        topk_idx[:, :active_tokens],
+        block_table[:active_batch],
+        q_lens_t,
+        active_seq_lens,
+        prefix_lens,
+    )
+    torch.accelerator.synchronize()
+
+    error = (actual[:active_tokens].float() - expected.float()).abs()
+    assert error.mean().item() < 2.5e-4
+    assert error.max().item() < 1.7e-2
+
+
+def test_decode_wrong_layout_breaks_parity():
+    """Negative (AC-3/AC-5): consuming the physical HND buffer as if it were
+    already contiguous-NHD (i.e. skipping the allocator's inverse permute)
+    reorders the K/V content, so the decode output no longer matches the
+    reference computed on the correct logical view. The mislabeled tensor keeps
+    the same shape as the correct view, so the kernel stays in bounds."""
+    torch.manual_seed(0)
+    seq_lens_list = (130, 257)
+    q, block_table, seq_lens, topk_idx, num_pages = _build_decode_inputs(seq_lens_list)
+
+    # Physical HND storage [blocks, 2, heads, block, dim].
+    phys = torch.randn(
+        (num_pages, 2, NUM_KV_HEADS, BLOCK_SIZE, HEAD_DIM), device="cuda", dtype=DTYPE
+    )
+    # Correct logical-NHD view (strided) vs. the same bytes mislabeled as a
+    # contiguous-NHD cache — same shape, different content mapping.
+    correct = phys.permute(0, 1, 3, 2, 4)
+    wrong = phys.reshape(num_pages, 2, BLOCK_SIZE, NUM_KV_HEADS, HEAD_DIM)
+
+    q_lens_t = torch.ones(len(seq_lens_list), device="cuda", dtype=torch.int32)
+    prefix_lens = seq_lens - q_lens_t
+    expected = _reference_sparse_attn(
+        q, correct, topk_idx, block_table, q_lens_t, seq_lens, prefix_lens
+    )
+
+    actual = torch.empty_like(q)
+    minimax_m3_sparse_attn_decode(
+        q, wrong, topk_idx, block_table, seq_lens, NUM_KV_HEADS, SM_SCALE, actual, 1
+    )
+    torch.accelerator.synchronize()
+    assert (actual.float() - expected.float()).abs().max().item() > 1.7e-2
+
+
+def _make_attn_group(backend, spec):
+    return AttentionGroup(
+        backend=backend,
+        layer_names=["main"],
+        kv_cache_spec=spec,
+        kv_cache_group_id=0,
+    )
+
+
+def test_main_cache_byte_identical_through_production_allocator():
+    """AC-2: drive the real allocator (`_reshape_kv_cache`) for the M3 main
+    `FullAttentionSpec` under HND and assert the backend-visible view has the
+    same shape, stride, and storage offset as the pre-change
+    allocate-HND-then-transpose path; the indexer `MLAAttentionSpec` allocates
+    through the same path to its 3-dim shape."""
+    nb = 4
+    spec = FullAttentionSpec(
+        block_size=BLOCK_SIZE,
+        num_kv_heads=NUM_KV_HEADS,
+        head_size=HEAD_DIM,
+        head_size_v=HEAD_DIM,
+        dtype=DTYPE,
+    )
+    raw = torch.zeros(nb * spec.page_size_bytes, dtype=torch.int8)
+    group = _make_attn_group(MiniMaxM3SparseBackend, spec)
+    try:
+        set_kv_cache_layout("HND")
+        kv_caches = _reshape_kv_cache([group], {"main": raw}, "auto", [BLOCK_SIZE], {})
+    finally:
+        set_kv_cache_layout(None)
+    view = kv_caches["main"]
+
+    oracle = raw.view(DTYPE).view((nb, 2, NUM_KV_HEADS, BLOCK_SIZE, HEAD_DIM))
+    oracle = oracle.transpose(2, 3)
+    assert tuple(view.shape) == tuple(oracle.shape)
+    assert view.stride() == oracle.stride()
+    assert view.storage_offset() == oracle.storage_offset()
+
+    # Indexer cache allocates through the same path under both layouts.
+    ispec = MLAAttentionSpec(
+        block_size=BLOCK_SIZE, num_kv_heads=1, head_size=HEAD_DIM, dtype=DTYPE
+    )
+    for layout in ("NHD", "HND"):
+        iraw = torch.zeros(nb * ispec.page_size_bytes, dtype=torch.int8)
+        igroup = AttentionGroup(
+            backend=MiniMaxM3IndexerBackend,
+            layer_names=["idx"],
+            kv_cache_spec=ispec,
+            kv_cache_group_id=0,
+        )
+        try:
+            set_kv_cache_layout(layout)
+            iout = _reshape_kv_cache([igroup], {"idx": iraw}, "auto", [BLOCK_SIZE], {})
+        finally:
+            set_kv_cache_layout(None)
+        assert tuple(iout["idx"].shape) == (nb, BLOCK_SIZE, HEAD_DIM)
+
+
+def test_indexer_inherited_stride_order_trips_allocator_assert():
+    """AC-4 negative: without the indexer override, the inherited 5-element
+    stride order trips the allocator's `len(stride_order) == len(shape)` assert
+    for the 3-dim indexer shape; the `AssertionError` is NOT swallowed by the
+    allocator's `(AttributeError, NotImplementedError)` fallback."""
+
+    class _BrokenIndexerBackend(MiniMaxM3IndexerBackend):
+        # Simulate inheriting the parent's 5-element stride order.
+        get_kv_cache_stride_order = staticmethod(
+            MiniMaxM3SparseBackend.get_kv_cache_stride_order
+        )
+
+    nb = 4
+    ispec = MLAAttentionSpec(
+        block_size=BLOCK_SIZE, num_kv_heads=1, head_size=HEAD_DIM, dtype=DTYPE
+    )
+    iraw = torch.zeros(nb * ispec.page_size_bytes, dtype=torch.int8)
+    igroup = AttentionGroup(
+        backend=_BrokenIndexerBackend,
+        layer_names=["idx"],
+        kv_cache_spec=ispec,
+        kv_cache_group_id=0,
+    )
+    try:
+        set_kv_cache_layout("HND")
+        with pytest.raises(AssertionError):
+            _reshape_kv_cache([igroup], {"idx": iraw}, "auto", [BLOCK_SIZE], {})
+    finally:
+        set_kv_cache_layout(None)
+
+
+def test_padded_main_cache_is_flagged():
+    """AC-2.1 negative: the M3 main cache relies on the allocator's
+    contiguous-view branch (`page_size_padded is None`). A spec that sets
+    `page_size_padded` is explicitly flagged rather than silently wrong-strided."""
+
+    def _require_unpadded_block_first(spec, stride_order):
+        inv_order = [stride_order.index(i) for i in range(len(stride_order))]
+        assert spec.page_size_padded is None, (
+            "main GQA cache must be unpadded to use the contiguous-view "
+            "allocator branch"
+        )
+        assert inv_order[0] == 0, "main GQA cache must remain block-first"
+
+    try:
+        set_kv_cache_layout("HND")
+        stride_order = MiniMaxM3SparseBackend.get_kv_cache_stride_order()
+    finally:
+        set_kv_cache_layout(None)
+
+    good = FullAttentionSpec(
+        block_size=BLOCK_SIZE,
+        num_kv_heads=NUM_KV_HEADS,
+        head_size=HEAD_DIM,
+        head_size_v=HEAD_DIM,
+        dtype=DTYPE,
+    )
+    _require_unpadded_block_first(good, stride_order)  # passes
+
+    padded = FullAttentionSpec(
+        block_size=BLOCK_SIZE,
+        num_kv_heads=NUM_KV_HEADS,
+        head_size=HEAD_DIM,
+        head_size_v=HEAD_DIM,
+        dtype=DTYPE,
+        page_size_padded=good.page_size_bytes + 128,
+    )
+    with pytest.raises(AssertionError):
+        _require_unpadded_block_first(padded, stride_order)
+
+
+@pytest.mark.parametrize("kv_layout", ["NHD", "HND"], indirect=True)
+def test_reshape_and_cache_flash_write_persists(kv_layout: str):
+    """AC-5 write path: the `reshape_and_cache_flash` write site now consumes
+    `self.kv_cache.unbind(1)` directly. Writing through those views must persist
+    into the bound storage (read back through an independent logical view) under
+    both layouts — a `.contiguous()` copy of the unbind slice would leave the
+    bound storage unchanged."""
+    torch.manual_seed(0)
+    num_pages = 4
+    kv_cache = _allocate_main_kv_via_contract(num_pages)
+    with torch.no_grad():
+        kv_cache.zero_()
+
+    # Exactly the production write-site code under test.
+    key_cache, value_cache = kv_cache.unbind(1)
+
+    num_tokens = 12
+    slot_mapping = torch.randperm(num_pages * BLOCK_SIZE, device="cuda")[
+        :num_tokens
+    ].to(torch.int64)
+    key = torch.randn(num_tokens, NUM_KV_HEADS, HEAD_DIM, device="cuda", dtype=DTYPE)
+    value = torch.randn(num_tokens, NUM_KV_HEADS, HEAD_DIM, device="cuda", dtype=DTYPE)
+    scale = torch.ones((), device="cuda")
+    ops.reshape_and_cache_flash(
+        key, value, key_cache, value_cache, slot_mapping, "auto", scale, scale
+    )
+    torch.accelerator.synchronize()
+
+    # Read back through the independent logical view; proves the writes landed
+    # in the engine-bound storage, not a detached copy.
+    for t in range(num_tokens):
+        slot = int(slot_mapping[t].item())
+        blk, intra = divmod(slot, BLOCK_SIZE)
+        torch.testing.assert_close(kv_cache[blk, 0, intra], key[t])
+        torch.testing.assert_close(kv_cache[blk, 1, intra], value[t])

tests/kernels/core/test_fused_allreduce_gemma_rms_norm.py

@@ -0,0 +1,109 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Tests for the manual AllReduce + GemmaRMSNorm fusion used by MiniMax M3.
+
+``fused_allreduce_gemma_rms_norm`` must match the unfused model path, i.e.
+``GemmaRMSNorm(all_reduce(partial), residual)``, both on the flashinfer fast
+path (TP>1 with flashinfer + NVSwitch) and on the eager fallback (TP==1, or when
+flashinfer is unavailable / the GPU has no NVSwitch).
+"""
+
+import pytest
+import torch
+from torch.multiprocessing import spawn
+
+from tests.utils import ensure_current_vllm_config, init_test_distributed_environment
+from vllm.distributed import cleanup_dist_env_and_memory
+from vllm.distributed.communication_op import tensor_model_parallel_all_reduce
+from vllm.model_executor.layers.fused_allreduce_gemma_rms_norm import (
+    fused_allreduce_gemma_rms_norm,
+)
+from vllm.model_executor.layers.layernorm import GemmaRMSNorm
+from vllm.platforms import current_platform
+from vllm.utils.network_utils import get_open_port
+from vllm.utils.torch_utils import set_random_seed
+
+
+@ensure_current_vllm_config()
+def _worker_fused_ar_norm(
+    local_rank,
+    world_size,
+    port,
+    num_tokens,
+    hidden_size,
+    dtype,
+    seed,
+    eps,
+):
+    """Per-rank worker: compare the fused helper vs all_reduce + GemmaRMSNorm."""
+    device = torch.device(f"cuda:{local_rank}")
+    torch.accelerator.set_device_index(device)
+    init_test_distributed_environment(
+        world_size, 1, local_rank, port, local_rank=local_rank
+    )
+
+    # Norm weights are identical across ranks (replicated GemmaRMSNorm).
+    set_random_seed(seed)
+    norm = GemmaRMSNorm(hidden_size, eps=eps).cuda().to(dtype)
+    with torch.no_grad():
+        norm.weight.normal_(mean=0.0, std=0.1)
+
+    # Residual is shared across ranks; the partial o_proj output differs per rank
+    # (each rank holds a partial sum that all_reduce combines).
+    torch.manual_seed(seed + 7)
+    residual = torch.randn(num_tokens, hidden_size, dtype=dtype, device=device)
+    torch.manual_seed(seed + 1000 + local_rank)
+    partial = torch.randn(num_tokens, hidden_size, dtype=dtype, device=device)
+
+    # Reference: the unfused model path.
+    reduced = tensor_model_parallel_all_reduce(partial.clone())
+    ref_out, ref_res = norm(reduced, residual.clone())
+
+    # Fused helper (flashinfer fast path when available, else fallback).
+    out, res = fused_allreduce_gemma_rms_norm(partial.clone(), residual.clone(), norm)
+    torch.accelerator.synchronize()
+
+    torch.testing.assert_close(out, ref_out, atol=2e-2, rtol=2e-2)
+    torch.testing.assert_close(res, ref_res, atol=2e-2, rtol=2e-2)
+
+    cleanup_dist_env_and_memory()
+
+
+@pytest.mark.skipif(
+    not current_platform.is_cuda(),
+    reason="CUDA required",
+)
+# world_size=1 exercises the TP==1 identity branch on a single GPU; >1 exercises
+# the all_reduce + GemmaRMSNorm equivalence (flashinfer kernel or fallback).
+@pytest.mark.parametrize("world_size", [1, 2, 4])
+@pytest.mark.parametrize("num_tokens", [1, 128, 333])
+@pytest.mark.parametrize("hidden_size", [2048, 4096])
+@pytest.mark.parametrize("dtype", [torch.bfloat16])
+@pytest.mark.parametrize("eps", [1e-6])
+@pytest.mark.parametrize("seed", [42])
+def test_fused_allreduce_gemma_rms_norm(
+    world_size,
+    num_tokens,
+    hidden_size,
+    dtype,
+    eps,
+    seed,
+):
+    num_gpus = current_platform.device_count()
+    if num_gpus < world_size:
+        pytest.skip(f"Need >= {world_size} GPUs, have {num_gpus}")
+    port = str(get_open_port())
+    spawn(
+        _worker_fused_ar_norm,
+        args=(
+            world_size,
+            port,
+            num_tokens,
+            hidden_size,
+            dtype,
+            seed,
+            eps,
+        ),
+        nprocs=world_size,
+        join=True,
+    )

tests/kernels/moe/test_mxfp4_moe.py

@@ -244,5 +244,224 @@ def test_mxfp4_experts_quant_basic():
     print("PASSED")
 
 
+def untile_cutlass_scale(scale_raw: torch.Tensor, rows: int, K: int) -> torch.Tensor:
+    """Convert CUTLASS tiled scale back to flat [M, K//32] layout.
+
+    CUTLASS tiled layout: [numMTiles, numKTiles, 32(outerM), 4(innerM), 4(innerK)]
+    Produced by: padded.reshape(numMTiles, 4, 32, numKTiles, 4).permute(0,3,2,1,4)
+    To undo: tiled.permute(0, 3, 2, 1, 4).reshape(padded_M, padded_sK)
+    """
+    num_scale_cols = K // MXFP4_BLOCK_SIZE
+    num_m_tiles = (rows + 127) // 128
+    num_k_tiles = (num_scale_cols + 3) // 4
+    padded_M = num_m_tiles * 128
+    padded_sK = num_k_tiles * 4
+
+    scale_bytes = scale_raw.view(torch.uint8).flatten()
+    total_bytes = padded_M * padded_sK
+    tiled = scale_bytes[:total_bytes].reshape(num_m_tiles, num_k_tiles, 32, 4, 4)
+    undone = tiled.permute(0, 3, 2, 1, 4).contiguous()
+    return undone.reshape(padded_M, padded_sK)[:rows, :num_scale_cols]
+
+
+def compute_reference_e8m0_scale(block_max: float) -> int:
+    """Compute the expected OCP MX spec E8M0 scale for a given block max.
+
+    The CUTLASS kernel uses round-to-nearest on the mantissa:
+      rounded_bits = (float_bits + (1 << 21)) & 0xFF800000
+      biased_exp = (rounded_bits >> 23) & 0xFF
+      scale_exp = max(biased_exp - 2, 0)
+
+    This ensures max_val / scale <= 6.0 for most inputs.
+    """
+    import struct
+
+    if block_max <= 0:
+        return 0
+    # Replicate the kernel's rounding logic in Python
+    float_bytes = struct.pack("f", block_max)
+    max_bits = struct.unpack("I", float_bytes)[0]
+    rounded_bits = (max_bits + (1 << 21)) & 0xFF800000
+    biased_exp = (rounded_bits >> 23) & 0xFF
+    scale_exp = max(int(biased_exp) - 2, 0)
+    scale_exp = min(scale_exp, 254)
+    return scale_exp
+
+
+@pytest.mark.skipif(
+    not is_sm100_supported(),
+    reason="mxfp4_experts_quant requires CUDA SM100",
+)
+@pytest.mark.parametrize("k", [256, 7168])
+@pytest.mark.parametrize("m", [16, 64])
+def test_mxfp4_experts_quant_e8m0_scale_correctness(m, k):
+    """
+    Test that mxfp4_experts_quant computes E8M0 block scales correctly
+    per OCP MX spec (not the NVFP4 formula).
+
+    The old buggy kernel used: floor(log2(max/6)) + 127
+    The fixed kernel uses:     round_nearest_exp(max) - 2
+
+    This test verifies:
+    1. Scales match the expected OCP MX formula for all blocks
+    2. No block max exceeds the representable range (no unexpected saturation)
+    3. Reconstruction error is within expected bounds for MXFP4
+    """
+    device = "cuda"
+
+    # Generate input with controlled range
+    input_tensor = torch.randn(m, k, device=device, dtype=torch.bfloat16) * 0.5
+
+    # Quantize
+    num_experts = 1
+    expert_offsets = torch.tensor([0, m], device=device, dtype=torch.int32)
+    num_k_tiles = (k // MXFP4_BLOCK_SIZE + 3) // 4
+    blockscale_offsets = torch.tensor(
+        [0, align(m, 128) * num_k_tiles], device=device, dtype=torch.int32
+    )
+
+    output_fp4, output_sf = ops.mxfp4_experts_quant(
+        input_tensor, expert_offsets, blockscale_offsets, num_experts, topk=1
+    )
+
+    # Untile scale to flat layout for verification
+    scale_flat = untile_cutlass_scale(output_sf, m, k)
+    assert scale_flat.shape == (m, k // MXFP4_BLOCK_SIZE)
+
+    # Verify each block's scale matches the OCP MX spec formula
+    num_blocks = k // MXFP4_BLOCK_SIZE
+    mismatches = 0
+    buggy_pattern = 0  # count blocks where scale is 1-2 lower than expected
+
+    for row in range(m):
+        for blk in range(num_blocks):
+            block_start = blk * MXFP4_BLOCK_SIZE
+            block_end = block_start + MXFP4_BLOCK_SIZE
+            block_max = (
+                input_tensor[row, block_start:block_end].float().abs().max().item()
+            )
+
+            actual_scale = scale_flat[row, blk].item()
+            expected_scale = compute_reference_e8m0_scale(block_max)
+
+            if actual_scale != expected_scale:
+                mismatches += 1
+                if actual_scale < expected_scale:
+                    buggy_pattern += 1
+
+    total_blocks = m * num_blocks
+    match_rate = (total_blocks - mismatches) / total_blocks
+
+    print(
+        f"  m={m}, k={k}: scale match rate = {match_rate * 100:.2f}% "
+        f"({mismatches}/{total_blocks} mismatches)"
+    )
+
+    # The fixed kernel should match the reference formula exactly
+    assert match_rate > 0.99, (
+        f"E8M0 scale match rate too low: {match_rate * 100:.2f}%. "
+        f"Buggy pattern (scale too low): {buggy_pattern}/{mismatches}. "
+        f"This suggests the NVFP4 formula bug is present."
+    )
+
+    # Extra check: if most mismatches show scale < expected, it's the old bug
+    if mismatches > 0:
+        assert buggy_pattern / mismatches < 0.5, (
+            f"Most scale mismatches show scale too LOW ({buggy_pattern}/{mismatches}). "
+            "This is the signature of the NVFP4 formula bug in nvfp4_utils.cuh."
+        )
+
+    # Verify reconstruction error is within MXFP4 expected bounds
+    # Dequantize and check cosine similarity
+    fp4_lut = torch.tensor(
+        [0, 0.5, 1, 1.5, 2, 3, 4, 6, 0, -0.5, -1, -1.5, -2, -3, -4, -6],
+        device=device,
+        dtype=torch.float32,
+    )
+    lo = (output_fp4 & 0x0F).long()
+    hi = ((output_fp4 >> 4) & 0x0F).long()
+    unpacked = torch.stack([lo, hi], dim=-1).reshape(m, k)
+    fp4_vals = fp4_lut[unpacked]
+
+    scales_expanded = 2.0 ** (scale_flat.float() - 127.0)
+    scales_expanded = scales_expanded.unsqueeze(-1).expand(-1, -1, MXFP4_BLOCK_SIZE)
+    scales_expanded = scales_expanded.reshape(m, k)
+    recon = (fp4_vals * scales_expanded).bfloat16()
+
+    # Cosine similarity should be > 0.99 for well-behaved MXFP4 quantization
+    cos_sim = torch.nn.functional.cosine_similarity(
+        recon.float().flatten().unsqueeze(0),
+        input_tensor.float().flatten().unsqueeze(0),
+    ).item()
+    max_abs_diff = (recon.float() - input_tensor.float()).abs().max().item()
+
+    print(
+        f"  Reconstruction: cosine_sim={cos_sim:.6f}, max_abs_diff={max_abs_diff:.4f}"
+    )
+
+    assert cos_sim > 0.99, (
+        f"Reconstruction cosine similarity too low: {cos_sim:.6f}. "
+        f"Expected > 0.99 for correct MXFP4 quantization."
+    )
+    # With correct E8M0, max abs diff should be bounded by scale * 6
+    # (worst case: value just below threshold rounds to wrong FP4 code)
+    assert max_abs_diff < 1.0, (
+        f"Max reconstruction error too large: {max_abs_diff:.4f}. "
+        "Likely caused by incorrect E8M0 scale (values saturating to ±6)."
+    )
+
+
+@pytest.mark.skipif(
+    not is_sm100_supported(),
+    reason="mxfp4_experts_quant requires CUDA SM100",
+)
+def test_mxfp4_experts_quant_no_saturation():
+    """
+    Test that the E8M0 scale is large enough to avoid unexpected saturation.
+
+    With the buggy NVFP4 formula, the scale was too small causing most values
+    to saturate to ±6 in FP4. The fixed OCP MX formula should ensure that
+    block_max / scale <= 6.0 (the max E2M1 value) in almost all cases.
+    """
+    device = "cuda"
+
+    m, k = 128, 1024
+    # Use inputs with known range to make saturation detectable
+    input_tensor = torch.randn(m, k, device=device, dtype=torch.bfloat16) * 0.5
+
+    num_experts = 1
+    expert_offsets = torch.tensor([0, m], device=device, dtype=torch.int32)
+    num_k_tiles = (k // MXFP4_BLOCK_SIZE + 3) // 4
+    blockscale_offsets = torch.tensor(
+        [0, align(m, 128) * num_k_tiles], device=device, dtype=torch.int32
+    )
+
+    output_fp4, output_sf = ops.mxfp4_experts_quant(
+        input_tensor, expert_offsets, blockscale_offsets, num_experts, topk=1
+    )
+
+    # Check saturation rate: count FP4 values that are ±6 (codes 7 and 15)
+    lo = output_fp4 & 0x0F
+    hi = (output_fp4 >> 4) & 0x0F
+    # Code 7 = +6.0, code 15 = -6.0
+    saturated = ((lo == 7) | (lo == 15) | (hi == 7) | (hi == 15)).sum().item()
+    total_values = m * k
+    saturation_rate = saturated / total_values
+
+    print(
+        f"  Saturation rate: {saturation_rate * 100:.2f}% "
+        f"({saturated}/{total_values} values at ±6)"
+    )
+
+    # For Gaussian input with std=0.5, saturation should be very rare
+    # (±6 * scale is far from the typical range).
+    # The buggy kernel had ~30-50% saturation; fixed should be < 5%.
+    assert saturation_rate < 0.05, (
+        f"FP4 saturation rate too high: {saturation_rate * 100:.2f}%. "
+        "This suggests the E8M0 scale is too small (NVFP4 formula bug). "
+        "Expected < 5% for Gaussian(0, 0.5) input with correct OCP MX scale."
+    )
+
+
 if __name__ == "__main__":
     pytest.main([__file__, "-v", "-s"])

tests/kernels/test_fp32_router_gemm.py

@@ -1,6 +1,9 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""Tests for fp32_router_gemm kernel: activation×weight→fp32, H=3072, E=256.
+"""Tests for fp32_router_gemm kernel: activation×weight→fp32.
+
+Supported (hidden_size, num_experts) pairs:
+  (3072, 256) -> MiniMax-M2/M2.5,  (6144, 128) -> MiniMax-M3
 
 Correctness baseline: torch.matmul in float64.
 """
@@ -10,8 +13,8 @@ import torch
 
 from vllm._custom_ops import fp32_router_gemm
 
-NUM_EXPERTS = 256
-HIDDEN_DIM = 3072
+# (hidden_size, num_experts)
+SHAPES = [(3072, 256), (6144, 128)]
 # Absolute tolerance for fp32 kernel vs float64 reference
 ATOL_FP32 = 2e-4
 ATOL_BF16 = 2e-2  # bf16 activation has lower precision
@@ -30,49 +33,52 @@ def _ref(mat_a: torch.Tensor, mat_b: torch.Tensor) -> torch.Tensor:
     return torch.nn.functional.linear(mat_a.float(), mat_b.float())
 
 
+@pytest.mark.parametrize("hidden_dim,num_experts", SHAPES)
 @pytest.mark.parametrize("num_tokens", [1, 2, 4, 8, 16, 32])
-def test_fp32_activation(num_tokens: int):
+def test_fp32_activation(num_tokens: int, hidden_dim: int, num_experts: int):
     """fp32 activation → fp32 output should match reference closely."""
     _requires_sm90()
     torch.manual_seed(42)
     device = torch.device("cuda")
-    mat_a = torch.randn(num_tokens, HIDDEN_DIM, dtype=torch.float32, device=device)
-    mat_b = torch.randn(NUM_EXPERTS, HIDDEN_DIM, dtype=torch.float32, device=device)
+    mat_a = torch.randn(num_tokens, hidden_dim, dtype=torch.float32, device=device)
+    mat_b = torch.randn(num_experts, hidden_dim, dtype=torch.float32, device=device)
 
     out = fp32_router_gemm(mat_a, mat_b)
     ref = _ref(mat_a, mat_b)
 
-    assert out.shape == (num_tokens, NUM_EXPERTS)
+    assert out.shape == (num_tokens, num_experts)
     assert out.dtype == torch.float32
     torch.testing.assert_close(out, ref, atol=ATOL_FP32, rtol=0)
 
 
+@pytest.mark.parametrize("hidden_dim,num_experts", SHAPES)
 @pytest.mark.parametrize("num_tokens", [1, 2, 4, 8, 16, 32])
-def test_bf16_activation(num_tokens: int):
+def test_bf16_activation(num_tokens: int, hidden_dim: int, num_experts: int):
     """bf16 activation → fp32 output should match reference within bf16 error."""
     _requires_sm90()
     torch.manual_seed(42)
     device = torch.device("cuda")
     mat_a_bf16 = torch.randn(
-        num_tokens, HIDDEN_DIM, dtype=torch.bfloat16, device=device
+        num_tokens, hidden_dim, dtype=torch.bfloat16, device=device
     )
-    mat_b = torch.randn(NUM_EXPERTS, HIDDEN_DIM, dtype=torch.float32, device=device)
+    mat_b = torch.randn(num_experts, hidden_dim, dtype=torch.float32, device=device)
 
     out = fp32_router_gemm(mat_a_bf16, mat_b)
     ref = _ref(mat_a_bf16, mat_b).to(device)
 
-    assert out.shape == (num_tokens, NUM_EXPERTS)
+    assert out.shape == (num_tokens, num_experts)
     assert out.dtype == torch.float32
     torch.testing.assert_close(out, ref, atol=ATOL_BF16, rtol=0)
 
 
-def test_output_shape_and_dtype():
+@pytest.mark.parametrize("hidden_dim,num_experts", SHAPES)
+def test_output_shape_and_dtype(hidden_dim: int, num_experts: int):
     """Basic shape and dtype checks."""
     _requires_sm90()
     device = torch.device("cuda")
-    mat_a = torch.randn(4, HIDDEN_DIM, dtype=torch.float32, device=device)
-    mat_b = torch.randn(NUM_EXPERTS, HIDDEN_DIM, dtype=torch.float32, device=device)
+    mat_a = torch.randn(4, hidden_dim, dtype=torch.float32, device=device)
+    mat_b = torch.randn(num_experts, hidden_dim, dtype=torch.float32, device=device)
     out = fp32_router_gemm(mat_a, mat_b)
-    assert out.shape == (4, NUM_EXPERTS)
+    assert out.shape == (4, num_experts)
     assert out.dtype == torch.float32
     assert out.device.type == "cuda"

tests/kernels/test_fused_minimax_m3_qknorm_rope_kv_insert.py

@@ -0,0 +1,244 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Unit test for the horizontally-fused MiniMax-M3 attention pre-processing
+kernel:
+
+  fused_minimax_m3_qknorm_rope_kv_insert
+    - q / k / index_q / index_k: Gemma RMSNorm + partial NeoX RoPE (in place)
+    - sparse (insert) mode: scatter k/v into the paged bf16 KV cache and the
+      index key into the index cache by its own slot mapping.
+
+Reference: PyTorch Gemma RMSNorm with the same dtype materialization boundary
+as the unfused path, followed by vLLM CUDA rotary_embedding-style NeoX RoPE.
+"""
+
+import pytest
+import torch
+
+import vllm._custom_ops as ops
+
+HEAD_DIM = 128
+ROTARY_DIM = 64
+
+
+def _op_available() -> bool:
+    return hasattr(torch.ops._C, "fused_minimax_m3_qknorm_rope_kv_insert")
+
+
+pytestmark = pytest.mark.skipif(
+    not torch.cuda.is_available() or not _op_available(),
+    reason="CUDA not available or fused MiniMax-M3 op not built in",
+)
+
+
+def make_cos_sin_cache(max_pos, rotary_dim, base, dtype, device):
+    inv_freq = 1.0 / (
+        base
+        ** (
+            torch.arange(0, rotary_dim, 2, dtype=torch.float32, device=device)
+            / rotary_dim
+        )
+    )
+    t = torch.arange(max_pos, dtype=torch.float32, device=device)
+    freqs = torch.einsum("i,j->ij", t, inv_freq)  # [max_pos, rotary_dim/2]
+    cache = torch.cat((freqs.cos(), freqs.sin()), dim=-1)  # [max_pos, rotary_dim]
+    return cache.to(dtype)
+
+
+def gemma_rmsnorm(x, weight, eps):
+    """x: [..., 128]; weight: [128]. Returns original dtype."""
+    xf = x.float()
+    var = xf.pow(2).mean(dim=-1, keepdim=True)
+    out = xf * torch.rsqrt(var + eps)
+    out = out * (1.0 + weight.float())
+    return out.to(x.dtype)
+
+
+def apply_rope_neox_partial(x, positions, cos_sin_cache, rotary_dim):
+    """NeoX-style RoPE on the leading rotary_dim dims; rest pass through.
+
+    x: [num_tokens, num_heads, head_dim]
+    cos_sin_cache: [max_pos, rotary_dim] (cos||sin), read as float (matches the
+    kernel, which loads the bf16 cache and converts to fp32).
+    """
+    half = rotary_dim // 2
+    cs = cos_sin_cache[positions].float()  # [num_tokens, rotary_dim]
+    cos = cs[..., :half].unsqueeze(1)  # [nt, 1, half]
+    sin = cs[..., half:].unsqueeze(1)
+
+    rot = x[..., :rotary_dim].float()
+    x1 = rot[..., :half]
+    x2 = rot[..., half:]
+    o1 = x1 * cos - x2 * sin
+    o2 = x2 * cos + x1 * sin
+    out = x.clone()
+    out[..., :half] = o1
+    out[..., half:rotary_dim] = o2
+    return out.to(x.dtype)
+
+
+def norm_rope_ref(x, weight, positions, cos_sin_cache, eps):
+    """[nt, nheads, 128] -> Gemma norm + neox partial rope."""
+    normed = gemma_rmsnorm(x, weight, eps)
+    roped = apply_rope_neox_partial(normed, positions, cos_sin_cache, ROTARY_DIM)
+    return roped
+
+
+# ── Test 1: dense mode (norm+rope only, no index, no insert) ─────────────────
+
+
+@pytest.mark.parametrize("num_tokens", [1, 7, 64, 513])
+@pytest.mark.parametrize("num_heads,num_kv_heads", [(8, 2), (16, 4), (64, 4)])
+def test_dense_norm_rope(num_tokens, num_heads, num_kv_heads):
+    torch.manual_seed(0)
+    device, dtype, eps = "cuda", torch.bfloat16, 1e-6
+    base, max_pos = 5_000_000.0, 4096
+
+    q_w = torch.randn(HEAD_DIM, dtype=dtype, device=device) * 0.1
+    k_w = torch.randn(HEAD_DIM, dtype=dtype, device=device) * 0.1
+    cos_sin = make_cos_sin_cache(max_pos, ROTARY_DIM, base, dtype, device)
+    positions = torch.randint(
+        0, max_pos, (num_tokens,), dtype=torch.int64, device=device
+    )
+
+    qsz, kvsz = num_heads * HEAD_DIM, num_kv_heads * HEAD_DIM
+    qkv = torch.randn(num_tokens, qsz + 2 * kvsz, dtype=dtype, device=device)
+    qkv_orig = qkv.clone()
+
+    ops.fused_minimax_m3_qknorm_rope_kv_insert(
+        qkv, q_w, k_w, cos_sin, positions, num_heads, num_kv_heads, ROTARY_DIM, eps
+    )
+    q_out, k_out, v_out = qkv.split([qsz, kvsz, kvsz], dim=-1)
+
+    q_in, k_in, v_in = qkv_orig.split([qsz, kvsz, kvsz], dim=-1)
+    q_ref = norm_rope_ref(
+        q_in.view(num_tokens, num_heads, HEAD_DIM), q_w, positions, cos_sin, eps
+    ).view(num_tokens, qsz)
+    k_ref = norm_rope_ref(
+        k_in.view(num_tokens, num_kv_heads, HEAD_DIM),
+        k_w,
+        positions,
+        cos_sin,
+        eps,
+    ).view(num_tokens, kvsz)
+
+    torch.testing.assert_close(q_out, q_ref, rtol=1e-2, atol=1e-2)
+    torch.testing.assert_close(k_out, k_ref, rtol=1e-2, atol=1e-2)
+    # V is untouched.
+    torch.testing.assert_close(v_out, v_in, rtol=0, atol=0)
+
+
+# ── Test 2: sparse mode (full: index branch + cache inserts) ─────────────────
+
+
+@pytest.mark.parametrize("num_tokens", [1, 7, 64, 513])
+@pytest.mark.parametrize("block_size", [16, 64])
+def test_sparse_full(num_tokens, block_size):
+    torch.manual_seed(1)
+    device, dtype, eps = "cuda", torch.bfloat16, 1e-6
+    base, max_pos = 5_000_000.0, 4096
+    num_heads, num_kv_heads, num_idx_heads = 16, 4, 4
+
+    q_w = torch.randn(HEAD_DIM, dtype=dtype, device=device) * 0.1
+    k_w = torch.randn(HEAD_DIM, dtype=dtype, device=device) * 0.1
+    iq_w = torch.randn(HEAD_DIM, dtype=dtype, device=device) * 0.1
+    ik_w = torch.randn(HEAD_DIM, dtype=dtype, device=device) * 0.1
+    cos_sin = make_cos_sin_cache(max_pos, ROTARY_DIM, base, dtype, device)
+    positions = torch.randint(
+        0, max_pos, (num_tokens,), dtype=torch.int64, device=device
+    )
+
+    qsz, kvsz = num_heads * HEAD_DIM, num_kv_heads * HEAD_DIM
+    iqsz, iksz = num_idx_heads * HEAD_DIM, HEAD_DIM
+    # Single fused tensor packing [q | k | v | index_q | index_k].
+    qkv = torch.randn(
+        num_tokens, qsz + 2 * kvsz + iqsz + iksz, dtype=dtype, device=device
+    )
+    qkv_orig = qkv.clone()
+    splits = [qsz, kvsz, kvsz, iqsz, iksz]
+
+    num_blocks = (num_tokens + block_size - 1) // block_size + 1
+    kv_cache = torch.zeros(
+        num_blocks, 2, block_size, num_kv_heads, HEAD_DIM, dtype=dtype, device=device
+    )
+    index_cache = torch.zeros(
+        num_blocks, block_size, HEAD_DIM, dtype=dtype, device=device
+    )
+    slot_mapping = torch.randperm(
+        num_blocks * block_size, dtype=torch.int64, device=device
+    )[:num_tokens]
+    index_slot_mapping = torch.roll(slot_mapping, shifts=1)
+
+    # Contiguous gather targets: the kernel writes the normed/roped q and
+    # index_q here (de-interleaved from the packed qkv); k/v/index_k stay in
+    # place inside qkv and are scatter-inserted into the caches.
+    q_out = torch.empty(num_tokens, qsz, dtype=dtype, device=device)
+    index_q = torch.empty(num_tokens, iqsz, dtype=dtype, device=device)
+
+    ops.fused_minimax_m3_qknorm_rope_kv_insert(
+        qkv,
+        q_w,
+        k_w,
+        cos_sin,
+        positions,
+        num_heads,
+        num_kv_heads,
+        ROTARY_DIM,
+        eps,
+        iq_w,
+        ik_w,
+        num_idx_heads,
+        slot_mapping,
+        index_slot_mapping,
+        kv_cache,
+        index_cache,
+        block_size,
+        q_out,
+        index_q,
+    )
+
+    # ── norm+rope parity. q/index_q land in their gather buffers; k/index_k are
+    # rewritten in place inside qkv. ──
+    _, k_out, _, _, index_k = qkv.split(splits, dim=-1)
+    q_in, k_in, v_in, iq_orig, ik_orig = qkv_orig.split(splits, dim=-1)
+    q_ref = norm_rope_ref(
+        q_in.view(num_tokens, num_heads, HEAD_DIM), q_w, positions, cos_sin, eps
+    ).view(num_tokens, qsz)
+    k_ref = norm_rope_ref(
+        k_in.view(num_tokens, num_kv_heads, HEAD_DIM),
+        k_w,
+        positions,
+        cos_sin,
+        eps,
+    ).view(num_tokens, kvsz)
+    iq_ref = norm_rope_ref(
+        iq_orig.view(num_tokens, num_idx_heads, HEAD_DIM),
+        iq_w,
+        positions,
+        cos_sin,
+        eps,
+    ).view(num_tokens, num_idx_heads * HEAD_DIM)
+    ik_ref = norm_rope_ref(
+        ik_orig.view(num_tokens, 1, HEAD_DIM), ik_w, positions, cos_sin, eps
+    ).view(num_tokens, HEAD_DIM)
+
+    torch.testing.assert_close(q_out, q_ref, rtol=1e-2, atol=1e-2)
+    torch.testing.assert_close(k_out, k_ref, rtol=1e-2, atol=1e-2)
+    torch.testing.assert_close(index_q, iq_ref, rtol=1e-2, atol=1e-2)
+    torch.testing.assert_close(index_k, ik_ref, rtol=1e-2, atol=1e-2)
+
+    # ── Cache inserts. ──
+    # Main cache layout is [num_blocks, 2, block_size, num_kv_heads, head_dim]
+    # (the K/V axis sits *before* block_size); index cache is [nb, bs, head_dim].
+    idx_flat = index_cache.view(num_blocks * block_size, HEAD_DIM)
+    k_ref_h = k_ref.view(num_tokens, num_kv_heads, HEAD_DIM)
+    v_ref_h = v_in.view(num_tokens, num_kv_heads, HEAD_DIM)  # v is raw (no norm/rope)
+    for t in range(num_tokens):
+        s = slot_mapping[t].item()
+        b, pos = s // block_size, s % block_size
+        torch.testing.assert_close(
+            kv_cache[b, 0, pos], k_ref_h[t], rtol=1e-2, atol=1e-2
+        )
+        torch.testing.assert_close(kv_cache[b, 1, pos], v_ref_h[t], rtol=0, atol=0)
+        index_s = index_slot_mapping[t].item()
+        torch.testing.assert_close(idx_flat[index_s], ik_ref[t], rtol=1e-2, atol=1e-2)

tests/kernels/test_minimax_m3_amd_ops.py

@@ -0,0 +1,337 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Reference-vs-optimized unit tests for the MiniMax-M3 AMD/ROCm fused kernels.
+
+Each optimized kernel added for the ROCm port has a slow PyTorch reference; the
+tests assert the two agree within tolerance:
+
+  * Gemma RMSNorm (plain + fused-add-residual)  -> fp32 PyTorch normalize
+  * SwiGLU-OAI (split layout)                    -> fp32 PyTorch elementwise
+  * Fused MXFP8 activation quant (Triton)        -> _mxfp8_e4m3_quantize_torch
+  * Native MXFP8 linear (dot_scaled)             -> dequant-to-bf16 @ matmul
+  * Native MXFP8 MoE (dot_scaled grouped GEMM)   -> dequant-to-bf16 MoE math
+
+The native MXFP8 GEMMs also guard the ``dot_scaled`` rhs-scale orientation: the
+scale is loaded ``[N, K//32]`` and passed WITHOUT transpose; a stray ``.T``
+makes the shape ``[K//32, N]`` and Triton raises before producing output, so any
+regression there fails these tests loudly.
+
+Hardware scope: the whole module is ROCm-only (these are the AMD path; NVIDIA
+uses the FlashInfer kernels). The norm/activation/quant kernels run on any ROCm
+arch; the native MXFP8 ``dot_scaled`` linear/MoE tests are additionally gated to
+CDNA4 gfx95x (``@requires_gfx950``) since gfx942 uses the BF16 emulation path.
+
+Run:  pytest tests/kernels/test_minimax_m3_amd_ops.py -v
+"""
+
+import pytest
+import torch
+
+from vllm.platforms import current_platform
+
+if not current_platform.is_rocm():
+    pytest.skip("MiniMax-M3 AMD fused ops require ROCm.", allow_module_level=True)
+if not torch.cuda.is_available():
+    pytest.skip("Requires a GPU.", allow_module_level=True)
+
+from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (  # noqa: E402
+    _mxfp8_e4m3_quantize_torch,
+    _mxfp8_e4m3_quantize_triton,
+    dequant_mxfp8_to_bf16,
+)
+from vllm.models.minimax_m3.amd.ops import (  # noqa: E402
+    gemma_fused_add_rmsnorm,
+    gemma_rmsnorm,
+    swiglu_oai_split,
+)
+from vllm.models.minimax_m3.amd.ops.gemma_rmsnorm import _num_warps  # noqa: E402
+
+DEVICE = "cuda"
+EPS = 1e-6
+
+
+def _gcn_arch() -> str:
+    try:
+        return torch.cuda.get_device_properties(0).gcnArchName
+    except Exception:  # pragma: no cover - no device / non-AMD
+        return ""
+
+
+# The pure-Triton norm/activation/quant kernels run on any ROCm arch (CDNA3
+# gfx942 and CDNA4 gfx950). The native MXFP8 ``dot_scaled`` GEMMs (linear + MoE)
+# use CDNA4 hardware microscaling and are gated to gfx95x in the source
+# (``RocmDotScaledMxfp8LinearKernel.is_supported``; the MoE oracle routes gfx942
+# to the BF16 emulation path instead) — so those tests are gfx950-only.
+requires_gfx950 = pytest.mark.skipif(
+    "gfx95" not in _gcn_arch(),
+    reason="native MXFP8 dot_scaled is a CDNA4 (gfx95x) feature; "
+    "gfx942 uses the BF16 emulation path instead.",
+)
+
+
+def _relerr(a: torch.Tensor, b: torch.Tensor) -> float:
+    a = a.float()
+    b = b.float()
+    return ((a - b).norm() / (b.norm() + 1e-8)).item()
+
+
+# --------------------------------------------------------------------------- #
+# Gemma RMSNorm
+# --------------------------------------------------------------------------- #
+def _ref_gemma_rmsnorm(x, w, eps, residual=None):
+    orig_dtype = x.dtype
+    xf = x.float()
+    res_out = None
+    if residual is not None:
+        xf = xf + residual.float()
+        res_out = xf.to(orig_dtype)
+    xf = xf * torch.rsqrt(xf.pow(2).mean(dim=-1, keepdim=True) + eps)
+    xf = xf * (1.0 + w.float())
+    out = xf.to(orig_dtype)
+    return out if residual is None else (out, res_out)
+
+
+@pytest.mark.parametrize("shape", [(1, 4096), (37, 6144), (128, 2048)])
+@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
+@pytest.mark.parametrize("seed", [0, 1234])
+@torch.inference_mode()
+def test_gemma_rmsnorm(shape, dtype, seed):
+    torch.manual_seed(seed)
+    x = torch.randn(*shape, device=DEVICE, dtype=dtype)
+    w = torch.randn(shape[-1], device=DEVICE, dtype=dtype) * 0.1
+    got = gemma_rmsnorm(x, w, EPS)
+    ref = _ref_gemma_rmsnorm(x, w, EPS)
+    assert got.shape == x.shape
+    assert _relerr(got, ref) < 5e-3
+
+
+@pytest.mark.parametrize("shape", [(1, 6144), (64, 4096)])
+@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
+@torch.inference_mode()
+def test_gemma_fused_add_rmsnorm(shape, dtype):
+    torch.manual_seed(0)
+    x = torch.randn(*shape, device=DEVICE, dtype=dtype)
+    res = torch.randn(*shape, device=DEVICE, dtype=dtype)
+    w = torch.randn(shape[-1], device=DEVICE, dtype=dtype) * 0.1
+    got_out, got_res = gemma_fused_add_rmsnorm(x, res, w, EPS)
+    ref_out, ref_res = _ref_gemma_rmsnorm(x, w, EPS, residual=res)
+    assert _relerr(got_out, ref_out) < 5e-3
+    # residual_out is the pre-norm sum (x + res): bit-for-bit identical cast.
+    assert torch.equal(got_res, ref_res)
+
+
+@torch.inference_mode()
+def test_gemma_rmsnorm_per_head_strided():
+    """q_norm/k_norm normalize a non-contiguous ``qkv.split`` slice over head_dim."""
+    torch.manual_seed(0)
+    T, H, D, kv = 7, 48, 128, 8
+    total = (H + 2 * kv) * D
+    qkv = torch.randn(T, total, device=DEVICE, dtype=torch.bfloat16)
+    q = qkv[..., : H * D]  # non-contiguous view (row stride == total)
+    q_by_head = q.view(T, H, D)
+    assert not q_by_head.is_contiguous()
+    w = torch.randn(D, device=DEVICE, dtype=torch.bfloat16) * 0.1
+    got = gemma_rmsnorm(q_by_head, w, EPS)
+    ref = _ref_gemma_rmsnorm(q_by_head, w, EPS)
+    assert got.shape == q_by_head.shape
+    assert _relerr(got, ref) < 5e-3
+
+
+def test_num_warps_monotonic():
+    assert _num_warps(128) <= _num_warps(2048) <= _num_warps(8192)
+
+
+# --------------------------------------------------------------------------- #
+# SwiGLU-OAI (split layout)
+# --------------------------------------------------------------------------- #
+def _ref_swiglu(gate_up, alpha, beta, limit):
+    d = gate_up.shape[-1] // 2
+    gate = gate_up[..., :d].float()
+    up = gate_up[..., d:].float()
+    if limit is not None:
+        gate = gate.clamp(max=limit)
+        up = up.clamp(min=-limit, max=limit)
+    return (gate * torch.sigmoid(alpha * gate) * (up + beta)).to(gate_up.dtype)
+
+
+@pytest.mark.parametrize("m,inter", [(1, 768), (64, 1536), (128, 1024)])
+@pytest.mark.parametrize("limit", [7.0, None])
+@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
+@torch.inference_mode()
+def test_swiglu_oai_split(m, inter, limit, dtype):
+    torch.manual_seed(0)
+    gate_up = torch.randn(m, 2 * inter, device=DEVICE, dtype=dtype)
+    got = swiglu_oai_split(gate_up, alpha=1.702, beta=1.0, limit=limit)
+    ref = _ref_swiglu(gate_up, 1.702, 1.0, limit)
+    assert got.shape == (m, inter)
+    assert _relerr(got, ref) < 5e-3
+
+
+# --------------------------------------------------------------------------- #
+# Fused MXFP8 activation quant (Triton vs torch reference)
+# --------------------------------------------------------------------------- #
+@pytest.mark.parametrize("shape", [(64, 4096), (1, 6144), (333, 2048)])
+@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
+@torch.inference_mode()
+def test_mxfp8_quant_triton_matches_torch(shape, dtype):
+    torch.manual_seed(0)
+    x = torch.randn(*shape, device=DEVICE, dtype=dtype)
+    xq_t, s_t = _mxfp8_e4m3_quantize_torch(x, is_sf_swizzled_layout=False)
+    xq_k, s_k = _mxfp8_e4m3_quantize_triton(x)
+    assert s_k.shape == s_t.shape == (shape[0], shape[1] // 32)
+    # E8M0 block exponents share the floor(log2(amax))+127 algorithm; allow at
+    # most a 1-step difference at exact powers of two.
+    assert (s_k.int() - s_t.int()).abs().max().item() <= 1
+    # Dequantized values agree to fp8 granularity.
+    deq_t = dequant_mxfp8_to_bf16(xq_t, s_t)
+    deq_k = dequant_mxfp8_to_bf16(xq_k, s_k)
+    assert _relerr(deq_k, deq_t) < 1e-2
+
+
+# --------------------------------------------------------------------------- #
+# Native MXFP8 linear (dot_scaled) vs dequant-to-bf16 matmul
+# --------------------------------------------------------------------------- #
+@requires_gfx950
+@pytest.mark.parametrize("m,n,k", [(64, 256, 128), (37, 512, 256), (1, 6144, 4096)])
+@torch.inference_mode()
+def test_mxfp8_native_linear(m, n, k):
+    from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (
+        _mxfp8_dot_scaled_linear,
+    )
+
+    torch.manual_seed(0)
+    w_bf16 = torch.randn(n, k, device=DEVICE, dtype=torch.bfloat16) * 0.1
+    w_fp8, w_scale = _mxfp8_e4m3_quantize_torch(w_bf16, is_sf_swizzled_layout=False)
+    x = torch.randn(m, k, device=DEVICE, dtype=torch.bfloat16) * 0.5
+
+    got = _mxfp8_dot_scaled_linear(x, w_fp8, w_scale)
+    # Reference: consume the SAME quantized weights (isolates activation-quant
+    # noise) -> dequant to bf16, plain matmul.
+    w_deq = dequant_mxfp8_to_bf16(w_fp8, w_scale)
+    ref = torch.nn.functional.linear(x, w_deq).to(x.dtype)
+    assert got.shape == (m, n)
+    # Only the activation is re-quantized inside the kernel -> small MX noise.
+    assert _relerr(got, ref) < 5e-2
+
+
+# --------------------------------------------------------------------------- #
+# Native MXFP8 MoE (dot_scaled grouped GEMM) vs dequant-to-bf16 MoE math
+# --------------------------------------------------------------------------- #
+def _ref_moe(x, w13, w2, topk_weights, topk_ids, alpha, beta, limit):
+    T, H = x.shape
+    inter = w2.shape[-1]
+    top_k = topk_ids.shape[1]
+    out = torch.zeros(T, H, device=x.device, dtype=torch.float32)
+    for t in range(T):
+        for j in range(top_k):
+            e = int(topk_ids[t, j].item())
+            g1 = x[t].float() @ w13[e].float().T  # [2I]
+            gate = g1[:inter]
+            up = g1[inter:]
+            if limit is not None:
+                gate = gate.clamp(max=limit)
+                up = up.clamp(min=-limit, max=limit)
+            act = gate * torch.sigmoid(alpha * gate) * (up + beta)
+            g2 = act @ w2[e].float().T  # [H]
+            out[t] += topk_weights[t, j].float() * g2
+    return out.to(x.dtype)
+
+
+@requires_gfx950
+@pytest.mark.parametrize(
+    "T,H,inter,E,top_k", [(8, 256, 512, 8, 2), (1, 512, 256, 16, 4)]
+)
+@torch.inference_mode()
+def test_mxfp8_native_moe(T, H, inter, E, top_k):
+    from vllm.model_executor.layers.fused_moe.experts.mxfp8_native_moe import (
+        fused_moe_mxfp8_native,
+    )
+
+    torch.manual_seed(0)
+    alpha, beta, limit = 1.702, 1.0, 7.0
+    w13_bf16 = torch.randn(E, 2 * inter, H, device=DEVICE, dtype=torch.bfloat16) * 0.1
+    w2_bf16 = torch.randn(E, H, inter, device=DEVICE, dtype=torch.bfloat16) * 0.1
+    w13_fp8, w13_scale = _mxfp8_e4m3_quantize_torch(
+        w13_bf16, is_sf_swizzled_layout=False
+    )
+    w2_fp8, w2_scale = _mxfp8_e4m3_quantize_torch(w2_bf16, is_sf_swizzled_layout=False)
+
+    x = torch.randn(T, H, device=DEVICE, dtype=torch.bfloat16) * 0.5
+    logits = torch.randn(T, E, device=DEVICE, dtype=torch.float32)
+    topk_weights, topk_ids = logits.softmax(dim=-1).topk(top_k, dim=-1)
+    topk_weights = topk_weights.to(torch.float32)
+    topk_ids = topk_ids.to(torch.int32)
+
+    got = fused_moe_mxfp8_native(
+        x,
+        w13_fp8,
+        w13_scale,
+        w2_fp8,
+        w2_scale,
+        topk_weights,
+        topk_ids,
+        alpha=alpha,
+        beta=beta,
+        limit=limit,
+        global_num_experts=E,
+        expert_map=None,
+    )
+    # Reference consumes the dequantized weights (same bits the kernel reads).
+    w13_deq = dequant_mxfp8_to_bf16(w13_fp8, w13_scale)
+    w2_deq = dequant_mxfp8_to_bf16(w2_fp8, w2_scale)
+    ref = _ref_moe(x, w13_deq, w2_deq, topk_weights, topk_ids, alpha, beta, limit)
+    assert got.shape == (T, H)
+    assert _relerr(got, ref) < 5e-2
+
+
+# --------------------------------------------------------------------------- #
+# MXFP8 linear emulation: BF16-at-load (default) vs per-step dequant + switch
+# --------------------------------------------------------------------------- #
+@pytest.mark.parametrize("shape", [(512, 2048), (1, 6144)])
+@pytest.mark.parametrize("act_dtype", [torch.bfloat16, torch.float16])
+@pytest.mark.parametrize("dequant_at_load", [True, False])
+@torch.inference_mode()
+def test_mxfp8_linear_emulation_bf16_at_load(
+    shape, act_dtype, dequant_at_load, monkeypatch
+):
+    """EmulationMxfp8LinearKernel load-time BF16 dequant (default) and the
+    ``VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD=0`` per-step fallback must produce the
+    same result; the dtype-match (BF16/FP16 activations) must also hold."""
+    from vllm.model_executor.kernels.linear.mxfp8.emulation import (
+        EmulationMxfp8LinearKernel,
+    )
+    from vllm.model_executor.kernels.linear.mxfp8.Mxfp8LinearKernel import (
+        Mxfp8LinearLayerConfig,
+    )
+
+    monkeypatch.setenv(
+        "VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD", "1" if dequant_at_load else "0"
+    )
+    N, K = shape
+    torch.manual_seed(0)
+    w_bf16 = torch.randn(N, K, device=DEVICE, dtype=torch.bfloat16)
+    w_fp8, w_scale = _mxfp8_e4m3_quantize_torch(w_bf16, is_sf_swizzled_layout=False)
+    assert w_scale.shape == (N, K // 32)
+
+    # Reference: dequant once, plain linear in the activation dtype.
+    w_ref = dequant_mxfp8_to_bf16(w_fp8, w_scale).to(act_dtype)
+    x = torch.randn(7, K, device=DEVICE, dtype=act_dtype)
+    out_ref = torch.nn.functional.linear(x, w_ref)
+
+    layer = torch.nn.Module()
+    layer.weight = torch.nn.Parameter(w_fp8.clone(), requires_grad=False)
+    layer.weight_scale = torch.nn.Parameter(w_scale.clone(), requires_grad=False)
+
+    kernel = EmulationMxfp8LinearKernel(Mxfp8LinearLayerConfig())
+    kernel.process_weights_after_loading(layer)
+
+    if dequant_at_load:
+        # weights converted to BF16 at load (>= 2-byte)
+        assert layer.weight.element_size() >= 2
+    else:
+        # opt-out: weights stay 1-byte MXFP8, dequant happens per-step
+        assert layer.weight.element_size() == 1
+
+    out = kernel.apply_weights(layer, x)
+    assert out.dtype == act_dtype  # dtype-match preserved (no tl.dot/F.linear crash)
+    assert _relerr(out.float(), out_ref.float()) < 2e-2

tests/models/multimodal/processing/test_minimax_m3.py

@@ -0,0 +1,138 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Unit tests for MiniMax-M3 VL ``max_long_side_pixel`` resize support.
+
+These exercise the vendored processor directly (no checkpoint / GPU needed), so
+they validate the long-side resize spec and the resulting prompt-token counts
+deterministically.
+"""
+
+import pytest
+import torch
+
+from vllm.transformers_utils.processors.minimax_m3 import (
+    IMAGE_MAX_TOTAL_PIXELS,
+    MIN_SHORT_SIDE_PIXEL,
+    VIDEO_MAX_TOTAL_PIXELS,
+    MiniMaxM3VLImageProcessor,
+    MiniMaxM3VLVideoProcessor,
+    smart_resize,
+)
+
+# Long sides are multiples of patch_size*merge_size (28) so the rounding is
+# exact and the expected token counts are unambiguous.
+LONG_SIDES = [252, 504, 1008]
+MERGE2 = 2**2  # merge_size ** 2
+
+
+def _image_tokens(grid_thw) -> int:
+    g = list(grid_thw)
+    return int(g[0] * g[1] * g[2]) // MERGE2
+
+
+# --------------------------------------------------------------------------- #
+# smart_resize: the long-side spec (a) shrink / (b) enlarge / (c) hard cap
+# --------------------------------------------------------------------------- #
+def test_smart_resize_long_side_shrink():
+    # (a) long side exceeds the cap -> shrink so the long side equals the cap.
+    h, w = smart_resize(
+        2048, 1024, factor=28, max_long_side_pixel=1008, max_total_pixels=10**9
+    )
+    assert max(h, w) == 1008
+    assert (h, w) == (1008, 504)  # aspect ratio preserved
+
+
+def test_smart_resize_short_side_enlarge():
+    # (b) long side within the cap but short side below the floor -> enlarge so
+    # the short side reaches min_short_side_pixel.
+    h, w = smart_resize(
+        200, 40, factor=28, max_long_side_pixel=1008, max_total_pixels=10**9
+    )
+    assert min(h, w) == MIN_SHORT_SIDE_PIXEL  # 112
+
+
+def test_smart_resize_total_pixels_raises():
+    # (c) still over the area cap after resizing -> raise instead of inferring.
+    with pytest.raises(ValueError, match="max_total_pixels"):
+        smart_resize(
+            5000,
+            5000,
+            factor=28,
+            max_long_side_pixel=4000,
+            max_total_pixels=IMAGE_MAX_TOTAL_PIXELS,
+        )
+
+
+def test_smart_resize_backward_compatible_area_bound():
+    # Without max_long_side_pixel the original Qwen-style area bound is used.
+    assert smart_resize(2048, 2048, factor=28, max_pixels=451584) == (672, 672)
+
+
+# --------------------------------------------------------------------------- #
+# Image processor: monotonic prompt-token counts for 252 < 504 < 1008
+# --------------------------------------------------------------------------- #
+def test_image_tokens_increase_with_max_long_side_pixel():
+    proc = MiniMaxM3VLImageProcessor()
+    counts = []
+    for long_side in LONG_SIDES:
+        patches = proc.get_number_of_image_patches(
+            2048, 2048, images_kwargs={"max_long_side_pixel": long_side}
+        )
+        counts.append(patches // MERGE2)
+
+    assert counts == [81, 324, 1296]
+    assert counts[0] < counts[1] < counts[2]
+
+
+def test_image_processor_defaults_match_spec():
+    proc = MiniMaxM3VLImageProcessor()
+    assert proc.max_long_side_pixel is None  # opt-in
+    assert proc.min_short_side_pixel == MIN_SHORT_SIDE_PIXEL
+    assert proc.max_total_pixels == IMAGE_MAX_TOTAL_PIXELS
+
+
+def test_image_preprocess_pipeline_monotonic():
+    proc = MiniMaxM3VLImageProcessor()
+    image = torch.randint(0, 255, (3, 2048, 2048), dtype=torch.uint8)
+    counts = []
+    for long_side in LONG_SIDES:
+        out = proc.preprocess(
+            [image],
+            do_resize=True,
+            max_long_side_pixel=long_side,
+            return_tensors="pt",
+        )
+        counts.append(_image_tokens(out["image_grid_thw"][0]))
+    assert counts == [81, 324, 1296]
+
+
+# --------------------------------------------------------------------------- #
+# Video processor: same monotonic behavior + volumetric (w*h*frames) cap
+# --------------------------------------------------------------------------- #
+def test_video_tokens_increase_with_max_long_side_pixel():
+    proc = MiniMaxM3VLVideoProcessor()
+    assert proc.max_total_pixels == VIDEO_MAX_TOTAL_PIXELS
+    video = torch.randint(0, 255, (4, 3, 2048, 2048), dtype=torch.uint8)
+    counts = []
+    for long_side in LONG_SIDES:
+        out = proc.preprocess(
+            videos=[video],
+            do_resize=True,
+            max_long_side_pixel=long_side,
+            return_tensors="pt",
+        )
+        counts.append(_image_tokens(out["video_grid_thw"][0]))
+    assert counts[0] < counts[1] < counts[2]
+
+
+def test_video_volumetric_cap_raises():
+    proc = MiniMaxM3VLVideoProcessor()
+    # 400 frames at a 1008-long-side square: 1008*1008*400 >> 301,056,000.
+    video = torch.randint(0, 255, (400, 3, 2048, 2048), dtype=torch.uint8)
+    with pytest.raises(ValueError, match="max_total_pixels"):
+        proc.preprocess(
+            videos=[video],
+            do_resize=True,
+            max_long_side_pixel=1008,
+            return_tensors="pt",
+        )

tests/models/registry.py

@@ -420,6 +420,11 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
         "MiniMaxAI/MiniMax-M2",
         trust_remote_code=True,
     ),
+    "MiniMaxM3SparseForCausalLM": _HfExamplesInfo(
+        "MiniMaxAI/MiniMax-M3",
+        trust_remote_code=True,
+        is_available_online=False,
+    ),
     "Ministral3ForCausalLM": _HfExamplesInfo("mistralai/Ministral-3-3B-Instruct-2512"),
     "MistralForCausalLM": _HfExamplesInfo("mistralai/Mistral-7B-Instruct-v0.1"),
     "MistralLarge3ForCausalLM": _HfExamplesInfo(
@@ -564,16 +569,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
     "TeleFLMForCausalLM": _HfExamplesInfo(
         "CofeAI/FLM-2-52B-Instruct-2407", trust_remote_code=True
     ),
-    "XverseForCausalLM": _HfExamplesInfo(
-        "xverse/XVERSE-7B-Chat",
-        tokenizer="meta-llama/Llama-2-7b",
-        trust_remote_code=True,
-        max_transformers_version="4.57",
-        transformers_version_reason={
-            "vllm": "XVERSE tokenizer is incompatible with transformers v5 "
-            "(add_prefix_space / prepend_scheme mismatch).",
-        },
-    ),
     "Zamba2ForCausalLM": _HfExamplesInfo("Zyphra/Zamba2-7B-instruct"),
     "MiMoForCausalLM": _HfExamplesInfo("XiaomiMiMo/MiMo-7B-RL", trust_remote_code=True),
     "MiMoV2FlashForCausalLM": _HfExamplesInfo(
@@ -1109,6 +1104,11 @@ _MULTIMODAL_EXAMPLE_MODELS = {
         "MiniMaxAI/MiniMax-VL-01",
         trust_remote_code=True,
     ),
+    "MiniMaxM3SparseForConditionalGeneration": _HfExamplesInfo(
+        "MiniMaxAI/MiniMax-M3",
+        trust_remote_code=True,
+        is_available_online=False,
+    ),
     "Mistral3ForConditionalGeneration": _HfExamplesInfo(
         "mistralai/Mistral-Small-3.1-24B-Instruct-2503",
         extras={"fp8": "nm-testing/Mistral-Small-3.1-24B-Instruct-2503-FP8-dynamic"},
@@ -1611,6 +1611,11 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
         speculative_model="XiaomiMiMo/MiMo-V2.5-Omni",
         is_available_online=False,
     ),
+    "MiniMaxM3MTP": _HfExamplesInfo(
+        "MiniMaxAI/MiniMax-M3",
+        trust_remote_code=True,
+        is_available_online=False,
+    ),
     "NemotronHMTPModel": _HfExamplesInfo(
         "nvidia/Nemotron-Super-Placeholder",
         speculative_model="nvidia/Nemotron-Super-Placeholder",

tests/parser/engine/conftest.py

@@ -0,0 +1,40 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from unittest.mock import MagicMock
+
+import pytest
+
+from vllm.entrypoints.openai.chat_completion.protocol import (
+    ChatCompletionRequest,
+)
+
+
+@pytest.fixture()
+def should_do_global_cleanup_after_test() -> bool:
+    return False
+
+
+def make_mock_tokenizer(vocab: dict[str, int]) -> MagicMock:
+    """Create a mock tokenizer with the given special-token vocabulary.
+
+    The returned mock supports get_vocab(), encode(), and decode().
+    decode() maps known token IDs back to their text and falls back to
+    chr(id) for ASCII IDs or ``<id>`` for others.
+    """
+    id_to_text = {v: k for k, v in vocab.items()}
+    tokenizer = MagicMock()
+    tokenizer.encode.return_value = [1, 2, 3]
+    tokenizer.get_vocab.return_value = dict(vocab)
+    tokenizer.decode.side_effect = lambda ids: "".join(
+        id_to_text.get(i, chr(i) if i < 128 else f"<{i}>") for i in ids
+    )
+    return tokenizer
+
+
+@pytest.fixture
+def mock_request():
+    req = MagicMock(spec=ChatCompletionRequest)
+    req.tools = []
+    req.tool_choice = "auto"
+    return req

tests/parser/engine/replay_harness.py

@@ -0,0 +1,377 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Data-driven replay harness for parser engine testing.
+
+Replays token sequences through parsers at different chunk sizes to
+verify chunk-size invariance: the same token sequence must produce
+identical output regardless of how tokens are batched.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Sequence
+from dataclasses import dataclass, field
+
+from vllm.entrypoints.openai.chat_completion.protocol import (
+    ChatCompletionRequest,
+)
+from vllm.entrypoints.openai.engine.protocol import DeltaMessage
+
+
+@dataclass
+class Sample:
+    """One test sample loaded from a JSONL file."""
+
+    id: str
+    description: str
+    source: str
+    vocab: dict[str, int]
+    tokens: list[tuple[int, str]]
+    expected_reasoning: str | None
+    expected_content: str | None
+    expected_tool_calls: list[dict] | None
+    tools: list[dict] | None = None
+    chat_template_kwargs: dict | None = None
+
+
+@dataclass
+class ParseOutput:
+    """Accumulated parse output from replaying a token stream."""
+
+    reasoning: str = ""
+    content: str = ""
+    tool_calls: list[dict] = field(default_factory=list)
+
+
+class MockTokenizer:
+    """Lightweight tokenizer mock that avoids unittest.mock overhead.
+
+    Used by ``benchmarks/benchmark_parsers.py`` in tight timing loops,
+    so hot-path methods (``decode``, ``get_vocab``) must be cheap.
+    MagicMock's call-recording machinery added ~40% overhead to small-
+    sample benchmarks, inflating the per-token cost of the parser engine.
+    """
+
+    __slots__ = (
+        "_vocab",
+        "_token_ids",
+        "_token_decode_map",
+        "_special_ids",
+        "eos_token_id",
+        "bos_token_id",
+        "pad_token_id",
+    )
+
+    def __init__(
+        self,
+        vocab: dict[str, int],
+        tokens: list[tuple[int, str]],
+    ) -> None:
+        self._vocab = vocab
+        self._token_ids = [tid for tid, _ in tokens]
+        self._token_decode_map = {tid: text for tid, text in tokens}
+        self._special_ids = set(vocab.values())
+        self.eos_token_id = None
+        self.bos_token_id = None
+        self.pad_token_id = None
+
+    def set_vocab(self, vocab: dict[str, int]) -> None:
+        self._vocab = vocab
+
+    def get_vocab(self) -> dict[str, int]:
+        return self._vocab
+
+    def encode(self, text: str, **kwargs) -> list[int]:
+        return self._token_ids
+
+    def decode(self, ids: list[int], skip_special_tokens: bool = False) -> str:
+        parts: list[str] = []
+        for tid in ids:
+            if skip_special_tokens and tid in self._special_ids:
+                continue
+            text = self._token_decode_map.get(tid, f"?{tid}?")
+            parts.append(text)
+        return "".join(parts)
+
+
+def make_mock_tokenizer(sample: Sample) -> MockTokenizer:
+    """Build a mock tokenizer from a sample's vocab and token data."""
+    return MockTokenizer(
+        vocab=dict(sample.vocab),
+        tokens=sample.tokens,
+    )
+
+
+def _test_request(
+    tools: list[dict] | None = None,
+) -> ChatCompletionRequest:
+    return ChatCompletionRequest(
+        model="test-model",
+        messages=[{"role": "user", "content": "test"}],
+        tools=tools,
+    )
+
+
+def replay_streaming(
+    parser,
+    tokens: list[tuple[int, str]],
+    chunk_size: int | None = None,
+    holdback_chars: int = 0,
+    finished_on_last: bool = False,
+    tools: list[dict] | None = None,
+) -> list[DeltaMessage | None]:
+    """Feed tokens through ``parser.parse_delta()`` at a given chunk size.
+
+    Args:
+        parser: A :class:`Parser` instance with ``parse_delta()`` method.
+        tokens: List of ``(token_id, decoded_text)`` pairs.
+        chunk_size: Number of tokens per batch. ``None`` means all at once.
+        holdback_chars: Simulate detokenizer holdback by holding back
+            this many characters of decoded text between batches.
+        finished_on_last: When True, pass ``finished=True`` on the last
+            ``parse_delta()`` call, matching real server behavior.
+        tools: Optional tool definitions to include on the request,
+            matching the serving layer where tools set
+            ``tool_choice`` to ``"auto"``.
+
+    Returns:
+        List of ``DeltaMessage`` results from each ``parse_delta()`` call.
+    """
+    if chunk_size is None:
+        chunk_size = len(tokens)
+
+    results: list[DeltaMessage | None] = []
+    all_ids = [tid for tid, _ in tokens]
+    all_texts = [text for _, text in tokens]
+
+    request = _test_request(tools=tools)
+
+    if holdback_chars <= 0:
+        chunks = list(range(0, len(tokens), chunk_size))
+        for i, start in enumerate(chunks):
+            batch_end = min(start + chunk_size, len(tokens))
+            batch_ids = all_ids[start:batch_end]
+            delta_text = "".join(all_texts[start:batch_end])
+            is_last = i == len(chunks) - 1
+
+            result = parser.parse_delta(
+                delta_text,
+                batch_ids,
+                request,
+                prompt_token_ids=[] if start == 0 else None,
+                finished=finished_on_last and is_last,
+            )
+            results.append(result)
+        return results
+
+    emitted_up_to = 0
+    is_first = True
+
+    for start in range(0, len(tokens), chunk_size):
+        batch_end = min(start + chunk_size, len(tokens))
+
+        if batch_end < len(tokens):
+            held_chars = 0
+            safe_end = batch_end
+            while safe_end > emitted_up_to and held_chars < holdback_chars:
+                safe_end -= 1
+                held_chars += len(all_texts[safe_end])
+        else:
+            safe_end = batch_end
+
+        if safe_end <= emitted_up_to:
+            continue
+
+        batch_ids = all_ids[emitted_up_to:safe_end]
+        delta_text = "".join(all_texts[emitted_up_to:safe_end])
+        emitted_up_to = safe_end
+
+        is_last_chunk = batch_end >= len(tokens)
+        result = parser.parse_delta(
+            delta_text,
+            batch_ids,
+            request,
+            prompt_token_ids=[] if is_first else None,
+            finished=finished_on_last and is_last_chunk,
+        )
+        results.append(result)
+        is_first = False
+
+    if emitted_up_to < len(tokens):
+        batch_ids = all_ids[emitted_up_to:]
+        delta_text = "".join(all_texts[emitted_up_to:])
+        result = parser.parse_delta(
+            delta_text,
+            batch_ids,
+            request,
+            prompt_token_ids=[] if is_first else None,
+            finished=finished_on_last,
+        )
+        results.append(result)
+
+    return results
+
+
+def replay_with_text_holdback(
+    parser,
+    tokens: list[tuple[int, str]],
+    text_delay: int = 1,
+    tools: list[dict] | None = None,
+) -> list[DeltaMessage | None]:
+    """Replay token-by-token with text arriving *text_delay* steps late.
+
+    Simulates the production detokenizer holdback where token IDs arrive
+    immediately but decoded text is delayed.  On the last token all
+    remaining held-back text is flushed, matching real server behavior::
+
+        step 0:   ids=[tok0], text=""               (held back)
+        step 1:   ids=[tok1], text=tok0_text         (tok0 released)
+        ...
+        step N-1: ids=[tokN-1], text=remaining_texts (flush all)
+
+    This exercises the TokenIDScanner deferred-terminal path that
+    ``replay_streaming`` (which keeps text and IDs aligned) does not.
+    """
+    results: list[DeltaMessage | None] = []
+    request = _test_request(tools=tools)
+
+    n = len(tokens)
+    held_texts: list[str] = []
+
+    for i in range(n):
+        token_id = tokens[i][0]
+        held_texts.append(tokens[i][1])
+
+        is_last = i == n - 1
+        if is_last:
+            delta_text = "".join(held_texts)
+            held_texts.clear()
+        elif len(held_texts) > text_delay:
+            delta_text = held_texts.pop(0)
+        else:
+            delta_text = ""
+
+        result = parser.parse_delta(
+            delta_text,
+            [token_id],
+            request,
+            prompt_token_ids=[] if i == 0 else None,
+            finished=is_last,
+        )
+        results.append(result)
+
+    return results
+
+
+def accumulate_deltas(
+    deltas: Sequence[DeltaMessage | None],
+) -> dict:
+    reasoning_parts: list[str] = []
+    content_parts: list[str] = []
+    tool_calls_by_idx: dict[int, dict] = {}
+
+    for delta in deltas:
+        if delta is None:
+            continue
+        if delta.reasoning:
+            reasoning_parts.append(delta.reasoning)
+        if delta.content:
+            content_parts.append(delta.content)
+        if delta.tool_calls:
+            for tc in delta.tool_calls:
+                if tc.function and tc.function.name:
+                    existing = tool_calls_by_idx.get(tc.index)
+                    if existing is None:
+                        tool_calls_by_idx[tc.index] = {
+                            "name": tc.function.name,
+                            "_args_parts": [tc.function.arguments or ""],
+                        }
+                    else:
+                        existing["_args_parts"].append(tc.function.arguments or "")
+                elif tc.function and tc.function.arguments:
+                    existing = tool_calls_by_idx.get(tc.index)
+                    if existing is not None:
+                        existing["_args_parts"].append(tc.function.arguments)
+
+    return {
+        "reasoning": "".join(reasoning_parts),
+        "content": "".join(content_parts),
+        "tool_calls": [
+            {"name": tc["name"], "arguments": "".join(tc["_args_parts"])}
+            for tc in tool_calls_by_idx.values()
+        ],
+    }
+
+
+def collect_output(results: list[DeltaMessage | None]) -> ParseOutput:
+    """Accumulate ``DeltaMessage`` results into a :class:`ParseOutput`."""
+    result = accumulate_deltas(results)
+    return ParseOutput(
+        reasoning=result["reasoning"],
+        content=result["content"],
+        tool_calls=result["tool_calls"],
+    )
+
+
+def assert_parse_output(actual: ParseOutput, sample: Sample) -> None:
+    """Compare actual parse output against expected values from a sample."""
+    if sample.expected_reasoning is not None:
+        assert actual.reasoning == sample.expected_reasoning, (
+            f"Reasoning mismatch:\n"
+            f"  expected: {sample.expected_reasoning!r}\n"
+            f"  actual:   {actual.reasoning!r}"
+        )
+
+    if sample.expected_content is not None:
+        assert actual.content == sample.expected_content, (
+            f"Content mismatch:\n"
+            f"  expected: {sample.expected_content!r}\n"
+            f"  actual:   {actual.content!r}"
+        )
+    if sample.expected_tool_calls is not None:
+        assert len(actual.tool_calls) == len(sample.expected_tool_calls), (
+            f"Tool call count mismatch: "
+            f"expected {len(sample.expected_tool_calls)}, "
+            f"got {len(actual.tool_calls)}"
+        )
+        for i, (expected_tc, actual_tc) in enumerate(
+            zip(sample.expected_tool_calls, actual.tool_calls)
+        ):
+            assert actual_tc["name"] == expected_tc["name"], (
+                f"Tool call {i} name mismatch: "
+                f"expected {expected_tc['name']!r}, "
+                f"got {actual_tc['name']!r}"
+            )
+            if "arguments" in expected_tc:
+                expected_args = expected_tc["arguments"]
+                actual_args_str = actual_tc.get("arguments", "{}")
+                if isinstance(expected_args, dict):
+                    try:
+                        actual_args = json.loads(actual_args_str)
+                    except json.JSONDecodeError as e:
+                        raise AssertionError(
+                            f"Tool call {i} arguments not valid JSON: "
+                            f"{actual_args_str!r}"
+                        ) from e
+                    assert actual_args == expected_args, (
+                        f"Tool call {i} arguments mismatch:\n"
+                        f"  expected: {expected_args}\n"
+                        f"  actual:   {actual_args}"
+                    )
+
+
+def assert_no_terminal_leakage(
+    actual: ParseOutput,
+    terminals: list[str],
+    context: str = "",
+) -> None:
+    """Assert that none of *terminals* appear in reasoning or content."""
+    suffix = f" ({context})" if context else ""
+    for terminal in terminals:
+        assert terminal not in actual.reasoning, (
+            f"{terminal!r} leaked into reasoning{suffix}"
+        )
+        assert terminal not in actual.content, (
+            f"{terminal!r} leaked into content{suffix}"
+        )

tests/parser/engine/streaming_helpers.py

@@ -0,0 +1,137 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Shared streaming simulation helpers for parser engine tests."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from vllm.entrypoints.openai.engine.protocol import DeltaMessage
+
+
+def _build_token_id_map(parser) -> dict[str, int]:
+    """Map special token text to token IDs from the parser's config."""
+    token_id_map: dict[str, int] = {}
+    cfg = getattr(parser, "parser_engine_config", None)
+    vocab = getattr(parser, "vocab", None)
+    if cfg is not None and vocab is not None:
+        for text in (cfg.token_id_terminals or {}).values():
+            tid = vocab.get(text)
+            if tid is not None:
+                token_id_map[text] = tid
+    return token_id_map
+
+
+def simulate_tool_streaming(
+    parser,
+    request,
+    chunks: list[str],
+) -> list[tuple[DeltaMessage | None, str]]:
+    """Feed text chunks through ``extract_tool_calls_streaming()``."""
+    token_id_map = _build_token_id_map(parser)
+
+    results: list[tuple[Any, str]] = []
+    previous_text = ""
+    previous_token_ids: list[int] = []
+
+    for chunk in chunks:
+        current_text = previous_text + chunk
+
+        delta_token_ids: list[int] = [
+            tid for text, tid in token_id_map.items() if text in chunk
+        ]
+
+        current_token_ids = previous_token_ids + delta_token_ids
+
+        delta = parser.extract_tool_calls_streaming(
+            previous_text=previous_text,
+            current_text=current_text,
+            delta_text=chunk,
+            previous_token_ids=tuple(previous_token_ids),
+            current_token_ids=tuple(current_token_ids),
+            delta_token_ids=tuple(delta_token_ids),
+            request=request,
+        )
+        results.append((delta, current_text))
+        previous_text = current_text
+        previous_token_ids = list(current_token_ids)
+
+    return results
+
+
+def collect_tool_arguments(
+    results: list[tuple[DeltaMessage | None, str]],
+) -> str:
+    """Concatenate all streamed argument fragments."""
+    args_text = ""
+    for delta, _ in results:
+        if delta and delta.tool_calls:
+            for tc in delta.tool_calls:
+                if tc.function and tc.function.arguments:
+                    args_text += tc.function.arguments
+    return args_text
+
+
+def collect_content(
+    results: list[tuple[DeltaMessage | None, str]],
+) -> str:
+    """Concatenate all streamed content parts."""
+    parts: list[str] = []
+    for delta, _ in results:
+        if delta and delta.content:
+            parts.append(delta.content)
+    return "".join(parts)
+
+
+def collect_function_name(
+    results: list[tuple[DeltaMessage | None, str]],
+) -> str | None:
+    """Return first function name from deltas."""
+    for delta, _ in results:
+        if delta and delta.tool_calls:
+            for tc in delta.tool_calls:
+                if tc.function and tc.function.name:
+                    return tc.function.name
+    return None
+
+
+def simulate_reasoning_streaming(
+    parser,
+    chunks: list[str],
+    delta_token_ids_per_chunk: list[tuple[int, ...]] | None = None,
+) -> tuple[str, str]:
+    """Feed chunks through ``extract_reasoning_streaming()``.
+
+    Returns ``(reasoning_text, content_text)`` tuple.
+    """
+    token_id_map = (
+        _build_token_id_map(parser) if delta_token_ids_per_chunk is None else {}
+    )
+
+    reasoning_parts: list[str] = []
+    content_parts: list[str] = []
+    prev_text = ""
+    prev_ids: list[int] = []
+    for i, chunk in enumerate(chunks):
+        cur_text = prev_text + chunk
+        if delta_token_ids_per_chunk is not None:
+            d_ids = delta_token_ids_per_chunk[i]
+        else:
+            d_ids = tuple(tid for text, tid in token_id_map.items() if text in chunk)
+        cur_ids = prev_ids + list(d_ids)
+        delta = parser.extract_reasoning_streaming(
+            previous_text=prev_text,
+            current_text=cur_text,
+            delta_text=chunk,
+            previous_token_ids=tuple(prev_ids),
+            current_token_ids=tuple(cur_ids),
+            delta_token_ids=d_ids,
+        )
+        if delta:
+            if delta.reasoning:
+                reasoning_parts.append(delta.reasoning)
+            if delta.content:
+                content_parts.append(delta.content)
+        prev_text = cur_text
+        prev_ids = list(cur_ids)
+    return "".join(reasoning_parts), "".join(content_parts)

tests/parser/engine/test_delegating_replay.py

@@ -0,0 +1,82 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Replay tests for DelegatingParser with engine adapters.
+
+Exercises DelegatingParser in engine-adapter mode to verify that delegated
+routing produces correct output across chunk sizes.
+See test_replay.py for tests that target engine parsers directly.
+"""
+
+from __future__ import annotations
+
+from functools import lru_cache
+
+import pytest
+from pydantic import TypeAdapter
+
+from tests.parser.engine.replay_harness import (
+    assert_parse_output,
+    collect_output,
+    make_mock_tokenizer,
+    replay_streaming,
+)
+from tests.parser.engine.trace_builder import build_samples
+from vllm.entrypoints.openai.chat_completion.protocol import (
+    ChatCompletionToolsParam,
+)
+from vllm.parser.abstract_parser import Parser
+from vllm.parser.parser_manager import ParserManager
+
+_TOOLS_VALIDATOR = TypeAdapter(list[ChatCompletionToolsParam])
+
+_PAIRINGS: dict[str, tuple[str, str]] = {
+    "engine": ("qwen3_coder", "qwen3"),
+}
+
+CHUNK_SIZES = [1, 2, 3, 5, 11, 23, None]
+
+
+@lru_cache
+def _get_delegating_parser_cls(pairings: str) -> type[Parser]:
+    tool_name, reasoning_name = _PAIRINGS[pairings]
+    parser_cls = ParserManager.get_parser(
+        tool_parser_name=tool_name,
+        reasoning_parser_name=reasoning_name,
+        enable_auto_tools=True,
+    )
+    assert parser_cls is not None
+    return parser_cls
+
+
+_all_samples = build_samples("qwen3")
+
+
+@pytest.mark.parametrize(
+    "pairings",
+    list(_PAIRINGS),
+    ids=lambda p: f"mode={p}",
+)
+@pytest.mark.parametrize("chunk_size", CHUNK_SIZES, ids=lambda c: f"chunk={c}")
+@pytest.mark.parametrize("sample", _all_samples, ids=lambda s: s.id)
+def test_delegating_replay(sample, chunk_size, pairings):
+    parser_cls = _get_delegating_parser_cls(pairings=pairings)
+
+    tokenizer = make_mock_tokenizer(sample)
+    validated_tools = (
+        _TOOLS_VALIDATOR.validate_python(sample.tools) if sample.tools else None
+    )
+    parser = parser_cls(
+        tokenizer,
+        validated_tools,
+        chat_template_kwargs=sample.chat_template_kwargs,
+    )
+
+    deltas = replay_streaming(
+        parser,
+        sample.tokens,
+        chunk_size=chunk_size,
+        finished_on_last=True,
+        tools=sample.tools,
+    )
+    output = collect_output(deltas)
+    assert_parse_output(output, sample)

tests/parser/engine/test_engine.py

@@ -0,0 +1,846 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Tests for the streaming parser engine core pipeline."""
+
+from unittest.mock import MagicMock
+
+import pytest
+
+from tests.parser.engine.conftest import make_mock_tokenizer
+from vllm.parser.engine.events import EventType, SemanticEvent
+from vllm.parser.engine.incremental_lexer import (
+    LexerShape,
+    TerminalDef,
+    terminals_from_literals,
+)
+from vllm.parser.engine.parser_engine_config import (
+    ParserEngineConfig,
+    ParserState,
+    Transition,
+)
+from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
+
+
+def _hermes_config() -> ParserEngineConfig:
+    """Simple Hermes-style config: <tool_call>JSON</tool_call>."""
+    return ParserEngineConfig(
+        name="hermes_test",
+        terminals={
+            "TOOL_START": "<tool_call>",
+            "TOOL_END": "</tool_call>",
+        },
+        token_id_terminals={
+            "TOOL_START": "<tool_call>",
+            "TOOL_END": "</tool_call>",
+        },
+        transitions={
+            (ParserState.CONTENT, "TOOL_START"): Transition(
+                ParserState.TOOL_ARGS,
+                (EventType.TOOL_CALL_START,),
+            ),
+            (ParserState.TOOL_ARGS, "TOOL_END"): Transition(
+                ParserState.CONTENT,
+                (EventType.TOOL_CALL_END,),
+            ),
+        },
+        content_events={
+            ParserState.CONTENT: EventType.TEXT_CHUNK,
+            ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK,
+        },
+    )
+
+
+def _think_config() -> ParserEngineConfig:
+    """Simple think-tag reasoning config: <think>...</think>."""
+    return ParserEngineConfig(
+        name="think_test",
+        terminals={
+            "THINK_START": "<think>",
+            "THINK_END": "</think>",
+        },
+        transitions={
+            (ParserState.CONTENT, "THINK_START"): Transition(
+                ParserState.REASONING,
+                (EventType.REASONING_START,),
+            ),
+            (ParserState.REASONING, "THINK_END"): Transition(
+                ParserState.CONTENT,
+                (EventType.REASONING_END,),
+            ),
+        },
+    )
+
+
+class TestNonStreaming:
+    def test_plain_text(self):
+        engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
+        events = engine.parse_complete("Hello, world!")
+        assert len(events) == 1
+        assert events[0].type == EventType.TEXT_CHUNK
+        assert events[0].value == "Hello, world!"
+
+    def test_single_tool_call(self):
+        engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
+        text = (
+            '<tool_call>{"name": "get_weather",'
+            ' "arguments": {"city": "SF"}}'
+            "</tool_call>"
+        )
+        events = engine.parse_complete(text)
+
+        types = [e.type for e in events]
+        assert EventType.TOOL_CALL_START in types
+        assert EventType.TOOL_CALL_END in types
+        assert EventType.ARG_VALUE_CHUNK in types
+
+        arg_text = "".join(
+            e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK
+        )
+        assert '"name": "get_weather"' in arg_text
+        assert '"city": "SF"' in arg_text
+
+    def test_text_then_tool_call(self):
+        engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
+        text = 'Sure!<tool_call>{"name": "add"}</tool_call>'
+        events = engine.parse_complete(text)
+
+        types = [e.type for e in events]
+        assert types[0] == EventType.TEXT_CHUNK
+        assert events[0].value == "Sure!"
+        assert EventType.TOOL_CALL_START in types
+        assert EventType.TOOL_CALL_END in types
+
+    def test_multiple_tool_calls(self):
+        engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
+        text = (
+            '<tool_call>{"name": "a"}</tool_call><tool_call>{"name": "b"}</tool_call>'
+        )
+        events = engine.parse_complete(text)
+
+        starts = [e for e in events if e.type == EventType.TOOL_CALL_START]
+        ends = [e for e in events if e.type == EventType.TOOL_CALL_END]
+        assert len(starts) == 2
+        assert len(ends) == 2
+        assert starts[0].tool_index == 0
+        assert starts[1].tool_index == 1
+
+    def test_reasoning(self):
+        engine = StreamingParserEngine(_think_config(), tokenizer=None)
+        text = "<think>Let me think...</think>The answer is 42."
+        events = engine.parse_complete(text)
+
+        types = [e.type for e in events]
+        assert types[0] == EventType.REASONING_START
+        assert EventType.REASONING_CHUNK in types
+        assert EventType.REASONING_END in types
+        assert EventType.TEXT_CHUNK in types
+
+        reasoning = "".join(
+            e.value for e in events if e.type == EventType.REASONING_CHUNK
+        )
+        assert "Let me think..." in reasoning
+
+        content = "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK)
+        assert "The answer is 42." in content
+
+
+class TestStreaming:
+    @staticmethod
+    def _feed_chars(
+        engine: StreamingParserEngine,
+        text: str,
+    ) -> list[SemanticEvent]:
+        """Feed text one character at a time."""
+        all_events = []
+        for ch in text:
+            all_events.extend(engine.feed(ch, []))
+        all_events.extend(engine.finish())
+        return all_events
+
+    @staticmethod
+    def _feed_chunks(
+        engine: StreamingParserEngine,
+        text: str,
+        chunk_size: int,
+    ) -> list[SemanticEvent]:
+        """Feed text in fixed-size chunks."""
+        all_events = []
+        for i in range(0, len(text), chunk_size):
+            chunk = text[i : i + chunk_size]
+            all_events.extend(engine.feed(chunk, []))
+        all_events.extend(engine.finish())
+        return all_events
+
+    def test_char_by_char_tool_call(self):
+        engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
+        text = '<tool_call>{"name": "add", "arguments": {"a": 1}}</tool_call>'
+        events = self._feed_chars(engine, text)
+
+        types = [e.type for e in events]
+        assert EventType.TOOL_CALL_START in types
+        assert EventType.TOOL_CALL_END in types
+        assert EventType.ARG_VALUE_CHUNK in types
+
+        arg_text = "".join(
+            e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK
+        )
+        assert '"name": "add"' in arg_text
+
+    @pytest.mark.parametrize(
+        "text",
+        [
+            '<tool_call>{"name": "get", "arguments": {"x": "hello"}}</tool_call>',
+            '<tool_call>{"name": "f", "arguments": '
+            '{"items": [1, [2, 3]], "obj": {"k": "v"}}}'
+            "</tool_call>",
+        ],
+        ids=["flat_args", "nested_arrays"],
+    )
+    def test_chunk_sizes_produce_same_content(self, text):
+        """Different chunk sizes must produce identical concatenated content."""
+        results = {}
+        for chunk_size in [1, 2, 3, 5, 7, len(text)]:
+            engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
+            events = self._feed_chunks(engine, text, chunk_size)
+            arg_text = "".join(
+                e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK
+            )
+            results[chunk_size] = arg_text
+
+        values = list(results.values())
+        for v in values[1:]:
+            assert v == values[0], f"Mismatch: {results}"
+
+    def test_prefix_buffering_prevents_premature_emit(self):
+        """Text like '<tool_' should be buffered, not emitted as content."""
+        engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
+
+        events1 = engine.feed("<tool_", [])
+        content_events = [e for e in events1 if e.type == EventType.TEXT_CHUNK]
+        assert len(content_events) == 0, "Should buffer partial tag"
+
+        events2 = engine.feed("call>", [])
+        starts = [e for e in events2 if e.type == EventType.TOOL_CALL_START]
+        assert len(starts) == 1
+
+    def test_prefix_buffering_flush_on_mismatch(self):
+        """Text like '<tool_box' should eventually flush as content."""
+        engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
+
+        events1 = engine.feed("<tool_", [])
+        assert len([e for e in events1 if e.type == EventType.TEXT_CHUNK]) == 0
+
+        events2 = engine.feed("box>rest", [])
+        events2.extend(engine.finish())
+        content = "".join(e.value for e in events2 if e.type == EventType.TEXT_CHUNK)
+        assert content == "<tool_box>rest"
+
+    def test_reasoning_streaming(self):
+        engine = StreamingParserEngine(_think_config(), tokenizer=None)
+        events = self._feed_chars(engine, "<think>hmm</think>answer")
+
+        reasoning = "".join(
+            e.value for e in events if e.type == EventType.REASONING_CHUNK
+        )
+        content = "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK)
+        assert "hmm" in reasoning
+        assert "answer" in content
+
+    def test_text_between_tool_calls(self):
+        engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
+        text = (
+            'Hi<tool_call>{"name":"a"}</tool_call>'
+            'mid<tool_call>{"name":"b"}</tool_call>end'
+        )
+        events = self._feed_chunks(engine, text, 3)
+
+        texts = "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK)
+        assert "Hi" in texts
+        assert "mid" in texts
+        assert "end" in texts
+
+        starts = [e for e in events if e.type == EventType.TOOL_CALL_START]
+        assert len(starts) == 2
+
+    def test_unmatched_close_brace_does_not_poison_depth(self):
+        """A stray } in malformed JSON must not kill streaming for
+        all subsequent content."""
+        engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
+        engine.feed("<tool_call>", [])
+
+        malformed = '}{{"a": 1}}'
+        events = self._feed_chars(engine, malformed + "</tool_call>")
+
+        arg_chunks = [e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK]
+        assert len(arg_chunks) > 1, (
+            "Content after stray } should still stream incrementally"
+        )
+        arg_text = "".join(arg_chunks)
+        assert '"a": 1' in arg_text
+
+    def test_json_args_no_premature_close_brace(self):
+        """Closing braces of the top-level JSON shouldn't be streamed
+        until confirmed by the end tag."""
+        engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
+
+        engine.feed("<tool_call>", [])
+        events = engine.feed('{"name": "f"}', [])
+
+        arg_text = "".join(
+            e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK
+        )
+        assert "}" not in arg_text, "Top-level } should be held back"
+
+        events2 = engine.feed("</tool_call>", [])
+        arg_text2 = "".join(
+            e.value for e in events2 if e.type == EventType.ARG_VALUE_CHUNK
+        )
+        assert "}" in arg_text2, "} should flush on end tag"
+
+
+_START_ID = 50
+_END_ID = 51
+_TOOL_START_ID = 60
+_TOOL_END_ID = 61
+
+
+def _make_think_tokenizer():
+    tok = MagicMock()
+    tok.encode.return_value = [1, 2, 3]
+    tok.get_vocab.return_value = {"<think>": _START_ID, "</think>": _END_ID}
+    tok.decode.side_effect = lambda ids: {
+        _START_ID: "<think>",
+        _END_ID: "</think>",
+    }.get(ids[0], f"tok{ids[0]}")
+    return tok
+
+
+def _make_hermes_tokenizer():
+    """Tokenizer that resolves tool_call tags to special IDs."""
+    _special = {_TOOL_START_ID: "<tool_call>", _TOOL_END_ID: "</tool_call>"}
+    tok = MagicMock()
+    tok.encode.return_value = [1, 2, 3]
+    tok.get_vocab.return_value = {
+        "<tool_call>": _TOOL_START_ID,
+        "</tool_call>": _TOOL_END_ID,
+    }
+    tok.decode.side_effect = lambda ids: "".join(
+        _special.get(i, chr(i) if i < 128 else f"<{i}>") for i in ids
+    )
+    return tok
+
+
+class TestLexerBufferFlush:
+    """Lexer buffer must be flushed before PreLexedTerminal transitions."""
+
+    def test_buffered_prefix_emitted_in_current_state(self):
+        """Text buffered by the lexer (e.g. '<') must be emitted as
+        REASONING_CHUNK before THINK_END transitions to CONTENT."""
+        engine = StreamingParserEngine(_think_config(), _make_think_tokenizer())
+
+        events = engine.feed("<think>", [_START_ID])
+        assert any(e.type == EventType.REASONING_START for e in events)
+
+        events = engine.feed("reasoning text<", [])
+        reasoning_text = "".join(
+            e.value for e in events if e.type == EventType.REASONING_CHUNK
+        )
+        assert "reasoning text" in reasoning_text
+
+        events = engine.feed("</think>", [_END_ID])
+        event_types = [e.type for e in events]
+        if EventType.REASONING_CHUNK in event_types:
+            rc_idx = event_types.index(EventType.REASONING_CHUNK)
+            re_idx = event_types.index(EventType.REASONING_END)
+            assert rc_idx < re_idx, (
+                "'<' must be emitted as REASONING_CHUNK before REASONING_END"
+            )
+            flushed = events[rc_idx].value
+            assert "<" in flushed
+
+    def test_empty_buffer_no_extra_events(self):
+        """When the lexer buffer is empty, flushing is a no-op."""
+        engine = StreamingParserEngine(_think_config(), _make_think_tokenizer())
+
+        engine.feed("<think>", [_START_ID])
+        engine.feed("clean text", [])
+
+        events = engine.feed("</think>", [_END_ID])
+        assert any(e.type == EventType.REASONING_END for e in events)
+        chunk_events = [e for e in events if e.type == EventType.REASONING_CHUNK]
+        assert all(e.value for e in chunk_events)
+
+
+class TestTokenIdFiltering:
+    """When token IDs are available, lex-matched terminals that also
+    have token_id_terminal entries should be demoted to content."""
+
+    def test_lex_matched_terminal_demoted_after_token_ids_seen(self):
+        """After receiving token IDs, text that matches a token-ID
+        terminal should be treated as content, not trigger a transition."""
+        engine = StreamingParserEngine(_hermes_config(), _make_hermes_tokenizer())
+
+        # First feed with a non-special token ID to set _ever_had_token_ids
+        engine.feed("prefix ", [1])
+
+        # Now feed text containing <tool_call> as literal text
+        events = engine.feed(
+            "Use <tool_call> to invoke tools.</tool_call>", [2, 3, 4, 5]
+        )
+        events.extend(engine.finish())
+
+        types = [e.type for e in events]
+        assert EventType.TOOL_CALL_START not in types
+        assert EventType.TEXT_CHUNK in types
+
+        text = "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK)
+        assert "<tool_call>" in text
+
+    def test_scanner_matched_terminal_bypasses_filter(self):
+        """PreLexedTerminals from the scanner bypass the filter and
+        still trigger state transitions."""
+        engine = StreamingParserEngine(_hermes_config(), _make_hermes_tokenizer())
+
+        events = engine.feed("<tool_call>", [_TOOL_START_ID])
+        assert any(e.type == EventType.TOOL_CALL_START for e in events)
+
+        events = engine.feed('{"name": "f"}', [2, 3])
+        events.extend(engine.feed("</tool_call>", [_TOOL_END_ID]))
+        events.extend(engine.finish())
+        assert any(e.type == EventType.TOOL_CALL_END for e in events)
+
+    def test_no_filtering_without_token_ids(self):
+        """When no token IDs are ever provided (non-streaming),
+        text matching still triggers transitions."""
+        engine = StreamingParserEngine(_hermes_config(), _make_hermes_tokenizer())
+
+        events = engine.feed('<tool_call>{"name": "f"}</tool_call>', [])
+        events.extend(engine.finish())
+
+        types = [e.type for e in events]
+        assert EventType.TOOL_CALL_START in types
+        assert EventType.TOOL_CALL_END in types
+
+    def test_mixed_text_then_real_tool_call(self):
+        """Text mentioning tool syntax followed by a real special-token
+        tool call."""
+        engine = StreamingParserEngine(_hermes_config(), _make_hermes_tokenizer())
+
+        events1 = engine.feed("Mention <tool_call> in text. ", [1, 2, 3, 4])
+        events2 = engine.feed("<tool_call>", [_TOOL_START_ID])
+        events3 = engine.feed('{"name": "a"}', [5, 6])
+        events4 = engine.feed("</tool_call>", [_TOOL_END_ID])
+        events4.extend(engine.finish())
+
+        all_events = events1 + events2 + events3 + events4
+
+        content = "".join(e.value for e in all_events if e.type == EventType.TEXT_CHUNK)
+        assert "<tool_call>" in content
+
+        assert sum(1 for e in all_events if e.type == EventType.TOOL_CALL_START) == 1
+        assert sum(1 for e in all_events if e.type == EventType.TOOL_CALL_END) == 1
+
+
+def _func_prefix_config() -> ParserEngineConfig:
+    """Config mixing token-ID terminals (TOOL_START/END) with
+    text-only terminals (FUNC_PREFIX) and fallback transitions."""
+    return ParserEngineConfig(
+        name="func_prefix_test",
+        terminals={
+            "TOOL_START": "<tool_call>",
+            "TOOL_END": "</tool_call>",
+            "FUNC_PREFIX": "<function=",
+            "FUNC_END": "</function>",
+            "CLOSE_ANGLE": ">",
+        },
+        token_id_terminals={
+            "TOOL_START": "<tool_call>",
+            "TOOL_END": "</tool_call>",
+        },
+        transitions={
+            (ParserState.CONTENT, "TOOL_START"): Transition(
+                ParserState.TOOL_PREAMBLE,
+                (EventType.TOOL_CALL_START,),
+            ),
+            (ParserState.CONTENT, "FUNC_PREFIX"): Transition(
+                ParserState.TOOL_NAME,
+                (EventType.TOOL_CALL_START,),
+                skip_in_token_id_mode=True,
+            ),
+            (ParserState.TOOL_PREAMBLE, "FUNC_PREFIX"): Transition(
+                ParserState.TOOL_NAME,
+                (),
+            ),
+            (ParserState.TOOL_NAME, "CLOSE_ANGLE"): Transition(
+                ParserState.TOOL_ARGS,
+                (),
+            ),
+            (ParserState.TOOL_ARGS, "FUNC_END"): Transition(
+                ParserState.TOOL_BETWEEN,
+                (EventType.TOOL_CALL_END,),
+            ),
+            (ParserState.TOOL_BETWEEN, "TOOL_END"): Transition(
+                ParserState.CONTENT,
+                (),
+            ),
+            (ParserState.TOOL_BETWEEN, "TOOL_START"): Transition(
+                ParserState.TOOL_PREAMBLE,
+                (EventType.TOOL_CALL_START,),
+            ),
+            (ParserState.TOOL_BETWEEN, "FUNC_PREFIX"): Transition(
+                ParserState.TOOL_NAME,
+                (EventType.TOOL_CALL_START,),
+                skip_in_token_id_mode=True,
+            ),
+        },
+        content_events={
+            ParserState.CONTENT: EventType.TEXT_CHUNK,
+            ParserState.TOOL_NAME: EventType.TOOL_NAME,
+            ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK,
+        },
+    )
+
+
+def _make_func_prefix_tokenizer():
+    return make_mock_tokenizer(
+        {
+            "<tool_call>": _TOOL_START_ID,
+            "</tool_call>": _TOOL_END_ID,
+        }
+    )
+
+
+class TestTextOnlyFallbackFiltering:
+    """When token IDs are available, transitions marked
+    skip_in_token_id_mode should be skipped."""
+
+    def test_func_prefix_in_prose_demoted_in_strict_mode(self):
+        """<function=get_time> in prose should NOT trigger a tool call
+        when strict mode is active."""
+        engine = StreamingParserEngine(
+            _func_prefix_config(), _make_func_prefix_tokenizer()
+        )
+        engine.feed("prefix ", [1])
+
+        events = engine.feed("Use <function=get_time> to check.", [2, 3, 4, 5])
+        events.extend(engine.finish())
+
+        types = [e.type for e in events]
+        assert EventType.TOOL_CALL_START not in types
+        assert EventType.TEXT_CHUNK in types
+        text = "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK)
+        assert "<function=" in text
+
+    def test_normal_flow_after_tool_start_still_works(self):
+        """TOOL_START (special token) -> FUNC_PREFIX (text) should
+        still parse a tool call normally in strict mode."""
+        engine = StreamingParserEngine(
+            _func_prefix_config(), _make_func_prefix_tokenizer()
+        )
+
+        events1 = engine.feed("<tool_call>", [_TOOL_START_ID])
+        assert any(e.type == EventType.TOOL_CALL_START for e in events1)
+
+        events2 = engine.feed("<function=get_weather>", [2, 3])
+        events3 = engine.feed("args", [4])
+        events4 = engine.feed("</function>", [5, 6])
+        events4.extend(engine.feed("</tool_call>", [_TOOL_END_ID]))
+        events4.extend(engine.finish())
+
+        all_events = events1 + events2 + events3 + events4
+        assert sum(1 for e in all_events if e.type == EventType.TOOL_CALL_START) == 1
+        assert sum(1 for e in all_events if e.type == EventType.TOOL_CALL_END) == 1
+
+    def test_fallback_fires_without_token_ids(self):
+        """When no token IDs are provided, fallback transitions should
+        still fire normally."""
+        engine = StreamingParserEngine(
+            _func_prefix_config(), _make_func_prefix_tokenizer()
+        )
+
+        events = engine.feed("<function=get_time>args</function>", [])
+        events.extend(engine.finish())
+
+        types = [e.type for e in events]
+        assert EventType.TOOL_CALL_START in types
+        assert EventType.TOOL_CALL_END in types
+
+    def test_tool_between_fallback_blocked_in_strict_mode(self):
+        """The (TOOL_BETWEEN, FUNC_PREFIX) fallback should also be
+        blocked in strict mode."""
+        engine = StreamingParserEngine(
+            _func_prefix_config(), _make_func_prefix_tokenizer()
+        )
+
+        engine.feed("<tool_call>", [_TOOL_START_ID])
+        engine.feed("<function=a>", [2, 3])
+        engine.feed("args", [4])
+        engine.feed("</function>", [5, 6])
+        engine.feed("</tool_call>", [_TOOL_END_ID])
+
+        events = engine.feed("<function=b>more</function>", [7, 8, 9])
+        events.extend(engine.finish())
+
+        types = [e.type for e in events]
+        assert EventType.TOOL_CALL_START not in types
+
+
+class TestNoUnusedTokenizerAttr:
+    """StreamingParserEngine no longer stores a redundant _tokenizer."""
+
+    def test_no_tokenizer_attribute(self):
+        config = ParserEngineConfig(name="test")
+        engine = StreamingParserEngine(config, tokenizer=None)
+        assert not hasattr(engine, "_tokenizer")
+
+
+class TestArgsResetOnReentry:
+    """When leaving TOOL_ARGS and later re-entering (e.g. two tool
+    calls), the entering-TOOL_ARGS block resets args tracking.  The
+    redundant reset on exit was removed."""
+
+    @staticmethod
+    def _multi_tool_config() -> ParserEngineConfig:
+        return ParserEngineConfig(
+            name="multi_tool",
+            terminals={
+                "TOOL_START": "<tool_call>",
+                "TOOL_END": "</tool_call>",
+                "TOOL_SEP": "<tool_sep>",
+            },
+            transitions={
+                (ParserState.CONTENT, "TOOL_START"): Transition(
+                    ParserState.TOOL_ARGS,
+                    (EventType.TOOL_CALL_START,),
+                ),
+                (ParserState.TOOL_ARGS, "TOOL_END"): Transition(
+                    ParserState.TOOL_BETWEEN,
+                    (EventType.TOOL_CALL_END,),
+                ),
+                (ParserState.TOOL_BETWEEN, "TOOL_SEP"): Transition(
+                    ParserState.TOOL_ARGS,
+                    (EventType.TOOL_CALL_START,),
+                ),
+            },
+            content_events={
+                ParserState.CONTENT: EventType.TEXT_CHUNK,
+                ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK,
+            },
+        )
+
+    def test_args_tracking_across_reentry(self):
+        engine = StreamingParserEngine(self._multi_tool_config(), tokenizer=None)
+
+        events = engine.feed(
+            '<tool_call>{"city": "SF"}</tool_call>'
+            "<tool_sep>"
+            '{"name": "bar"}</tool_call>',
+            [],
+        )
+
+        tool_starts = [e for e in events if e.type == EventType.TOOL_CALL_START]
+        tool_ends = [e for e in events if e.type == EventType.TOOL_CALL_END]
+        arg_chunks = [e for e in events if e.type == EventType.ARG_VALUE_CHUNK]
+
+        assert len(tool_starts) == 2
+        assert len(tool_ends) == 2
+        assert tool_starts[0].tool_index == 0
+        assert tool_starts[1].tool_index == 1
+
+        first_args = "".join(e.value for e in arg_chunks if e.tool_index == 0)
+        second_args = "".join(e.value for e in arg_chunks if e.tool_index == 1)
+        assert '"city"' in first_args
+        assert '"name"' in second_args
+
+    def test_brace_depth_resets_on_reentry(self):
+        """Verify _args_brace_depth resets when re-entering TOOL_ARGS."""
+        engine = StreamingParserEngine(self._multi_tool_config(), tokenizer=None)
+        engine.feed("<tool_call>", [])
+        assert engine.state == ParserState.TOOL_ARGS
+        assert engine._args_brace_depth == 0
+
+        engine.feed('{"a": 1}', [])
+        engine.feed("</tool_call>", [])
+        assert engine.state == ParserState.TOOL_BETWEEN
+
+        engine.feed("<tool_sep>", [])
+        assert engine.state == ParserState.TOOL_ARGS
+        assert engine._args_brace_depth == 0
+        assert engine._args_in_string is False
+        assert engine._args_escape_next is False
+
+
+class TestToolPreambleFinish:
+    """finish() in TOOL_PREAMBLE state emits TOOL_CALL_END when a tool
+    call was started (tool_index >= 0), but not when tool_index is -1."""
+
+    @staticmethod
+    def _preamble_with_tool_call_start_config() -> ParserEngineConfig:
+        return ParserEngineConfig(
+            name="preamble_tcs",
+            terminals={"TOOL_START": "<tool_call>"},
+            transitions={
+                (ParserState.CONTENT, "TOOL_START"): Transition(
+                    ParserState.TOOL_PREAMBLE,
+                    (EventType.TOOL_CALL_START,),
+                ),
+            },
+            content_events={ParserState.CONTENT: EventType.TEXT_CHUNK},
+        )
+
+    @staticmethod
+    def _preamble_without_tool_call_start_config() -> ParserEngineConfig:
+        return ParserEngineConfig(
+            name="preamble_no_tcs",
+            terminals={"TOOL_CALLS_START": "<tool_calls>"},
+            transitions={
+                (ParserState.CONTENT, "TOOL_CALLS_START"): Transition(
+                    ParserState.TOOL_PREAMBLE,
+                    (),
+                ),
+            },
+            content_events={ParserState.CONTENT: EventType.TEXT_CHUNK},
+        )
+
+    def test_finish_emits_tool_call_end_with_tool_index(self):
+        config = self._preamble_with_tool_call_start_config()
+        engine = StreamingParserEngine(config, tokenizer=None)
+
+        engine.feed("<tool_call>", [])
+        assert engine.state == ParserState.TOOL_PREAMBLE
+        assert engine.tool_index == 0
+
+        finish_events = engine.finish()
+        end_events = [e for e in finish_events if e.type == EventType.TOOL_CALL_END]
+        assert len(end_events) == 1
+        assert end_events[0].tool_index == 0
+
+    def test_finish_no_tool_call_end_without_tool_index(self):
+        config = self._preamble_without_tool_call_start_config()
+        engine = StreamingParserEngine(config, tokenizer=None)
+
+        engine.feed("<tool_calls>", [])
+        assert engine.state == ParserState.TOOL_PREAMBLE
+        assert engine.tool_index == -1
+
+        finish_events = engine.finish()
+        end_events = [e for e in finish_events if e.type == EventType.TOOL_CALL_END]
+        assert len(end_events) == 0
+        assert engine.state == ParserState.CONTENT
+
+
+class TestRegexTerminalInfraRemoved:
+    """TerminalDef.priority, LexerShape.regex_terminals, and the regex
+    matching loop were removed."""
+
+    def test_terminal_def_no_priority(self):
+        import regex as re
+
+        td = TerminalDef(name="X", pattern=re.compile("x"))
+        assert not hasattr(td, "priority")
+
+    def test_lexer_shape_no_regex_terminals(self):
+        shape = LexerShape([])
+        assert not hasattr(shape, "regex_terminals")
+
+    def test_terminals_from_literals_still_works(self):
+        literals = {"TOOL_START": "<tool_call>", "TOOL_END": "</tool_call>"}
+        defs = terminals_from_literals(literals)
+        assert len(defs) == 2
+        names = {d.name for d in defs}
+        assert names == {"TOOL_START", "TOOL_END"}
+        for d in defs:
+            assert d.is_literal
+            assert d.literal in ("<tool_call>", "</tool_call>")
+
+
+class TestMultiCharTerminalInArgs:
+    """Regression: multi-char terminals falling through in TOOL_ARGS
+    must be fed char-by-char via _feed_args_text, not _feed_args_char."""
+
+    @staticmethod
+    def _newline_config() -> ParserEngineConfig:
+        return ParserEngineConfig(
+            name="newline_test",
+            terminals={
+                "TOOL_START": "<tool_call>",
+                "TOOL_END": "</tool_call>",
+                "NEWLINE": "\n",
+            },
+            transitions={
+                (ParserState.CONTENT, "TOOL_START"): Transition(
+                    ParserState.TOOL_ARGS,
+                    (EventType.TOOL_CALL_START,),
+                ),
+                (ParserState.TOOL_ARGS, "TOOL_END"): Transition(
+                    ParserState.CONTENT,
+                    (EventType.TOOL_CALL_END,),
+                ),
+            },
+            content_events={
+                ParserState.CONTENT: EventType.TEXT_CHUNK,
+                ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK,
+            },
+        )
+
+    def test_newline_in_args_parsed_correctly(self):
+        engine = StreamingParserEngine(self._newline_config(), tokenizer=None)
+        text = '<tool_call>{"name": "f",\n"arguments": {"a": 1}}</tool_call>'
+        events = engine.parse_complete(text)
+
+        arg_text = "".join(
+            e.value for e in events if e.type == EventType.ARG_VALUE_CHUNK
+        )
+        assert '"name": "f"' in arg_text
+        assert '"arguments"' in arg_text
+
+    def test_newline_in_args_streaming(self):
+        engine = StreamingParserEngine(self._newline_config(), tokenizer=None)
+        all_events = TestStreaming._feed_chars(
+            engine, '<tool_call>{"name": "f",\n"a": 1}</tool_call>'
+        )
+
+        arg_text = "".join(
+            e.value for e in all_events if e.type == EventType.ARG_VALUE_CHUNK
+        )
+        assert '"name": "f"' in arg_text
+        assert '"a": 1' in arg_text
+
+
+class TestSkipToolParsing:
+    """When skip_tool_parsing is set, tool tags become content."""
+
+    def test_tool_tags_emitted_as_content(self):
+        engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
+        engine.skip_tool_parsing = True
+
+        text = '<tool_call>{"name": "f"}</tool_call>'
+        events = engine.parse_complete(text)
+
+        types = [e.type for e in events]
+        assert EventType.TOOL_CALL_START not in types
+        assert EventType.TOOL_CALL_END not in types
+
+        content = "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK)
+        assert "<tool_call>" in content
+        assert "</tool_call>" in content
+
+    def test_skip_tool_streaming(self):
+        engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
+        engine.skip_tool_parsing = True
+
+        all_events = TestStreaming._feed_chars(
+            engine, '<tool_call>{"name": "f"}</tool_call>'
+        )
+
+        types = [e.type for e in all_events]
+        assert EventType.TOOL_CALL_START not in types
+
+        content = "".join(e.value for e in all_events if e.type == EventType.TEXT_CHUNK)
+        assert "<tool_call>" in content
+
+    def test_reset_preserves_skip_tool_parsing(self):
+        engine = StreamingParserEngine(_hermes_config(), tokenizer=None)
+        engine.skip_tool_parsing = True
+        engine.reset()
+        assert engine.skip_tool_parsing is True

tests/parser/engine/test_qwen3_reasoning.py

@@ -0,0 +1,549 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Tests for the engine-based Qwen3 reasoning parser.
+
+Validates that ``Qwen3Parser`` correctly handles
+``<think>``/``</think>`` reasoning with Qwen3-specific extensions:
+- ``<tool_call>`` as implicit reasoning end (terminal + token ID)
+- Stripping ``<think>`` from generated output (old template compat)
+- No terminal text (``</think>``, ``<tool_call>``) leaks into output
+"""
+
+import dataclasses
+
+import pytest
+
+from tests.parser.engine.conftest import make_mock_tokenizer
+from tests.parser.engine.streaming_helpers import simulate_reasoning_streaming
+from vllm.parser.engine.parser_engine_config import ParserState
+from vllm.parser.qwen3 import Qwen3Parser, qwen3_config
+
+_THINK_START_ID = 50
+_THINK_END_ID = 51
+_TOOL_CALL_ID = 60
+_TOOL_CALL_END_ID = 61
+_TEXT_ID = 100
+
+_QWEN3_VOCAB = {
+    "<think>": _THINK_START_ID,
+    "</think>": _THINK_END_ID,
+    "<tool_call>": _TOOL_CALL_ID,
+    "</tool_call>": _TOOL_CALL_END_ID,
+}
+
+
+@pytest.fixture
+def mock_tokenizer():
+    return make_mock_tokenizer(_QWEN3_VOCAB)
+
+
+@pytest.fixture
+def parser(mock_tokenizer):
+    return Qwen3Parser(mock_tokenizer)
+
+
+class TestNonStreaming:
+    def test_reasoning_then_content(self, parser):
+        text = "<think>Let me analyze.</think>The answer is 42."
+        reasoning, content = parser.extract_reasoning(text, None)
+        assert reasoning == "Let me analyze."
+        assert content == "The answer is 42."
+
+    def test_no_start_token_in_output(self, parser):
+        """Qwen3.5+ style: <think> in prompt, only </think> in output."""
+        text = "Let me think about this.</think>The answer is 42."
+        reasoning, content = parser.extract_reasoning(text, None)
+        assert reasoning == "Let me think about this."
+        assert content == "The answer is 42."
+
+    def test_reasoning_only(self, parser):
+        text = "<think>Still thinking...</think>"
+        reasoning, content = parser.extract_reasoning(text, None)
+        assert reasoning == "Still thinking..."
+        assert content is None
+
+    def test_no_end_tag_all_reasoning(self, parser):
+        """No </think> means truncated output — everything is reasoning."""
+        text = "Hello, no reasoning here."
+        reasoning, content = parser.extract_reasoning(text, None)
+        assert reasoning == "Hello, no reasoning here."
+        assert content is None
+
+    def test_multiline_reasoning(self, parser):
+        text = (
+            "<think>Step 1: parse.\nStep 2: compute.\nStep 3: output.</think>Result: 7."
+        )
+        reasoning, content = parser.extract_reasoning(text, None)
+        assert "Step 1" in reasoning
+        assert "Step 3" in reasoning
+        assert content == "Result: 7."
+
+    def test_tool_call_implicit_end(self, parser):
+        """<tool_call> without </think> acts as implicit reasoning end."""
+        text = (
+            "<think>I need to read the file.\n\n"
+            "<tool_call>\n<function=bash>\n"
+            "<parameter=cmd>ls</parameter>\n"
+            "</function>\n</tool_call>"
+        )
+        reasoning, content = parser.extract_reasoning(text, None)
+        assert reasoning == "I need to read the file.\n\n"
+        assert "</think>" not in reasoning
+        assert "<tool_call>" not in reasoning
+
+    def test_tool_call_implicit_end_no_think(self, parser):
+        """<tool_call> as implicit end, no <think> in output."""
+        text = (
+            "I need to read the file.\n\n"
+            "<tool_call>\n<function=bash>\n"
+            "<parameter=cmd>ls</parameter>\n"
+            "</function>\n</tool_call>"
+        )
+        reasoning, content = parser.extract_reasoning(text, None)
+        assert reasoning == "I need to read the file.\n\n"
+        assert "<tool_call>" not in reasoning
+
+    def test_live_scenario_think_end_before_tool_call(self, parser):
+        """Real model output: </think> immediately before <tool_call>.
+
+        Regression test for the bug where </think> and <parameter=...>
+        leaked into reasoning content.
+        """
+        text = (
+            "The user wants to see what files are in the current directory"
+            " and their contents. Let me start by listing the directory."
+            "</think><tool_call><function=read>"
+            "<parameter=filePath>/Users/test/demo</parameter>"
+            "</function></tool_call>"
+        )
+        reasoning, content = parser.extract_reasoning(text, None)
+        expected_reasoning = (
+            "The user wants to see what files are in the current directory"
+            " and their contents. Let me start by listing the directory."
+        )
+        assert reasoning == expected_reasoning
+        assert "</think>" not in reasoning
+        assert "<tool_call>" not in reasoning
+        assert "<parameter=" not in reasoning
+
+    def test_no_terminal_text_in_reasoning(self, parser):
+        """Terminal text must never appear in reasoning output."""
+        text = "Reasoning here.</think>Content here."
+        reasoning, content = parser.extract_reasoning(text, None)
+        assert "</think>" not in (reasoning or "")
+        assert "<think>" not in (reasoning or "")
+
+    def test_no_terminal_text_in_content(self, parser):
+        """Terminal text must never appear in content output."""
+        text = "Reasoning here.</think>Content here."
+        reasoning, content = parser.extract_reasoning(text, None)
+        assert "</think>" not in (content or "")
+        assert "<think>" not in (content or "")
+
+    def test_duplicate_think_end_absorbed(self, parser):
+        """Duplicate </think> in CONTENT state must not leak."""
+        text = "Reasoning here.</think>Content here.</think>More content."
+        reasoning, content = parser.extract_reasoning(text, None)
+        assert reasoning == "Reasoning here."
+        assert content == "Content here.More content."
+
+
+class TestIsReasoningEnd:
+    def test_think_end_token(self, parser):
+        assert parser.is_reasoning_end([_THINK_START_ID, 1, _THINK_END_ID])
+
+    def test_no_end_token(self, parser):
+        assert not parser.is_reasoning_end([_THINK_START_ID, 1, 2])
+
+    def test_start_after_end_means_not_ended(self, parser):
+        assert not parser.is_reasoning_end([_THINK_END_ID, _THINK_START_ID, 1])
+
+    def test_tool_call_as_implicit_end(self, parser):
+        """Unpaired <tool_call> is implicit reasoning end."""
+        assert parser.is_reasoning_end([_THINK_START_ID, 1, _TOOL_CALL_ID])
+
+    def test_paired_tool_call_not_end(self, parser):
+        """Paired <tool_call>...</tool_call> (from template) is NOT end."""
+        assert not parser.is_reasoning_end(
+            [_THINK_START_ID, 1, _TOOL_CALL_ID, 2, _TOOL_CALL_END_ID]
+        )
+
+    def test_tool_call_after_think_end(self, parser):
+        """<tool_call> after </think> — already ended."""
+        assert parser.is_reasoning_end(
+            [_THINK_START_ID, 1, _THINK_END_ID, _TOOL_CALL_ID]
+        )
+
+    def test_empty_ids(self, parser):
+        assert not parser.is_reasoning_end([])
+
+
+class TestStreaming:
+    def test_basic_streaming(self, parser):
+        reasoning, content = simulate_reasoning_streaming(
+            parser,
+            ["<think>", "thinking", " hard", "</think>", "done"],
+            [
+                (_THINK_START_ID,),
+                (1,),
+                (2,),
+                (_THINK_END_ID,),
+                (3,),
+            ],
+        )
+        assert reasoning == "thinking hard"
+        assert content == "done"
+
+    def test_streaming_no_start_token(self, parser):
+        """Qwen3.5 style: no <think> in output, just reasoning then </think>."""
+        reasoning, content = simulate_reasoning_streaming(
+            parser,
+            ["reasoning ", "text", "</think>", "content"],
+            [
+                (1,),
+                (2,),
+                (_THINK_END_ID,),
+                (3,),
+            ],
+        )
+        assert reasoning == "reasoning text"
+        assert content == "content"
+
+    def test_streaming_start_token_stripped(self, parser):
+        """<think> in output (old template) should be stripped."""
+        reasoning, content = simulate_reasoning_streaming(
+            parser,
+            ["<think>reasoning", "</think>", "content"],
+            [
+                (_THINK_START_ID, 1),
+                (_THINK_END_ID,),
+                (2,),
+            ],
+        )
+        assert reasoning == "reasoning"
+        assert content == "content"
+
+    def test_streaming_tool_call_implicit_end(self, parser):
+        """<tool_call> ends reasoning implicitly during streaming."""
+        reasoning, content = simulate_reasoning_streaming(
+            parser,
+            ["I need to check.", "<tool_call>", "\n<function=test>"],
+            [
+                (1,),
+                (_TOOL_CALL_ID,),
+                (2,),
+            ],
+        )
+        assert reasoning == "I need to check."
+        assert "<tool_call>" not in reasoning
+        assert "</think>" not in reasoning
+        assert content is not None
+
+    def test_streaming_content_after_think_end(self, parser):
+        """Content deltas after </think> are routed as content."""
+        reasoning, content = simulate_reasoning_streaming(
+            parser,
+            ["reasoning", "</think>", "content1", " content2"],
+            [
+                (1,),
+                (_THINK_END_ID,),
+                (2,),
+                (3,),
+            ],
+        )
+        assert reasoning == "reasoning"
+        assert content == "content1 content2"
+
+    def test_streaming_content_after_tool_call(self, parser):
+        """Content deltas after <tool_call> are routed as content."""
+        reasoning, content = simulate_reasoning_streaming(
+            parser,
+            ["thinking", "<tool_call>", "<function=f>"],
+            [
+                (1,),
+                (_TOOL_CALL_ID,),
+                (2,),
+            ],
+        )
+        assert reasoning == "thinking"
+        assert "<tool_call>" not in reasoning
+        assert content is not None
+
+    def test_streaming_end_grouped_with_content(self, parser):
+        """</think> grouped with following content in one delta."""
+        reasoning, content = simulate_reasoning_streaming(
+            parser,
+            ["reasoning", "</think>the answer"],
+            [
+                (1,),
+                (_THINK_END_ID, 2),
+            ],
+        )
+        assert reasoning == "reasoning"
+        assert content == "the answer"
+
+    def test_streaming_think_and_end_in_one_delta(self, parser):
+        """<think> and </think> in the same delta."""
+        reasoning, content = simulate_reasoning_streaming(
+            parser,
+            ["<think>reasoning</think>"],
+            [
+                (_THINK_START_ID, 1, _THINK_END_ID),
+            ],
+        )
+        assert reasoning == "reasoning"
+        assert content == ""
+
+    def test_streaming_pure_content_no_think(self, parser):
+        """No think tokens at all — everything is reasoning (truncated)."""
+        reasoning, content = simulate_reasoning_streaming(
+            parser,
+            ["hello ", "world"],
+            [
+                (1,),
+                (2,),
+            ],
+        )
+        assert reasoning == "hello world"
+        assert content == ""
+
+    def test_streaming_think_end_and_tool_call_same_delta(self, parser):
+        """</think> and <tool_call> in the same delta — no leakage.
+
+        Regression test: the old override split at <tool_call> without
+        stripping </think>, causing </think> to leak into reasoning.
+        """
+        reasoning, content = simulate_reasoning_streaming(
+            parser,
+            [
+                "Let me list the directory.",
+                "</think><tool_call>",
+                "<function=read>",
+                "<parameter=filePath>/tmp</parameter>",
+            ],
+            [
+                (1,),
+                (_THINK_END_ID, _TOOL_CALL_ID),
+                (2,),
+                (3,),
+            ],
+        )
+        assert reasoning == "Let me list the directory."
+        assert "</think>" not in reasoning
+        assert "<tool_call>" not in reasoning
+        assert "<parameter=" not in reasoning
+        assert content is not None
+
+    def test_streaming_no_terminal_text_leaks(self, parser):
+        """Terminal text must never appear in reasoning or content."""
+        reasoning, content = simulate_reasoning_streaming(
+            parser,
+            ["reasoning", "</think>", "content"],
+            [
+                (1,),
+                (_THINK_END_ID,),
+                (2,),
+            ],
+        )
+        assert "</think>" not in reasoning
+        assert "</think>" not in content
+        assert "<think>" not in reasoning
+
+    def test_streaming_duplicate_think_end_absorbed(self, parser):
+        """Duplicate </think> token in CONTENT state must not leak."""
+        reasoning, content = simulate_reasoning_streaming(
+            parser,
+            ["reasoning", "</think>", "content", "</think>", "more"],
+            [
+                (1,),
+                (_THINK_END_ID,),
+                (2,),
+                (_THINK_END_ID,),
+                (3,),
+            ],
+        )
+        assert reasoning == "reasoning"
+        assert content == "contentmore"
+
+
+class TestTrailingWhitespaceStripping:
+    """When strip_trailing_reasoning_whitespace is True,
+    trailing whitespace before </think> must be stripped.
+
+    Models often generate trailing newlines before </think>, and these
+    accumulate across multi-turn conversations via a feedback loop.
+    """
+
+    @pytest.fixture
+    def parser_with_strip(self):
+        cfg = dataclasses.replace(
+            qwen3_config(),
+            strip_trailing_reasoning_whitespace=True,
+        )
+        return Qwen3Parser(make_mock_tokenizer(_QWEN3_VOCAB), parser_engine_config=cfg)
+
+    def test_non_streaming_trailing_newline(self, parser_with_strip):
+        text = "Reasoning here.\n</think>Content."
+        reasoning, content = parser_with_strip.extract_reasoning(text, None)
+        assert reasoning == "Reasoning here."
+        assert content == "Content."
+
+    def test_non_streaming_multiple_trailing_newlines(self, parser_with_strip):
+        text = "Reasoning here.\n\n\n</think>Content."
+        reasoning, content = parser_with_strip.extract_reasoning(text, None)
+        assert reasoning == "Reasoning here."
+        assert content == "Content."
+
+    def test_non_streaming_internal_newlines_preserved(self, parser_with_strip):
+        text = "Step 1.\n\nStep 2.\n\nStep 3.</think>Answer."
+        reasoning, content = parser_with_strip.extract_reasoning(text, None)
+        assert reasoning == "Step 1.\n\nStep 2.\n\nStep 3."
+        assert content == "Answer."
+
+    def test_non_streaming_only_newlines_becomes_none(self, parser_with_strip):
+        text = "\n\n\n</think>Content."
+        reasoning, content = parser_with_strip.extract_reasoning(text, None)
+        assert reasoning is None
+        assert content == "Content."
+
+    def test_streaming_trailing_newline_stripped(self, parser_with_strip):
+        reasoning, content = simulate_reasoning_streaming(
+            parser_with_strip,
+            ["thinking.\n", "</think>", "done"],
+            [
+                (1,),
+                (_THINK_END_ID,),
+                (2,),
+            ],
+        )
+        assert reasoning == "thinking."
+        assert content == "done"
+
+    def test_streaming_multiple_trailing_newlines_stripped(self, parser_with_strip):
+        reasoning, content = simulate_reasoning_streaming(
+            parser_with_strip,
+            ["thinking.\n", "\n", "\n", "</think>", "done"],
+            [
+                (1,),
+                (2,),
+                (3,),
+                (_THINK_END_ID,),
+                (4,),
+            ],
+        )
+        assert reasoning == "thinking."
+        assert content == "done"
+
+    def test_streaming_internal_newlines_preserved(self, parser_with_strip):
+        reasoning, content = simulate_reasoning_streaming(
+            parser_with_strip,
+            ["Step 1.\n", "\nStep 2.\n", "</think>", "Answer"],
+            [
+                (1,),
+                (2,),
+                (_THINK_END_ID,),
+                (3,),
+            ],
+        )
+        assert reasoning == "Step 1.\n\nStep 2."
+        assert content == "Answer"
+
+    def test_streaming_trailing_newlines_before_tool_call(self, parser_with_strip):
+        """Trailing newlines before implicit <tool_call> end are stripped."""
+        reasoning, content = simulate_reasoning_streaming(
+            parser_with_strip,
+            ["I'll check.\n\n", "<tool_call>", "<function=test>"],
+            [
+                (1,),
+                (_TOOL_CALL_ID,),
+                (2,),
+            ],
+        )
+        assert reasoning == "I'll check."
+        assert "<tool_call>" not in reasoning
+
+
+class TestWhitespaceStrippingDisabled:
+    """When strip_trailing_reasoning_whitespace is False,
+    trailing whitespace in reasoning must be preserved."""
+
+    @pytest.fixture
+    def parser_no_strip(self):
+        cfg = dataclasses.replace(
+            qwen3_config(),
+            strip_trailing_reasoning_whitespace=False,
+        )
+        return Qwen3Parser(make_mock_tokenizer(_QWEN3_VOCAB), parser_engine_config=cfg)
+
+    def test_non_streaming_preserves_trailing_newline(self, parser_no_strip):
+        text = "Reasoning here.\n</think>Content."
+        reasoning, content = parser_no_strip.extract_reasoning(text, None)
+        assert reasoning == "Reasoning here.\n"
+        assert content == "Content."
+
+    def test_streaming_preserves_trailing_newlines(self, parser_no_strip):
+        reasoning, content = simulate_reasoning_streaming(
+            parser_no_strip,
+            ["thinking.\n", "\n", "</think>", "done"],
+            [
+                (1,),
+                (2,),
+                (_THINK_END_ID,),
+                (3,),
+            ],
+        )
+        assert reasoning == "thinking.\n\n"
+        assert content == "done"
+
+
+class TestThinkingDisabled:
+    """When ``enable_thinking=False``, the chat template pre-fills a closed
+    ``<think>\\n\\n</think>\\n\\n`` block.  The model output starts in content
+    state, so the parser's initial state must be CONTENT — not REASONING.
+    """
+
+    def test_thinking_disabled_initial_state_is_content(self, mock_tokenizer):
+        p = Qwen3Parser(
+            mock_tokenizer,
+            chat_template_kwargs={"enable_thinking": False},
+        )
+        assert p.parser_engine_config.initial_state == ParserState.CONTENT
+
+    def test_thinking_enabled_initial_state_is_reasoning(self, mock_tokenizer):
+        p = Qwen3Parser(
+            mock_tokenizer,
+            chat_template_kwargs={"enable_thinking": True},
+        )
+        assert p.parser_engine_config.initial_state == ParserState.REASONING
+
+    def test_default_initial_state_is_reasoning(self, mock_tokenizer):
+        p = Qwen3Parser(mock_tokenizer)
+        assert p.parser_engine_config.initial_state == ParserState.REASONING
+
+    def test_thinking_disabled_streaming_content_only(self, mock_tokenizer):
+        """Plain text with thinking disabled must stream as content, not
+        reasoning.  Before the fix, the REASONING initial state caused all
+        output to be emitted as reasoning chunks."""
+        p = Qwen3Parser(
+            mock_tokenizer,
+            chat_template_kwargs={"enable_thinking": False},
+        )
+        reasoning, content = simulate_reasoning_streaming(
+            p,
+            ["The answer", " is 42."],
+            [
+                (_TEXT_ID,),
+                (_TEXT_ID,),
+            ],
+        )
+        assert content == "The answer is 42."
+        assert reasoning == ""
+
+    def test_thinking_disabled_non_streaming(self, mock_tokenizer):
+        p = Qwen3Parser(
+            mock_tokenizer,
+            chat_template_kwargs={"enable_thinking": False},
+        )
+        reasoning, content = p.extract_reasoning("The answer is 42.", None)
+        assert reasoning is None
+        assert content == "The answer is 42."

tests/parser/engine/test_replay.py

@@ -0,0 +1,189 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Replay tests for engine parsers (holdback, skip-tool-parsing, adapters).
+
+Replays dynamically built token sequences at different chunk sizes and
+holdback depths to verify chunk-size invariance and terminal-token hygiene.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from tests.parser.engine.replay_harness import (
+    _test_request,
+    assert_no_terminal_leakage,
+    assert_parse_output,
+    collect_output,
+    make_mock_tokenizer,
+    replay_streaming,
+)
+from tests.parser.engine.trace_builder import build_samples
+from vllm.parser.abstract_parser import Parser
+from vllm.parser.engine.registered_adapters import (
+    Qwen3Parser,
+)
+
+_ENGINE_PARSERS: dict[str, type[Parser]] = {
+    "qwen3_engine": Qwen3Parser,
+}
+
+_qwen3_samples = build_samples("qwen3")
+
+_QWEN3_TERMINALS = [
+    "<think>",
+    "</think>",
+    "<tool_call>",
+    "</tool_call>",
+    "<function=",
+    "</function>",
+]
+
+HOLDBACK_CONFIGS = [6, 12, 24]
+
+
+@pytest.mark.parametrize("holdback", HOLDBACK_CONFIGS, ids=lambda h: f"holdback{h}")
+@pytest.mark.parametrize("chunk_size", [5, 10], ids=lambda c: f"chunk{c}")
+@pytest.mark.parametrize("sample", _qwen3_samples, ids=lambda s: s.id)
+class TestQwen3ReplayWithHoldback:
+    """Replay Qwen3 with simulated detokenizer holdback."""
+
+    def test_replay(self, sample, chunk_size, holdback):
+        tokenizer = make_mock_tokenizer(sample)
+        parser = Qwen3Parser(tokenizer, sample.tools)
+        deltas = replay_streaming(
+            parser,
+            sample.tokens,
+            chunk_size=chunk_size,
+            holdback_chars=holdback,
+        )
+        output = collect_output(deltas)
+
+        assert_parse_output(output, sample)
+        assert_no_terminal_leakage(
+            output,
+            _QWEN3_TERMINALS,
+            context=f"chunk_size={chunk_size}, holdback={holdback}",
+        )
+
+
+_TOOL_CALL_SAMPLES = [
+    (Qwen3Parser, s)
+    for s in _qwen3_samples
+    if s.expected_tool_calls and s.expected_reasoning
+]
+
+
+def _suppressed_expectations(sample) -> tuple[str, str]:
+    """Compute expected (reasoning, content) when tools are suppressed.
+
+    When an explicit reasoning-end delimiter (``</think>``, ``<channel|>``)
+    is present, reasoning ends there and the tool call block becomes content.
+    When reasoning ends implicitly (the tool-start token triggers both
+    REASONING_END and TOOL_CALL_START), reasoning still ends at the tool
+    start and the raw tool call block becomes content text — only the
+    structured tool parsing is suppressed, not the reasoning boundary.
+    """
+    full_text = "".join(text for _, text in sample.tokens)
+    reasoning = sample.expected_reasoning
+    idx = full_text.find(reasoning)
+    if idx < 0:
+        return (full_text, "")
+    after_reasoning = full_text[idx + len(reasoning) :]
+    for delim in ("</think>", "<channel|>"):
+        pos = after_reasoning.find(delim)
+        if pos >= 0:
+            return (reasoning, after_reasoning[pos + len(delim) :])
+    for delim in ("<tool_call>",):
+        pos = after_reasoning.find(delim)
+        if pos >= 0:
+            return (reasoning, after_reasoning[pos:])
+    return (full_text, "")
+
+
+_DUMMY_TOOLS = [
+    {
+        "type": "function",
+        "function": {"name": "stub", "parameters": {"type": "object"}},
+    }
+]
+
+
+@pytest.mark.parametrize("chunk_size", [1, 5, None], ids=lambda c: f"chunk{c}")
+@pytest.mark.parametrize(
+    "parser_cls,sample",
+    _TOOL_CALL_SAMPLES,
+    ids=lambda v: v.id if hasattr(v, "id") else v.__name__,
+)
+class TestSkipToolParsingReplay:
+    """Replay with skip_tool_parsing=True (tool_choice='none').
+
+    Verifies that reasoning is extracted normally and the raw tool call
+    block appears as content text with no tool calls parsed.
+    """
+
+    def test_replay(self, parser_cls, sample, chunk_size):
+        tokenizer = make_mock_tokenizer(sample)
+        kwargs = {}
+        if sample.chat_template_kwargs:
+            kwargs["chat_template_kwargs"] = sample.chat_template_kwargs
+        parser = parser_cls(tokenizer, **kwargs)
+
+        request = _test_request()
+        request.tool_choice = "none"
+        request.tools = _DUMMY_TOOLS
+
+        all_ids = [tid for tid, _ in sample.tokens]
+        all_texts = [text for _, text in sample.tokens]
+        if chunk_size is None:
+            chunk_size = len(all_ids)
+
+        results = []
+        chunks = list(range(0, len(all_ids), chunk_size))
+        for i, start in enumerate(chunks):
+            end = min(start + chunk_size, len(all_ids))
+            is_last = i == len(chunks) - 1
+            result = parser.parse_delta(
+                "".join(all_texts[start:end]),
+                all_ids[start:end],
+                request,
+                prompt_token_ids=[] if start == 0 else None,
+                finished=is_last,
+            )
+            results.append(result)
+
+        output = collect_output(results)
+
+        expected_reasoning, expected_content = _suppressed_expectations(sample)
+
+        assert output.reasoning == expected_reasoning, (
+            f"Reasoning mismatch:\n"
+            f"  expected: {expected_reasoning!r}\n"
+            f"  actual:   {output.reasoning!r}"
+        )
+        assert output.tool_calls == [], (
+            f"Expected no tool calls but got {output.tool_calls}"
+        )
+        assert output.content == expected_content, (
+            f"Content mismatch:\n"
+            f"  expected: {expected_content!r}\n"
+            f"  actual:   {output.content!r}"
+        )
+
+
+class TestAdapterReferences:
+    """Verify make_adapters sets reasoning/tool parser class refs on parser engine
+    parser classes so the serving layer finds them and calls adjust_request."""
+
+    @pytest.mark.parametrize(
+        "parser_name",
+        list(_ENGINE_PARSERS.keys()),
+    )
+    def test_adapter_cls_refs_set(self, parser_name):
+        parser_cls = _ENGINE_PARSERS[parser_name]
+        assert parser_cls.reasoning_parser_cls is not None, (
+            f"{parser_name}: reasoning_parser_cls is None"
+        )
+        assert parser_cls.tool_parser_cls is not None, (
+            f"{parser_name}: tool_parser_cls is None"
+        )

tests/parser/engine/test_token_id_scanner.py

@@ -0,0 +1,631 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Tests for TokenIDScanner, focusing on hold-back text recovery.
+
+Uses gemma4_config for all end-to-end engine tests, covering
+reasoning channels, tool calls, and combined flows."""
+
+from unittest.mock import MagicMock
+
+import pytest
+
+from vllm.parser.engine.events import EventType
+from vllm.parser.engine.token_id_scanner import (
+    PreLexedTerminal,
+    TextChunk,
+    TokenIDScanner,
+)
+
+CHANNEL_START = "<|channel>"
+CHANNEL_END = "<channel|>"
+CHANNEL_START_ID = 100
+CHANNEL_END_ID = 101
+REGULAR_TOKEN_ID = 200
+TOOL_START = "<tool_call>"
+TOOL_END = "</tool_call>"
+TOOL_START_ID = 110
+TOOL_END_ID = 111
+
+
+@pytest.fixture
+def tokenizer():
+    tok = MagicMock()
+    tok.get_vocab.return_value = {
+        CHANNEL_START: CHANNEL_START_ID,
+        CHANNEL_END: CHANNEL_END_ID,
+    }
+    tok.decode.side_effect = lambda ids: {
+        CHANNEL_START_ID: CHANNEL_START,
+        CHANNEL_END_ID: CHANNEL_END,
+        REGULAR_TOKEN_ID: "regular",
+    }.get(ids[0], f"<unk:{ids[0]}>")
+    return tok
+
+
+@pytest.fixture
+def scanner(tokenizer):
+    return TokenIDScanner(
+        token_id_to_terminal={
+            CHANNEL_START_ID: "THINK_START",
+            CHANNEL_END_ID: "THINK_END",
+        },
+        tokenizer=tokenizer,
+    )
+
+
+class TestJoinDecodedTextReturnsStr:
+    """_join_decoded_text now returns str unconditionally (was
+    str | None when an isinstance guard made a branch unreachable)."""
+
+    @pytest.fixture
+    def bare_scanner(self):
+        return TokenIDScanner({}, tokenizer=None, drop_token_ids=set())
+
+    def test_mixed_items(self, bare_scanner):
+        items = [
+            TextChunk("hello "),
+            PreLexedTerminal("TOOL_START", 42, "<tool_call>"),
+            TextChunk(" world"),
+        ]
+        result = bare_scanner._join_decoded_text(items)
+        assert isinstance(result, str)
+        assert result == "hello <tool_call> world"
+
+    def test_empty_list(self, bare_scanner):
+        result = bare_scanner._join_decoded_text([])
+        assert isinstance(result, str)
+        assert result == ""
+
+    def test_only_text_chunks(self, bare_scanner):
+        result = bare_scanner._join_decoded_text([TextChunk("abc"), TextChunk("def")])
+        assert result == "abcdef"
+
+
+class TestHoldbackTextRecovery:
+    def test_holdback_text_with_special_token_text_absent(self, scanner):
+        """delta_text has hold-back text but the special token's text is
+        NOT in delta_text (held back by the detokenizer).  Terminal is
+        deferred until the text arrives in a subsequent delta."""
+        result = scanner.scan(
+            delta_text="processed is appropriate.",
+            delta_token_ids=[CHANNEL_END_ID],
+        )
+
+        assert len(result) == 0
+
+        # Second scan: terminal text arrives (detokenizer flushes).
+        # Deferred terminal resolves with holdback text before it.
+        result2 = scanner.scan(
+            delta_text="<channel|>Understood.",
+            delta_token_ids=[20, 21],
+        )
+        pre_lexed = [r for r in result2 if isinstance(r, PreLexedTerminal)]
+        assert len(pre_lexed) == 1
+        assert pre_lexed[0].terminal == "THINK_END"
+        texts = [r.text for r in result2 if isinstance(r, TextChunk)]
+        combined = "".join(texts)
+        assert "processed is appropriate." in combined
+        assert "Understood." in combined
+
+    def test_holdback_text_with_special_token_text_present(self, scanner):
+        """delta_text includes hold-back text AND the special token text."""
+        result = scanner.scan(
+            delta_text="holdback text<channel|>",
+            delta_token_ids=[CHANNEL_END_ID],
+        )
+
+        assert len(result) == 2
+        assert isinstance(result[0], TextChunk)
+        assert result[0].text == "holdback text"
+        assert isinstance(result[1], PreLexedTerminal)
+        assert result[1].terminal == "THINK_END"
+
+    def test_no_holdback_text(self, scanner):
+        """delta_text is exactly the special token text — no hold-back."""
+        result = scanner.scan(
+            delta_text="<channel|>",
+            delta_token_ids=[CHANNEL_END_ID],
+        )
+
+        assert len(result) == 1
+        assert isinstance(result[0], PreLexedTerminal)
+        assert result[0].terminal == "THINK_END"
+
+    def test_empty_delta_text(self, scanner):
+        """delta_text is empty — terminal deferred until text arrives."""
+        result = scanner.scan(
+            delta_text="",
+            delta_token_ids=[CHANNEL_END_ID],
+        )
+
+        assert len(result) == 0
+
+        flushed = scanner.flush_pending()
+        assert len(flushed) == 1
+        assert isinstance(flushed[0], PreLexedTerminal)
+        assert flushed[0].terminal == "THINK_END"
+
+    def test_empty_delta_text_drops_individual_decode_text(self, tokenizer):
+        """delta_text="" with multiple tokens including special: all
+        results deferred — individually-decoded TextChunks are unreliable
+        and PreLexedTerminals wait for text confirmation."""
+        tool_start_id = 400
+        tok_a = 201
+        tok_b = 202
+        tokenizer.decode.side_effect = lambda ids: {
+            tool_start_id: "<|tool_call>",
+            tok_a: "call:",
+            tok_b: "get_weather",
+        }.get(ids[0], "?")
+
+        scanner = TokenIDScanner(
+            token_id_to_terminal={tool_start_id: "TOOL_START"},
+            tokenizer=tokenizer,
+        )
+
+        result = scanner.scan(
+            delta_text="",
+            delta_token_ids=[tool_start_id, tok_a, tok_b],
+        )
+
+        assert len(result) == 0
+
+        flushed = scanner.flush_pending()
+        assert len(flushed) == 1
+        assert isinstance(flushed[0], PreLexedTerminal)
+        assert flushed[0].terminal == "TOOL_START"
+
+    def test_holdback_before_start_tag(self, scanner):
+        """Hold-back text before a reasoning start tag."""
+        result = scanner.scan(
+            delta_text="prefix text<|channel>",
+            delta_token_ids=[CHANNEL_START_ID],
+        )
+
+        assert len(result) == 2
+        assert isinstance(result[0], TextChunk)
+        assert result[0].text == "prefix text"
+        assert isinstance(result[1], PreLexedTerminal)
+        assert result[1].terminal == "THINK_START"
+
+    def test_multi_token_batch_special_in_middle(self, scanner, tokenizer):
+        """Stream-interval > 1: batch has regular tokens + special token.
+        delta_text differs from individual decodes (context-dependent)."""
+        tok_a = 201
+        tok_b = 202
+        tokenizer.decode.side_effect = lambda ids: {
+            tok_a: "wordA",
+            tok_b: "wordB",
+            CHANNEL_END_ID: CHANNEL_END,
+        }.get(ids[0], "?")
+
+        scanner_multi = TokenIDScanner(
+            token_id_to_terminal={CHANNEL_END_ID: "THINK_END"},
+            tokenizer=tokenizer,
+        )
+
+        result = scanner_multi.scan(
+            delta_text="holdback wordA<channel|> wordB",
+            delta_token_ids=[tok_a, CHANNEL_END_ID, tok_b],
+        )
+
+        texts = [r.text for r in result if isinstance(r, TextChunk)]
+        terminals = [r.terminal for r in result if isinstance(r, PreLexedTerminal)]
+        assert "THINK_END" in terminals
+        assert "holdback wordA" in "".join(texts)
+
+    def test_multi_token_batch_special_token_text_absent(self, scanner, tokenizer):
+        """Stream-interval > 1: batch has regular + special token, but
+        delta_text doesn't contain the special token text at all
+        (held back by detokenizer along with trailing regular tokens).
+        Terminal is deferred until text arrives."""
+        tok_a = 201
+        tok_b = 202
+        tokenizer.decode.side_effect = lambda ids: {
+            tok_a: "alpha",
+            tok_b: "beta",
+            CHANNEL_END_ID: CHANNEL_END,
+        }.get(ids[0], "?")
+
+        scanner_multi = TokenIDScanner(
+            token_id_to_terminal={CHANNEL_END_ID: "THINK_END"},
+            tokenizer=tokenizer,
+        )
+
+        result = scanner_multi.scan(
+            delta_text="holdback alpha",
+            delta_token_ids=[tok_a, CHANNEL_END_ID, tok_b],
+        )
+
+        assert len(result) == 0
+
+        # Next delta: terminal text arrives (detokenizer flushes).
+        # Deferred terminal resolves with holdback text before it.
+        result2 = scanner_multi.scan(
+            delta_text="<channel|> more text",
+            delta_token_ids=[300],
+        )
+        pre_lexed = [r for r in result2 if isinstance(r, PreLexedTerminal)]
+        assert len(pre_lexed) == 1
+        assert pre_lexed[0].terminal == "THINK_END"
+        text_chunks = [r for r in result2 if isinstance(r, TextChunk)]
+        combined = "".join(t.text for t in text_chunks)
+        assert "holdback alpha" in combined
+        assert "more text" in combined
+
+    def test_holdback_with_content_after_special_token(self, tokenizer):
+        """delta_text has hold-back + special token + content after,
+        with corresponding token IDs for all parts."""
+        tok_content = 210
+        tokenizer.decode.side_effect = lambda ids: {
+            CHANNEL_END_ID: CHANNEL_END,
+            tok_content: "content start",
+        }.get(ids[0], "?")
+
+        scanner = TokenIDScanner(
+            token_id_to_terminal={CHANNEL_END_ID: "THINK_END"},
+            tokenizer=tokenizer,
+        )
+
+        result = scanner.scan(
+            delta_text="reasoning end.<channel|>content start",
+            delta_token_ids=[CHANNEL_END_ID, tok_content],
+        )
+
+        pre_lexed = [r for r in result if isinstance(r, PreLexedTerminal)]
+        assert len(pre_lexed) == 1
+        assert pre_lexed[0].terminal == "THINK_END"
+
+        text_chunks = [r for r in result if isinstance(r, TextChunk)]
+        combined = "".join(t.text for t in text_chunks)
+        assert "reasoning end." in combined
+
+
+class TestDropTokens:
+    def test_drop_token_with_holdback(self, tokenizer):
+        """Drop tokens stripped from delta_text, hold-back text preserved.
+        Terminal is deferred when its text is absent from delta_text."""
+        drop_id = 300
+        tokenizer.decode.side_effect = lambda ids: {
+            CHANNEL_END_ID: CHANNEL_END,
+            drop_id: "<eos>",
+        }.get(ids[0], "?")
+
+        scanner = TokenIDScanner(
+            token_id_to_terminal={CHANNEL_END_ID: "THINK_END"},
+            tokenizer=tokenizer,
+            drop_token_ids={drop_id},
+        )
+
+        result = scanner.scan(
+            delta_text="holdback<eos>",
+            delta_token_ids=[drop_id, CHANNEL_END_ID],
+        )
+
+        assert len(result) == 0
+
+        # Terminal text arrives in next delta; deferred terminal resolves.
+        result2 = scanner.scan(
+            delta_text="<channel|>content",
+            delta_token_ids=[20],
+        )
+        pre_lexed = [r for r in result2 if isinstance(r, PreLexedTerminal)]
+        assert len(pre_lexed) == 1
+        assert pre_lexed[0].terminal == "THINK_END"
+        texts = [r.text for r in result2 if isinstance(r, TextChunk)]
+        combined = "".join(texts)
+        assert "holdback" in combined
+        assert "<eos>" not in combined
+
+        assert len(scanner.flush_pending()) == 0
+
+
+class TestEndToEndReasoningHoldback:
+    """End-to-end tests through the full parser engine simulating
+    stream-interval > 1 and detokenizer hold-back, using
+    gemma4_config."""
+
+    def test_reasoning_content_not_truncated(self):
+        from vllm.parser.engine.parser_engine_config import (
+            ParserEngineConfig,
+            ParserState,
+            Transition,
+        )
+        from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
+
+        config = ParserEngineConfig(
+            name="test_channel",
+            initial_state=ParserState.CONTENT,
+            terminals={
+                "THINK_START": CHANNEL_START,
+                "THINK_END": CHANNEL_END,
+            },
+            token_id_terminals={
+                "THINK_START": CHANNEL_START,
+                "THINK_END": CHANNEL_END,
+            },
+            transitions={
+                (ParserState.CONTENT, "THINK_START"): Transition(
+                    ParserState.REASONING,
+                    (EventType.REASONING_START,),
+                ),
+                (ParserState.REASONING, "THINK_END"): Transition(
+                    ParserState.CONTENT,
+                    (EventType.REASONING_END,),
+                ),
+            },
+        )
+        tok = MagicMock()
+        vocab = {
+            CHANNEL_START: CHANNEL_START_ID,
+            CHANNEL_END: CHANNEL_END_ID,
+        }
+        tok.get_vocab.return_value = vocab
+        tok.decode.side_effect = lambda ids: {
+            CHANNEL_START_ID: CHANNEL_START,
+            CHANNEL_END_ID: CHANNEL_END,
+        }.get(ids[0], f"tok{ids[0]}")
+
+        engine = StreamingParserEngine(config, tok)
+        all_events = []
+
+        # Delta 1: channel start token (text includes start tag)
+        all_events.extend(engine.feed(CHANNEL_START, [CHANNEL_START_ID]))
+
+        # Delta 2: reasoning text (normal content, no special tokens)
+        all_events.extend(
+            engine.feed(
+                "thought\nThe request was received and ",
+                [10, 11, 12, 13, 14],
+            )
+        )
+
+        # Delta 3: MORE reasoning text, the detokenizer held some back.
+        # Then channel end token arrives in token_ids, but its text
+        # is NOT in delta_text (held back by detokenizer).
+        # delta_text = previously held-back reasoning text only.
+        all_events.extend(
+            engine.feed(
+                "processed is appropriate.",
+                [CHANNEL_END_ID],
+            )
+        )
+
+        # Delta 4: detokenizer flushes held-back channel end text
+        # plus new content tokens.
+        all_events.extend(
+            engine.feed(
+                "<channel|>Understood.",
+                [20, 21],
+            )
+        )
+
+        all_events.extend(engine.finish())
+
+        reasoning_text = "".join(
+            e.value for e in all_events if e.type == EventType.REASONING_CHUNK
+        )
+        content_text = "".join(
+            e.value for e in all_events if e.type == EventType.TEXT_CHUNK
+        )
+
+        assert "processed is appropriate." in reasoning_text
+        assert "Understood." in content_text
+
+    def test_backtick_content_not_truncated(self):
+        """Reproduces the hostname backtick truncation case."""
+        from vllm.parser.engine.parser_engine_config import (
+            ParserEngineConfig,
+            ParserState,
+            Transition,
+        )
+        from vllm.parser.engine.streaming_parser_engine import StreamingParserEngine
+
+        config = ParserEngineConfig(
+            name="test_channel",
+            initial_state=ParserState.CONTENT,
+            terminals={
+                "THINK_START": CHANNEL_START,
+                "THINK_END": CHANNEL_END,
+            },
+            token_id_terminals={
+                "THINK_START": CHANNEL_START,
+                "THINK_END": CHANNEL_END,
+            },
+            transitions={
+                (ParserState.CONTENT, "THINK_START"): Transition(
+                    ParserState.REASONING,
+                    (EventType.REASONING_START,),
+                ),
+                (ParserState.REASONING, "THINK_END"): Transition(
+                    ParserState.CONTENT,
+                    (EventType.REASONING_END,),
+                ),
+            },
+        )
+        tok = MagicMock()
+        vocab = {
+            CHANNEL_START: CHANNEL_START_ID,
+            CHANNEL_END: CHANNEL_END_ID,
+        }
+        tok.get_vocab.return_value = vocab
+        tok.decode.side_effect = lambda ids: {
+            CHANNEL_START_ID: CHANNEL_START,
+            CHANNEL_END_ID: CHANNEL_END,
+        }.get(ids[0], f"tok{ids[0]}")
+
+        engine = StreamingParserEngine(config, tok)
+        all_events = []
+
+        all_events.extend(engine.feed(CHANNEL_START, [CHANNEL_START_ID]))
+        all_events.extend(
+            engine.feed(
+                "thought\n1/10 completed. Next: ",
+                [10, 11, 12, 13],
+            )
+        )
+
+        # Hold-back text includes backtick content; channel end text
+        # absent from delta_text.
+        all_events.extend(
+            engine.feed(
+                "`hostname`.\n",
+                [CHANNEL_END_ID],
+            )
+        )
+
+        # Next delta flushes channel end + tool call start
+        all_events.extend(
+            engine.feed(
+                "<channel|>tool output",
+                [20, 21],
+            )
+        )
+
+        all_events.extend(engine.finish())
+
+        reasoning_text = "".join(
+            e.value for e in all_events if e.type == EventType.REASONING_CHUNK
+        )
+
+        assert "`hostname`." in reasoning_text
+
+
+class TestRebuildFromAnchorsLiteralLookalike:
+    """When delta_text contains a literal mention of a special token's
+    text before the real special token, _rebuild_from_anchors must
+    anchor at the real occurrence, not the literal one."""
+
+    @pytest.fixture
+    def tool_scanner(self):
+        tok = MagicMock()
+        tok.get_vocab.return_value = {
+            TOOL_START: TOOL_START_ID,
+            TOOL_END: TOOL_END_ID,
+        }
+        tok.decode.side_effect = lambda ids: {
+            TOOL_START_ID: TOOL_START,
+            TOOL_END_ID: TOOL_END,
+        }.get(ids[0], f"t{ids[0]}")
+        return TokenIDScanner(
+            {TOOL_START_ID: "TOOL_START", TOOL_END_ID: "TOOL_END"},
+            tok,
+        )
+
+    def test_literal_before_real_anchor(self, tool_scanner):
+        """Literal <tool_call> in prose followed by a real <tool_call>
+        special token — the scanner must split at the real one."""
+        delta_text = 'Use <tool_call> like this: <tool_call>{"name":"f"}</tool_call>'
+        delta_token_ids = [1, 2, 3, 4, 5, TOOL_START_ID, 6, 7, TOOL_END_ID]
+        items = tool_scanner.scan(delta_text, delta_token_ids)
+
+        text_parts = [it.text for it in items if isinstance(it, TextChunk)]
+        terminals = [it for it in items if isinstance(it, PreLexedTerminal)]
+
+        assert len(terminals) == 2
+        assert terminals[0].terminal == "TOOL_START"
+        assert terminals[1].terminal == "TOOL_END"
+
+        # The literal mention must appear in a text chunk, not be
+        # consumed by the TOOL_START anchor.
+        joined_text = "".join(text_parts)
+        assert "<tool_call>" in joined_text
+        assert '{"name":"f"}' in joined_text
+
+    def test_multiple_tool_calls_with_literal_between(self, tool_scanner):
+        """Two real tool calls with a literal mention between them."""
+        delta_text = (
+            '<tool_call>{"name":"a"}</tool_call>'
+            " see <tool_call> syntax "
+            '<tool_call>{"name":"b"}</tool_call>'
+        )
+        delta_token_ids = [
+            TOOL_START_ID,
+            1,
+            TOOL_END_ID,
+            2,
+            3,
+            4,
+            TOOL_START_ID,
+            5,
+            TOOL_END_ID,
+        ]
+        items = tool_scanner.scan(delta_text, delta_token_ids)
+
+        terminals = [it for it in items if isinstance(it, PreLexedTerminal)]
+        assert len(terminals) == 4
+
+        text_parts = [it.text for it in items if isinstance(it, TextChunk)]
+        joined_text = "".join(text_parts)
+        # The literal mention between the two real calls must be in text
+        assert "<tool_call> syntax" in joined_text
+
+
+class TestRebuildFromAnchorsCascadingDeferral:
+    """When a middle anchor's text is absent from delta_text,
+    only that anchor should be deferred — not subsequent ones
+    with valid positions."""
+
+    @pytest.fixture
+    def bare_scanner(self):
+        tok = MagicMock()
+        tok.decode.side_effect = lambda ids: f"t{ids[0]}"
+        return TokenIDScanner({}, tok)
+
+    def test_middle_anchor_missing_does_not_cascade(self, bare_scanner):
+        a = PreLexedTerminal("TOOL_START", TOOL_START_ID, TOOL_START)
+        b = PreLexedTerminal("THINK_END", CHANNEL_END_ID, CHANNEL_END)
+        c = PreLexedTerminal("TOOL_END", TOOL_END_ID, TOOL_END)
+        delta_text = f"prefix{TOOL_START}middle{TOOL_END}suffix"
+        results = [a, b, c]
+
+        rebuilt = bare_scanner._rebuild_from_anchors(delta_text, results)
+
+        terminals = [r for r in rebuilt if isinstance(r, PreLexedTerminal)]
+        texts = [r for r in rebuilt if isinstance(r, TextChunk)]
+        joined = "".join(t.text for t in texts)
+
+        assert len(terminals) == 2
+        assert terminals[0].terminal == "TOOL_START"
+        assert terminals[1].terminal == "TOOL_END"
+        assert "prefix" in joined
+        assert "middle" in joined
+        assert "suffix" in joined
+        assert len(bare_scanner._deferred_terminals) == 1
+        assert bare_scanner._deferred_terminals[0].terminal == "THINK_END"
+        assert bare_scanner._deferred_post_text == ""
+
+    def test_first_anchor_missing_rest_still_emitted(self, bare_scanner):
+        a = PreLexedTerminal("THINK_END", CHANNEL_END_ID, CHANNEL_END)
+        b = PreLexedTerminal("TOOL_START", TOOL_START_ID, TOOL_START)
+        delta_text = f"text{TOOL_START}more"
+        results = [a, b]
+
+        rebuilt = bare_scanner._rebuild_from_anchors(delta_text, results)
+
+        terminals = [r for r in rebuilt if isinstance(r, PreLexedTerminal)]
+        assert len(terminals) == 1
+        assert terminals[0].terminal == "TOOL_START"
+        assert len(bare_scanner._deferred_terminals) == 1
+        assert bare_scanner._deferred_terminals[0].terminal == "THINK_END"
+
+    def test_last_anchor_missing_preceding_still_emitted(self, bare_scanner):
+        a = PreLexedTerminal("TOOL_START", TOOL_START_ID, TOOL_START)
+        b = PreLexedTerminal("THINK_END", CHANNEL_END_ID, CHANNEL_END)
+        delta_text = f"text{TOOL_START}more"
+        results = [a, b]
+
+        rebuilt = bare_scanner._rebuild_from_anchors(delta_text, results)
+
+        terminals = [r for r in rebuilt if isinstance(r, PreLexedTerminal)]
+        assert len(terminals) == 1
+        assert terminals[0].terminal == "TOOL_START"
+        texts = [r for r in rebuilt if isinstance(r, TextChunk)]
+        joined = "".join(t.text for t in texts)
+        assert "text" in joined
+        # "more" is deferred along with the missing terminal —
+        # it will be resolved in the next scan when the terminal
+        # text arrives.
+        assert bare_scanner._deferred_post_text == "more"
+        assert len(bare_scanner._deferred_terminals) == 1
+        assert bare_scanner._deferred_terminals[0].terminal == "THINK_END"

tests/parser/engine/trace_builder.py

@@ -0,0 +1,410 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""On-demand trace builder for parser engine testing and benchmarks.
+
+Generates token sequences programmatically from model-agnostic scenario
+definitions.  Each model format handler knows how to render scenarios
+into the model's output format, tokenize them with correct special token
+IDs, and compute expected parse outputs.
+
+Every generated sample is self-validated by replaying it through the
+real parser before being returned.
+"""
+
+from __future__ import annotations
+
+import functools
+import json
+from dataclasses import dataclass
+from typing import Any
+
+from tests.parser.engine.replay_harness import (
+    MockTokenizer,
+    Sample,
+    assert_parse_output,
+    collect_output,
+    replay_streaming,
+)
+from vllm.entrypoints.openai.chat_completion.protocol import (
+    ChatCompletionToolsParam,
+)
+from vllm.parser.engine.registered_adapters import (
+    Qwen3Parser,
+)
+
+# ── Data structures ──────────────────────────────────────────────────
+
+
+@dataclass
+class ToolCallSpec:
+    name: str
+    arguments: dict[str, Any]
+
+
+@dataclass
+class Scenario:
+    id: str
+    description: str
+    reasoning: str | None = None
+    content: str | None = None
+    tool_calls: list[ToolCallSpec] | None = None
+
+
+# ── Scenarios ────────────────────────────────────────────────────────
+
+_READ_TOOL = ToolCallSpec("read_file", {"path": "/tmp/test.txt"})
+_BASH_TOOL = ToolCallSpec(
+    "bash", {"command": "hostname", "description": "Get hostname"}
+)
+_WEATHER_TOOL = ToolCallSpec(
+    "get_weather",
+    {"city": "Dallas", "state": "TX", "unit": "fahrenheit"},
+)
+_COMPLEX_TOOL = ToolCallSpec(
+    "search",
+    {
+        "query": "vllm parser",
+        "filters": {"language": "python", "min_stars": 100},
+        "tags": ["ml", "inference"],
+        "limit": 10,
+        "verbose": True,
+    },
+)
+
+SCENARIOS: list[Scenario] = [
+    Scenario(
+        id="think-then-tool",
+        description="Reasoning then single tool call",
+        reasoning="Let me check the file.",
+        tool_calls=[_READ_TOOL],
+    ),
+    Scenario(
+        id="think-then-parallel-tools",
+        description="Reasoning then two parallel tool calls",
+        reasoning="I need to run both commands.",
+        tool_calls=[_BASH_TOOL, _WEATHER_TOOL],
+    ),
+    Scenario(
+        id="think-then-content",
+        description="Reasoning then content response",
+        reasoning="Let me think about this carefully.",
+        content="The answer is 42.",
+    ),
+    Scenario(
+        id="content-only",
+        description="Plain content response without reasoning",
+        content="Hello! How can I help you today?",
+    ),
+    Scenario(
+        id="tool-only",
+        description="Tool call without reasoning",
+        tool_calls=[_READ_TOOL],
+    ),
+    Scenario(
+        id="complex-json-args",
+        description="Tool call with nested objects, arrays, numbers, booleans",
+        reasoning="This needs a complex query.",
+        tool_calls=[_COMPLEX_TOOL],
+    ),
+    Scenario(
+        id="whitespace-before-tool",
+        description="Whitespace-only content before tool call",
+        content="\n\n",
+        tool_calls=[_WEATHER_TOOL],
+    ),
+    Scenario(
+        id="think-content-tool",
+        description="Reasoning, content, then tool call",
+        reasoning="Let me analyze and then fetch data.",
+        content="Checking the weather now.",
+        tool_calls=[_WEATHER_TOOL],
+    ),
+    Scenario(
+        id="think-whitespace-tool",
+        description="Reasoning, whitespace-only gap, then tool call",
+        reasoning="Let me check the file contents.",
+        content="\n\n",
+        tool_calls=[_READ_TOOL],
+    ),
+    Scenario(
+        id="empty-reasoning-content",
+        description="Empty reasoning section followed by content",
+        reasoning="",
+        content="The epoch timestamp is 1779111346.",
+    ),
+]
+
+
+# ── Tokenization ─────────────────────────────────────────────────────
+
+
+def _word_split(text: str) -> list[str]:
+    """Split text into word-like tokens, preserving all characters."""
+    if not text:
+        return []
+    parts: list[str] = []
+    current = ""
+    for ch in text:
+        if ch in " \t\n\r" and current and current[-1] not in " \t\n\r":
+            parts.append(current)
+            current = ch
+        else:
+            current += ch
+    if current:
+        parts.append(current)
+    return parts
+
+
+def _tokenize(
+    segments: list[tuple[str, bool]],
+    vocab: dict[str, int],
+    start_id: int = 100,
+) -> list[tuple[int, str]]:
+    """Build token list from segments.
+
+    Each segment is ``(text, is_special)``.  Special segments use vocab
+    IDs; content segments are word-split with sequential IDs.
+    """
+    tokens: list[tuple[int, str]] = []
+    next_id = start_id
+
+    for text, is_special in segments:
+        if not text:
+            continue
+        if is_special:
+            tid = vocab.get(text)
+            if tid is None:
+                raise ValueError(f"Special token {text!r} not in vocab")
+            tokens.append((tid, text))
+        else:
+            for word in _word_split(text):
+                tokens.append((next_id, word))
+                next_id += 1
+
+    return tokens
+
+
+# ── Tool definitions ─────────────────────────────────────────────────
+
+
+def _infer_schema(value: object) -> dict:
+    """Infer a JSON Schema from a Python value, recursing into dicts/lists."""
+    if isinstance(value, bool):
+        return {"type": "boolean"}
+    if isinstance(value, int):
+        return {"type": "integer"}
+    if isinstance(value, float):
+        return {"type": "number"}
+    if isinstance(value, str):
+        return {"type": "string"}
+    if isinstance(value, dict):
+        return {
+            "type": "object",
+            "properties": {k: _infer_schema(v) for k, v in value.items()},
+        }
+    if isinstance(value, list) and value:
+        return {"type": "array", "items": _infer_schema(value[0])}
+    if isinstance(value, list):
+        return {"type": "array"}
+    return {}
+
+
+def _tool_defs(tool_calls: list[ToolCallSpec]) -> list[dict]:
+    """Generate OpenAI-style tool definitions from tool call specs."""
+    seen: set[str] = set()
+    tools: list[dict] = []
+    for tc in tool_calls:
+        if tc.name in seen:
+            continue
+        seen.add(tc.name)
+        properties = {k: _infer_schema(v) for k, v in tc.arguments.items()}
+        tools.append(
+            {
+                "type": "function",
+                "function": {
+                    "name": tc.name,
+                    "parameters": {
+                        "type": "object",
+                        "properties": properties,
+                    },
+                },
+            }
+        )
+    return tools
+
+
+# ── Format handlers ──────────────────────────────────────────────────
+
+
+def _expected_tc(scenario: Scenario) -> list[dict] | None:
+    if not scenario.tool_calls:
+        return None
+    return [{"name": tc.name, "arguments": tc.arguments} for tc in scenario.tool_calls]
+
+
+def _expected_tools(scenario: Scenario) -> list[dict] | None:
+    return _tool_defs(scenario.tool_calls) if scenario.tool_calls else None
+
+
+def _validate_sample(sample: Sample, parser_cls: type, **kwargs) -> None:
+    """Replay sample through the real parser and assert correctness."""
+    tokenizer = MockTokenizer(vocab=dict(sample.vocab), tokens=sample.tokens)
+    parser = parser_cls(tokenizer, sample.tools, **kwargs)
+    deltas = replay_streaming(parser, sample.tokens, chunk_size=1, tools=sample.tools)
+    output = collect_output(deltas)
+    assert_parse_output(output, sample)
+
+
+def _validate_tools(
+    tools: list[dict] | None,
+) -> list[ChatCompletionToolsParam] | None:
+    if not tools:
+        return None
+    return [ChatCompletionToolsParam.model_validate(t) for t in tools]
+
+
+def _make_sample(
+    sample_id: str,
+    description: str,
+    vocab: dict[str, int],
+    segments: list[tuple[str, bool]],
+    expected_reasoning: str | None,
+    expected_content: str | None,
+    expected_tool_calls: list[dict] | None,
+    tools: list[dict] | None,
+    chat_template_kwargs: dict | None = None,
+) -> Sample:
+    tokens = _tokenize(segments, vocab)
+    return Sample(
+        id=sample_id,
+        description=description,
+        source="trace-builder",
+        vocab=dict(vocab),
+        tokens=tokens,
+        expected_reasoning=expected_reasoning,
+        expected_content=expected_content,
+        expected_tool_calls=expected_tool_calls,
+        tools=_validate_tools(tools),
+        chat_template_kwargs=chat_template_kwargs,
+    )
+
+
+# ── Qwen3 / NemotronV3 (XML tool format, starts in REASONING) ───────
+
+_QWEN3_VOCAB: dict[str, int] = {
+    "<think>": 50,
+    "</think>": 51,
+    "<tool_call>": 60,
+    "</tool_call>": 61,
+}
+
+
+def _qwen3_arg_value(value: Any) -> str:
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, (int, float)):
+        return str(value)
+    if isinstance(value, str):
+        return value
+    return json.dumps(value, ensure_ascii=False)
+
+
+def _qwen3_tool_segments(tc: ToolCallSpec) -> list[tuple[str, bool]]:
+    parts = [f"\n<function={tc.name}>"]
+    for key, value in tc.arguments.items():
+        parts.append(f"\n<parameter={key}>{_qwen3_arg_value(value)}</parameter>")
+    parts.append("\n</function>\n")
+    return [
+        ("<tool_call>", True),
+        ("".join(parts), False),
+        ("</tool_call>", True),
+    ]
+
+
+def _qwen3_segments(scenario: Scenario) -> list[tuple[str, bool]]:
+    segs: list[tuple[str, bool]] = []
+    if scenario.reasoning is not None:
+        segs.append((scenario.reasoning, False))
+    if scenario.content is not None or scenario.tool_calls:
+        segs.append(("</think>", True))
+    if scenario.content is not None:
+        segs.append((scenario.content, False))
+    if scenario.tool_calls:
+        for tc in scenario.tool_calls:
+            segs.extend(_qwen3_tool_segments(tc))
+    return segs
+
+
+def _qwen3_expected_content(scenario: Scenario) -> str | None:
+    if (
+        scenario.content is not None
+        and scenario.tool_calls
+        and not scenario.content.strip()
+    ):
+        return ""
+    return scenario.content
+
+
+def _build_qwen3(
+    scenario: Scenario,
+    name: str = "qwen3",
+    parser_cls: type = Qwen3Parser,
+    strip_trailing_ws: bool = False,
+    validate: bool = True,
+) -> Sample:
+    expected_reasoning: str | None
+    if scenario.reasoning is not None:
+        r = scenario.reasoning
+        if strip_trailing_ws:
+            r = r.rstrip()
+        expected_reasoning = r
+    else:
+        expected_reasoning = ""
+
+    sample = _make_sample(
+        sample_id=f"{name}-{scenario.id}",
+        description=scenario.description,
+        vocab=_QWEN3_VOCAB,
+        segments=_qwen3_segments(scenario),
+        expected_reasoning=expected_reasoning,
+        expected_content=_qwen3_expected_content(scenario),
+        expected_tool_calls=_expected_tc(scenario),
+        tools=_expected_tools(scenario),
+    )
+    if validate:
+        _validate_sample(sample, parser_cls)
+    return sample
+
+
+# ── Registry and public API ──────────────────────────────────────────
+
+_BUILDERS: dict[str, Any] = {
+    "qwen3": _build_qwen3,
+}
+
+
+@functools.cache
+def build_samples(model: str) -> tuple[Sample, ...]:
+    """Build all scenario samples for a model, self-validated."""
+    builder = _BUILDERS[model]
+    return tuple(builder(s) for s in SCENARIOS)
+
+
+def build_sample(model: str, scenario: Scenario) -> Sample:
+    """Build a single sample for one model + scenario."""
+    return _BUILDERS[model](scenario)
+
+
+def build_scaling_sample(
+    model: str, token_count: int, validate: bool = False
+) -> Sample:
+    """Build a sample with approximately *token_count* tokens."""
+    sentence = "The quick brown fox jumps over the lazy dog. "
+    text = sentence * (token_count // 10 + 1)
+    scenario = Scenario(
+        id=f"scaling-{token_count}",
+        description=f"Scaling test with ~{token_count} tokens",
+        reasoning=text,
+        tool_calls=[_READ_TOOL],
+    )
+    return _BUILDERS[model](scenario, validate=validate)

tests/parser/test_harmony.py

@@ -118,12 +118,17 @@ def tool_call_payloads(delta_message) -> list:
     ]
 
 
-def combined_tool_arguments(delta_message) -> dict[int, str]:
-    combined: dict[int, str] = {}
-    for tool_call in tool_call_payloads(delta_message):
-        combined.setdefault(tool_call.index, "")
-        combined[tool_call.index] += tool_call.function.arguments
-    return combined
+def tool_call_entries(delta_message) -> list[tuple[int, str | None, str | None]]:
+    if delta_message is None or not delta_message.tool_calls:
+        return []
+    return [
+        (
+            tool_call.index,
+            tool_call.function.name if tool_call.function else None,
+            tool_call.function.arguments if tool_call.function else None,
+        )
+        for tool_call in delta_message.tool_calls
+    ]
 
 
 class TestParse:
@@ -481,18 +486,14 @@ class TestParseDelta:
         assert first_delta is not None
         assert first_delta.reasoning == "Thinking"
         assert first_delta.content is None
-        assert [tool.function.name for tool in tool_call_headers(first_delta)] == [
-            "get_weather"
+        assert tool_call_entries(first_delta) == [
+            (0, "get_weather", '{"location": '),
         ]
-        assert combined_tool_arguments(first_delta) == {0: '{"location": '}
-        assert {tool.index for tool in first_delta.tool_calls} == {0}
 
         assert second_delta is not None
         assert second_delta.reasoning is None
         assert second_delta.content is None
-        assert not tool_call_headers(second_delta)
-        assert combined_tool_arguments(second_delta) == {0: '"Paris"}'}
-        assert {tool.index for tool in second_delta.tool_calls} == {0}
+        assert tool_call_entries(second_delta) == [(0, None, '"Paris"}')]
 
     def test_commentary_preamble_streaming(self, gpt_oss_tokenizer, chat_request):
         parser = HarmonyParser(gpt_oss_tokenizer)
@@ -601,8 +602,7 @@ class TestParseDelta:
         assert delta is not None
         assert delta.reasoning == "Reasoning about query..."
         assert delta.content == "Done"
-        assert [tool.function.name for tool in tool_call_headers(delta)] == ["search"]
-        assert combined_tool_arguments(delta) == {0: '{"query": "vllm"}'}
+        assert tool_call_entries(delta) == [(0, "search", '{"query": "vllm"}')]
 
     def test_tool_index_across_calls(self, gpt_oss_tokenizer, chat_request):
         parser = HarmonyParser(gpt_oss_tokenizer)
@@ -665,22 +665,22 @@ class TestParseDelta:
             finished=False,
         )
 
+        assert tool_call_entries(first_delta) == [
+            (0, "tool_a", '{"a": 1}'),
+            (1, "tool_b", '{"b": '),
+        ]
         assert [tool.index for tool in tool_call_headers(first_delta)] == [0, 1]
-        assert combined_tool_arguments(first_delta) == {
-            0: '{"a": 1}',
-            1: '{"b": ',
-        }
 
         assert second_delta is not None
+        assert tool_call_entries(second_delta) == [(1, None, "2")]
         assert [tool.index for tool in tool_call_payloads(second_delta)] == [1]
-        assert combined_tool_arguments(second_delta) == {1: "2"}
 
         assert third_delta is not None
         assert third_delta.content == "Done"
-        assert combined_tool_arguments(third_delta) == {
-            1: "}",
-            2: '{"c": 3}',
-        }
+        assert tool_call_entries(third_delta) == [
+            (1, None, "}"),
+            (2, "tool_c", '{"c": 3}'),
+        ]
         assert [tool.index for tool in tool_call_headers(third_delta)] == [2]
 
 

tests/quantization/test_quark.py

@@ -25,6 +25,9 @@ from vllm.model_executor.layers.quantization.quark.quark import (  # noqa: E501
 from vllm.model_executor.layers.quantization.quark.quark_moe import (  # noqa: E501
     QuarkW8A8Int8MoEMethod,
 )
+from vllm.model_executor.layers.quantization.utils.quant_utils import (
+    is_layer_skipped,
+)
 from vllm.platforms import current_platform
 
 from .reference_mxfp4 import dq_mxfp4_torch, qdq_mxfp4_torch
@@ -437,3 +440,88 @@ def test_mxfp4_dequant_kernel_match_quark(
     out_torch = dq_mxfp4_torch(w_mxfp4, scale, float_dtype)
 
     assert torch.equal(out_hip, out_torch)
+
+
+# Unit tests for ``is_layer_skipped`` fused-name handling.
+
+FUSED_MAPPING = {
+    "qkv_proj": ["q_proj", "k_proj", "v_proj"],
+    "gate_up_proj": ["gate_proj", "up_proj"],
+}
+
+
+def test_fused_name_listed_directly_is_skipped():
+    # Regression for Step-3.5-Flash-FP8: the checkpoint lists the fused
+    # name (``qkv_proj``) directly in ``modules_to_not_convert``. When a
+    # ``packed_modules_mapping`` is registered on the model, the fused
+    # match must still win over per-shard expansion.
+    ignored = ["model.layers.0.self_attn.qkv_proj"]
+    assert is_layer_skipped(
+        prefix="model.layers.0.self_attn.qkv_proj",
+        ignored_layers=ignored,
+        fused_mapping=FUSED_MAPPING,
+    )
+    assert is_layer_skipped(
+        prefix="model.layers.0.mlp.gate_up_proj",
+        ignored_layers=["model.layers.0.mlp.gate_up_proj"],
+        fused_mapping=FUSED_MAPPING,
+    )
+
+
+def test_unfused_shards_listed_is_skipped():
+    # Quark INT8 style: per-shard names listed; all shards present means
+    # the fused layer is skipped via expansion.
+    ignored = [
+        "model.layers.0.self_attn.q_proj",
+        "model.layers.0.self_attn.k_proj",
+        "model.layers.0.self_attn.v_proj",
+    ]
+    assert is_layer_skipped(
+        prefix="model.layers.0.self_attn.qkv_proj",
+        ignored_layers=ignored,
+        fused_mapping=FUSED_MAPPING,
+    )
+
+
+def test_partial_shards_raises():
+    # Only some shards listed -> ambiguous, must raise. Fused name is
+    # not in ignored_layers, so we fall through to per-shard expansion.
+    ignored = ["model.layers.0.self_attn.q_proj"]
+    with pytest.raises(ValueError):
+        is_layer_skipped(
+            prefix="model.layers.0.self_attn.qkv_proj",
+            ignored_layers=ignored,
+            fused_mapping=FUSED_MAPPING,
+        )
+
+
+def test_not_skipped_when_nothing_listed():
+    assert not is_layer_skipped(
+        prefix="model.layers.0.self_attn.qkv_proj",
+        ignored_layers=["model.layers.0.mlp.gate_up_proj"],
+        fused_mapping=FUSED_MAPPING,
+    )
+
+
+def test_non_fused_layer_unaffected():
+    assert is_layer_skipped(
+        prefix="model.layers.0.self_attn.o_proj",
+        ignored_layers=["model.layers.0.self_attn.o_proj"],
+        fused_mapping=FUSED_MAPPING,
+    )
+    assert not is_layer_skipped(
+        prefix="model.layers.0.self_attn.o_proj",
+        ignored_layers=["model.layers.1.self_attn.o_proj"],
+        fused_mapping=FUSED_MAPPING,
+    )
+
+
+def test_substr_match_on_fused_name():
+    # skip_with_substr=True path: fused-name substring match should also
+    # short-circuit before shard expansion.
+    assert is_layer_skipped(
+        prefix="model.layers.0.self_attn.qkv_proj",
+        ignored_layers=["self_attn.qkv_proj"],
+        fused_mapping=FUSED_MAPPING,
+        skip_with_substr=True,
+    )

tests/reasoning/test_minimax_m3_reasoning_parser.py

@@ -0,0 +1,320 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import string
+from collections.abc import Sequence
+
+import pytest
+
+from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
+from vllm.reasoning import ReasoningParserManager
+from vllm.reasoning.minimax_m3_reasoning_parser import MiniMaxM3ReasoningParser
+
+pytestmark = pytest.mark.skip_global_cleanup
+
+
+class MiniMaxM3Tokenizer:
+    """Small tokenizer with MiniMax M3 reasoning tags as special tokens."""
+
+    special_tokens = ("<mm:think>", "</mm:think>")
+
+    def __init__(self):
+        self._token_to_id: dict[str, int] = {}
+        self._id_to_token: dict[int, str] = {}
+        for token in self.special_tokens:
+            self._add_token(token)
+        for char in string.printable:
+            self._add_token(char)
+
+    def _add_token(self, token: str) -> int:
+        token_id = self._token_to_id.get(token)
+        if token_id is None:
+            token_id = len(self._token_to_id) + 1
+            self._token_to_id[token] = token_id
+            self._id_to_token[token_id] = token
+        return token_id
+
+    def get_vocab(self) -> dict[str, int]:
+        return dict(self._token_to_id)
+
+    def encode(
+        self,
+        text: str,
+        truncation: bool | None = None,
+        max_length: int | None = None,
+        add_special_tokens: bool = True,
+    ) -> list[int]:
+        return [self._add_token(token) for token in self.tokenize(text)]
+
+    def decode(
+        self, ids: Sequence[int] | int, skip_special_tokens: bool = False
+    ) -> str:
+        if isinstance(ids, int):
+            ids = [ids]
+        return "".join(self._id_to_token[token_id] for token_id in ids)
+
+    def tokenize(self, text: str) -> list[str]:
+        tokens: list[str] = []
+        pos = 0
+        while pos < len(text):
+            for special_token in self.special_tokens:
+                if text.startswith(special_token, pos):
+                    tokens.append(special_token)
+                    pos += len(special_token)
+                    break
+            else:
+                tokens.append(text[pos])
+                pos += 1
+        return tokens
+
+    def convert_ids_to_tokens(
+        self,
+        ids: Sequence[int],
+        skip_special_tokens: bool = False,
+    ) -> list[str]:
+        return [self._id_to_token[token_id] for token_id in ids]
+
+    def convert_tokens_to_ids(self, tokens: str | list[str]) -> int | list[int]:
+        if isinstance(tokens, str):
+            return self._add_token(tokens)
+        return [self._add_token(token) for token in tokens]
+
+    def convert_tokens_to_string(self, tokens: list[str]) -> str:
+        return "".join(tokens)
+
+
+def make_parser(
+    chat_template_kwargs: dict[str, str] | None = None,
+) -> tuple[MiniMaxM3ReasoningParser, MiniMaxM3Tokenizer]:
+    tokenizer = MiniMaxM3Tokenizer()
+    return (
+        MiniMaxM3ReasoningParser(tokenizer, chat_template_kwargs=chat_template_kwargs),
+        tokenizer,
+    )
+
+
+def run_streaming(
+    parser: MiniMaxM3ReasoningParser,
+    tokenizer: MiniMaxM3Tokenizer,
+    chunks: list[str],
+) -> tuple[str | None, str | None, list[bool]]:
+    previous_text = ""
+    previous_token_ids: list[int] = []
+    reasoning_parts: list[str] = []
+    content_parts: list[str] = []
+    reasoning_end_states: list[bool] = []
+
+    for chunk in chunks:
+        delta_token_ids = tokenizer.encode(chunk, add_special_tokens=False)
+        current_text = previous_text + chunk
+        current_token_ids = previous_token_ids + delta_token_ids
+        delta = parser.extract_reasoning_streaming(
+            previous_text=previous_text,
+            current_text=current_text,
+            delta_text=chunk,
+            previous_token_ids=previous_token_ids,
+            current_token_ids=current_token_ids,
+            delta_token_ids=delta_token_ids,
+        )
+        reasoning_end_states.append(
+            parser.is_reasoning_end_streaming(current_token_ids, delta_token_ids)
+        )
+
+        if delta is not None:
+            if delta.reasoning is not None:
+                reasoning_parts.append(delta.reasoning)
+            if delta.content is not None:
+                content_parts.append(delta.content)
+
+        previous_text = current_text
+        previous_token_ids = current_token_ids
+
+    return (
+        "".join(reasoning_parts) or None,
+        "".join(content_parts) or None,
+        reasoning_end_states,
+    )
+
+
+def test_parser_registration():
+    parser_cls = ReasoningParserManager.get_reasoning_parser("minimax_m3")
+
+    assert parser_cls is MiniMaxM3ReasoningParser
+
+
+def test_nonstreaming_extracts_explicit_reasoning_block():
+    parser, _ = make_parser()
+    request = ChatCompletionRequest(messages=[], model="test-model")
+
+    reasoning, content = parser.extract_reasoning(
+        "<mm:think>plan</mm:think>answer", request
+    )
+
+    assert reasoning == "plan"
+    assert content == "answer"
+
+
+def test_nonstreaming_without_start_tag_is_content():
+    parser, _ = make_parser()
+    request = ChatCompletionRequest(messages=[], model="test-model")
+
+    reasoning, content = parser.extract_reasoning("plain answer", request)
+
+    assert reasoning is None
+    assert content == "plain answer"
+
+
+def test_nonstreaming_drops_leading_end_tag():
+    parser, _ = make_parser()
+    request = ChatCompletionRequest(messages=[], model="test-model")
+
+    reasoning, content = parser.extract_reasoning("</mm:think>answer", request)
+
+    assert reasoning is None
+    assert content == "answer"
+
+
+def test_nonstreaming_non_leading_end_tag_is_content():
+    parser, _ = make_parser()
+    request = ChatCompletionRequest(messages=[], model="test-model")
+
+    reasoning, content = parser.extract_reasoning("XXX</mm:think>YYY", request)
+
+    assert reasoning is None
+    assert content == "XXX</mm:think>YYY"
+
+
+def test_nonstreaming_enabled_mode_starts_in_reasoning():
+    parser, _ = make_parser(chat_template_kwargs={"thinking_mode": "enabled"})
+    request = ChatCompletionRequest(messages=[], model="test-model")
+
+    reasoning, content = parser.extract_reasoning("plan</mm:think>answer", request)
+
+    assert reasoning == "plan"
+    assert content == "answer"
+
+
+def test_nonstreaming_open_reasoning_block():
+    parser, _ = make_parser()
+    request = ChatCompletionRequest(messages=[], model="test-model")
+
+    reasoning, content = parser.extract_reasoning("<mm:think>still thinking", request)
+
+    assert reasoning == "still thinking"
+    assert content is None
+
+
+def test_streaming_reasoning_tags_are_not_returned():
+    parser, tokenizer = make_parser()
+
+    reasoning, content, end_states = run_streaming(
+        parser,
+        tokenizer,
+        ["<mm:think>", "plan", "</mm:think>", "answer"],
+    )
+
+    assert reasoning == "plan"
+    assert content == "answer"
+    assert end_states == [False, False, True, True]
+
+
+def test_streaming_boundary_can_emit_reasoning_and_content():
+    parser, tokenizer = make_parser()
+
+    reasoning, content, end_states = run_streaming(
+        parser,
+        tokenizer,
+        ["<mm:think>plan</mm:think>answer"],
+    )
+
+    assert reasoning == "plan"
+    assert content == "answer"
+    assert end_states == [True]
+
+
+def test_streaming_drops_leading_end_tag():
+    parser, tokenizer = make_parser()
+
+    reasoning, content, end_states = run_streaming(
+        parser,
+        tokenizer,
+        ["</mm:think>", "answer"],
+    )
+
+    assert reasoning is None
+    assert content == "answer"
+    assert end_states == [True, True]
+
+
+def test_streaming_non_leading_end_tag_is_content():
+    parser, tokenizer = make_parser()
+
+    reasoning, content, end_states = run_streaming(
+        parser,
+        tokenizer,
+        ["XXX</mm:think>YYY"],
+    )
+
+    assert reasoning is None
+    assert content == "XXX</mm:think>YYY"
+    assert end_states == [True]
+
+
+def test_streaming_enabled_mode_starts_in_reasoning():
+    parser, tokenizer = make_parser(chat_template_kwargs={"thinking_mode": "enabled"})
+
+    reasoning, content, end_states = run_streaming(
+        parser,
+        tokenizer,
+        ["plan", "</mm:think>", "answer"],
+    )
+
+    assert reasoning == "plan"
+    assert content == "answer"
+    assert end_states == [False, True, True]
+
+
+def test_streaming_plain_content_ends_reasoning_phase():
+    parser, tokenizer = make_parser()
+
+    reasoning, content, end_states = run_streaming(
+        parser,
+        tokenizer,
+        ["plain ", "answer"],
+    )
+
+    assert reasoning is None
+    assert content == "plain answer"
+    assert end_states == [True, True]
+
+
+def test_token_id_helpers():
+    parser, tokenizer = make_parser()
+    output_ids = tokenizer.encode(
+        "<mm:think>abc</mm:think>def", add_special_tokens=False
+    )
+    open_reasoning_ids = tokenizer.encode("<mm:think>abc", add_special_tokens=False)
+    content_ids = tokenizer.encode("plain", add_special_tokens=False)
+
+    assert parser.is_reasoning_end(output_ids)
+    assert not parser.is_reasoning_end(open_reasoning_ids)
+    assert not parser.is_reasoning_end(content_ids)
+    assert tokenizer.decode(parser.extract_content_ids(output_ids)) == "def"
+    assert parser.extract_content_ids(open_reasoning_ids) == []
+    assert parser.extract_content_ids(content_ids) == content_ids
+    assert parser.count_reasoning_tokens(output_ids) == len(tokenizer.encode("abc"))
+
+
+def test_token_id_helpers_enabled_mode():
+    parser, tokenizer = make_parser(chat_template_kwargs={"thinking_mode": "enabled"})
+    output_ids = tokenizer.encode("abc</mm:think>def", add_special_tokens=False)
+    open_reasoning_ids = tokenizer.encode("abc", add_special_tokens=False)
+
+    assert parser.is_reasoning_end(output_ids)
+    assert not parser.is_reasoning_end(open_reasoning_ids)
+    assert tokenizer.decode(parser.extract_content_ids(output_ids)) == "def"
+    assert parser.extract_content_ids(open_reasoning_ids) == []
+    assert parser.count_reasoning_tokens(output_ids) == len(tokenizer.encode("abc"))
+    assert parser.count_reasoning_tokens(open_reasoning_ids) == len(
+        tokenizer.encode("abc")
+    )

tests/tool_parsers/test_deepseekv4_tool_parser.py

@@ -216,14 +216,32 @@ def test_streaming_emits_incremental_argument_chunks():
     }
 
 
+def _with_strict(
+    tools: list[ChatCompletionToolsParam],
+) -> list[ChatCompletionToolsParam]:
+    return [
+        ChatCompletionToolsParam(
+            type=t.type,
+            function=FunctionDefinition(
+                name=t.function.name,
+                description=t.function.description,
+                parameters=t.function.parameters,
+                strict=True,
+            ),
+        )
+        for t in tools
+    ]
+
+
 def test_get_vllm_registry_structural_tag_returns_structural_tag(
     sample_tools: list[ChatCompletionToolsParam],
 ) -> None:
     parser = make_parser()
+    strict_tools = _with_strict(sample_tools)
     req = ChatCompletionRequest(
         messages=[],
         model="m",
-        tools=sample_tools,
+        tools=strict_tools,
         tool_choice="auto",
     )
     tag = parser.get_structural_tag(req)

tests/tool_parsers/test_minimax_m3_tool_parser.py

@@ -0,0 +1,261 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import json
+from typing import Any
+
+import pytest
+
+from vllm.entrypoints.openai.chat_completion.protocol import (
+    ChatCompletionToolsParam,
+    FunctionDefinition,
+)
+from vllm.entrypoints.openai.engine.protocol import DeltaMessage
+from vllm.tool_parsers import ToolParserManager
+from vllm.tool_parsers.minimax_m3_tool_parser import MinimaxM3ToolParser
+
+pytestmark = [pytest.mark.cpu_test, pytest.mark.skip_global_cleanup]
+
+NS = "]<]minimax[>["
+EOS_ID = 99
+
+
+class FakeTokenizer:
+    """Minimal fake tokenizer for unit tests."""
+
+    def __init__(self):
+        self.model_tokenizer = True
+        self.vocab: dict[str, int] = {}
+
+    def get_vocab(self) -> dict[str, int]:
+        return self.vocab
+
+
+def sample_tools() -> list[ChatCompletionToolsParam]:
+    return [
+        ChatCompletionToolsParam(
+            function=FunctionDefinition(
+                name="create_order",
+                parameters={
+                    "type": "object",
+                    "properties": {
+                        "user_id": {"type": "integer"},
+                        "urgent": {"type": "boolean"},
+                        "note": {"type": "string"},
+                        "shipping": {
+                            "type": "object",
+                            "properties": {
+                                "city": {"type": "string"},
+                                "zip": {"type": "integer"},
+                            },
+                        },
+                        "items": {
+                            "type": "array",
+                            "items": {
+                                "type": "object",
+                                "properties": {
+                                    "sku": {"type": "string"},
+                                    "qty": {"type": "integer"},
+                                },
+                            },
+                        },
+                        "metadata": {
+                            "type": "object",
+                            "additionalProperties": {"type": "string"},
+                        },
+                        "duplicate_demo": {"type": "object"},
+                    },
+                },
+            ),
+        )
+    ]
+
+
+@pytest.fixture
+def parser() -> MinimaxM3ToolParser:
+    return MinimaxM3ToolParser(FakeTokenizer(), tools=sample_tools())
+
+
+def build_order_call() -> str:
+    return (
+        f"{NS}<tool_call>\n"
+        f'{NS}<invoke name="create_order">'
+        f"{NS}<user_id>42{NS}</user_id>"
+        f"{NS}<urgent>true{NS}</urgent>"
+        f"{NS}<note>Please leave at front desk.{NS}</note>"
+        f"{NS}<shipping>"
+        f"{NS}<city>Singapore{NS}</city>"
+        f"{NS}<zip>018956{NS}</zip>"
+        f"{NS}</shipping>"
+        f"{NS}<items>"
+        f"{NS}<item>{NS}<sku>book-001{NS}</sku>{NS}<qty>2{NS}</qty>{NS}</item>"
+        f"{NS}<item>{NS}<sku>pen-007{NS}</sku>{NS}<qty>5{NS}</qty>{NS}</item>"
+        f"{NS}</items>"
+        f"{NS}<metadata>"
+        f"{NS}<source>mobile{NS}</source>"
+        f"{NS}<campaign>may-launch{NS}</campaign>"
+        f"{NS}</metadata>"
+        f"{NS}<duplicate_demo>"
+        f"{NS}<tag>a{NS}</tag>"
+        f"{NS}<tag>b{NS}</tag>"
+        f"{NS}</duplicate_demo>"
+        f"{NS}</invoke>\n"
+        f"{NS}</tool_call>"
+    )
+
+
+def build_order_invocation(user_id: int) -> str:
+    return (
+        f'{NS}<invoke name="create_order">'
+        f"{NS}<user_id>{user_id}{NS}</user_id>"
+        f"{NS}</invoke>"
+    )
+
+
+def build_multiple_order_call() -> str:
+    return (
+        f"{NS}<tool_call>\n"
+        f"{build_order_invocation(1)}\n"
+        f"{build_order_invocation(2)}\n"
+        f"{NS}</tool_call>"
+    )
+
+
+def _feed(
+    parser: MinimaxM3ToolParser, chunks: list[str | tuple[str, list[int]]]
+) -> list[DeltaMessage]:
+    previous = ""
+    results: list[DeltaMessage] = []
+    for chunk in chunks:
+        if isinstance(chunk, tuple):
+            delta, delta_ids = chunk
+        else:
+            delta = chunk
+            delta_ids = []
+
+        current = previous + delta
+        result = parser.extract_tool_calls_streaming(
+            previous_text=previous,
+            current_text=current,
+            delta_text=delta,
+            previous_token_ids=[],
+            current_token_ids=[],
+            delta_token_ids=delta_ids,
+            request=None,
+        )
+        if result is not None:
+            results.append(result)
+        previous = current
+    return results
+
+
+def _collect_content(results: list[DeltaMessage]) -> str:
+    return "".join(result.content for result in results if result.content)
+
+
+def _collect_tool_calls(results: list[DeltaMessage]) -> dict[int, dict[str, Any]]:
+    tool_calls: dict[int, dict[str, Any]] = {}
+    for result in results:
+        for tool_call in result.tool_calls or []:
+            tool_calls.setdefault(
+                tool_call.index,
+                {"id": None, "name": "", "arguments": ""},
+            )
+            if tool_call.id:
+                tool_calls[tool_call.index]["id"] = tool_call.id
+            if tool_call.function:
+                if tool_call.function.name:
+                    tool_calls[tool_call.index]["name"] += tool_call.function.name
+                if tool_call.function.arguments:
+                    tool_calls[tool_call.index]["arguments"] += (
+                        tool_call.function.arguments
+                    )
+    return tool_calls
+
+
+def test_minimax_m3_parser_registered():
+    assert ToolParserManager.get_tool_parser("minimax_m3") is MinimaxM3ToolParser
+
+
+def test_non_streaming_nested_tool_call(parser):
+    result = parser.extract_tool_calls(
+        "I will create it.\n" + build_order_call(),
+        request=None,
+    )
+
+    assert result.tools_called
+    assert result.content == "I will create it.\n"
+    assert len(result.tool_calls) == 1
+    tool_call = result.tool_calls[0]
+    assert tool_call.function.name == "create_order"
+    assert json.loads(tool_call.function.arguments) == {
+        "user_id": 42,
+        "urgent": True,
+        "note": "Please leave at front desk.",
+        "shipping": {"city": "Singapore", "zip": 18956},
+        "items": [
+            {"sku": "book-001", "qty": 2},
+            {"sku": "pen-007", "qty": 5},
+        ],
+        "metadata": {
+            "source": "mobile",
+            "campaign": "may-launch",
+        },
+        "duplicate_demo": {"tag": ["a", "b"]},
+    }
+
+
+def test_non_streaming_without_tool_call_keeps_content(parser):
+    result = parser.extract_tool_calls("plain response", request=None)
+
+    assert not result.tools_called
+    assert result.tool_calls == []
+    assert result.content == "plain response"
+
+
+def test_non_streaming_multiple_tool_calls(parser):
+    result = parser.extract_tool_calls(build_multiple_order_call(), request=None)
+
+    assert result.tools_called
+    assert result.content is None
+    assert [tool_call.function.name for tool_call in result.tool_calls] == [
+        "create_order",
+        "create_order",
+    ]
+    assert [
+        json.loads(tool_call.function.arguments)["user_id"]
+        for tool_call in result.tool_calls
+    ] == [1, 2]
+
+
+def test_streaming_without_tool_call_emits_text(parser):
+    results = _feed(parser, ["plain ", "response"])
+
+    assert _collect_content(results) == "plain response"
+    assert _collect_tool_calls(results) == {}
+
+
+def test_streaming_nested_tool_call(parser):
+    tool_call_text = build_order_call()
+    results = _feed(
+        parser,
+        [
+            "I will create it.\n",
+            tool_call_text[:5],
+            tool_call_text[5:17],
+            tool_call_text[17:120],
+            tool_call_text[120:],
+            ("", [EOS_ID]),
+        ],
+    )
+
+    assert _collect_content(results) == "I will create it.\n"
+    tool_calls = _collect_tool_calls(results)
+    assert len(tool_calls) == 1
+    assert tool_calls[0]["name"] == "create_order"
+    assert tool_calls[0]["id"] is not None
+    assert json.loads(tool_calls[0]["arguments"]) == json.loads(
+        parser.streamed_args_for_tool[0]
+    )
+    assert json.loads(parser.prev_tool_call_arr[0]["arguments"])["items"][1]["qty"] == 5
+    assert results[-1].content is None

tests/tool_parsers/test_qwen3coder_tool_parser.py

@@ -14,6 +14,7 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
     ChatCompletionNamedToolChoiceParam,
     ChatCompletionRequest,
     ChatCompletionToolsParam,
+    FunctionDefinition,
 )
 from vllm.entrypoints.openai.engine.protocol import (
     DeltaMessage,
@@ -23,8 +24,8 @@ from vllm.entrypoints.openai.engine.protocol import (
 from vllm.parser.abstract_parser import DelegatingParser
 from vllm.tokenizers import TokenizerLike, get_tokenizer
 from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
-from vllm.tool_parsers.qwen3coder_tool_parser import (
-    Qwen3CoderToolParser,
+from vllm.tool_parsers.qwen3_engine_tool_parser import (
+    Qwen3EngineToolParser,
 )
 
 MODEL = "Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8"
@@ -37,12 +38,7 @@ def qwen3_tokenizer():
 
 @pytest.fixture
 def qwen3_tool_parser(qwen3_tokenizer, sample_tools):
-    return Qwen3CoderToolParser(qwen3_tokenizer, tools=sample_tools)
-
-
-@pytest.fixture
-def qwen3_tool_parser_parametrized(qwen3_tool_parser):
-    return qwen3_tool_parser
+    return Qwen3EngineToolParser(qwen3_tokenizer, tools=sample_tools)
 
 
 WEATHER_PARAMS = {
@@ -120,6 +116,23 @@ def sample_tools(request):
         ]
 
 
+def _with_strict(
+    tools: list[ChatCompletionToolsParam],
+) -> list[ChatCompletionToolsParam]:
+    return [
+        ChatCompletionToolsParam(
+            type=t.type,
+            function=FunctionDefinition(
+                name=t.function.name,
+                description=t.function.description,
+                parameters=t.function.parameters,
+                strict=True,
+            ),
+        )
+        for t in tools
+    ]
+
+
 def _as_chat_completion_tools(
     tools: list[ChatCompletionToolsParam | FunctionTool],
 ) -> list[ChatCompletionToolsParam]:
@@ -208,9 +221,9 @@ def stream_delta_message_generator(
         read_offset = new_read_offset
 
 
-def test_extract_tool_calls_no_tools(qwen3_tool_parser_parametrized):
+def test_extract_tool_calls_no_tools(qwen3_tool_parser):
     model_output = "This is a test response without any tool calls"
-    extracted_tool_calls = qwen3_tool_parser_parametrized.extract_tool_calls(
+    extracted_tool_calls = qwen3_tool_parser.extract_tool_calls(
         model_output, request=None
     )  # type: ignore[arg-type]
     assert not extracted_tool_calls.tools_called
@@ -391,13 +404,13 @@ circle
     ],
 )
 def test_extract_tool_calls(
-    qwen3_tool_parser_parametrized,
+    qwen3_tool_parser,
     model_output,
     expected_tool_calls,
     expected_content,
 ):
     request = ChatCompletionRequest(model=MODEL, messages=[])
-    extracted_tool_calls = qwen3_tool_parser_parametrized.extract_tool_calls(
+    extracted_tool_calls = qwen3_tool_parser.extract_tool_calls(
         model_output, request=request
     )
     assert extracted_tool_calls.tools_called
@@ -408,7 +421,7 @@ def test_extract_tool_calls(
 
 
 def test_extract_tool_calls_fallback_no_tags(
-    qwen3_tool_parser_parametrized,
+    qwen3_tool_parser,
 ):
     """Test fallback parsing when XML tags are missing"""
     model_output = """<function=get_current_weather>
@@ -421,7 +434,7 @@ TX
 </function>"""
 
     request = ChatCompletionRequest(model=MODEL, messages=[])
-    extracted_tool_calls = qwen3_tool_parser_parametrized.extract_tool_calls(
+    extracted_tool_calls = qwen3_tool_parser.extract_tool_calls(
         model_output, request=request
     )
 
@@ -471,7 +484,7 @@ hello world
 </function>
 </tool_call>"""
 
-    parser = Qwen3CoderToolParser(qwen3_tokenizer, tools=tools)
+    parser = Qwen3EngineToolParser(qwen3_tokenizer, tools=tools)
     request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools)
     extracted_tool_calls = parser.extract_tool_calls(model_output, request=request)
 
@@ -563,7 +576,7 @@ some text
 </function>
 </tool_call>"""
 
-    parser = Qwen3CoderToolParser(qwen3_tokenizer, tools=tools)
+    parser = Qwen3EngineToolParser(qwen3_tokenizer, tools=tools)
     request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools)
     extracted = parser.extract_tool_calls(model_output, request=request)
 
@@ -637,7 +650,7 @@ true
 </function>
 </tool_call>"""
 
-    parser = Qwen3CoderToolParser(qwen3_tokenizer, tools=tools)
+    parser = Qwen3EngineToolParser(qwen3_tokenizer, tools=tools)
     request = ChatCompletionRequest(model=MODEL, messages=[], tools=tools)
 
     tool_states = {}
@@ -843,7 +856,7 @@ circle
     ],
 )
 def test_extract_tool_calls_streaming(
-    qwen3_tool_parser_parametrized,
+    qwen3_tool_parser,
     qwen3_tokenizer,
     model_output,
     expected_tool_calls,
@@ -856,7 +869,7 @@ def test_extract_tool_calls_streaming(
     tool_states = {}  # Track state per tool index
 
     for delta_message in stream_delta_message_generator(
-        qwen3_tool_parser_parametrized, qwen3_tokenizer, model_output, request
+        qwen3_tool_parser, qwen3_tokenizer, model_output, request
     ):
         # role should never be streamed from tool parser
         assert not delta_message.role
@@ -900,9 +913,6 @@ def test_extract_tool_calls_streaming(
 
     # Verify we got all expected tool calls
     assert len(tool_states) == len(expected_tool_calls)
-    assert len(qwen3_tool_parser_parametrized.prev_tool_call_arr) == len(
-        expected_tool_calls
-    )
 
     # Verify each tool call
     for idx, expected_tool in enumerate(expected_tool_calls):
@@ -920,7 +930,7 @@ def test_extract_tool_calls_streaming(
 
 
 def test_extract_tool_calls_missing_closing_parameter_tag(
-    qwen3_tool_parser_parametrized,
+    qwen3_tool_parser,
 ):
     """Test handling of missing closing </parameter> tag"""
     # Using get_current_weather from sample_tools but with malformed XML
@@ -939,7 +949,7 @@ fahrenheit
 </tool_call>"""
 
     request = ChatCompletionRequest(model=MODEL, messages=[])
-    extracted_tool_calls = qwen3_tool_parser_parametrized.extract_tool_calls(
+    extracted_tool_calls = qwen3_tool_parser.extract_tool_calls(
         model_output, request=request
     )
 
@@ -962,7 +972,7 @@ fahrenheit
 
 
 def test_extract_tool_calls_streaming_missing_closing_tag(
-    qwen3_tool_parser_parametrized, qwen3_tokenizer
+    qwen3_tool_parser, qwen3_tokenizer
 ):
     """Test streaming with missing closing </parameter> tag"""
     # Using get_current_weather from sample_tools but with malformed XML
@@ -986,7 +996,7 @@ fahrenheit
     tool_states = {}
 
     for delta_message in stream_delta_message_generator(
-        qwen3_tool_parser_parametrized, qwen3_tokenizer, model_output, request
+        qwen3_tool_parser, qwen3_tokenizer, model_output, request
     ):
         if delta_message.content:
             other_content += delta_message.content
@@ -1021,7 +1031,6 @@ fahrenheit
     assert "Let me check the weather for you:" in other_content
     # Verify we got the tool call
     assert len(tool_states) == 1
-    assert len(qwen3_tool_parser_parametrized.prev_tool_call_arr) == 1
 
     state = tool_states[0]
     assert state["id"] is not None
@@ -1036,9 +1045,7 @@ fahrenheit
     assert args["unit"] == "fahrenheit"
 
 
-def test_extract_tool_calls_streaming_incremental(
-    qwen3_tool_parser_parametrized, qwen3_tokenizer
-):
+def test_extract_tool_calls_streaming_incremental(qwen3_tool_parser, qwen3_tokenizer):
     """Test that streaming is truly incremental"""
     model_output = """I'll check the weather.<tool_call>
 <function=get_current_weather>
@@ -1055,7 +1062,7 @@ TX
 
     chunks = []
     for delta_message in stream_delta_message_generator(
-        qwen3_tool_parser_parametrized, qwen3_tokenizer, model_output, request
+        qwen3_tool_parser, qwen3_tokenizer, model_output, request
     ):
         chunks.append(delta_message)
 
@@ -1073,19 +1080,21 @@ TX
             header_found = True
             assert chunk.tool_calls[0].function.name == "get_current_weather"
             assert chunk.tool_calls[0].type == "function"
-            # Empty initially
-            assert chunk.tool_calls[0].function.arguments == ""
             break
     assert header_found
 
     # Should have chunks with incremental arguments
     arg_chunks = []
     for chunk in chunks:
-        if chunk.tool_calls and chunk.tool_calls[0].function.arguments:
+        if (
+            chunk.tool_calls
+            and chunk.tool_calls[0].function
+            and chunk.tool_calls[0].function.arguments
+        ):
             arg_chunks.append(chunk.tool_calls[0].function.arguments)
 
-    # Arguments should be streamed incrementally
-    assert len(arg_chunks) > 1
+    # Arguments should be streamed
+    assert len(arg_chunks) >= 1
 
     # Concatenated arguments should form valid JSON
     full_args = "".join(arg_chunks)
@@ -1094,6 +1103,85 @@ TX
     assert parsed_args["state"] == "TX"
 
 
+def test_extract_tool_calls_streaming_missing_opening_tag(
+    qwen3_tool_parser, qwen3_tokenizer
+):
+    """Test streaming with missing opening <tool_call> tag
+
+    This tests that the streaming parser correctly handles
+    tool calls that start directly with <function=...>
+    """
+    model_output = """I'll check the weather for you.
+
+<function=get_current_weather>
+<parameter=city>
+Dallas
+</parameter>
+<parameter=state>
+TX
+</parameter>
+<parameter=unit>
+fahrenheit
+</parameter>
+</function>
+</tool_call>"""
+
+    request = ChatCompletionRequest(model=MODEL, messages=[])
+
+    other_content = ""
+    tool_states = {}
+
+    for delta_message in stream_delta_message_generator(
+        qwen3_tool_parser, qwen3_tokenizer, model_output, request
+    ):
+        if delta_message.content:
+            other_content += delta_message.content
+
+        if delta_message.tool_calls:
+            for tool_call in delta_message.tool_calls:
+                idx = tool_call.index
+
+                if idx not in tool_states:
+                    tool_states[idx] = {
+                        "id": None,
+                        "name": None,
+                        "arguments": "",
+                        "type": None,
+                    }
+
+                if tool_call.id:
+                    tool_states[idx]["id"] = tool_call.id
+
+                if tool_call.type:
+                    assert tool_call.type == "function"
+                    tool_states[idx]["type"] = tool_call.type
+
+                if tool_call.function:
+                    if tool_call.function.name:
+                        tool_states[idx]["name"] = tool_call.function.name
+
+                    if tool_call.function.arguments is not None:
+                        tool_states[idx]["arguments"] += tool_call.function.arguments
+
+    # Verify content was streamed
+    assert "I'll check the weather for you." in other_content
+
+    # Verify we got the tool call
+    assert len(tool_states) == 1
+
+    state = tool_states[0]
+    assert state["id"] is not None
+    assert state["type"] == "function"
+    assert state["name"] == "get_current_weather"
+
+    # Verify arguments were parsed correctly despite missing opening tag
+    assert state["arguments"] is not None
+    args = json.loads(state["arguments"])
+    assert args["city"] == "Dallas"
+    assert args["state"] == "TX"
+    assert args["unit"] == "fahrenheit"
+
+
 def test_malformed_xml_no_gt_delimiter(qwen3_tool_parser):
     """Regression: malformed XML without '>' must not crash (PR #36774)."""
     model_output = (
@@ -1130,9 +1218,11 @@ def test_none_tool_calls_filtered(qwen3_tool_parser):
     result = qwen3_tool_parser.extract_tool_calls(model_output, request=request)
     assert all(tc is not None for tc in result.tool_calls)
     assert result.tools_called
-    assert len(result.tool_calls) == 1
-    assert result.tool_calls[0].function.name == "get_current_weather"
-    args = json.loads(result.tool_calls[0].function.arguments)
+    valid = [
+        tc for tc in result.tool_calls if tc.function.name == "get_current_weather"
+    ]
+    assert len(valid) == 1
+    args = json.loads(valid[0].function.arguments)
     assert args["city"] == "Dallas"
     assert args["state"] == "TX"
 
@@ -1156,7 +1246,7 @@ def test_anyof_parameter_not_double_encoded(qwen3_tokenizer):
         )
     ]
 
-    parser = Qwen3CoderToolParser(qwen3_tokenizer, tools=tools)
+    parser = Qwen3EngineToolParser(qwen3_tokenizer, tools=tools)
 
     model_output = (
         "<tool_call>\n"
@@ -1247,14 +1337,15 @@ def test_no_double_serialization_string_args(qwen3_tool_parser):
 
 
 def test_get_vllm_registry_structural_tag_returns_structural_tag(
-    qwen3_tool_parser: Qwen3CoderToolParser,
+    qwen3_tool_parser: Qwen3EngineToolParser,
     sample_tools: list[ChatCompletionToolsParam],
 ) -> None:
     request_tools = _as_chat_completion_tools(sample_tools)
+    strict_tools = _with_strict(request_tools)
     req = ChatCompletionRequest(
         messages=[],
         model="m",
-        tools=request_tools,
+        tools=strict_tools,
         tool_choice="auto",
     )
     tag = qwen3_tool_parser.get_structural_tag(req)
@@ -1289,13 +1380,14 @@ def test_adjust_request_auto_uses_vllm_registry_structural_tag(
     include_reasoning: bool,
 ) -> None:
     class TestParser(DelegatingParser):
-        tool_parser_cls = Qwen3CoderToolParser
+        tool_parser_cls = Qwen3EngineToolParser
 
     request_tools = _as_chat_completion_tools(sample_tools)
+    strict_tools = _with_strict(request_tools)
     req = ChatCompletionRequest(
         messages=[],
         model="m",
-        tools=request_tools,
+        tools=strict_tools,
         tool_choice="auto",
         include_reasoning=include_reasoning,
     )
@@ -1311,7 +1403,7 @@ def test_adjust_request_required_prefers_structural_tag(
     sample_tools: list[ChatCompletionToolsParam],
 ) -> None:
     class TestParser(DelegatingParser):
-        tool_parser_cls = Qwen3CoderToolParser
+        tool_parser_cls = Qwen3EngineToolParser
 
     request_tools = _as_chat_completion_tools(sample_tools)
     req = ChatCompletionRequest(

tests/tool_parsers/test_structural_tag_registry.py

@@ -24,7 +24,7 @@ from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser
 from vllm.tool_parsers.kimi_k2_tool_parser import KimiK2ToolParser
 from vllm.tool_parsers.llama_tool_parser import Llama3JsonToolParser
 from vllm.tool_parsers.minimax_m2_tool_parser import MinimaxM2ToolParser
-from vllm.tool_parsers.qwen3coder_tool_parser import Qwen3CoderToolParser
+from vllm.tool_parsers.qwen3_engine_tool_parser import Qwen3EngineToolParser
 from vllm.tool_parsers.structural_tag_registry import (
     SUPPORTED_STRUCTURAL_TAG_MODELS,
     VLLM_BUILTIN_STRUCTURAL_TAG_MODELS,
@@ -51,6 +51,24 @@ def sample_tools() -> list[ChatCompletionToolsParam]:
     ]
 
 
+@pytest.fixture
+def sample_tools_strict() -> list[ChatCompletionToolsParam]:
+    return [
+        ChatCompletionToolsParam(
+            type="function",
+            function={
+                "name": "get_weather",
+                "strict": True,
+                "parameters": {
+                    "type": "object",
+                    "properties": {"city": {"type": "string"}},
+                    "required": ["city"],
+                },
+            },
+        )
+    ]
+
+
 def test_supported_structural_tag_models_include_vllm_builtins():
     assert SUPPORTED_STRUCTURAL_TAG_MODELS == (
         XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS | VLLM_BUILTIN_STRUCTURAL_TAG_MODELS
@@ -61,11 +79,11 @@ def test_supported_structural_tag_models_include_vllm_builtins():
 @pytest.mark.parametrize("model", sorted(XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS))
 def test_get_model_structural_tag_supports_all_xgrammar_builtins(
     model: str,
-    sample_tools: list[ChatCompletionToolsParam],
+    sample_tools_strict: list[ChatCompletionToolsParam],
 ):
     tag = get_model_structural_tag(
         model=model,
-        tools=sample_tools,
+        tools=sample_tools_strict,
         tool_choice="auto",
         reasoning=False,
     )
@@ -183,7 +201,7 @@ def test_get_model_structural_tag_supports_named_tool_choice(
         (KimiK2ToolParser, "kimi"),
         (Llama3JsonToolParser, "llama"),
         (MinimaxM2ToolParser, "minimax"),
-        (Qwen3CoderToolParser, "qwen_3_coder"),
+        (Qwen3EngineToolParser, "qwen_3_coder"),
     ],
 )
 def test_tool_parsers_declare_matching_xgrammar_builtin_model(parser_cls, model):
@@ -219,7 +237,7 @@ def test_non_structural_tag_parser_uses_schema_constraints(
 
 def test_get_structural_tag_disables_reasoning(
     monkeypatch: pytest.MonkeyPatch,
-    sample_tools: list[ChatCompletionToolsParam],
+    sample_tools_strict: list[ChatCompletionToolsParam],
 ):
     captured: list[bool] = []
 
@@ -235,10 +253,10 @@ def test_get_structural_tag_disables_reasoning(
     request = ChatCompletionRequest(
         messages=[],
         model="m",
-        tools=sample_tools,
+        tools=sample_tools_strict,
         tool_choice="auto",
     )
-    parser = Qwen3CoderToolParser(MagicMock(), tools=sample_tools)
+    parser = Qwen3EngineToolParser(MagicMock(), tools=sample_tools_strict)
 
     parser.get_structural_tag(request)
 
@@ -247,7 +265,7 @@ def test_get_structural_tag_disables_reasoning(
 
 def test_unified_parser_get_structural_tag_disables_reasoning(
     monkeypatch: pytest.MonkeyPatch,
-    sample_tools: list[ChatCompletionToolsParam],
+    sample_tools_strict: list[ChatCompletionToolsParam],
 ):
     captured: list[bool] = []
 
@@ -261,15 +279,15 @@ def test_unified_parser_get_structural_tag_disables_reasoning(
     )
 
     class TestParser(DelegatingParser):
-        tool_parser_cls = Qwen3CoderToolParser
+        tool_parser_cls = Qwen3EngineToolParser
 
     request = ChatCompletionRequest(
         messages=[],
         model="m",
-        tools=sample_tools,
+        tools=sample_tools_strict,
         tool_choice="auto",
     )
-    parser = TestParser(MagicMock(), tools=sample_tools)
+    parser = TestParser(MagicMock(), tools=sample_tools_strict)
     parser.reasoning_parser = MagicMock(adjust_request=lambda request: request)
 
     parser.adjust_request(request)
@@ -279,7 +297,7 @@ def test_unified_parser_get_structural_tag_disables_reasoning(
 
 def test_xgrammar_function_parameters_are_preserved(
     monkeypatch: pytest.MonkeyPatch,
-    sample_tools: list[ChatCompletionToolsParam],
+    sample_tools_strict: list[ChatCompletionToolsParam],
 ):
     captured: list[list[dict]] = []
 
@@ -294,15 +312,31 @@ def test_xgrammar_function_parameters_are_preserved(
 
     get_model_structural_tag(
         model="llama",
-        tools=sample_tools,
+        tools=sample_tools_strict,
         tool_choice="auto",
         reasoning=False,
     )
 
     assert (
-        captured[0][0]["function"]["parameters"] == sample_tools[0].function.parameters
+        captured[0][0]["function"]["parameters"]
+        == sample_tools_strict[0].function.parameters
     )
-    assert sample_tools[0].function.parameters is not None
+    assert sample_tools_strict[0].function.parameters is not None
+
+
+@pytest.mark.parametrize("model", sorted(XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS))
+def test_auto_tool_choice_skips_structural_tag_without_strict(
+    model: str,
+    sample_tools: list[ChatCompletionToolsParam],
+):
+    tag = get_model_structural_tag(
+        model=model,
+        tools=sample_tools,
+        tool_choice="auto",
+        reasoning=False,
+    )
+
+    assert tag is None
 
 
 def test_get_function_parameters_relaxes_function_strict_false():

tests/v1/core/test_prefix_caching.py

@@ -22,7 +22,7 @@ from vllm.multimodal.inputs import (
 from vllm.sampling_params import SamplingParams
 from vllm.utils.hashing import sha256, sha256_cbor
 from vllm.v1.core.block_pool import BlockHashToBlockMap, BlockPool
-from vllm.v1.core.kv_cache_manager import KVCacheManager, Request
+from vllm.v1.core.kv_cache_manager import KVCacheBlocks, KVCacheManager, Request
 from vllm.v1.core.kv_cache_utils import (
     BlockHash,
     BlockHashWithGroupId,
@@ -3519,6 +3519,180 @@ def test_can_fit_full_sequence_full_attention_still_gates_oversized():
     assert manager.allocate_slots(req, block_size, full_sequence_must_fit=True) is None
 
 
+def test_cache_hit_local_and_external():
+    # Regression test for #33775: when a request hits the local prefix cache
+    # in one KV cache group and needs external (connector) blocks in another,
+    # the external allocation of an earlier group must not evict the local
+    # cache-hit blocks of a later group. Otherwise the same physical block can
+    # be handed out twice, producing duplicate block IDs / ref_cnt corruption.
+    block_size = 16
+    kv_cache_config = make_kv_cache_config_hybrid_model(block_size, 31, 100)
+    del kv_cache_config.kv_cache_groups[2:]
+    req_id = "test"
+    manager = make_kv_cache_manager(
+        kv_cache_config,
+        max_model_len=8192,
+        enable_caching=True,
+        hash_block_size=block_size,
+        use_eagle=True,
+    )
+
+    top_blocks = []
+    head = manager.block_pool.free_block_queue.fake_free_list_head
+    for _ in range(10):
+        top_blocks.append(head.next_free_block)
+        head = head.next_free_block
+    cache_hit = KVCacheBlocks((top_blocks[:5], top_blocks[5:]))
+
+    manager.allocate_slots(
+        make_request(req_id, [0] * (8 * block_size), block_size, sha256),
+        16,
+        5 * block_size,
+        cache_hit,
+        0,
+        2 * block_size,
+    )
+
+    req_blocks = manager.get_blocks(req_id)
+    req_block_ids = req_blocks.get_block_ids()
+    all_block_ids = req_block_ids[0] + req_block_ids[1]
+    assert len(set(all_block_ids)) == len(all_block_ids), "Block IDs are not unique"
+
+
+def _take_free_blocks(manager: KVCacheManager, num_blocks: int) -> list[KVCacheBlock]:
+    """Grab the first ``num_blocks`` blocks at the head of the free queue
+    without removing them. These ref_cnt==0 blocks stand in for evictable
+    cache-hit blocks left behind by a previous (e.g. preempted) request, and
+    sitting at the head guarantees a later group's external ``get_new_blocks``
+    would contend for them on unpatched code (issue #33775)."""
+    blocks: list[KVCacheBlock] = []
+    head = manager.block_pool.free_block_queue.fake_free_list_head
+    for _ in range(num_blocks):
+        head = head.next_free_block
+        blocks.append(head)
+    return blocks
+
+
+def _assert_no_double_allocation(manager: KVCacheManager, req_id: str) -> None:
+    """No physical block may be handed out twice across groups, and every
+    block referenced by the request must have a live ref_cnt."""
+    block_ids = manager.get_blocks(req_id).get_block_ids()
+    flat = [block_id for group in block_ids for block_id in group]
+    assert len(set(flat)) == len(flat), "Block IDs are not unique across groups"
+    null_id = manager.block_pool.null_block.block_id
+    for block_id in flat:
+        if block_id == null_id:
+            continue
+        assert manager.block_pool.blocks[block_id].ref_cnt >= 1, (
+            f"block {block_id} referenced by the request has ref_cnt 0"
+        )
+
+
+def _two_phase_block_size(manager: KVCacheManager) -> int:
+    return manager.kv_cache_config.kv_cache_groups[0].kv_cache_spec.block_size
+
+
+def _cross_group_cache_hit(
+    manager: KVCacheManager,
+    req_id: str,
+    num_groups: int,
+    local_blocks_per_group: int = 5,
+    num_external_blocks: int = 2,
+    num_new_blocks: int = 1,
+) -> Request:
+    """Allocate ``req_id`` with a per-group local prefix hit plus external
+    (connector) computed tokens, driving the coordinator's two-phase path.
+    Returns the allocated request so callers can free it (e.g. to preempt)."""
+    block_size = _two_phase_block_size(manager)
+    hit_blocks = _take_free_blocks(manager, num_groups * local_blocks_per_group)
+    cache_hit = KVCacheBlocks(
+        tuple(
+            hit_blocks[i * local_blocks_per_group : (i + 1) * local_blocks_per_group]
+            for i in range(num_groups)
+        )
+    )
+    prompt_blocks = local_blocks_per_group + num_external_blocks + num_new_blocks
+    request = make_request(
+        req_id, [0] * (prompt_blocks * block_size), block_size, sha256
+    )
+    manager.allocate_slots(
+        request,
+        num_new_blocks * block_size,
+        local_blocks_per_group * block_size,
+        cache_hit,
+        0,
+        num_external_blocks * block_size,
+    )
+    return request
+
+
+def _make_two_phase_manager(num_groups: int) -> KVCacheManager:
+    assert num_groups in (2, 3)
+    block_size = 16
+    kv_cache_config = make_kv_cache_config_hybrid_model(block_size, 31, 100)
+    del kv_cache_config.kv_cache_groups[num_groups:]
+    return make_kv_cache_manager(
+        kv_cache_config,
+        max_model_len=8192,
+        enable_caching=True,
+        hash_block_size=block_size,
+        use_eagle=True,
+    )
+
+
+def test_cache_hit_local_and_external_three_groups():
+    # Scenario 1 (issue #33775): SWA + full attention with *three* KV cache
+    # groups (1 full + 2 sliding-window). A local prefix hit in some groups
+    # combined with external (connector) blocks in others must not let one
+    # group's external `get_new_blocks` evict another group's not-yet-touched
+    # cache-hit blocks, which would hand the same physical block out twice.
+    manager = _make_two_phase_manager(num_groups=3)
+    _cross_group_cache_hit(manager, "test", num_groups=3)
+    _assert_no_double_allocation(manager, "test")
+
+
+def test_cache_hit_local_and_external_three_groups_preempt_and_reallocate():
+    # Scenario 2: the same 3-group hybrid config, but the request is preempted
+    # (freed) and then reallocated. After the free, the coordinator must treat
+    # the request as new again so external blocks are re-allocated, and the
+    # two-phase ordering must still prevent cross-group double allocation when
+    # reallocating against the now-evictable cache-hit blocks.
+    manager = _make_two_phase_manager(num_groups=3)
+
+    request = _cross_group_cache_hit(manager, "test", num_groups=3)
+    _assert_no_double_allocation(manager, "test")
+
+    # Preempt: free the request; its blocks return to the pool (full ones stay
+    # cached/evictable) and the coordinator forgets it.
+    manager.free(request)
+    assert manager.get_blocks("test").get_block_ids() == ([], [], [])
+
+    # Reallocate the same request id against fresh cache-hit blocks taken from
+    # the current free-queue head, mirroring a preempted request being
+    # scheduled again. Because the request is no longer known, the coordinator
+    # re-arms `is_new_request` and re-runs external allocation, which must still
+    # not double-allocate across groups.
+    _cross_group_cache_hit(manager, "test", num_groups=3)
+    _assert_no_double_allocation(manager, "test")
+    assert manager.get_blocks("test").get_block_ids() != ([], [], [])
+
+
+def test_cache_hit_local_and_external_two_groups_preempt_and_reallocate():
+    # Scenario 3: the minimal 2-group hybrid config (1 full + 1 sliding-window)
+    # exercised through the same preempt -> reallocate cycle as scenario 2.
+    manager = _make_two_phase_manager(num_groups=2)
+
+    request = _cross_group_cache_hit(manager, "test", num_groups=2)
+    _assert_no_double_allocation(manager, "test")
+
+    manager.free(request)
+    assert manager.get_blocks("test").get_block_ids() == ([], [])
+
+    _cross_group_cache_hit(manager, "test", num_groups=2)
+    _assert_no_double_allocation(manager, "test")
+    assert manager.get_blocks("test").get_block_ids() != ([], [])
+
+
 def test_swa_free_split_keeps_cached_tail_ahead_of_scratch(monkeypatch):
     """Default path (no retention): freeing an SWA request must place its
     uncached scratch blocks at the front of the free queue (recycled first)