[cherry-pick][Bugfix] Disable monolithic TRTLLM MoE for Renormalize routing (#37591 )#37605

Signed-off-by: khluu <khluu000@gmail.com>
[Bugfix] Zero-init MLA attention output buffers to prevent NaN from CUDA graph padding (#37442 )
2026-03-19 15:06:38 -07:00 · 2026-03-18 18:44:16 -07:00 · 2026-03-18 18:43:52 -07:00 · 2026-03-18 11:04:33 -07:00 · 2026-03-18 01:48:32 -07:00 · 2026-03-18 01:41:42 -07:00
239 changed files with 4379 additions and 8325 deletions
@@ -333,15 +333,15 @@ apply_rocm_test_overrides() {
  # --- Entrypoint ignores ---
  if [[ $cmds == *" entrypoints/openai "* ]]; then
    cmds=${cmds//" entrypoints/openai "/" entrypoints/openai \
-    --ignore=entrypoints/openai/chat_completion/test_audio.py \
-    --ignore=entrypoints/openai/completion/test_shutdown.py \
+    --ignore=entrypoints/openai/test_audio.py \
+    --ignore=entrypoints/openai/test_shutdown.py \
    --ignore=entrypoints/openai/test_completion.py \
    --ignore=entrypoints/openai/test_models.py \
    --ignore=entrypoints/openai/test_lora_adapters.py \
    --ignore=entrypoints/openai/test_return_tokens_as_ids.py \
-    --ignore=entrypoints/openai/chat_completion/test_root_path.py \
+    --ignore=entrypoints/openai/test_root_path.py \
    --ignore=entrypoints/openai/test_tokenization.py \
-    --ignore=entrypoints/openai/completion/test_prompt_validation.py "}
+    --ignore=entrypoints/openai/test_prompt_validation.py "}
  fi

  if [[ $cmds == *" entrypoints/llm "* ]]; then
@@ -162,7 +162,7 @@ steps:
  - tests/entrypoints/test_chat_utils
  commands:
  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-  - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/chat_completion/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/chat_completion/test_oot_registration.py --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/tool_parsers/ --ignore=entrypoints/openai/responses
+  - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/test_oot_registration.py --ignore=entrypoints/openai/test_tensorizer_entrypoint.py --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/tool_parsers/ --ignore=entrypoints/openai/responses
  - pytest -v -s entrypoints/test_chat_utils.py

 - label: Entrypoints Integration Test (API Server 2)
@@ -674,12 +674,12 @@ steps:
  - vllm/config/model.py
  - vllm/model_executor
  - tests/model_executor
-  - tests/entrypoints/openai/completion/test_tensorizer_entrypoint.py
+  - tests/entrypoints/openai/test_tensorizer_entrypoint.py
  commands:
    - apt-get update && apt-get install -y curl libsodium23
    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
    - pytest -v -s model_executor
-    - pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py
+    - pytest -v -s entrypoints/openai/test_tensorizer_entrypoint.py

 - label: Benchmarks # 11min
  timeout_in_minutes: 20
@@ -1143,7 +1143,7 @@ steps:
  - pytest -v -s plugins_tests/test_scheduler_plugins.py
  - pip install -e ./plugins/vllm_add_dummy_model
  - pytest -v -s distributed/test_distributed_oot.py
-  - pytest -v -s entrypoints/openai/chat_completion/test_oot_registration.py
+  - pytest -v -s entrypoints/openai/test_oot_registration.py
  - pytest -v -s models/test_oot_registration.py
  - pytest -v -s plugins/lora_resolvers

@@ -1502,7 +1502,7 @@ steps:
  - tests/entrypoints/test_chat_utils
  commands:
  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-  - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/chat_completion/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/chat_completion/test_oot_registration.py --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/tool_parsers/ --ignore=entrypoints/openai/responses
+  - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/test_oot_registration.py --ignore=entrypoints/openai/test_tensorizer_entrypoint.py --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/tool_parsers/ --ignore=entrypoints/openai/responses
  - pytest -v -s entrypoints/test_chat_utils.py

 - label: Entrypoints Integration Test (API Server 2)
@@ -2133,12 +2133,12 @@ steps:
  - vllm/config/model.py
  - vllm/model_executor
  - tests/model_executor
-  - tests/entrypoints/openai/completion/test_tensorizer_entrypoint.py
+  - tests/entrypoints/openai/test_tensorizer_entrypoint.py
  commands:
    - apt-get update && apt-get install -y curl libsodium23
    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
    - pytest -v -s model_executor
-    - pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py
+    - pytest -v -s entrypoints/openai/test_tensorizer_entrypoint.py

 - label: Benchmarks # 11min
  timeout_in_minutes: 20
@@ -2735,7 +2735,7 @@ steps:
  - pytest -v -s plugins_tests/test_scheduler_plugins.py
  - pip install -e ./plugins/vllm_add_dummy_model
  - pytest -v -s distributed/test_distributed_oot.py
-  - pytest -v -s entrypoints/openai/chat_completion/test_oot_registration.py # it needs a clean process
+  - pytest -v -s entrypoints/openai/test_oot_registration.py # it needs a clean process
  - pytest -v -s models/test_oot_registration.py # it needs a clean process
  - pytest -v -s plugins/lora_resolvers # unit tests for in-tree lora resolver plugins

@@ -3257,7 +3257,7 @@ steps:
  - tests/entrypoints/test_chat_utils
  commands:
  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-  - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/chat_completion/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/chat_completion/test_oot_registration.py --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/tool_parsers/ --ignore=entrypoints/openai/responses
+  - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/test_oot_registration.py --ignore=entrypoints/openai/test_tensorizer_entrypoint.py --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/tool_parsers/ --ignore=entrypoints/openai/responses
  - pytest -v -s entrypoints/test_chat_utils.py

 - label: Entrypoints Integration Test (API Server 2)
@@ -3872,12 +3872,12 @@ steps:
  - vllm/config/model.py
  - vllm/model_executor
  - tests/model_executor
-  - tests/entrypoints/openai/completion/test_tensorizer_entrypoint.py
+  - tests/entrypoints/openai/test_tensorizer_entrypoint.py
  commands:
    - apt-get update && apt-get install -y curl libsodium23
    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
    - pytest -v -s model_executor
-    - pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py
+    - pytest -v -s entrypoints/openai/test_tensorizer_entrypoint.py

 - label: Benchmarks # 11min
  timeout_in_minutes: 20
@@ -4508,7 +4508,7 @@ steps:
  - pytest -v -s plugins_tests/test_scheduler_plugins.py
  - pip install -e ./plugins/vllm_add_dummy_model
  - pytest -v -s distributed/test_distributed_oot.py
-  - pytest -v -s entrypoints/openai/chat_completion/test_oot_registration.py # it needs a clean process
+  - pytest -v -s entrypoints/openai/test_oot_registration.py # it needs a clean process
  - pytest -v -s models/test_oot_registration.py # it needs a clean process
  - pytest -v -s plugins/lora_resolvers # unit tests for in-tree lora resolver plugins

@@ -15,29 +15,8 @@ steps:
  - pytest -v -s distributed/test_shm_buffer.py
  - pytest -v -s distributed/test_shm_storage.py

- label: Distributed DP Tests (2 GPUs)
-  timeout_in_minutes: 20
-  working_dir: "/vllm-workspace/tests"
-  num_devices: 2
-  source_file_dependencies:
-  - vllm/distributed/
-  - vllm/engine/
-  - vllm/executor/
-  - vllm/worker/worker_base.py
-  - vllm/v1/engine/
-  - vllm/v1/worker/
-  - tests/v1/distributed
-  - tests/v1/entrypoints/openai/test_multi_api_servers.py
-  commands:
-  # https://github.com/NVIDIA/nccl/issues/1838
-  - export NCCL_CUMEM_HOST_ENABLE=0
-  - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py
-  - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
-  - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py
-  - DP_SIZE=2 pytest -v -s v1/entrypoints/openai/test_multi_api_servers.py
-
- label: Distributed Compile + RPC Tests (2 GPUs)
-  timeout_in_minutes: 20
+- label: Distributed (2 GPUs)
+  timeout_in_minutes: 60
  working_dir: "/vllm-workspace/tests"
  num_devices: 2
  source_file_dependencies:
@@ -50,31 +29,22 @@ steps:
  - vllm/v1/worker/
  - tests/compile/fullgraph/test_basic_correctness.py
  - tests/compile/test_wrapper.py
-  - tests/entrypoints/llm/test_collective_rpc.py
-  commands:
-  # https://github.com/NVIDIA/nccl/issues/1838
-  - export NCCL_CUMEM_HOST_ENABLE=0
-  - pytest -v -s entrypoints/llm/test_collective_rpc.py
-  - pytest -v -s ./compile/fullgraph/test_basic_correctness.py
-  - pytest -v -s ./compile/test_wrapper.py
-
- label: Distributed Torchrun + Shutdown Tests (2 GPUs)
-  timeout_in_minutes: 20
-  working_dir: "/vllm-workspace/tests"
-  num_devices: 2
-  source_file_dependencies:
-  - vllm/distributed/
-  - vllm/engine/
-  - vllm/executor/
-  - vllm/worker/worker_base.py
-  - vllm/v1/engine/
-  - vllm/v1/worker/
  - tests/distributed/
+  - tests/entrypoints/llm/test_collective_rpc.py
+  - tests/v1/distributed
+  - tests/v1/entrypoints/openai/test_multi_api_servers.py
  - tests/v1/shutdown
  - tests/v1/worker/test_worker_memory_snapshot.py
  commands:
  # https://github.com/NVIDIA/nccl/issues/1838
  - export NCCL_CUMEM_HOST_ENABLE=0
+  - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py
+  - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
+  - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py
+  - DP_SIZE=2 pytest -v -s v1/entrypoints/openai/test_multi_api_servers.py
+  - pytest -v -s entrypoints/llm/test_collective_rpc.py
+  - pytest -v -s ./compile/fullgraph/test_basic_correctness.py
+  - pytest -v -s ./compile/test_wrapper.py
  - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed'
  - VLLM_TEST_SAME_HOST=1 VLLM_TEST_WITH_DEFAULT_DEVICE_SET=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed'
  - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s v1/shutdown
@@ -34,7 +34,7 @@ steps:
  - tests/entrypoints/test_chat_utils
  commands:
  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-  - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/chat_completion/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/chat_completion/test_oot_registration.py --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py --ignore=entrypoints/openai/correctness/  --ignore=entrypoints/openai/tool_parsers/ --ignore=entrypoints/openai/responses
+  - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/chat_completion/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/test_oot_registration.py --ignore=entrypoints/openai/test_tensorizer_entrypoint.py --ignore=entrypoints/openai/correctness/  --ignore=entrypoints/openai/tool_parsers/ --ignore=entrypoints/openai/responses
  - pytest -v -s entrypoints/test_chat_utils.py
  mirror:
    amd:
@@ -35,7 +35,7 @@ steps:
  parallelism: 2

 - label: Kernels MoE Test %N
-  timeout_in_minutes: 25
+  timeout_in_minutes: 60
  source_file_dependencies:
  - csrc/quantization/cutlass_w8a8/moe/
  - csrc/moe/
@@ -47,7 +47,7 @@ steps:
  commands:
    - pytest -v -s kernels/moe --ignore=kernels/moe/test_modular_oai_triton_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
    - pytest -v -s kernels/moe/test_modular_oai_triton_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
-  parallelism: 5
+  parallelism: 2

 - label: Kernels Mamba Test
  timeout_in_minutes: 45
@@ -45,6 +45,22 @@ steps:
  commands:
  - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell.txt

+- label: LM Eval Qwen3.5 Models (B200)
+  timeout_in_minutes: 120
+  device: b200
+  optional: true
+  num_devices: 2
+  source_file_dependencies:
+  - vllm/model_executor/models/qwen3_5.py
+  - vllm/model_executor/models/qwen3_5_mtp.py
+  - vllm/transformers_utils/configs/qwen3_5.py
+  - vllm/transformers_utils/configs/qwen3_5_moe.py
+  - vllm/model_executor/models/qwen3_next.py
+  - vllm/model_executor/models/qwen3_next_mtp.py
+  - vllm/model_executor/layers/fla/ops/
+  commands:
+    - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-qwen35-blackwell.txt
+
 - label: LM Eval Large Models (H200)
  timeout_in_minutes: 60
  device: h200
@@ -9,9 +9,9 @@ steps:
  - vllm/config/model.py
  - vllm/model_executor
  - tests/model_executor
-  - tests/entrypoints/openai/completion/test_tensorizer_entrypoint.py
+  - tests/entrypoints/openai/test_tensorizer_entrypoint.py
  commands:
    - apt-get update && apt-get install -y curl libsodium23
    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
    - pytest -v -s model_executor
-    - pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py
+    - pytest -v -s entrypoints/openai/test_tensorizer_entrypoint.py
@@ -36,6 +36,6 @@ steps:
  - pytest -v -s plugins_tests/test_scheduler_plugins.py
  - pip install -e ./plugins/vllm_add_dummy_model
  - pytest -v -s distributed/test_distributed_oot.py
-  - pytest -v -s entrypoints/openai/chat_completion/test_oot_registration.py # it needs a clean process
+  - pytest -v -s entrypoints/openai/test_oot_registration.py # it needs a clean process
  - pytest -v -s models/test_oot_registration.py # it needs a clean process
  - pytest -v -s plugins/lora_resolvers # unit tests for in-tree lora resolver plugins
@@ -35,7 +35,7 @@ steps:
  # as it is a heavy test that is covered in other steps.
  # Use `find` to launch multiple instances of pytest so that
  # they do not suffer from https://github.com/vllm-project/vllm/issues/28965
-  - "find compile/fullgraph/ -name 'test_*.py' -not -name 'test_full_graph.py' -print0 | xargs -0 -n1 -I{} pytest -s -v '{}'"
+  - "find compile/fullgraph/ -name 'test_*.py' -not -name 'test_full_graph.py' -exec pytest -s -v {} \\;"

 - label: PyTorch Fullgraph
  timeout_in_minutes: 30
@@ -381,7 +381,7 @@ pull_request_rules:
    - or:
      - files~=^vllm/model_executor/model_loader/tensorizer.py
      - files~=^vllm/model_executor/model_loader/tensorizer_loader.py
-      - files~=^tests/entrypoints/openai/completion/test_tensorizer_entrypoint.py
+      - files~=^tests/entrypoints/openai/test_tensorizer_entrypoint.py
      - files~=^tests/model_executor/model_loader/tensorizer_loader/
  actions:
    assign:
@@ -47,8 +47,6 @@ from common import (
    is_mla_backend,
 )

-from vllm.v1.worker.workspace import init_workspace_manager
-

 def run_standard_attention_benchmark(config: BenchmarkConfig) -> BenchmarkResult:
    """Run standard attention benchmark (Flash/Triton/FlashInfer)."""
@@ -464,7 +462,7 @@ def main():
    parser.add_argument(
        "--batch-specs",
        nargs="+",
-        default=None,
+        default=["q2k", "8q1s1k"],
        help="Batch specifications using extended grammar",
    )

@@ -480,21 +478,6 @@ def main():
    parser.add_argument("--repeats", type=int, default=1, help="Repetitions")
    parser.add_argument("--warmup-iters", type=int, default=3, help="Warmup iterations")
    parser.add_argument("--profile-memory", action="store_true", help="Profile memory")
-    parser.add_argument(
-        "--kv-cache-dtype",
-        default="auto",
-        choices=["auto", "fp8"],
-        help="KV cache dtype: auto or fp8",
-    )
-    parser.add_argument(
-        "--cuda-graphs",
-        action=argparse.BooleanOptionalAction,
-        default=True,
-        help=(
-            "Launch kernels with CUDA graphs to eliminate CPU overhead"
-            "in measurements (default: True)"
-        ),
-    )

    # Parameter sweep (use YAML config for advanced sweeps)
    parser.add_argument(
@@ -553,24 +536,21 @@ def main():

        # Batch specs and sizes
        # Support both explicit batch_specs and generated batch_spec_ranges
-        # CLI --batch-specs takes precedence over YAML when provided.
-        cli_batch_specs_provided = args.batch_specs is not None
-        if not cli_batch_specs_provided:
-            if "batch_spec_ranges" in yaml_config:
-                # Generate batch specs from ranges
-                generated_specs = generate_batch_specs_from_ranges(
-                    yaml_config["batch_spec_ranges"]
-                )
-                # Combine with any explicit batch_specs
-                if "batch_specs" in yaml_config:
-                    args.batch_specs = yaml_config["batch_specs"] + generated_specs
-                else:
-                    args.batch_specs = generated_specs
-                console.print(
-                    f"[dim]Generated {len(generated_specs)} batch specs from ranges[/]"
-                )
-            elif "batch_specs" in yaml_config:
-                args.batch_specs = yaml_config["batch_specs"]
+        if "batch_spec_ranges" in yaml_config:
+            # Generate batch specs from ranges
+            generated_specs = generate_batch_specs_from_ranges(
+                yaml_config["batch_spec_ranges"]
+            )
+            # Combine with any explicit batch_specs
+            if "batch_specs" in yaml_config:
+                args.batch_specs = yaml_config["batch_specs"] + generated_specs
+            else:
+                args.batch_specs = generated_specs
+            console.print(
+                f"[dim]Generated {len(generated_specs)} batch specs from ranges[/]"
+            )
+        elif "batch_specs" in yaml_config:
+            args.batch_specs = yaml_config["batch_specs"]

        if "batch_sizes" in yaml_config:
            args.batch_sizes = yaml_config["batch_sizes"]
@@ -595,10 +575,6 @@ def main():
            args.warmup_iters = yaml_config["warmup_iters"]
        if "profile_memory" in yaml_config:
            args.profile_memory = yaml_config["profile_memory"]
-        if "kv_cache_dtype" in yaml_config:
-            args.kv_cache_dtype = yaml_config["kv_cache_dtype"]
-        if "cuda_graphs" in yaml_config:
-            args.cuda_graphs = yaml_config["cuda_graphs"]

        # Parameter sweep configuration
        if "parameter_sweep" in yaml_config:
@@ -653,18 +629,12 @@ def main():
    # Determine backends
    backends = args.backends or ([args.backend] if args.backend else ["flash"])
    prefill_backends = getattr(args, "prefill_backends", None)
-    if not args.batch_specs:
-        args.batch_specs = ["q2k", "8q1s1k"]
    console.print(f"Backends: {', '.join(backends)}")
    if prefill_backends:
        console.print(f"Prefill backends: {', '.join(prefill_backends)}")
    console.print(f"Batch specs: {', '.join(args.batch_specs)}")
-    console.print(f"KV cache dtype: {args.kv_cache_dtype}")
-    console.print(f"CUDA graphs: {args.cuda_graphs}")
    console.print()

-    init_workspace_manager(args.device)
-
    # Run benchmarks
    all_results = []

@@ -717,8 +687,6 @@ def main():
                        repeats=args.repeats,
                        warmup_iters=args.warmup_iters,
                        profile_memory=args.profile_memory,
-                        kv_cache_dtype=args.kv_cache_dtype,
-                        use_cuda_graphs=args.cuda_graphs,
                    )

                    # Add decode pipeline config
@@ -871,8 +839,6 @@ def main():
            "repeats": args.repeats,
            "warmup_iters": args.warmup_iters,
            "profile_memory": args.profile_memory,
-            "kv_cache_dtype": args.kv_cache_dtype,
-            "use_cuda_graphs": args.cuda_graphs,
        }
        all_results = run_model_parameter_sweep(
            backends,
@@ -895,8 +861,6 @@ def main():
            "repeats": args.repeats,
            "warmup_iters": args.warmup_iters,
            "profile_memory": args.profile_memory,
-            "kv_cache_dtype": args.kv_cache_dtype,
-            "use_cuda_graphs": args.cuda_graphs,
        }
        all_results = run_parameter_sweep(
            backends, args.batch_specs, base_config_args, args.parameter_sweep, console
@@ -927,8 +891,6 @@ def main():
                            repeats=args.repeats,
                            warmup_iters=args.warmup_iters,
                            profile_memory=args.profile_memory,
-                            kv_cache_dtype=args.kv_cache_dtype,
-                            use_cuda_graphs=args.cuda_graphs,
                        )

                        result = run_benchmark(config)
@@ -213,9 +213,6 @@ class BenchmarkConfig:
    profile_memory: bool = False
    use_cuda_graphs: bool = False

-    # "auto" or "fp8"
-    kv_cache_dtype: str = "auto"
-
    # MLA-specific
    prefill_backend: str | None = None
    kv_lora_rank: int | None = None
@@ -372,7 +369,6 @@ class ResultsFormatter:
                    "backend",
                    "batch_spec",
                    "num_layers",
-                    "kv_cache_dtype",
                    "mean_time",
                    "std_time",
                    "throughput",
@@ -386,7 +382,6 @@ class ResultsFormatter:
                        "backend": r.config.backend,
                        "batch_spec": r.config.batch_spec,
                        "num_layers": r.config.num_layers,
-                        "kv_cache_dtype": r.config.kv_cache_dtype,
                        "mean_time": r.mean_time,
                        "std_time": r.std_time,
                        "throughput": r.throughput_tokens_per_sec or 0,
@@ -30,9 +30,9 @@ batch_specs:
  - "2q16k_32q1s4k"         # 2 very large prefill + 32 decode

  # Context extension + decode
-  - "2q1ks2k_16q1s1k"       # 2 extend + 16 decode
-  - "4q2ks4k_32q1s2k"       # 4 extend + 32 decode
-  - "2q1ks8k_32q1s2k"       # 2 large extend + 32 decode
+  - "2q1kkv2k_16q1s1k"       # 2 extend + 16 decode
+  - "4q2kkv4k_32q1s2k"       # 4 extend + 32 decode
+  - "2q1kkv8k_32q1s2k"       # 2 large extend + 32 decode

  # Explicitly chunked prefill
  - "q8k"           # 8k prefill with chunking hint
@@ -1,58 +0,0 @@
-# MLA decode-only benchmark configuration
-
-model:
-  name: "deepseek-v3"
-  num_layers: 60
-  num_q_heads: 128  # Base value, can be swept for TP simulation
-  num_kv_heads: 1  # MLA uses single latent KV
-  head_dim: 576
-  kv_lora_rank: 512
-  qk_nope_head_dim: 128
-  qk_rope_head_dim: 64
-  v_head_dim: 128
-  block_size: 128  # CUTLASS MLA and FlashAttn MLA use 128
-
-# Model parameter sweep: simulate tensor parallelism by varying num_q_heads
-# TP=1: 128 heads, TP=2: 64 heads, TP=4: 32 heads, TP=8: 16 heads
-model_parameter_sweep:
-  param_name: "num_q_heads"
-  values: [128, 64, 32, 16]
-  label_format: "{backend}_{value}h"
-
-batch_specs:
-  # Small batches, varying sequence lengths
-  - "16q1s512"     # 16 requests, 512 KV cache
-  - "16q1s1k"      # 16 requests, 1k KV cache
-  - "16q1s2k"      # 16 requests, 2k KV cache
-  - "16q1s4k"      # 16 requests, 4k KV cache
-
-  # Medium batches
-  - "32q1s1k"      # 32 requests, 1k KV cache
-  - "32q1s2k"      # 32 requests, 2k KV cache
-  - "32q1s4k"      # 32 requests, 4k KV cache
-  - "32q1s8k"      # 32 requests, 8k KV cache
-
-  # Large batches
-  - "64q1s1k"      # 64 requests, 1k KV cache
-  - "64q1s2k"      # 64 requests, 2k KV cache
-  - "64q1s4k"      # 64 requests, 4k KV cache
-  - "64q1s8k"      # 64 requests, 8k KV cache
-
-  # Very large batches
-  - "128q1s1k"     # 128 requests, 1k KV cache
-  - "128q1s2k"     # 128 requests, 2k KV cache
-  - "128q1s4k"     # 128 requests, 4k KV cache
-  - "128q1s8k"     # 128 requests, 8k KV cache
-
-  # Long context
-  - "32q1s16k"     # 32 requests, 16k KV cache
-  - "32q1s32k"     # 32 requests, 32k KV cache
-
-backends:
-  - FLASHMLA_SPARSE
-  - FLASHINFER_MLA_SPARSE
-
-device: "cuda:0"
-repeats: 100
-warmup_iters: 10
-profile_memory: true
@@ -60,11 +60,9 @@ def create_minimal_vllm_config(
    model_name: str = "deepseek-v3",
    block_size: int = 128,
    max_num_seqs: int = 256,
-    max_num_batched_tokens: int = 8192,
    mla_dims: dict | None = None,
    index_topk: int | None = None,
    prefill_backend: str | None = None,
-    kv_cache_dtype: str = "auto",
 ) -> VllmConfig:
    """
    Create minimal VllmConfig for MLA benchmarks.
@@ -151,13 +149,13 @@ def create_minimal_vllm_config(
    cache_config = CacheConfig(
        block_size=block_size,
        gpu_memory_utilization=0.9,
-        cache_dtype=kv_cache_dtype,
+        cache_dtype="auto",
        enable_prefix_caching=False,
    )

    scheduler_config = SchedulerConfig(
        max_num_seqs=max_num_seqs,
-        max_num_batched_tokens=max(max_num_batched_tokens, max_num_seqs),
+        max_num_batched_tokens=8192,
        max_model_len=32768,
        is_encoder_decoder=False,
        enable_chunked_prefill=True,
@@ -537,7 +535,6 @@ def _create_backend_impl(
    device: torch.device,
    max_num_tokens: int = 8192,
    index_topk: int | None = None,
-    kv_cache_dtype: str = "auto",
 ):
    """
    Create backend implementation instance.
@@ -586,7 +583,7 @@ def _create_backend_impl(
        "num_kv_heads": mla_dims["num_kv_heads"],
        "alibi_slopes": None,
        "sliding_window": None,
-        "kv_cache_dtype": kv_cache_dtype,
+        "kv_cache_dtype": "auto",
        "logits_soft_cap": None,
        "attn_type": "decoder",
        "kv_sharing_target_layer_name": None,
@@ -704,7 +701,6 @@ def _run_single_benchmark(
    mla_dims: dict,
    device: torch.device,
    indexer=None,
-    kv_cache_dtype: str | None = None,
 ) -> BenchmarkResult:
    """
    Run a single benchmark iteration.
@@ -738,124 +734,49 @@ def _run_single_benchmark(
    )

    # Create KV cache
-    if kv_cache_dtype is None:
-        kv_cache_dtype = getattr(config, "kv_cache_dtype", "auto")
-    head_size = mla_dims["kv_lora_rank"] + mla_dims["qk_rope_head_dim"]
-    if kv_cache_dtype == "fp8_ds_mla":
-        # FlashMLA sparse custom format: 656 bytes per token, stored as uint8.
-        # Layout: kv_lora_rank fp8 bytes + 4 float32 tile scales
-        #         + 2*rope_dim bf16 bytes
-        # = 512 + 16 + 128 = 656 bytes for DeepSeek dims.
-        kv_cache = torch.zeros(
-            num_blocks,
-            block_size,
-            656,
-            device=device,
-            dtype=torch.uint8,
-        )
-    elif kv_cache_dtype == "fp8":
-        from vllm.platforms import current_platform
+    kv_cache = torch.zeros(
+        num_blocks,
+        block_size,
+        mla_dims["kv_lora_rank"] + mla_dims["qk_rope_head_dim"],
+        device=device,
+        dtype=torch.bfloat16,
+    )

-        kv_cache = torch.zeros(
-            num_blocks,
-            block_size,
-            head_size,
-            device=device,
-            dtype=torch.uint8,
-        ).view(current_platform.fp8_dtype())
-    else:
-        kv_cache = torch.zeros(
-            num_blocks,
-            block_size,
-            head_size,
-            device=device,
-            dtype=torch.bfloat16,
-        )
+    # Create input tensors for both decode and prefill modes
+    decode_inputs, prefill_inputs = _create_input_tensors(
+        total_q,
+        mla_dims,
+        backend_cfg["query_format"],
+        device,
+        torch.bfloat16,
+    )

    # Fill indexer with random indices for sparse backends
    is_sparse = backend_cfg.get("is_sparse", False)
    if is_sparse and indexer is not None:
        indexer.fill_random_indices(total_q, max_kv_len)

-    # Determine which forward methods to use based on metadata.
-    # Sparse MLA backends always use forward_mqa
-    has_decode = is_sparse or getattr(metadata, "decode", None) is not None
-    has_prefill = not is_sparse and getattr(metadata, "prefill", None) is not None
-    if not has_decode and not has_prefill:
+    # Determine which forward method to use based on metadata
+    if metadata.decode is not None:
+        forward_fn = lambda: impl.forward_mqa(decode_inputs, kv_cache, metadata, layer)
+    elif metadata.prefill is not None:
+        forward_fn = lambda: impl.forward_mha(
+            prefill_inputs["q"],
+            prefill_inputs["k_c_normed"],
+            prefill_inputs["k_pe"],
+            kv_cache,
+            metadata,
+            prefill_inputs["k_scale"],
+            prefill_inputs["output"],
+        )
+    else:
        raise RuntimeError("Metadata has neither decode nor prefill metadata")

-    num_decode = (
-        metadata.num_decode_tokens
-        if (has_decode and has_prefill)
-        else total_q
-        if has_decode
-        else 0
-    )
-    num_prefill = total_q - num_decode
-
-    # Some backends requires fp8 queries when using fp8 KV cache.
-    is_fp8_kvcache = kv_cache_dtype.startswith("fp8")
-    quantize_query = is_fp8_kvcache and getattr(
-        impl, "supports_quant_query_input", False
-    )
-
-    # quantize_query forces concat format
-    query_fmt = "concat" if quantize_query else backend_cfg["query_format"]
-
-    # Create decode query tensors
-    if has_decode:
-        decode_inputs, _ = _create_input_tensors(
-            num_decode, mla_dims, query_fmt, device, torch.bfloat16
-        )
-        # Cast decode query to fp8 if the backend supports it
-        if quantize_query:
-            from vllm.platforms import current_platform
-
-            if isinstance(decode_inputs, tuple):
-                decode_inputs = torch.cat(list(decode_inputs), dim=-1)
-            decode_inputs = decode_inputs.to(current_platform.fp8_dtype())
-
-    # Create prefill input tensors
-    if has_prefill:
-        _, prefill_inputs = _create_input_tensors(
-            num_prefill, mla_dims, query_fmt, device, torch.bfloat16
-        )
-
-    # Build forward function
-    def forward_fn():
-        results = []
-        if has_decode:
-            results.append(impl.forward_mqa(decode_inputs, kv_cache, metadata, layer))
-        if has_prefill:
-            results.append(
-                impl.forward_mha(
-                    prefill_inputs["q"],
-                    prefill_inputs["k_c_normed"],
-                    prefill_inputs["k_pe"],
-                    kv_cache,
-                    metadata,
-                    prefill_inputs["k_scale"],
-                    prefill_inputs["output"],
-                )
-            )
-        return results[0] if len(results) == 1 else tuple(results)
-
    # Warmup
    for _ in range(config.warmup_iters):
        forward_fn()
    torch.accelerator.synchronize()

-    # Optionally capture a CUDA graph after warmup.
-    # Graph replay eliminates CPU launch overhead so timings reflect pure
-    # kernel time.
-    if config.use_cuda_graphs:
-        graph = torch.cuda.CUDAGraph()
-        with torch.cuda.graph(graph):
-            forward_fn()
-        benchmark_fn = graph.replay
-    else:
-        benchmark_fn = forward_fn
-
    # Benchmark
    times = []
    for _ in range(config.repeats):
@@ -864,7 +785,7 @@ def _run_single_benchmark(

        start.record()
        for _ in range(config.num_layers):
-            benchmark_fn()
+            forward_fn()
        end.record()

        torch.accelerator.synchronize()
@@ -931,30 +852,13 @@ def _run_mla_benchmark_batched(
    # Determine if this is a sparse backend
    is_sparse = backend_cfg.get("is_sparse", False)

-    # Extract kv_cache_dtype from the first config
-    kv_cache_dtype = getattr(first_config, "kv_cache_dtype", "auto")
-
-    # FlashMLA sparse only supports "fp8_ds_mla" internally (not generic "fp8").
-    # Remap here so the user can pass --kv-cache-dtype fp8 regardless of backend.
-    if backend.upper() == "FLASHMLA_SPARSE" and kv_cache_dtype == "fp8":
-        kv_cache_dtype = "fp8_ds_mla"
-
-    # Compute max total_q across all configs so the metadata builder buffer
-    # and scheduler config are large enough for all batch specs.
-    max_total_q = max(
-        sum(r.q_len for r in parse_batch_spec(cfg.batch_spec))
-        for cfg, *_ in configs_with_params
-    )
-
    # Create and set vLLM config for MLA (reused across all benchmarks)
    vllm_config = create_minimal_vllm_config(
        model_name="deepseek-v3",  # Used only for model path
        block_size=block_size,
-        max_num_batched_tokens=max_total_q,
        mla_dims=mla_dims,  # Use custom dims from config or default
        index_topk=index_topk if is_sparse else None,
        prefill_backend=prefill_backend,
-        kv_cache_dtype=kv_cache_dtype,
    )

    results = []
@@ -979,9 +883,7 @@ def _run_mla_benchmark_batched(
            mla_dims,
            vllm_config,
            device,
-            max_num_tokens=max_total_q,
            index_topk=index_topk if is_sparse else None,
-            kv_cache_dtype=kv_cache_dtype,
        )

        # Verify the actual prefill backend matches what was requested
@@ -1040,7 +942,6 @@ def _run_mla_benchmark_batched(
                    mla_dims,
                    device,
                    indexer=indexer,
-                    kv_cache_dtype=kv_cache_dtype,
                )
                results.append(result)

@@ -140,7 +140,7 @@ def _create_vllm_config(

    cache_config = CacheConfig(
        block_size=config.block_size,
-        cache_dtype=config.kv_cache_dtype,
+        cache_dtype="auto",
    )
    cache_config.num_gpu_blocks = max_num_blocks
    cache_config.num_cpu_blocks = 0
@@ -215,7 +215,7 @@ def _create_backend_impl(
        num_kv_heads=config.num_kv_heads,
        alibi_slopes=None,
        sliding_window=None,
-        kv_cache_dtype=config.kv_cache_dtype,
+        kv_cache_dtype="auto",
    )

    kv_cache_spec = FullAttentionSpec(
@@ -288,22 +288,12 @@ def _create_input_tensors(
    total_q: int,
    device: torch.device,
    dtype: torch.dtype,
-    quantize_query: bool = False,
 ) -> tuple:
-    """Create Q, K, V input tensors for all layers.
-
-    When quantize_query is True, queries are cast to fp8 to match backends
-    that require query/key/value dtype consistency.
-    """
-    q_dtype = dtype
-    if quantize_query:
-        from vllm.platforms import current_platform
-
-        q_dtype = current_platform.fp8_dtype()
+    """Create Q, K, V input tensors for all layers."""
    q_list = [
        torch.randn(
            total_q, config.num_q_heads, config.head_dim, device=device, dtype=dtype
-        ).to(q_dtype)
+        )
        for _ in range(config.num_layers)
    ]
    k_list = [
@@ -354,17 +344,10 @@ def _create_kv_cache(
    # Compute inverse permutation to get back to logical view
    inv_order = [stride_order.index(i) for i in range(len(stride_order))]

-    # Use fp8 dtype for cache when requested.
-    cache_dtype = dtype
-    if config.kv_cache_dtype == "fp8":
-        from vllm.platforms import current_platform
-
-        cache_dtype = current_platform.fp8_dtype()
-
    cache_list = []
    for _ in range(config.num_layers):
        # Allocate in physical layout order (contiguous in memory)
-        cache = torch.zeros(*physical_shape, device=device, dtype=cache_dtype)
+        cache = torch.zeros(*physical_shape, device=device, dtype=dtype)
        # Permute to logical view
        cache = cache.permute(*inv_order)
        cache_list.append(cache)
@@ -409,37 +392,6 @@ def _run_single_benchmark(
            )
    torch.accelerator.synchronize()

-    # Optionally capture a CUDA graph after warmup.
-    # Graph replay eliminates CPU launch overhead so timings reflect pure
-    # kernel time.
-    if config.use_cuda_graphs:
-        graph = torch.cuda.CUDAGraph()
-        with torch.cuda.graph(graph):
-            for i in range(config.num_layers):
-                impl.forward(
-                    layer,
-                    q_list[i],
-                    k_list[i],
-                    v_list[i],
-                    cache_list[i],
-                    attn_metadata,
-                    output=out,
-                )
-        benchmark_fn = graph.replay
-    else:
-
-        def benchmark_fn():
-            for i in range(config.num_layers):
-                impl.forward(
-                    layer,
-                    q_list[i],
-                    k_list[i],
-                    v_list[i],
-                    cache_list[i],
-                    attn_metadata,
-                    output=out,
-                )
-
    # Benchmark
    times = []
    for _ in range(config.repeats):
@@ -447,7 +399,16 @@ def _run_single_benchmark(
        end = torch.cuda.Event(enable_timing=True)

        start.record()
-        benchmark_fn()
+        for i in range(config.num_layers):
+            impl.forward(
+                layer,
+                q_list[i],
+                k_list[i],
+                v_list[i],
+                cache_list[i],
+                attn_metadata,
+                output=out,
+            )
        end.record()

        torch.accelerator.synchronize()
@@ -541,12 +502,8 @@ def run_attention_benchmark(config: BenchmarkConfig) -> BenchmarkResult:
                common_attn_metadata=common_metadata,
            )

-            # Only quantize queries when the impl supports it
-            quantize_query = config.kv_cache_dtype.startswith("fp8") and getattr(
-                impl, "supports_quant_query_input", False
-            )
            q_list, k_list, v_list = _create_input_tensors(
-                config, total_q, device, dtype, quantize_query=quantize_query
+                config, total_q, device, dtype
            )

            cache_list = _create_kv_cache(
@@ -27,7 +27,7 @@ def get_attn_isa(
    else:
        if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
            return "neon"
-        elif torch.cpu._is_amx_tile_supported():
+        elif torch._C._cpu._is_amx_tile_supported():
            return "amx"
        else:
            return "vec"
@@ -24,7 +24,7 @@ except (ImportError, AttributeError) as e:
    sys.exit(1)

 # ISA selection following test_cpu_fused_moe.py pattern
-ISA_CHOICES = ["amx", "vec"] if torch.cpu._is_amx_tile_supported() else ["vec"]
+ISA_CHOICES = ["amx", "vec"] if torch._C._cpu._is_amx_tile_supported() else ["vec"]


@torch.inference_mode()
@@ -39,7 +39,7 @@ else()
  FetchContent_Declare(
          vllm-flash-attn
          GIT_REPOSITORY https://github.com/vllm-project/flash-attention.git
-          GIT_TAG 29210221863736a08f71a866459e368ad1ac4a95
+          GIT_TAG 1488682bb545f7d020e958a33116b1419d1cfc83
          GIT_PROGRESS TRUE
          # Don't share the vllm-flash-attn build between build types
          BINARY_DIR ${CMAKE_BINARY_DIR}/vllm-flash-attn
@@ -286,15 +286,6 @@ void rms_norm_per_block_quant(torch::Tensor& out, torch::Tensor const& input,
                "Outer scale stride must be 1 when scales are not transposed");
  }

-  int64_t hidden_size = input.size(-1);
-  TORCH_CHECK(hidden_size > 0 && hidden_size % group_size == 0,
-              "hidden_size must be a positive multiple of group_size");
-  int64_t num_tokens = input.numel() / hidden_size;
-  int64_t num_groups = hidden_size / group_size;
-  TORCH_CHECK(scales.numel() >= num_tokens * num_groups,
-              "scales buffer too small: need ", num_tokens * num_groups,
-              " elements, got ", scales.numel());
-
  rms_norm_per_block_quant_dispatch(out, input, weight, scales, group_size,
                                    var_epsilon, scale_ub, residual,
                                    is_scale_transposed);
@@ -127,8 +127,8 @@ Priority is **1 = highest** (tried first).
 | 3 | `FLASH_ATTN_MLA` |
 | 4 | `FLASHMLA` |
 | 5 | `TRITON_MLA` |
-| 6 | `FLASHINFER_MLA_SPARSE`**\*** |
-| 7 | `FLASHMLA_SPARSE` |
+| 6 | `FLASHMLA_SPARSE` |
+| 7 | `FLASHINFER_MLA_SPARSE` |

 **Ampere/Hopper (SM 8.x-9.x):**

@@ -140,8 +140,6 @@ Priority is **1 = highest** (tried first).
 | 4 | `TRITON_MLA` |
 | 5 | `FLASHMLA_SPARSE` |

-> **\*** For sparse MLA, FP8 KV cache always prefers `FLASHINFER_MLA_SPARSE`. With BF16 KV cache, `FLASHINFER_MLA_SPARSE` is preferred for low query-head counts (<= 16), while `FLASHMLA_SPARSE` is preferred otherwise.
->
 > **Note:** ROCm and CPU platforms have their own selection logic. See the platform-specific documentation for details.

 ## Legend
@@ -389,17 +389,3 @@ vllm serve model --enable-lora --max-lora-rank 64
 # Bad: unnecessarily high, wastes memory
 vllm serve model --enable-lora --max-lora-rank 256
 ```
-
-### Restricting LoRA to Specific Modules
-
-The `--lora-target-modules` parameter allows you to restrict which model modules have LoRA applied at deployment time. This is useful for performance tuning when you only need LoRA on specific layers:
-
-```bash
-# Apply LoRA only to output projection layers
-vllm serve model --enable-lora --lora-target-modules o_proj
-
-# Apply LoRA to multiple specific modules
-vllm serve model --enable-lora --lora-target-modules o_proj qkv_proj down_proj
-```
-
-When `--lora-target-modules` is not specified, LoRA will be applied to all supported modules in the model. This parameter accepts module suffixes (the last component of the module name), such as `o_proj`, `qkv_proj`, `gate_proj`, etc.
@@ -107,27 +107,6 @@ vLLM supports the `tool_choice='none'` option in the chat completion API. When t
 !!! note
    When tools are specified in the request, vLLM includes tool definitions in the prompt by default, regardless of the `tool_choice` setting. To exclude tool definitions when `tool_choice='none'`, use the `--exclude-tools-when-tool-choice-none` option.

-## Constrained Decoding Behavior
-
-Whether vLLM enforces the tool parameter schema during generation depends on the `tool_choice` mode:
-
-| `tool_choice` value | Schema-constrained decoding | Behavior |
-| --- | --- | --- |
-| Named function | Yes (via structured outputs backend) | Arguments are guaranteed to be valid JSON conforming to the function's parameter schema. |
-| `"required"` | Yes (via structured outputs backend) | Same as named function. The model must produce at least one tool call. |
-| `"auto"` | No | The model generates freely. A tool-call parser extracts tool calls from the raw text. Arguments may be malformed or not match the schema. |
-| `"none"` | N/A | No tool calls are produced. |
-
-When schema conformance matters, prefer `tool_choice="required"` or named function calling over `"auto"`.
-
-### Strict Mode (`strict` parameter)
-
-The [OpenAI API](https://platform.openai.com/docs/guides/function-calling#strict-mode) supports a `strict` field on function definitions. When set to `true`, OpenAI uses constrained decoding to guarantee that tool-call arguments match the function schema, even in `tool_choice="auto"` mode.
-
-vLLM **does not implement** `strict` mode today. The `strict` field is accepted in requests (to avoid breaking clients that set it), but it has no effect on decoding behavior. In auto mode, argument validity depends entirely on the model's output quality and the parser's extraction logic.
-
-Tracking issues: [#15526](https://github.com/vllm-project/vllm/issues/15526), [#16313](https://github.com/vllm-project/vllm/issues/16313).
-
 ## Automatic Function Calling

 To enable this feature, you should set the following flags:
@@ -145,9 +124,6 @@ from HuggingFace; and you can find an example of this in a `tokenizer_config.jso

 If your favorite tool-calling model is not supported, please feel free to contribute a parser & tool use chat template!

-!!! note
-    With `tool_choice="auto"`, tool-call arguments are extracted from the model's raw text output by the selected parser. No schema-level constraint is applied during decoding, so arguments may occasionally be malformed or violate the function's parameter schema. See [Constrained Decoding Behavior](#constrained-decoding-behavior) for details.
-
 ### Hermes Models (`hermes`)

 All Nous Research Hermes-series models newer than Hermes 2 Pro should be supported.
@@ -20,9 +20,9 @@ run the script with
 python openai_chat_completion_client_for_multimodal.py --chat-type audio
 """

+import base64
 import os

-import pybase64 as base64
 import requests
 from openai import OpenAI
 from utils import get_first_model
@@ -24,11 +24,11 @@ The script:

 import argparse
 import asyncio
+import base64
 import json

 import librosa
 import numpy as np
-import pybase64 as base64
 import websockets

 from vllm.assets.audio import AudioAsset
@@ -18,13 +18,13 @@ Requirements: websockets, numpy, gradio

 import argparse
 import asyncio
+import base64
 import json
 import queue
 import threading

 import gradio as gr
 import numpy as np
-import pybase64 as base64
 import websockets

 SAMPLE_RATE = 16_000
@@ -7,8 +7,8 @@ NOTE:
 """

 import argparse
+import base64

-import pybase64 as base64
 import requests
 import torch

@@ -7,10 +7,10 @@ Refer to each `run_*` function for the command to run the server for that model.
 """

 import argparse
+import base64
 import io
 from typing import Literal

-import pybase64 as base64
 from openai import OpenAI
 from openai._types import NOT_GIVEN, NotGiven
 from openai.types.chat import ChatCompletionMessageParam
@@ -1,8 +1,8 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import base64
 import os

-import pybase64 as base64
 import torch

 from vllm import LLM
@@ -1,9 +1,9 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

+import base64
 import os

-import pybase64 as base64
 import requests

 # This example shows how to perform an online inference that generates
@@ -15,9 +15,9 @@ Then run this script:
    python colqwen3_rerank_online.py
 """

+import base64
 from io import BytesIO

-import pybase64 as base64
 import requests
 from PIL import Image

@@ -21,10 +21,10 @@ Then run this script:
 """

 import argparse
+import base64
 from io import BytesIO

 import numpy as np
-import pybase64 as base64
 import requests
 from PIL import Image

@@ -37,7 +37,7 @@ pyyaml
 six>=1.16.0; python_version > '3.11' # transitive dependency of pandas that needs to be the latest version for python 3.12
 setuptools>=77.0.3,<81.0.0; python_version > '3.11' # Setuptools is used by triton, we need to ensure a modern version is installed for 3.12+ so that it does not try to import distutils, which was removed in 3.12
 einops # Required for Qwen2-VL.
-compressed-tensors == 0.14.0.1 # required for compressed-tensors
+compressed-tensors == 0.13.0 # required for compressed-tensors
 depyf==0.20.0 # required for profiling and debugging with compilation config
 cloudpickle # allows pickling lambda functions in model_executor/models/registry.py
 watchfiles # required for http server to monitor the updates of TLS files
@@ -50,7 +50,7 @@ av==16.1.0
 blobfile==3.0.0
    # Multi-Modal Models Test
 decord==0.6.0
-    # video processing, required by entrypoints/openai/chat_completion/test_video.py
+    # video processing, required by entrypoints/openai/test_video.py
 rapidfuzz==3.12.1

 # OpenAI compatibility and testing
@@ -1,12 +1,12 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

+import base64
 import os
 from tempfile import NamedTemporaryFile
 from typing import Any, cast

 import cv2
-import pybase64 as base64
 import pytest
 from transformers import AutoTokenizer, PreTrainedTokenizerBase

@@ -1,9 +1,7 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

-from tests.entrypoints.openai.chat_completion.test_oot_registration import (
-    run_and_test_dummy_opt_api_server,
-)
+from ..entrypoints.openai.test_oot_registration import run_and_test_dummy_opt_api_server


 def test_distributed_oot(dummy_opt_path: str):
@@ -6,10 +6,10 @@ Unit tests for engine classes (parsing, validation, registry).
 Integration tests for NCCL and IPC weight transfer between processes using Ray.
 """

+import base64
 import pickle
 from unittest.mock import MagicMock

-import pybase64 as base64
 import pytest
 import ray
 import torch
@@ -4,11 +4,12 @@ import weakref

 import pytest

-from tests.entrypoints.openai.chat_completion.test_vision import TEST_IMAGE_ASSETS
 from vllm import LLM
 from vllm.distributed import cleanup_dist_env_and_memory
 from vllm.sampling_params import SamplingParams

+from ..openai.test_vision import TEST_IMAGE_ASSETS
+

@pytest.fixture(scope="function")
 def text_llm():
@@ -6,12 +6,13 @@ import logging
 import pytest
 import regex as re

-from tests.entrypoints.openai.chat_completion.test_vision import TEST_IMAGE_ASSETS
 from vllm import LLM
 from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
 from vllm.v1.metrics import loggers as stat_loggers
 from vllm.v1.metrics.reader import Counter, Metric

+from ..openai.test_vision import TEST_IMAGE_ASSETS
+

 def _make_messages(image_url: str) -> list[ChatCompletionMessageParam]:
    return [
@@ -111,7 +111,7 @@ def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat:
            [{"prompt_token_ids": [1, 2, 3]}],
        )

-    serving_chat.openai_serving_render.preprocess_chat = AsyncMock(
+    serving_chat.openai_serving_render._preprocess_chat = AsyncMock(
        side_effect=_fake_preprocess_chat
    )
    return serving_chat
@@ -8,9 +8,6 @@ from collections.abc import Callable
 from typing import Any

 import pytest
-import pytest_asyncio
-
-from tests.utils import RemoteOpenAIServer

 logger = logging.getLogger(__name__)

@@ -364,38 +361,3 @@ def log_response_diagnostics(
    )

    return diagnostics
-
-
-@pytest.fixture(scope="module")
-def default_server_args():
-    return [
-        "--max-model-len",
-        "8192",
-        "--enforce-eager",  # For faster startup.
-        "--enable-auto-tool-choice",
-        "--structured-outputs-config.backend",
-        "xgrammar",
-        "--tool-call-parser",
-        "hermes",
-        "--reasoning-parser",
-        "qwen3",
-    ]
-
-
-@pytest.fixture(scope="module")
-def server_with_store(default_server_args):
-    with RemoteOpenAIServer(
-        "Qwen/Qwen3-1.7B",
-        default_server_args,
-        env_dict={
-            "VLLM_ENABLE_RESPONSES_API_STORE": "1",
-            "VLLM_SERVER_DEV_MODE": "1",
-        },
-    ) as remote_server:
-        yield remote_server
-
-
-@pytest_asyncio.fixture
-async def client(server_with_store):
-    async with server_with_store.get_async_client() as async_client:
-        yield async_client
@@ -16,8 +16,7 @@ import requests
 from openai import InternalServerError, NotFoundError, OpenAI
 from openai_harmony import Message

-from tests.utils import RemoteOpenAIServer
-
+from ....utils import RemoteOpenAIServer
 from .conftest import (
    BASE_TEST_ENV,
    events_contain_type,
@@ -9,9 +9,9 @@ import pytest_asyncio
 from openai import OpenAI
 from openai_harmony import ToolDescription, ToolNamespaceConfig

-from tests.utils import RemoteOpenAIServer
 from vllm.entrypoints.mcp.tool_server import MCPToolServer

+from ....utils import RemoteOpenAIServer
 from .conftest import (
    BASE_TEST_ENV,
    events_contain_type,
@@ -9,8 +9,7 @@ import pytest
 import pytest_asyncio
 from openai import OpenAI

-from tests.utils import RemoteOpenAIServer
-
+from ....utils import RemoteOpenAIServer
 from .conftest import (
    BASE_TEST_ENV,
    has_output_type,
@@ -5,8 +5,7 @@ import pytest
 import pytest_asyncio
 from openai import OpenAI

-from tests.utils import RemoteOpenAIServer
-
+from ....utils import RemoteOpenAIServer
 from .conftest import validate_streaming_event_stack

 MODEL_NAME = "Qwen/Qwen3-8B"
@@ -7,10 +7,11 @@ import openai
 import pytest
 import pytest_asyncio

-from tests.utils import RemoteOpenAIServer
 from vllm.assets.audio import AudioAsset
 from vllm.multimodal.utils import encode_audio_base64, encode_audio_url, fetch_audio

+from ...utils import RemoteOpenAIServer
+
 MODEL_NAME = "fixie-ai/ultravox-v0_5-llama-3_2-1b"
 TEST_AUDIO_URLS = [
    AudioAsset("winning_call").url,
@@ -1,15 +1,15 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

+import base64
 import json

 import openai
-import pybase64 as base64
 import pytest
 import pytest_asyncio

-from tests.conftest import VideoTestAssets
-from tests.utils import RemoteOpenAIServer
+from ...conftest import VideoTestAssets
+from ...utils import RemoteOpenAIServer

 MODEL_NAME = "Qwen/Qwen2.5-Omni-3B"

@@ -291,32 +291,3 @@ def test_served_model_name_parsing(tmp_path, vllm_parser, args, raises):
    else:
        with pytest.raises(raises):
            vllm_parser.parse_args(args=args)
-
-
-### Tests for LoRA target modules parsing
-def test_lora_target_modules_single(serve_parser):
-    """Test parsing single lora-target-modules argument"""
-    args = serve_parser.parse_args(
-        args=["--enable-lora", "--lora-target-modules", "o_proj"]
-    )
-    assert args.lora_target_modules == ["o_proj"]
-
-
-def test_lora_target_modules_multiple(serve_parser):
-    """Test parsing multiple lora-target-modules arguments"""
-    args = serve_parser.parse_args(
-        args=[
-            "--enable-lora",
-            "--lora-target-modules",
-            "o_proj",
-            "qkv_proj",
-            "down_proj",
-        ]
-    )
-    assert args.lora_target_modules == ["o_proj", "qkv_proj", "down_proj"]
-
-
-def test_lora_target_modules_default_none(serve_parser):
-    """Test that lora-target-modules defaults to None"""
-    args = serve_parser.parse_args(args=[])
-    assert args.lora_target_modules is None
@@ -1,11 +1,11 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

+import base64
 import io
 import json

 import openai  # use the official client for correctness check
-import pybase64 as base64
 import pytest
 import pytest_asyncio
 import torch
@@ -14,7 +14,7 @@ import torch
 from openai import BadRequestError
 from transformers import AutoConfig

-from tests.utils import RemoteOpenAIServer
+from ...utils import RemoteOpenAIServer

 # any model with a chat template should work here
 MODEL_NAME = "facebook/opt-125m"
@@ -8,8 +8,8 @@ import pytest
 import pytest_asyncio
 from huggingface_hub import snapshot_download

-from tests.conftest import AudioTestAssets
-from tests.utils import RemoteOpenAIServer
+from ...conftest import AudioTestAssets
+from ...utils import RemoteOpenAIServer

 # NOTE - the tests in this module are currently analogous to test_chat, but are
 # separated to avoid OOM killing due to module-scoped servers, since we
@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

-from tests.utils import VLLM_PATH, RemoteOpenAIServer
+from ...utils import VLLM_PATH, RemoteOpenAIServer

 chatml_jinja_path = VLLM_PATH / "examples/template_chatml.jinja"
 assert chatml_jinja_path.exists()
@@ -11,10 +11,11 @@ import pytest
 import regex as re
 import torch

-from tests.utils import RemoteOpenAIServer
 from vllm.config import ModelConfig
 from vllm.renderers.embed_utils import safe_load_prompt_embeds

+from ...utils import RemoteOpenAIServer
+

@pytest.mark.asyncio
 async def test_empty_prompt():
@@ -2,12 +2,12 @@
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

 import asyncio
+import base64
 import json
 import warnings

 import librosa
 import numpy as np
-import pybase64 as base64
 import pytest
 import websockets

@@ -8,7 +8,7 @@ from typing import Any, NamedTuple
 import openai  # use the official client for correctness check
 import pytest

-from tests.utils import RemoteOpenAIServer
+from ...utils import RemoteOpenAIServer

 # # any model with a chat template should work here
 MODEL_NAME = "Qwen/Qwen2-1.5B-Instruct"
@@ -159,7 +159,6 @@ class TestInitializeToolSessions:
        instance = OpenAIServingResponses(
            engine_client=engine_client,
            models=models,
-            openai_serving_render=MagicMock(),
            request_logger=None,
            chat_template=None,
            chat_template_content_format="auto",
@@ -246,7 +245,6 @@ class TestValidateGeneratorInput:
        instance = OpenAIServingResponses(
            engine_client=engine_client,
            models=models,
-            openai_serving_render=MagicMock(),
            request_logger=None,
            chat_template=None,
            chat_template_content_format="auto",
@@ -310,7 +308,6 @@ async def test_reasoning_tokens_counted_for_text_reasoning_model(monkeypatch):
    serving = OpenAIServingResponses(
        engine_client=engine_client,
        models=models,
-        openai_serving_render=MagicMock(),
        request_logger=None,
        chat_template=None,
        chat_template_content_format="auto",
@@ -610,7 +607,6 @@ def _make_serving_instance_with_reasoning():
    serving = OpenAIServingResponses(
        engine_client=engine_client,
        models=models,
-        openai_serving_render=MagicMock(),
        request_logger=None,
        chat_template=None,
        chat_template_content_format="auto",
@@ -9,7 +9,6 @@ import pytest
 import pytest_asyncio
 import torch.cuda

-from tests.utils import RemoteOpenAIServer
 from vllm.engine.arg_utils import EngineArgs
 from vllm.model_executor.model_loader.tensorizer import (
    TensorizerConfig,
@@ -18,6 +17,8 @@ from vllm.model_executor.model_loader.tensorizer import (
 )
 from vllm.platforms import current_platform

+from ...utils import RemoteOpenAIServer
+
 MODEL_NAME = "unsloth/llama-3.2-1b-Instruct"
 LORA_PATH = "davzoku/finqa_adapter_1b"

@@ -6,10 +6,11 @@ import tempfile

 import pytest

-from tests.utils import RemoteOpenAIServer
 from vllm.model_executor.model_loader.weight_utils import download_weights_from_hf
 from vllm.tokenizers import get_tokenizer

+from ...utils import RemoteOpenAIServer
+
 MODEL_NAME = "Qwen/Qwen3-0.6B"
 MODEL_PATH = os.path.join(tempfile.gettempdir(), "qwen3_06b")

@@ -7,10 +7,11 @@ import openai
 import pytest
 import pytest_asyncio

-from tests.utils import RemoteOpenAIServer
 from vllm.multimodal.utils import encode_video_url, fetch_video
 from vllm.platforms import current_platform

+from ...utils import RemoteOpenAIServer
+
 MODEL_NAME = "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
 MAXIMUM_VIDEOS = 3

@@ -8,11 +8,12 @@ import pytest
 import pytest_asyncio
 from transformers import AutoProcessor

-from tests.utils import ROCM_ENV_OVERRIDES, ROCM_EXTRA_ARGS, RemoteOpenAIServer
 from vllm.multimodal.media import MediaWithBytes
 from vllm.multimodal.utils import encode_image_url, fetch_image
 from vllm.platforms import current_platform

+from ...utils import ROCM_ENV_OVERRIDES, ROCM_EXTRA_ARGS, RemoteOpenAIServer
+
 MODEL_NAME = "microsoft/Phi-3.5-vision-instruct"
 MAXIMUM_IMAGES = 2

@@ -1,15 +1,17 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

+import base64
+
 import numpy as np
-import pybase64 as base64
 import pytest
 import requests
 import torch

-from tests.utils import RemoteOpenAIServer
 from vllm.utils.serial_utils import tensor2base64

+from ...utils import RemoteOpenAIServer
+

@pytest.mark.parametrize(
    "model_name", ["ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11"]
@@ -5,7 +5,7 @@ import json

 import pytest

-from tests.tool_parsers.utils import (
+from tests.entrypoints.openai.tool_parsers.utils import (
    run_tool_extraction,
    run_tool_extraction_streaming,
 )
@@ -13,13 +13,6 @@ from vllm.entrypoints.openai.engine.protocol import FunctionCall
 from vllm.tokenizers import TokenizerLike
 from vllm.tool_parsers import ToolParser, ToolParserManager

-MSG_SEP_TOKEN = "<|message_sep|>\n\n"
-ROLE_SEP_TOKEN = "<|role_sep|>\n"
-EOS_TOKEN = "</s>"
-TOOL_HEADER_GIGACHAT3 = f"function call{ROLE_SEP_TOKEN}"
-TOOL_HEADER_GIGACHAT31 = "<|function_call|>"
-
-
 SIMPLE_ARGS_DICT = {
    "action": "create",
    "id": "preferences",
@@ -31,10 +24,7 @@ SIMPLE_FUNCTION_JSON = json.dumps(
    },
    ensure_ascii=False,
 )
-SIMPLE_FUNCTION_OUTPUT_GIGACHAT3 = (
-    f"{MSG_SEP_TOKEN}{TOOL_HEADER_GIGACHAT3}{SIMPLE_FUNCTION_JSON}"
-)
-SIMPLE_FUNCTION_OUTPUT_GIGACHAT31 = f"{TOOL_HEADER_GIGACHAT31}{SIMPLE_FUNCTION_JSON}"
+SIMPLE_FUNCTION_OUTPUT = "function call" + SIMPLE_FUNCTION_JSON
 SIMPLE_FUNCTION_CALL = FunctionCall(
    name="manage_user_memory",
    arguments=json.dumps(SIMPLE_ARGS_DICT, ensure_ascii=False),
@@ -48,12 +38,7 @@ PARAMETERLESS_FUNCTION_JSON = json.dumps(
    },
    ensure_ascii=False,
 )
-PARAMETERLESS_FUNCTION_OUTPUT_GIGACHAT3 = (
-    f"{MSG_SEP_TOKEN}{TOOL_HEADER_GIGACHAT3}{PARAMETERLESS_FUNCTION_JSON}"
-)
-PARAMETERLESS_FUNCTION_OUTPUT_GIGACHAT31 = (
-    f"{TOOL_HEADER_GIGACHAT31}{PARAMETERLESS_FUNCTION_JSON}"
-)
+PARAMETERLESS_FUNCTION_OUTPUT = "function call" + PARAMETERLESS_FUNCTION_JSON
 PARAMETERLESS_FUNCTION_CALL = FunctionCall(
    name="manage_user_memory",
    arguments=json.dumps({}, ensure_ascii=False),
@@ -77,38 +62,17 @@ COMPLEX_FUNCTION_JSON = json.dumps(
    },
    ensure_ascii=False,
 )
-COMPLEX_FUNCTION_OUTPUT_GIGACHAT3 = (
-    f"{MSG_SEP_TOKEN}{TOOL_HEADER_GIGACHAT3}{COMPLEX_FUNCTION_JSON}"
-)
-COMPLEX_FUNCTION_OUTPUT_GIGACHAT31 = f"{TOOL_HEADER_GIGACHAT31}{COMPLEX_FUNCTION_JSON}"
+COMPLEX_FUNCTION_OUTPUT = "function call" + COMPLEX_FUNCTION_JSON
 COMPLEX_FUNCTION_CALL = FunctionCall(
    name="manage_user_memory",
    arguments=json.dumps(COMPLEX_ARGS_DICT, ensure_ascii=False),
 )


-CONTENT_TEXT = "I'll check that for you."
-MIXED_OUTPUT_GIGACHAT3 = f"{CONTENT_TEXT}{SIMPLE_FUNCTION_OUTPUT_GIGACHAT3}"
-MIXED_OUTPUT_GIGACHAT31 = f"{CONTENT_TEXT}{SIMPLE_FUNCTION_OUTPUT_GIGACHAT31}"
-
-
-@pytest.fixture(name="gigachat_tokenizer")
-def fixture_gigachat_tokenizer(default_tokenizer: TokenizerLike):
-    default_tokenizer.add_tokens(
-        [
-            MSG_SEP_TOKEN,
-            ROLE_SEP_TOKEN,
-            TOOL_HEADER_GIGACHAT31,
-            EOS_TOKEN,
-        ]
-    )
-    return default_tokenizer
-
-
@pytest.mark.parametrize("streaming", [True, False])
-def test_no_tool_call(streaming: bool, gigachat_tokenizer: TokenizerLike):
+def test_no_tool_call(streaming: bool, default_tokenizer: TokenizerLike):
    tool_parser: ToolParser = ToolParserManager.get_tool_parser("gigachat3")(
-        gigachat_tokenizer
+        default_tokenizer
    )
    model_output = "How can I help you today?"
    content, tool_calls = run_tool_extraction(
@@ -121,143 +85,45 @@ def test_no_tool_call(streaming: bool, gigachat_tokenizer: TokenizerLike):
 TEST_CASES = [
    pytest.param(
        True,
-        SIMPLE_FUNCTION_OUTPUT_GIGACHAT3,
+        SIMPLE_FUNCTION_OUTPUT,
        [SIMPLE_FUNCTION_CALL],
        None,
-        id="simple_streaming_gigachat3",
+        id="simple_streaming",
    ),
    pytest.param(
        False,
-        SIMPLE_FUNCTION_OUTPUT_GIGACHAT3,
+        SIMPLE_FUNCTION_OUTPUT,
        [SIMPLE_FUNCTION_CALL],
        None,
-        id="simple_nonstreaming_gigachat3",
+        id="simple_nonstreaming",
    ),
    pytest.param(
        True,
-        PARAMETERLESS_FUNCTION_OUTPUT_GIGACHAT3,
+        PARAMETERLESS_FUNCTION_OUTPUT,
        [PARAMETERLESS_FUNCTION_CALL],
        None,
-        id="parameterless_streaming_gigachat3",
+        id="parameterless_streaming",
    ),
    pytest.param(
        False,
-        PARAMETERLESS_FUNCTION_OUTPUT_GIGACHAT3,
+        PARAMETERLESS_FUNCTION_OUTPUT,
        [PARAMETERLESS_FUNCTION_CALL],
        None,
-        id="parameterless_nonstreaming_gigachat3",
+        id="parameterless_nonstreaming",
    ),
    pytest.param(
        True,
-        COMPLEX_FUNCTION_OUTPUT_GIGACHAT3,
+        COMPLEX_FUNCTION_OUTPUT,
        [COMPLEX_FUNCTION_CALL],
        None,
-        id="complex_streaming_gigachat3",
+        id="complex_streaming",
    ),
    pytest.param(
        False,
-        COMPLEX_FUNCTION_OUTPUT_GIGACHAT3,
+        COMPLEX_FUNCTION_OUTPUT,
        [COMPLEX_FUNCTION_CALL],
        None,
-        id="complex_nonstreaming_gigachat3",
-    ),
-    pytest.param(
-        True,
-        MIXED_OUTPUT_GIGACHAT3,
-        [SIMPLE_FUNCTION_CALL],
-        CONTENT_TEXT,
-        id="mixed_content_streaming_gigachat3",
-    ),
-    pytest.param(
-        False,
-        MIXED_OUTPUT_GIGACHAT3,
-        [SIMPLE_FUNCTION_CALL],
-        CONTENT_TEXT,
-        id="mixed_content_nonstreaming_gigachat3",
-    ),
-    pytest.param(
-        True,
-        MIXED_OUTPUT_GIGACHAT3 + EOS_TOKEN,
-        [SIMPLE_FUNCTION_CALL],
-        CONTENT_TEXT,
-        id="mixed_content_streaming_with_eos_gigachat3",
-    ),
-    pytest.param(
-        False,
-        MIXED_OUTPUT_GIGACHAT3 + EOS_TOKEN,
-        [SIMPLE_FUNCTION_CALL],
-        CONTENT_TEXT,
-        id="mixed_content_nonstreaming_with_eos_gigachat3",
-    ),
-    pytest.param(
-        True,
-        SIMPLE_FUNCTION_OUTPUT_GIGACHAT31,
-        [SIMPLE_FUNCTION_CALL],
-        None,
-        id="simple_streaming_gigachat31",
-    ),
-    pytest.param(
-        False,
-        SIMPLE_FUNCTION_OUTPUT_GIGACHAT31,
-        [SIMPLE_FUNCTION_CALL],
-        None,
-        id="simple_nonstreaming_gigachat31",
-    ),
-    pytest.param(
-        True,
-        PARAMETERLESS_FUNCTION_OUTPUT_GIGACHAT31,
-        [PARAMETERLESS_FUNCTION_CALL],
-        None,
-        id="parameterless_streaming_gigachat31",
-    ),
-    pytest.param(
-        False,
-        PARAMETERLESS_FUNCTION_OUTPUT_GIGACHAT31,
-        [PARAMETERLESS_FUNCTION_CALL],
-        None,
-        id="parameterless_nonstreaming_gigachat31",
-    ),
-    pytest.param(
-        True,
-        COMPLEX_FUNCTION_OUTPUT_GIGACHAT31,
-        [COMPLEX_FUNCTION_CALL],
-        None,
-        id="complex_streaming_gigachat31",
-    ),
-    pytest.param(
-        False,
-        COMPLEX_FUNCTION_OUTPUT_GIGACHAT31,
-        [COMPLEX_FUNCTION_CALL],
-        None,
-        id="complex_nonstreaming_gigachat31",
-    ),
-    pytest.param(
-        True,
-        MIXED_OUTPUT_GIGACHAT31,
-        [SIMPLE_FUNCTION_CALL],
-        CONTENT_TEXT,
-        id="mixed_content_streaming_gigachat31",
-    ),
-    pytest.param(
-        False,
-        MIXED_OUTPUT_GIGACHAT31,
-        [SIMPLE_FUNCTION_CALL],
-        CONTENT_TEXT,
-        id="mixed_content_nonstreaming_gigachat31",
-    ),
-    pytest.param(
-        True,
-        MIXED_OUTPUT_GIGACHAT31 + EOS_TOKEN,
-        [SIMPLE_FUNCTION_CALL],
-        CONTENT_TEXT,
-        id="mixed_content_streaming_with_eos_gigachat31",
-    ),
-    pytest.param(
-        False,
-        MIXED_OUTPUT_GIGACHAT31 + EOS_TOKEN,
-        [SIMPLE_FUNCTION_CALL],
-        CONTENT_TEXT,
-        id="mixed_content_nonstreaming_with_eos_gigachat31",
+        id="complex_nonstreaming",
    ),
 ]

@@ -270,16 +136,14 @@ def test_tool_call(
    model_output: str,
    expected_tool_calls: list[FunctionCall],
    expected_content: str | None,
-    gigachat_tokenizer: TokenizerLike,
+    default_tokenizer: TokenizerLike,
 ):
    tool_parser: ToolParser = ToolParserManager.get_tool_parser("gigachat3")(
-        gigachat_tokenizer
+        default_tokenizer
    )
    content, tool_calls = run_tool_extraction(
        tool_parser, model_output, streaming=streaming
    )
-    if content == "":
-        content = None
    assert content == expected_content
    assert len(tool_calls) == len(expected_tool_calls)
    for actual, expected in zip(tool_calls, expected_tool_calls):
@@ -290,46 +154,15 @@ def test_tool_call(
        assert actual_args == expected_args


-@pytest.mark.parametrize(
-    "model_output_deltas",
-    [
-        pytest.param(
-            [
-                CONTENT_TEXT[:3],
-                CONTENT_TEXT[3:5],
-                CONTENT_TEXT[5:],
-                MSG_SEP_TOKEN,
-                TOOL_HEADER_GIGACHAT3,
-                COMPLEX_FUNCTION_JSON[:40],
-                COMPLEX_FUNCTION_JSON[40:-1],
-                COMPLEX_FUNCTION_JSON[-1],
-            ],
-            id="gigachat3",
-        ),
-        pytest.param(
-            [
-                CONTENT_TEXT[:3],
-                CONTENT_TEXT[3:5],
-                CONTENT_TEXT[5:],
-                TOOL_HEADER_GIGACHAT31,
-                COMPLEX_FUNCTION_JSON[:40],
-                COMPLEX_FUNCTION_JSON[40:-1],
-                COMPLEX_FUNCTION_JSON[-1],
-            ],
-            id="gigachat31",
-        ),
-    ],
-)
-def test_streaming_tool_call_with_large_steps(
-    model_output_deltas: list[str],
-    gigachat_tokenizer: TokenizerLike,
-):
-    """
-    Test that the closing braces are streamed correctly.
-    """
+def test_streaming_tool_call_with_large_steps(default_tokenizer: TokenizerLike):
    tool_parser: ToolParser = ToolParserManager.get_tool_parser("gigachat3")(
-        gigachat_tokenizer
+        default_tokenizer
    )
+    model_output_deltas = [
+        "function call",
+        COMPLEX_FUNCTION_JSON[:40],
+        COMPLEX_FUNCTION_JSON[40:],
+    ]
    reconstructor = run_tool_extraction_streaming(
        tool_parser,
        model_output_deltas,
@@ -7,7 +7,7 @@ from unittest.mock import MagicMock

 import pytest

-from tests.tool_parsers.utils import (
+from tests.entrypoints.openai.tool_parsers.utils import (
    run_tool_extraction,
    run_tool_extraction_streaming,
 )
@@ -5,7 +5,7 @@ from unittest.mock import MagicMock, patch

 import pytest

-from tests.tool_parsers.utils import (
+from tests.entrypoints.openai.tool_parsers.utils import (
    run_tool_extraction,
    run_tool_extraction_streaming,
 )
@@ -5,7 +5,7 @@ from unittest.mock import MagicMock, patch

 import pytest

-from tests.tool_parsers.utils import (
+from tests.entrypoints.openai.tool_parsers.utils import (
    run_tool_extraction,
    run_tool_extraction_streaming,
 )
@@ -5,7 +5,7 @@ from unittest.mock import MagicMock, patch

 import pytest

-from tests.tool_parsers.utils import (
+from tests.entrypoints.openai.tool_parsers.utils import (
    run_tool_extraction,
    run_tool_extraction_streaming,
 )
@@ -7,10 +7,10 @@ embedding models, covering text embedding, embedding type conversions,
 response structure, batching, normalisation, and semantic similarity.
 """

+import base64
 import struct

 import numpy as np
-import pybase64 as base64
 import pytest
 import requests

@@ -6,11 +6,11 @@ Validates image embedding, batching, normalisation, and embedding type
 conversions through the /v2/embed endpoint.
 """

+import base64
 import struct
 import zlib

 import numpy as np
-import pybase64 as base64
 import pytest
 import requests

@@ -1,11 +1,11 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

+import base64
 import json

 import numpy as np
 import openai
-import pybase64 as base64
 import pytest
 import pytest_asyncio
 import requests
@@ -3,10 +3,10 @@
 """Unit tests for Cohere embed protocol: build_typed_embeddings and its
 underlying packing helpers, plus Cohere-specific serving helpers."""

+import base64
 import struct

 import numpy as np
-import pybase64 as base64
 import pytest

 from vllm.entrypoints.pooling.embed.protocol import (
@@ -1,10 +1,10 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

+import base64
 import json

 import numpy as np
-import pybase64 as base64
 import pytest
 import requests
 import torch
@@ -0,0 +1,8 @@
+model_name: "Qwen/Qwen3.5-35B-A3B"
+accuracy_threshold: 0.86
+num_questions: 1319
+num_fewshot: 5
+server_args: >-
+  --max-model-len 4096
+  --data-parallel-size 2
+  --enable-expert-parallel
@@ -0,0 +1,9 @@
+model_name: "Qwen/Qwen3.5-35B-A3B-FP8"
+accuracy_threshold: 0.86
+num_questions: 1319
+num_fewshot: 5
+server_args: >-
+  --max-model-len 4096
+  --data-parallel-size 2
+  --enable-expert-parallel
+  --kv-cache-dtype fp8
@@ -0,0 +1,2 @@
+Qwen3.5-35B-A3B-DEP2.yaml
+Qwen3.5-35B-A3B-FP8-DEP2.yaml
@@ -48,7 +48,7 @@ def get_attn_isa(
    else:
        if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
            return "neon"
-        elif torch.cpu._is_amx_tile_supported():
+        elif torch._C._cpu._is_amx_tile_supported():
            return "amx"
        else:
            return "vec"
@@ -400,7 +400,9 @@ def test_varlen_with_paged_kv_normal_vec(
@pytest.mark.parametrize("use_alibi", [False])
@pytest.mark.parametrize("use_sink", [False])
@pytest.mark.parametrize("isa", ["amx"])
-@pytest.mark.skipif(not torch.cpu._is_amx_tile_supported(), reason="no AMX support.")
+@pytest.mark.skipif(
+    not torch._C._cpu._is_amx_tile_supported(), reason="no AMX support."
+)
 def test_varlen_with_paged_kv_normal_amx(
    seq_lens: list[tuple[int, int]],
    num_heads: tuple[int, int],
@@ -1,434 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""
-Standalone unit tests for trtllm_prefill_attn_kvfp8_dequant.
-
-Tests both contiguous and non-contiguous (cross-layer unified) KV cache
-layouts against a pure-PyTorch reference implementation.
-"""
-
-import pytest
-import torch
-
-from vllm.platforms import current_platform
-
-FP8_DTYPE = current_platform.fp8_dtype()
-
-NUM_BLOCKS = 128
-
-
-def to_float8(x, dtype=None):
-    if dtype is None:
-        dtype = FP8_DTYPE
-    finfo = torch.finfo(dtype)
-    min_val, max_val = x.aminmax()
-    amax = torch.maximum(min_val.abs(), max_val.abs()).clamp(min=1e-12)
-    scale = finfo.max / amax * 0.1
-    x_scl_sat = (x * scale).clamp(min=finfo.min, max=finfo.max)
-    return x_scl_sat.to(dtype), scale.float().reciprocal()
-
-
-def make_contiguous_kv_cache(num_blocks, num_kv_heads, block_size, head_size):
-    """Create a standard contiguous fp8 KV cache (HND layout)."""
-    raw = torch.randn(
-        num_blocks,
-        2,
-        num_kv_heads,
-        block_size,
-        head_size,
-        dtype=torch.bfloat16,
-        device="cuda",
-    )
-    kv_cache, scale = to_float8(raw)
-    return kv_cache, scale
-
-
-def make_cross_layer_kv_cache(
-    num_blocks,
-    num_kv_heads,
-    block_size,
-    head_size,
-    num_layers=4,
-):
-    """
-    Create a non-contiguous per-layer view mimicking cross-layer allocation.
-
-    Physical layout: (num_blocks, 2, num_kv_heads, num_layers, block_size, head_size)
-    Returned view:   (num_blocks, 2, num_kv_heads, block_size, head_size)
-    with non-contiguous strides on dims 0, 1, 2 (they skip over num_layers).
-    """
-    raw = torch.randn(
-        num_blocks,
-        2,
-        num_kv_heads,
-        num_layers,
-        block_size,
-        head_size,
-        dtype=torch.bfloat16,
-        device="cuda",
-    )
-    fp8_full, scale = to_float8(raw)
-    layer_view = fp8_full[:, :, :, 0, :, :]
-    assert not layer_view.is_contiguous(), (
-        f"Expected non-contiguous view, got strides {layer_view.stride()}"
-    )
-    return layer_view, scale
-
-
-def ref_dequant(kv_cache, block_tables, k_scale, v_scale, dequant_dtype):
-    """Pure PyTorch reference: gather pages and dequantize fp8 -> dequant_dtype."""
-    batch_size, num_pages_per_seq = block_tables.shape
-    s = kv_cache.shape
-    out = torch.zeros(
-        batch_size * num_pages_per_seq + 1,
-        s[1],
-        s[2],
-        s[3],
-        s[4],
-        dtype=dequant_dtype,
-        device=kv_cache.device,
-    )
-    for b in range(batch_size):
-        for p in range(num_pages_per_seq):
-            page_idx = block_tables[b, p].item()
-            if page_idx <= 0:
-                continue
-            mock_idx = b * num_pages_per_seq + p + 1
-            out[mock_idx, 0] = (kv_cache[page_idx, 0].float() * k_scale.item()).to(
-                dequant_dtype
-            )
-            out[mock_idx, 1] = (kv_cache[page_idx, 1].float() * v_scale.item()).to(
-                dequant_dtype
-            )
-    return out
-
-
-@pytest.mark.parametrize("num_kv_heads", [1, 8])
-@pytest.mark.parametrize("head_size", [64, 128])
-@pytest.mark.parametrize("block_size", [16, 32])
-@pytest.mark.parametrize("batch_size", [1, 4])
-@pytest.mark.parametrize("num_pages_per_seq", [3, 8])
-@pytest.mark.parametrize("contiguous", [True, False])
-@torch.inference_mode()
-def test_trtllm_kvfp8_dequant(
-    num_kv_heads: int,
-    head_size: int,
-    block_size: int,
-    batch_size: int,
-    num_pages_per_seq: int,
-    contiguous: bool,
-):
-    from vllm.v1.attention.backends.flashinfer import (
-        trtllm_prefill_attn_kvfp8_dequant,
-    )
-
-    torch.set_default_device("cuda")
-
-    if contiguous:
-        kv_cache, scale = make_contiguous_kv_cache(
-            NUM_BLOCKS,
-            num_kv_heads,
-            block_size,
-            head_size,
-        )
-    else:
-        kv_cache, scale = make_cross_layer_kv_cache(
-            NUM_BLOCKS,
-            num_kv_heads,
-            block_size,
-            head_size,
-        )
-
-    k_scale = scale.clone()
-    v_scale = scale.clone()
-
-    block_tables = torch.randint(
-        1,
-        NUM_BLOCKS,
-        (batch_size, num_pages_per_seq),
-        dtype=torch.int32,
-    )
-
-    mock_kv_cache, mock_block_table = trtllm_prefill_attn_kvfp8_dequant(
-        kv_cache,
-        block_tables,
-        k_scale,
-        v_scale,
-        torch.bfloat16,
-    )
-
-    ref = ref_dequant(kv_cache, block_tables, k_scale, v_scale, torch.bfloat16)
-
-    expected_bt = torch.arange(
-        1,
-        batch_size * num_pages_per_seq + 1,
-        dtype=torch.int32,
-        device="cuda",
-    ).reshape(batch_size, num_pages_per_seq)
-    torch.testing.assert_close(mock_block_table, expected_bt)
-
-    # Page 0 is padding (never written), compare only pages 1+
-    torch.testing.assert_close(mock_kv_cache[1:], ref[1:], atol=1e-3, rtol=1e-3)
-
-
-@torch.inference_mode()
-def test_block_tables_with_zero_pages():
-    """Pages with index <= 0 must be skipped (early return in kernel)."""
-    from vllm.v1.attention.backends.flashinfer import (
-        trtllm_prefill_attn_kvfp8_dequant,
-    )
-
-    torch.set_default_device("cuda")
-    num_kv_heads, block_size, head_size = 8, 16, 64
-
-    kv_cache, scale = make_contiguous_kv_cache(
-        NUM_BLOCKS,
-        num_kv_heads,
-        block_size,
-        head_size,
-    )
-    k_scale = v_scale = scale.clone()
-
-    # Mix of valid pages and zeros (padding)
-    block_tables = torch.tensor(
-        [[5, 0, 10], [0, 0, 0], [3, 7, 0]],
-        dtype=torch.int32,
-        device="cuda",
-    )
-
-    mock_kv_cache, _ = trtllm_prefill_attn_kvfp8_dequant(
-        kv_cache,
-        block_tables,
-        k_scale,
-        v_scale,
-        torch.bfloat16,
-    )
-    ref = ref_dequant(kv_cache, block_tables, k_scale, v_scale, torch.bfloat16)
-
-    # Only compare pages that were actually written (non-zero page indices)
-    for b in range(block_tables.shape[0]):
-        for p in range(block_tables.shape[1]):
-            if block_tables[b, p].item() > 0:
-                idx = b * block_tables.shape[1] + p + 1
-                torch.testing.assert_close(
-                    mock_kv_cache[idx],
-                    ref[idx],
-                    atol=1e-3,
-                    rtol=1e-3,
-                )
-
-
-@torch.inference_mode()
-def test_all_zero_block_tables():
-    """All-zero block_tables: kernel should write nothing."""
-    from vllm.v1.attention.backends.flashinfer import (
-        trtllm_prefill_attn_kvfp8_dequant,
-    )
-
-    torch.set_default_device("cuda")
-    num_kv_heads, block_size, head_size = 4, 16, 64
-
-    kv_cache, scale = make_contiguous_kv_cache(
-        NUM_BLOCKS,
-        num_kv_heads,
-        block_size,
-        head_size,
-    )
-    k_scale = v_scale = scale.clone()
-
-    block_tables = torch.zeros(2, 4, dtype=torch.int32, device="cuda")
-
-    # Should not crash even though no pages are valid
-    mock_kv_cache, mock_block_table = trtllm_prefill_attn_kvfp8_dequant(
-        kv_cache,
-        block_tables,
-        k_scale,
-        v_scale,
-        torch.bfloat16,
-    )
-    assert mock_kv_cache.shape[0] == 2 * 4 + 1
-    assert mock_block_table.shape == (2, 4)
-
-
-@torch.inference_mode()
-def test_different_k_v_scales():
-    """Verify K and V are dequantized with independent scales."""
-    from vllm.v1.attention.backends.flashinfer import (
-        trtllm_prefill_attn_kvfp8_dequant,
-    )
-
-    torch.set_default_device("cuda")
-    num_kv_heads, block_size, head_size = 8, 16, 64
-
-    kv_cache, _ = make_contiguous_kv_cache(
-        NUM_BLOCKS,
-        num_kv_heads,
-        block_size,
-        head_size,
-    )
-    k_scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
-    v_scale = torch.tensor([2.0], dtype=torch.float32, device="cuda")
-
-    block_tables = torch.tensor([[1, 2]], dtype=torch.int32, device="cuda")
-
-    mock_kv_cache, _ = trtllm_prefill_attn_kvfp8_dequant(
-        kv_cache,
-        block_tables,
-        k_scale,
-        v_scale,
-        torch.bfloat16,
-    )
-    ref = ref_dequant(kv_cache, block_tables, k_scale, v_scale, torch.bfloat16)
-
-    torch.testing.assert_close(mock_kv_cache[1:], ref[1:], atol=1e-3, rtol=1e-3)
-
-
-@torch.inference_mode()
-def test_single_page_per_seq():
-    """Minimum grid dim 1 = 1 page per sequence."""
-    from vllm.v1.attention.backends.flashinfer import (
-        trtllm_prefill_attn_kvfp8_dequant,
-    )
-
-    torch.set_default_device("cuda")
-    num_kv_heads, block_size, head_size = 8, 16, 128
-
-    kv_cache, scale = make_contiguous_kv_cache(
-        NUM_BLOCKS,
-        num_kv_heads,
-        block_size,
-        head_size,
-    )
-    k_scale = v_scale = scale.clone()
-
-    block_tables = torch.tensor([[5], [10], [20]], dtype=torch.int32, device="cuda")
-
-    mock_kv_cache, _ = trtllm_prefill_attn_kvfp8_dequant(
-        kv_cache,
-        block_tables,
-        k_scale,
-        v_scale,
-        torch.bfloat16,
-    )
-    ref = ref_dequant(kv_cache, block_tables, k_scale, v_scale, torch.bfloat16)
-
-    torch.testing.assert_close(mock_kv_cache[1:], ref[1:], atol=1e-3, rtol=1e-3)
-
-
-@torch.inference_mode()
-def test_large_page_indices():
-    """Page indices near the top of the buffer stress offset arithmetic."""
-    from vllm.v1.attention.backends.flashinfer import (
-        trtllm_prefill_attn_kvfp8_dequant,
-    )
-
-    torch.set_default_device("cuda")
-    num_kv_heads, block_size, head_size = 8, 16, 128
-    large_num_blocks = 32768
-
-    kv_cache, scale = make_contiguous_kv_cache(
-        large_num_blocks,
-        num_kv_heads,
-        block_size,
-        head_size,
-    )
-    k_scale = v_scale = scale.clone()
-
-    # Use page indices near the top of the buffer
-    block_tables = torch.tensor(
-        [[large_num_blocks - 1, large_num_blocks - 2, 1]],
-        dtype=torch.int32,
-        device="cuda",
-    )
-
-    mock_kv_cache, _ = trtllm_prefill_attn_kvfp8_dequant(
-        kv_cache,
-        block_tables,
-        k_scale,
-        v_scale,
-        torch.bfloat16,
-    )
-    ref = ref_dequant(kv_cache, block_tables, k_scale, v_scale, torch.bfloat16)
-
-    torch.testing.assert_close(mock_kv_cache[1:], ref[1:], atol=1e-3, rtol=1e-3)
-
-
-@torch.inference_mode()
-def test_large_block_size():
-    """block_size=64 -> HEAD_STRIDE=8192, large tl.arange per thread block."""
-    from vllm.v1.attention.backends.flashinfer import (
-        trtllm_prefill_attn_kvfp8_dequant,
-    )
-
-    torch.set_default_device("cuda")
-    num_kv_heads, block_size, head_size = 4, 64, 128
-
-    kv_cache, scale = make_contiguous_kv_cache(
-        NUM_BLOCKS,
-        num_kv_heads,
-        block_size,
-        head_size,
-    )
-    k_scale = v_scale = scale.clone()
-
-    block_tables = torch.randint(
-        1,
-        NUM_BLOCKS,
-        (2, 4),
-        dtype=torch.int32,
-        device="cuda",
-    )
-
-    mock_kv_cache, _ = trtllm_prefill_attn_kvfp8_dequant(
-        kv_cache,
-        block_tables,
-        k_scale,
-        v_scale,
-        torch.bfloat16,
-    )
-    ref = ref_dequant(kv_cache, block_tables, k_scale, v_scale, torch.bfloat16)
-
-    torch.testing.assert_close(mock_kv_cache[1:], ref[1:], atol=1e-3, rtol=1e-3)
-
-
-@torch.inference_mode()
-def test_cross_layer_many_layers():
-    """
-    Non-contiguous with 36 layers -- matches real gpt-oss-120b.
-    Strides are far from contiguous (factor of 36 in the gaps).
-    """
-    from vllm.v1.attention.backends.flashinfer import (
-        trtllm_prefill_attn_kvfp8_dequant,
-    )
-
-    torch.set_default_device("cuda")
-    num_kv_heads, block_size, head_size = 8, 16, 64
-    num_layers = 36
-
-    kv_cache, scale = make_cross_layer_kv_cache(
-        NUM_BLOCKS,
-        num_kv_heads,
-        block_size,
-        head_size,
-        num_layers=num_layers,
-    )
-    k_scale = v_scale = scale.clone()
-
-    block_tables = torch.randint(
-        1,
-        NUM_BLOCKS,
-        (4, 6),
-        dtype=torch.int32,
-        device="cuda",
-    )
-
-    mock_kv_cache, _ = trtllm_prefill_attn_kvfp8_dequant(
-        kv_cache,
-        block_tables,
-        k_scale,
-        v_scale,
-        torch.bfloat16,
-    )
-    ref = ref_dequant(kv_cache, block_tables, k_scale, v_scale, torch.bfloat16)
-
-    torch.testing.assert_close(mock_kv_cache[1:], ref[1:], atol=1e-3, rtol=1e-3)
@@ -280,22 +280,21 @@ def test_rms_norm(
        assert torch.allclose(ref_residual, ops_residual)

    output = torch.empty(x.shape, dtype=quant_dtype, device=x.device)
+    scales = torch.empty(
+        (x.numel() // x.shape[-1], 1), device=x.device, dtype=torch.float32
+    )
+
    if group_size is None:
-        scales = torch.empty(
-            (x.numel() // x.shape[-1], 1), device=x.device, dtype=torch.float32
-        )
        opcheck(
            torch.ops._C.rms_norm_dynamic_per_token_quant,
            (output, x, layer.weight, scales, 1e-5, scale_ub, residual),
        )
    else:
-        assert hidden_size % group_size[1] == 0
-        num_groups = hidden_size // group_size[1]
-        scales = torch.empty(
-            (num_groups, num_tokens),
-            device=x.device,
-            dtype=torch.float32,
-        ).transpose(0, 1)
+        # TODO(luka/eliza) opcheck is broken?
+        #  Somehow the cloned args are getting mutated in-place,
+        #  which causes the opcheck to fail.
+        # https://github.com/vllm-project/vllm/issues/36688
+        return
        opcheck(
            torch.ops._C.rms_norm_per_block_quant,
            (
@@ -22,7 +22,7 @@ INTERMEDIATE_DIM = [128, 2880]
 BATCH_SIZE = [1, 64, 256]
 ACT = [MoEActivation.SILU, MoEActivation.SWIGLUOAI]
 USE_BIAS = [True, False]
-ISA = ["amx", "vec"] if torch.cpu._is_amx_tile_supported() else ["vec"]
+ISA = ["amx", "vec"] if torch._C._cpu._is_amx_tile_supported() else ["vec"]
 DTYPE = [torch.bfloat16]


@@ -711,192 +711,3 @@ def test_packed_loras(default_vllm_config, dist_init, dummy_model_gate_up, devic
    torch.testing.assert_close(
        packed_lora1.lora_b[1], model_lora_clone1.get_lora("up_proj").lora_b
    )
-
-
-def _test_target_modules(
-    model,
-    target_modules: list[str] | None,
-    device: str,
-    expected_lora: list[tuple[str, type]],
-    expected_no_lora: list[tuple[str, type]],
-):
-    """Create a LoRAModelManager and assert which modules have LoRA applied."""
-    LoRAModelManager(
-        model,
-        2,
-        2,
-        2,
-        LoRAConfig(
-            max_lora_rank=8,
-            max_cpu_loras=2,
-            max_loras=2,
-            lora_dtype=DEFAULT_DTYPE,
-            target_modules=target_modules,
-        ),
-        device=device,
-    )
-    for module_path, lora_cls in expected_lora:
-        assert isinstance(model.get_submodule(module_path), lora_cls)
-    for module_path, lora_cls in expected_no_lora:
-        assert not isinstance(model.get_submodule(module_path), lora_cls)
-
-
-@pytest.mark.parametrize("device", DEVICES)
-def test_target_modules_config(default_vllm_config, dist_init, dummy_model, device):
-    """Test that target_modules config restricts which modules get LoRA applied."""
-    _test_target_modules(
-        dummy_model,
-        ["dense1"],
-        device,
-        expected_lora=[
-            ("dense1", ColumnParallelLinearWithLoRA),
-            ("layer1.dense1", ColumnParallelLinearWithLoRA),
-        ],
-        expected_no_lora=[
-            ("dense2", RowParallelLinearWithLoRA),
-            ("layer1.dense2", RowParallelLinearWithLoRA),
-        ],
-    )
-
-
-@pytest.mark.parametrize("device", DEVICES)
-def test_target_modules_multiple(default_vllm_config, dist_init, dummy_model, device):
-    """Test that multiple target_modules work correctly."""
-    _test_target_modules(
-        dummy_model,
-        ["dense1", "dense2"],
-        device,
-        expected_lora=[
-            ("dense1", ColumnParallelLinearWithLoRA),
-            ("layer1.dense1", ColumnParallelLinearWithLoRA),
-            ("dense2", RowParallelLinearWithLoRA),
-            ("layer1.dense2", RowParallelLinearWithLoRA),
-        ],
-        expected_no_lora=[],
-    )
-
-
-@pytest.mark.parametrize("device", DEVICES)
-def test_target_modules_none_uses_all(
-    default_vllm_config, dist_init, dummy_model, device
-):
-    """Test that target_modules=None uses all supported modules."""
-    _test_target_modules(
-        dummy_model,
-        None,
-        device,
-        expected_lora=[
-            ("dense1", ColumnParallelLinearWithLoRA),
-            ("layer1.dense1", ColumnParallelLinearWithLoRA),
-            ("dense2", RowParallelLinearWithLoRA),
-            ("layer1.dense2", RowParallelLinearWithLoRA),
-        ],
-        expected_no_lora=[],
-    )
-
-
-@pytest.mark.parametrize("device", DEVICES)
-def test_load_adapter_warns_on_unsupported_modules(
-    default_vllm_config, dist_init, dummy_model_gate_up, device, tmp_path
-):
-    """Test that _load_adapter warns when a LoRA adapter contains modules
-    not in the model's supported LoRA target modules."""
-    from unittest.mock import patch
-
-    import vllm.lora.worker_manager as wm_module
-
-    lora_config = LoRAConfig(
-        max_lora_rank=8, max_cpu_loras=4, max_loras=4, lora_dtype=DEFAULT_DTYPE
-    )
-
-    dummy_lora_files = f"{tmp_path}/lora_adapter"
-    os.makedirs(dummy_lora_files, exist_ok=True)
-    create_peft_lora(
-        dummy_model_gate_up,
-        save_dir=dummy_lora_files,
-        target_modules=["layer1.dense1", "dense2"],
-        lora_dtype=DEFAULT_DTYPE,
-    )
-
-    model_config = ModelConfig(max_model_len=16)
-    vllm_config = VllmConfig(model_config=model_config, lora_config=lora_config)
-    vllm_config.scheduler_config.max_num_seqs = 4
-    vllm_config.scheduler_config.max_num_batched_tokens = 2
-
-    worker_manager = WorkerLoRAManager(vllm_config, device, EMBEDDING_MODULES)
-    worker_manager.vocab_size = dummy_model_gate_up.unpadded_vocab_size
-    worker_manager.create_lora_manager(dummy_model_gate_up)
-
-    # Patch from_local_checkpoint to inject an unsupported module
-    original_from_checkpoint = LoRAModel.from_local_checkpoint
-
-    def patched_from_checkpoint(*args, **kwargs):
-        lora = original_from_checkpoint(*args, **kwargs)
-        lora.loras["unsupported_module"] = LoRALayerWeights(
-            module_name="unsupported_module",
-            rank=8,
-            lora_alpha=16,
-            lora_a=torch.randn(8, 10),
-            lora_b=torch.randn(10, 8),
-        )
-        return lora
-
-    lora_request = LoRARequest("test", 1, dummy_lora_files)
-    with (
-        patch.object(LoRAModel, "from_local_checkpoint", patched_from_checkpoint),
-        patch.object(wm_module.logger, "warning_once") as mock_warning,
-    ):
-        worker_manager._load_adapter(lora_request)
-        warning_args = mock_warning.call_args_list
-        found = any("unsupported_module" in str(call) for call in warning_args)
-        assert found, (
-            f"Expected warning about 'unsupported_module', got: {warning_args}"
-        )
-
-
-@pytest.mark.parametrize("device", DEVICES)
-def test_load_adapter_warns_on_target_modules_restriction(
-    default_vllm_config, dist_init, dummy_model_gate_up, device, tmp_path
-):
-    """Test that _load_adapter warns when a LoRA adapter contains modules
-    excluded by the deployment-time target_modules restriction."""
-    from unittest.mock import patch
-
-    import vllm.lora.worker_manager as wm_module
-
-    # Restrict to only dense2 — adapter has dense1 which will be excluded
-    lora_config = LoRAConfig(
-        max_lora_rank=8,
-        max_cpu_loras=4,
-        max_loras=4,
-        lora_dtype=DEFAULT_DTYPE,
-        target_modules=["dense2"],
-    )
-
-    dummy_lora_files = f"{tmp_path}/lora_adapter"
-    os.makedirs(dummy_lora_files, exist_ok=True)
-    create_peft_lora(
-        dummy_model_gate_up,
-        save_dir=dummy_lora_files,
-        target_modules=["layer1.dense1", "dense2"],
-        lora_dtype=DEFAULT_DTYPE,
-    )
-
-    model_config = ModelConfig(max_model_len=16)
-    vllm_config = VllmConfig(model_config=model_config, lora_config=lora_config)
-    vllm_config.scheduler_config.max_num_seqs = 4
-    vllm_config.scheduler_config.max_num_batched_tokens = 2
-
-    worker_manager = WorkerLoRAManager(vllm_config, device, EMBEDDING_MODULES)
-    worker_manager.vocab_size = dummy_model_gate_up.unpadded_vocab_size
-    worker_manager.create_lora_manager(dummy_model_gate_up)
-
-    lora_request = LoRARequest("test", 1, dummy_lora_files)
-    with patch.object(wm_module.logger, "warning_once") as mock_warning:
-        worker_manager._load_adapter(lora_request)
-        warning_args = mock_warning.call_args_list
-        # dense1 is supported by the model but excluded by target_modules
-        found = any("target_modules" in str(call) for call in warning_args)
-        assert found, (
-            f"Expected warning about target_modules restriction, got: {warning_args}"
-        )
@@ -1,60 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-
-from vllm.lora.utils import is_in_target_modules, is_supported_lora_module
-
-
-class TestIsSupportedLoraModule:
-    """Tests for is_supported_lora_module (model-definition check)."""
-
-    def test_suffix_match(self):
-        assert is_supported_lora_module(
-            "model.layers.0.self_attn.o_proj", ["o_proj", "q_proj"]
-        )
-
-    def test_no_match(self):
-        assert not is_supported_lora_module(
-            "model.layers.0.self_attn.o_proj", ["q_proj", "k_proj"]
-        )
-
-    def test_exact_match(self):
-        assert is_supported_lora_module("o_proj", ["o_proj"])
-
-    def test_regex_suffix_matching(self):
-        """Regex anchors to end — partial suffix should not match."""
-        assert not is_supported_lora_module("model.layers.0.self_attn.o_proj", ["proj"])
-
-    def test_empty_supported_modules(self):
-        assert not is_supported_lora_module("model.layers.0.self_attn.o_proj", [])
-
-    def test_multiple_supported_modules(self):
-        supported = ["q_proj", "k_proj", "v_proj", "o_proj"]
-        assert is_supported_lora_module("model.layers.0.self_attn.v_proj", supported)
-        assert not is_supported_lora_module("model.layers.0.mlp.gate_proj", supported)
-
-
-class TestIsInTargetModules:
-    """Tests for is_in_target_modules (deployment-time filter)."""
-
-    def test_none_allows_all(self):
-        assert is_in_target_modules("model.layers.0.self_attn.o_proj", None)
-
-    def test_suffix_in_target(self):
-        assert is_in_target_modules(
-            "model.layers.0.self_attn.o_proj", ["o_proj", "q_proj"]
-        )
-
-    def test_suffix_not_in_target(self):
-        assert not is_in_target_modules(
-            "model.layers.0.self_attn.o_proj", ["q_proj", "k_proj"]
-        )
-
-    def test_empty_target_modules(self):
-        assert not is_in_target_modules("model.layers.0.self_attn.o_proj", [])
-
-    def test_exact_name_match(self):
-        assert is_in_target_modules("dense1", ["dense1", "dense2"])
-
-    def test_exact_name_no_match(self):
-        assert not is_in_target_modules("dense3", ["dense1", "dense2"])
@@ -777,6 +777,7 @@ VLM_TEST_SETTINGS = {
        max_model_len=8192,
        max_num_seqs=2,
        auto_cls=AutoModelForCausalLM,
+        patch_hf_runner=model_utils.paddleocr_vl_patch_hf_runner,
        image_size_factors=[(0.25,)],
        marks=[
            pytest.mark.skipif(
@@ -1149,6 +1149,31 @@ def ovis2_5_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
    return hf_model


+def paddleocr_vl_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
+    """Patches the HfRunner to fix create_causal_mask API mismatch.
+
+    The PaddleOCR-VL HF model passes `inputs_embeds` to create_causal_mask,
+    but transformers renamed this parameter to `input_embeds`.
+    """
+    import sys
+
+    model_module = sys.modules.get(type(hf_model.model.model).__module__)
+    if model_module is None:
+        return hf_model
+
+    original_create_causal_mask = getattr(model_module, "create_causal_mask", None)
+    if original_create_causal_mask is None:
+        return hf_model
+
+    def patched_create_causal_mask(*args, **kwargs):
+        if "inputs_embeds" in kwargs:
+            kwargs["input_embeds"] = kwargs.pop("inputs_embeds")
+        return original_create_causal_mask(*args, **kwargs)
+
+    model_module.create_causal_mask = patched_create_causal_mask  # type: ignore[attr-defined]
+    return hf_model
+
+
 def qwen2_5_omni_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
    """Patches and returns an instance of the HfRunner for Qwen2.5-Omni."""
    thinker = hf_model.model.thinker
@@ -1235,9 +1260,9 @@ def voxtral_patch_hf_runner(hf_model: "HfRunner") -> "HfRunner":
    generated).
    """

+    import base64
    import io

-    import pybase64 as base64
    import soundfile as sf

    processor = hf_model.processor
@@ -7,9 +7,9 @@ ColPali is a multi-vector retrieval model based on PaliGemma backbone
 It produces per-token embeddings for both text and image inputs.
 """

+import base64
 from io import BytesIO

-import pybase64 as base64
 import pytest
 import torch
 from PIL import Image
@@ -7,9 +7,9 @@ ColBERT-style late interaction scoring (MaxSim). It produces per-token
 embeddings for both text and image inputs.
 """

+import base64
 from io import BytesIO

-import pybase64 as base64
 import pytest
 import torch
 from PIL import Image
@@ -9,10 +9,10 @@ Tests for the LlamaNemotronVL model family:
 Both variants share a SigLIP vision encoder with a bidirectional LLaMA backbone.
 """

+import base64
 from io import BytesIO
 from pathlib import Path

-import pybase64 as base64
 import pytest
 import torch
 from transformers import AutoModel, AutoModelForSequenceClassification, AutoProcessor
@@ -3,7 +3,6 @@

 import pytest
 import torch.nn.functional as F
-import transformers.utils
 from PIL import Image

 from vllm.assets.base import get_vllm_public_assets
@@ -13,12 +12,6 @@ from ....conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
 from ....utils import large_gpu_test
 from ...utils import check_embeddings_close

-# BC for method that was deleted in Transformers v5.
-# Only needed for generating the HF reference.
-transformers.utils.is_flash_attn_greater_or_equal_2_10 = (
-    lambda: transformers.utils.is_flash_attn_greater_or_equal("2.1.0")
-)
-
 HF_TEXT_PROMPTS = [
    # T -> X
    "Find me an everyday image that matches the given caption: The label of the object is stop sign",  # noqa: E501
@@ -23,7 +23,7 @@ def _get_expected_num_patches(
    min_num: int,
    max_num: int,
 ):
-    from vllm.transformers_utils.processors.h2ovl import (
+    from vllm.model_executor.models.h2ovl import (
        calculate_h2ovl_targets,
        get_h2ovl_target_ratios,
    )
@@ -23,7 +23,7 @@ def _get_expected_num_patches(
    min_num: int,
    max_num: int,
 ):
-    from vllm.transformers_utils.processors.internvl import (
+    from vllm.model_executor.models.internvl import (
        calculate_internvl_targets,
        get_internvl_target_ratios,
    )
@@ -23,7 +23,7 @@ def _get_expected_num_patches(
    min_num: int,
    max_num: int,
 ):
-    from vllm.transformers_utils.processors.nemotron_vl import (
+    from vllm.model_executor.models.nemotron_vl import (
        calculate_nemotron_vl_targets,
        get_nemotron_vl_target_ratios,
    )
@@ -185,16 +185,14 @@ def make_mock_model(hidden: int = 8):

    # super().embed_input_ids → use SupportsMultiModal.embed_input_ids
    def fake_super_embed(
-        ids,
-        mm_embs=None,
-        *,
-        is_multimodal=None,
+        ids, mm_embs=None, *, is_multimodal=None, handle_oov_mm_token=False
    ):
        return SupportsMultiModal.embed_input_ids(
            model,
            ids,
            mm_embs,
            is_multimodal=is_multimodal,
+            handle_oov_mm_token=handle_oov_mm_token,
        )

    # Bind embed_input_ids as the real method
@@ -1,104 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-"""E2E tests for online MXFP8 quantization.
-
-Loads a BF16 model with ``--quantization mxfp8`` (online quantization) and
-compares log-probabilities against the same model served in BF16 without
-quantization.  This exercises the full pipeline: config parsing,
-``Mxfp8OnlineLinearMethod``, ``Mxfp8OnlineMoEMethod``, weight loading,
-online quantization / shuffling, and inference through ``apply_monolithic``.
-
-Layer skipping (``modules_to_not_convert``) is configured in the model's
-``config.json`` under ``quantization_config`` and is not tested here.
-
-``example_prompts`` is a pytest fixture (from conftest.py) that loads 8
-diverse prompts from ``tests/prompts/example.txt``.
-"""
-
-import pytest
-
-from tests.quantization.utils import is_quant_method_supported
-
-from ..utils import check_logprobs_close
-
-# A small MoE model that fits on a single GPU and has both linear + MoE layers.
-MOE_MODEL = "Qwen/Qwen3-30B-A3B"
-# A small dense model (no MoE) to validate the linear-only path.
-DENSE_MODEL = "Qwen/Qwen3-0.6B"
-
-MAX_MODEL_LEN = 1024
-MAX_TOKENS = 4
-NUM_LOG_PROBS = 8
-
-
-@pytest.mark.skipif(
-    not is_quant_method_supported("mxfp8"),
-    reason="mxfp8 is not supported on this GPU type (requires sm_100+).",
-)
-@pytest.mark.quant_model
-@pytest.mark.parametrize("model", [DENSE_MODEL, MOE_MODEL], ids=["dense", "moe"])
-def test_mxfp8_logprobs(
-    vllm_runner,
-    example_prompts,
-    model: str,
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    """Compare BF16 baseline logprobs against online MXFP8-quantized model.
-
-    Runs the same model twice -- once in BF16 (baseline) and once with
-    online MXFP8 quantization -- then checks that the top log-probabilities
-    are close.  Only 4 tokens are generated to keep the test fast while
-    still catching numerical divergence.
-    """
-    with monkeypatch.context() as m:
-        m.setenv("TOKENIZERS_PARALLELISM", "true")
-
-        with vllm_runner(
-            model,
-            max_model_len=MAX_MODEL_LEN,
-            enforce_eager=True,
-        ) as vllm_model:
-            baseline_outputs = vllm_model.generate_greedy_logprobs(
-                example_prompts, MAX_TOKENS, NUM_LOG_PROBS
-            )
-
-        with vllm_runner(
-            model,
-            max_model_len=MAX_MODEL_LEN,
-            enforce_eager=True,
-            quantization="mxfp8",
-        ) as vllm_model:
-            test_outputs = vllm_model.generate_greedy_logprobs(
-                example_prompts, MAX_TOKENS, NUM_LOG_PROBS
-            )
-
-        check_logprobs_close(
-            outputs_0_lst=baseline_outputs,
-            outputs_1_lst=test_outputs,
-            name_0="bf16",
-            name_1="mxfp8",
-        )
-
-
-@pytest.mark.skipif(
-    not is_quant_method_supported("mxfp8"),
-    reason="mxfp8 is not supported on this GPU type (requires sm_100+).",
-)
-@pytest.mark.quant_model
-@pytest.mark.parametrize("model", [DENSE_MODEL, MOE_MODEL], ids=["dense", "moe"])
-def test_mxfp8_generation(vllm_runner, model: str) -> None:
-    """Smoke test: verify online MXFP8 model generates coherent text."""
-    prompt = "1 2 3 4 5"
-    with vllm_runner(
-        model,
-        enforce_eager=True,
-        quantization="mxfp8",
-        max_model_len=MAX_MODEL_LEN,
-    ) as vllm_model:
-        output = vllm_model.generate_greedy([prompt], max_tokens=5)
-
-    generated = output[0][1]
-    assert len(generated) > len(prompt), (
-        f"MXFP8 model produced no new tokens. Output: {generated!r}"
-    )
@@ -1,11 +1,11 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import base64
 from pathlib import Path
 from unittest.mock import patch

 import librosa
 import numpy as np
-import pybase64 as base64
 import pytest

 from vllm.multimodal.media import AudioMediaIO
@@ -2,13 +2,13 @@
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

 import asyncio
+import base64
 import mimetypes
 import os
 from tempfile import NamedTemporaryFile, TemporaryDirectory

 import aiohttp
 import numpy as np
-import pybase64 as base64
 import pytest
 import requests
 import torch
@@ -3,10 +3,10 @@

 from collections.abc import Sequence

-from vllm.config import ModelConfig, PoolerConfig, VllmConfig
+from vllm.config import VllmConfig
 from vllm.entrypoints.openai.engine.protocol import UsageInfo
-from vllm.entrypoints.pooling.base.protocol import EmbedRequestMixin
 from vllm.inputs.data import PromptType
+from vllm.logger import init_logger
 from vllm.outputs import PoolingRequestOutput
 from vllm.plugins.io_processors.interface import (
    IOProcessor,
@@ -16,13 +16,14 @@ from vllm.renderers import BaseRenderer
 from vllm.tokenizers.detokenizer_utils import convert_ids_list_to_tokens

 from .types import (
-    EMBED_TASKS,
    SparseEmbeddingCompletionRequestMixin,
    SparseEmbeddingResponse,
    SparseEmbeddingResponseData,
    SparseEmbeddingTokenWeight,
 )

+logger = init_logger(__name__)
+

 class BgeM3SparseEmbeddingsProcessor(
    IOProcessor[SparseEmbeddingCompletionRequestMixin, SparseEmbeddingResponse]
@@ -32,22 +33,6 @@ class BgeM3SparseEmbeddingsProcessor(
        self.offline_requests: list[SparseEmbeddingCompletionRequestMixin] = []
        self.online_requests: dict[str, SparseEmbeddingCompletionRequestMixin] = {}
        self.renderer: BaseRenderer = renderer
-        self.default_pooling_params = {}
-        pooler_config: PoolerConfig = vllm_config.model_config.pooler_config
-        if pooler_config is not None:
-            for param in ["use_activation", "dimensions"]:
-                if getattr(pooler_config, param, None) is None:
-                    continue
-                self.default_pooling_params[param] = getattr(pooler_config, param)
-        self.embed_dimensions = vllm_config.model_config.embedding_size
-        self.embed_request_queue: list[EmbedRequestMixin] = []
-
-    def __repr__(self) -> str:
-        return (
-            f"BgeM3SparseEmbeddingsProcessor("
-            f"embed_dimensions={self.embed_dimensions}, "
-            f"default_pooling_params={self.default_pooling_params})"
-        )

    def merge_pooling_params(
        self,
@@ -56,57 +41,7 @@ class BgeM3SparseEmbeddingsProcessor(
        if params is None:
            params = PoolingParams()
        # refer to PoolingCompletionRequest.to_pooling_params
-        # set and verify pooling params
-        params.skip_reading_prefix_cache = True
-
-        raw_embed_request = self.embed_request_queue.pop(0)
-        if raw_embed_request.embed_task not in EMBED_TASKS:
-            raise ValueError(
-                f"Unsupported task {raw_embed_request}, "
-                f"Supported tasks are {EMBED_TASKS}"
-            )
-        has_dense_embed = True
-        if raw_embed_request.embed_task == "dense":
-            params.task = "embed"
-            params.skip_reading_prefix_cache = False
-        elif raw_embed_request.embed_task == "sparse":
-            params.task = "token_classify"
-            has_dense_embed = False
-        else:
-            params.task = "embed&token_classify"
-        params.use_activation = raw_embed_request.use_activation
-        if params.use_activation is None:
-            params.use_activation = True
-        if not has_dense_embed:
-            params.dimensions = None
-            return params
-
-        params.dimensions = raw_embed_request.dimensions
-
-        model_config: ModelConfig = self.vllm_config.model_config
-        for param in self.default_pooling_params:
-            if getattr(params, param, None) is None:
-                setattr(params, param, self.default_pooling_params[param])
-
-        if params.dimensions is not None:
-            if not model_config.is_matryoshka:
-                raise ValueError(
-                    f'Model "{model_config.served_model_name}" does not '
-                    f"support matryoshka representation, "
-                    f"changing output dimensions will lead to poor results."
-                )
-
-            mds = model_config.matryoshka_dimensions
-            if mds is not None:
-                if params.dimensions not in mds:
-                    raise ValueError(
-                        f"Model {model_config.served_model_name!r} "
-                        f"only supports {str(mds)} matryoshka dimensions, "
-                        f"use other output dimensions will "
-                        f"lead to poor results."
-                    )
-            elif params.dimensions < 1:
-                raise ValueError("Dimensions must be greater than 0")
+        params.task = "token_classify"
        return params

    def parse_request(
@@ -126,16 +61,14 @@ class BgeM3SparseEmbeddingsProcessor(
        if request_id is not None:
            assert request_id not in self.online_requests, "request_id duplicated"
            self.online_requests[request_id] = prompt
-            self.embed_request_queue.extend(prompt.to_embed_requests_online())
        else:
            self.offline_requests.append(prompt)
-            self.embed_request_queue.extend(prompt.to_embed_requests_offline())
        return prompt.input

    def _get_sparse_embedding_request(self, request_id: str | None = None):
        if request_id:
            return self.online_requests.pop(request_id, None)
-        return self.offline_requests.pop(0)
+        return self.offline_requests.pop()

    def _build_sparse_embedding_token_weights(
        self,
@@ -167,45 +100,26 @@ class BgeM3SparseEmbeddingsProcessor(
    ) -> SparseEmbeddingResponse:
        num_prompt_tokens = 0
        response_data = []
-        raw_request = self._get_sparse_embedding_request(request_id)
-        has_dense_embed = raw_request.embed_task in ["dense", "dense&sparse"]
-        has_sparse_embed = raw_request.embed_task in ["sparse", "dense&sparse"]
-        embed_dimensions = 0
-        if has_dense_embed:
-            embed_dimensions = (
-                self.embed_dimensions
-                if raw_request.dimensions is None
-                else raw_request.dimensions
-            )
+        return_tokens = self._get_sparse_embedding_request(request_id).return_tokens
        for idx in range(len(model_output)):
            mo = model_output[idx]
-            sparse_embedding_dict: dict[int, float] = {}
+            sparse_embedding: dict[int, float] = {}
            num_prompt_tokens += len(mo.prompt_token_ids)
-            dense_embedding: list[float] | None = None
-            sparse_embedding: list[SparseEmbeddingTokenWeight] | None = None
-            if has_dense_embed:
-                dense_embedding = mo.outputs.data[:embed_dimensions].tolist()
-            if has_sparse_embed:
-                sparse_weights = mo.outputs.data[embed_dimensions:].tolist()
-                if len(mo.prompt_token_ids) != len(sparse_weights):
-                    # this is the case that add_special_tokens is True,
-                    # which means first token and last token are special tokens
-                    mo.prompt_token_ids = mo.prompt_token_ids[1:]
-                for token_id, weight in zip(mo.prompt_token_ids, sparse_weights):
-                    sparse_embedding_dict[token_id] = max(
-                        weight, sparse_embedding_dict.get(token_id, 0.0)
-                    )
-                sparse_embedding = self._build_sparse_embedding_token_weights(
-                    sparse_embedding_dict,
-                    raw_request.return_tokens,
+            if len(mo.prompt_token_ids) != len(mo.outputs.data):
+                # this is the case that add_special_tokens is True,
+                # which means first token and last token are special tokens
+                mo.prompt_token_ids = mo.prompt_token_ids[1:]
+            for token_id, weight in zip(mo.prompt_token_ids, mo.outputs.data.tolist()):
+                sparse_embedding[token_id] = max(
+                    weight, sparse_embedding.get(token_id, 0.0)
                )
-
            response_data.append(
                SparseEmbeddingResponseData(
                    index=idx,
-                    object=raw_request.embed_task,
-                    sparse_embedding=sparse_embedding,
-                    dense_embedding=dense_embedding,
+                    sparse_embedding=self._build_sparse_embedding_token_weights(
+                        sparse_embedding,
+                        return_tokens,
+                    ),
                )
            )

@@ -1,44 +1,18 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

-from typing import Literal, get_args
-
 from pydantic import BaseModel, Field

 from vllm.entrypoints.openai.engine.protocol import UsageInfo
-from vllm.entrypoints.pooling.base.protocol import (
-    CompletionRequestMixin,
-    EmbedRequestMixin,
-)
-
-EmbedTask = Literal[
-    "sparse",
-    "dense",
-    "dense&sparse",
-]
-
-EMBED_TASKS: tuple[EmbedTask, ...] = get_args(EmbedTask)
+from vllm.entrypoints.pooling.base.protocol import CompletionRequestMixin


-class SparseEmbeddingCompletionRequestMixin(CompletionRequestMixin, EmbedRequestMixin):
+class SparseEmbeddingCompletionRequestMixin(CompletionRequestMixin):
    return_tokens: bool | None = Field(
        default=None,
        description="Whether to return dict shows the mapping of token_id to text."
        "`None` or False means not return.",
    )
-    embed_task: EmbedTask = Field(
-        default="dense&sparse",
-        description="embed task, can be one of 'sparse', 'dense' , 'dense&sparse', "
-        "default to 'dense&sparse'",
-    )
-
-    def to_embed_requests_offline(self) -> list[EmbedRequestMixin]:
-        if isinstance(self.input, list):
-            return [self] * len(self.input)
-        return [self]
-
-    def to_embed_requests_online(self) -> list[EmbedRequestMixin]:
-        return [self]


 class SparseEmbeddingTokenWeight(BaseModel):
@@ -49,9 +23,8 @@ class SparseEmbeddingTokenWeight(BaseModel):

 class SparseEmbeddingResponseData(BaseModel):
    index: int
-    object: str = "dense&sparse"
-    sparse_embedding: list[SparseEmbeddingTokenWeight] | None
-    dense_embedding: list[float] | None
+    object: str = "sparse-embedding"
+    sparse_embedding: list[SparseEmbeddingTokenWeight]


 class SparseEmbeddingResponse(BaseModel):
@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

+import base64
 import datetime
 import os
 import tempfile
@@ -10,7 +11,6 @@ from typing import Any

 import albumentations
 import numpy as np
-import pybase64 as base64
 import rasterio
 import regex as re
 import torch
@@ -19,12 +19,6 @@ model_config = {
    ),
 }

-dense_embedding_sum = [
-    -0.7214539647102356,  # "What is the capital of France?"
-    -0.6926871538162231,  # "What is the capital of Germany?"
-    -0.7129564881324768,  # "What is the capital of Spain?"
-]
-

 def _float_close(expected: object, result: object):
    assert isinstance(expected, float) and isinstance(result, float), (
@@ -39,12 +33,6 @@ def _get_attr_or_val(obj: object | dict, key: str):
    return getattr(obj, key, None)


-def _check_dense_embedding(data, index=0):
-    assert _float_close(sum(data), dense_embedding_sum[index]), (
-        "dense-embedding result not match"
-    )
-
-
 def _check_sparse_embedding(data, check_tokens=False):
    expected_weights = [
        {"token_id": 32, "weight": 0.0552978515625, "token": "?"},
@@ -121,7 +109,7 @@ async def test_bge_m3_sparse_plugin_online(
    assert len(_get_attr_or_val(parsed_response, "data")) > 0

    data_entry = _get_attr_or_val(parsed_response, "data")[0]
-    assert _get_attr_or_val(data_entry, "object") == "dense&sparse"
+    assert _get_attr_or_val(data_entry, "object") == "sparse-embedding"
    assert _get_attr_or_val(data_entry, "sparse_embedding")

    # Verify sparse embedding format
@@ -129,11 +117,6 @@ async def test_bge_m3_sparse_plugin_online(
    assert isinstance(sparse_embedding, list)
    _check_sparse_embedding(sparse_embedding, return_tokens)

-    # Verify dense embedding format
-    dense_embedding = _get_attr_or_val(data_entry, "dense_embedding")
-    assert isinstance(dense_embedding, list)
-    _check_dense_embedding(dense_embedding)
-
    # Verify usage information
    usage = _get_attr_or_val(parsed_response, "usage")
    assert usage, f"usage not found for {parsed_response}"
@@ -181,9 +164,6 @@ def test_bge_m3_sparse_plugin_offline(vllm_runner, return_tokens: bool):
        sparse_embedding = output.sparse_embedding
        assert isinstance(sparse_embedding, list)
        _check_sparse_embedding(sparse_embedding, return_tokens)
-        dense_embedding = output.dense_embedding
-        assert isinstance(dense_embedding, list)
-        _check_dense_embedding(dense_embedding)

    # Verify usage
    assert response.usage.prompt_tokens > 0
@@ -226,9 +206,6 @@ def test_bge_m3_sparse_plugin_offline_multiple_inputs(vllm_runner):
        # Each output should have sparse embeddings
        sparse_embedding = output.sparse_embedding
        assert isinstance(sparse_embedding, list)
-        dense_embedding = output.dense_embedding
-        assert isinstance(dense_embedding, list)
-        _check_dense_embedding(dense_embedding, i)

    # Verify usage
    assert response.usage.prompt_tokens > 0
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
khluu	bcf2be9612	[cherry-pick][Bugfix] Disable monolithic TRTLLM MoE for Renormalize routing (#37591 )#37605 Signed-off-by: khluu <khluu000@gmail.com>	2026-03-19 15:06:38 -07:00
Elvir Crnčevićandkhluu	89138b21cc	[Bugfix] Zero-init MLA attention output buffers to prevent NaN from CUDA graph padding (#37442 ) Signed-off-by: Elvir Crncevic <elvircrn@gmail.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com> (cherry picked from commit `ef2c4f778d`)	2026-03-18 18:44:16 -07:00
JartXandkhluu	6edd43de3c	[Bugfix][ROCm] Fix worker startup OOM on ROCm by skipping unreliable cudagraph memory profiling (#36720 ) Signed-off-by: JartX <sagformas@epdcenter.es> (cherry picked from commit `e8f9dbc369`)	2026-03-18 18:43:52 -07:00
Andreas Karatzasandkhluu	16c971dbc7	[CI] Fix PaddleOCR-VL HF test failure due to create_causal_mask API rename (#37328 ) Signed-off-by: Andreas Karatzas <akaratza@amd.com> (cherry picked from commit `eaf7c9b976`)	2026-03-18 11:04:33 -07:00
khluu	262ddd0d81	[cherry-pick][Bugfix] Fix EP weight filter breaking EPLB and NVFP4 accuracy #37322 Signed-off-by: khluu <khluu000@gmail.com>	2026-03-18 01:48:32 -07:00
Li, Jiangandkhluu	e60c1674b3	[Bugfix] Avoid OpenMP thread reallocation in CPU torch compile (#37391 ) Signed-off-by: jiang1.li <jiang1.li@intel.com> (cherry picked from commit `261801242f`)	2026-03-18 01:41:42 -07:00
Roy Wangandkhluu	faa80947f5	[Performance] Add --enable-ep-weight-filter CLI option (#37351 ) Signed-off-by: esmeetu <jasonailu87@gmail.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com> (cherry picked from commit `761e0aa7a0`)	2026-03-18 01:41:25 -07:00
Terry Gaoandkhluu	eeabf740bb	[Custom Ops] Add functional + out variant for scaled_fp4_quant (#34389 ) Signed-off-by: tianrengao <terrygao87@gmail.com> (cherry picked from commit `3e6a1e1686`)	2026-03-18 01:41:09 -07:00
Elvir Crnčevićandkhluu	cdcffafef8	Fix eplb nvfp4 experts hook (#37217 ) Signed-off-by: Elvir Crncevic <elvircrn@gmail.com> Signed-off-by: Elvir Crncevic <elvir@anthropic.com> Co-authored-by: Tyler Michael Smith <tyler@neuralmagic.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com> (cherry picked from commit `fd4d96302a`)	2026-03-18 01:40:57 -07:00
Walter Beller-Moralesandkhluu	4d22667c32	[Feature][Frontend] add support for Cohere Embed v2 API (#37074 ) Signed-off-by: walterbm <walter.beller.morales@gmail.com> (cherry picked from commit `061980c36a`)	2026-03-16 22:05:47 -07:00
Andreas Karatzasandkhluu	1fe3932c8b	[ROCm] Fix AttributeError for torch.compiler.skip_all_guards_unsafe on older PyTorch (#37219 ) Signed-off-by: Andreas Karatzas <akaratza@amd.com> (cherry picked from commit `54a62a79f7`)	2026-03-16 21:03:49 -07:00