add pdl

add
2026-03-26 16:25:35 +00:00 · 2026-03-26 03:21:46 +00:00 · 2026-03-25 02:47:31 -07:00 · 2026-03-23 22:58:27 -07:00 · 2026-03-24 07:02:51 +02:00 · 2026-03-24 05:00:12 +00:00
567 changed files with 23801 additions and 16697 deletions
@@ -1,12 +0,0 @@
-# For vllm script, with -t option (tensor parallel size).
-# bash ./run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/SparseLlama-3.1-8B-gsm8k-pruned.2of4-chnl_wts_per_tok_dyn_act_fp8-BitM -b "auto" -t 2
-model_name: "nm-testing/SparseLlama-3.1-8B-gsm8k-pruned.2of4-chnl_wts_per_tok_dyn_act_fp8-BitM"
-tasks:
- name: "gsm8k"
-  metrics:
-  - name: "exact_match,strict-match"
-    value: 0.6353
-  - name: "exact_match,flexible-extract"
-    value: 0.637
-limit: null
-num_fewshot: null 
@@ -0,0 +1 @@
+Qwen3-235B-A22B-Instruct-2507-FP8.yaml
@@ -12,7 +12,7 @@ steps:
        depends_on: ~
        id: build-wheel-arm64-cuda-12-9
        agents:
-          queue: arm64_cpu_queue_postmerge
+          queue: arm64_cpu_queue_release
        commands:
          # #NOTE: torch_cuda_arch_list is derived from upstream PyTorch build files here:
          # https://github.com/pytorch/pytorch/blob/main/.ci/aarch64_linux/aarch64_ci_build.sh#L7
@@ -27,7 +27,7 @@ steps:
        depends_on: ~
        id: build-wheel-arm64-cuda-13-0
        agents:
-          queue: arm64_cpu_queue_postmerge
+          queue: arm64_cpu_queue_release
        commands:
          # #NOTE: torch_cuda_arch_list is derived from upstream PyTorch build files here:
          # https://github.com/pytorch/pytorch/blob/main/.ci/aarch64_linux/aarch64_ci_build.sh#L7
@@ -42,7 +42,7 @@ steps:
        depends_on: ~
        id: build-wheel-arm64-cpu
        agents:
-          queue: arm64_cpu_queue_postmerge
+          queue: arm64_cpu_queue_release
        commands:
          - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg GIT_REPO_CHECK=1 --build-arg VLLM_BUILD_ACL=ON --tag vllm-ci:build-image --target vllm-build --progress plain -f docker/Dockerfile.cpu ."
          - "mkdir artifacts"
@@ -55,7 +55,7 @@ steps:
        depends_on: ~
        id: build-wheel-x86-cuda-12-9
        agents:
-          queue: cpu_queue_postmerge
+          queue: cpu_queue_release
        commands:
          - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
          - "mkdir artifacts"
@@ -68,7 +68,7 @@ steps:
        depends_on: ~
        id: build-wheel-x86-cuda-13-0
        agents:
-          queue: cpu_queue_postmerge
+          queue: cpu_queue_release
        commands:
          - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu22.04 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
          - "mkdir artifacts"
@@ -81,7 +81,7 @@ steps:
        depends_on: ~
        id: build-wheel-x86-cpu
        agents:
-          queue: cpu_queue_postmerge
+          queue: cpu_queue_release
        commands:
          - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg GIT_REPO_CHECK=1 --build-arg VLLM_CPU_X86=true --tag vllm-ci:build-image --target vllm-build --progress plain -f docker/Dockerfile.cpu ."
          - "mkdir artifacts"
@@ -97,7 +97,7 @@ steps:
        depends_on: ~
        id: build-release-image-x86
        agents:
-          queue: cpu_queue_postmerge
+          queue: cpu_queue_release
        commands:
          - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
          - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg FLASHINFER_AOT_COMPILE=true --build-arg INSTALL_KV_CONNECTORS=true --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m) --target vllm-openai --progress plain -f docker/Dockerfile ."
@@ -110,7 +110,7 @@ steps:
        depends_on: ~
        id: build-release-image-arm64
        agents:
-          queue: arm64_cpu_queue_postmerge
+          queue: arm64_cpu_queue_release
        commands:
          - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
          - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg FLASHINFER_AOT_COMPILE=true --build-arg torch_cuda_arch_list='8.7 8.9 9.0 10.0+PTX 12.0' --build-arg INSTALL_KV_CONNECTORS=true --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m) --target vllm-openai --progress plain -f docker/Dockerfile ."
@@ -120,7 +120,7 @@ steps:
        depends_on: ~
        id: build-release-image-x86-cuda-13-0
        agents:
-          queue: cpu_queue_postmerge
+          queue: cpu_queue_release
        commands:
          - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
          - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu22.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130 --target vllm-openai --progress plain -f docker/Dockerfile ."
@@ -133,7 +133,7 @@ steps:
        depends_on: ~
        id: build-release-image-arm64-cuda-13-0
        agents:
-          queue: arm64_cpu_queue_postmerge
+          queue: arm64_cpu_queue_release
        commands:
          - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
          # compute capability 12.0 for RTX-50 series / RTX PRO 6000 Blackwell, 12.1 for DGX Spark
@@ -149,7 +149,7 @@ steps:
          - block-cpu-release-image-build
          - input-release-version
        agents:
-          queue: cpu_queue_postmerge
+          queue: cpu_queue_release
        commands:
          - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
          - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg GIT_REPO_CHECK=1 --build-arg VLLM_CPU_X86=true --tag public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:$(buildkite-agent meta-data get release-version) --tag public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:latest --progress plain --target vllm-openai -f docker/Dockerfile.cpu ."
@@ -167,7 +167,7 @@ steps:
          - block-arm64-cpu-release-image-build
          - input-release-version
        agents:
-          queue: arm64_cpu_queue_postmerge
+          queue: arm64_cpu_queue_release
        commands:
          - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
          - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg GIT_REPO_CHECK=1 --tag public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:$(buildkite-agent meta-data get release-version) --tag public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:latest --progress plain --target vllm-openai -f docker/Dockerfile.cpu ."
@@ -185,7 +185,7 @@ steps:
          - build-release-image-arm64
        id: create-multi-arch-manifest
        agents:
-          queue: small_cpu_queue_postmerge
+          queue: small_cpu_queue_release
        commands:
          - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
          - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64 --amend"
@@ -196,7 +196,7 @@ steps:
          - create-multi-arch-manifest
        id: annotate-release-workflow
        agents:
-          queue: small_cpu_queue_postmerge
+          queue: small_cpu_queue_release
        commands:
          - "bash .buildkite/scripts/annotate-release.sh"

@@ -206,7 +206,7 @@ steps:
          - build-release-image-arm64-cuda-13-0
        id: create-multi-arch-manifest-cuda-13-0
        agents:
-          queue: small_cpu_queue_postmerge
+          queue: small_cpu_queue_release
        commands:
          - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
          - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu130 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64-cu130 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64-cu130 --amend"
@@ -217,7 +217,7 @@ steps:
          - create-multi-arch-manifest
        if: build.env("NIGHTLY") == "1"
        agents:
-          queue: small_cpu_queue_postmerge
+          queue: small_cpu_queue_release
        commands:
          - "bash .buildkite/scripts/push-nightly-builds.sh"
          # Clean up old nightly builds (keep only last 14)
@@ -235,7 +235,7 @@ steps:
          - create-multi-arch-manifest-cuda-13-0
        if: build.env("NIGHTLY") == "1"
        agents:
-          queue: small_cpu_queue_postmerge
+          queue: small_cpu_queue_release
        commands:
          - "bash .buildkite/scripts/push-nightly-builds.sh cu130"
          # Clean up old nightly builds (keep only last 14)
@@ -262,7 +262,7 @@ steps:
          - block-upload-release-wheels
        id: upload-release-wheels
        agents:
-          queue: small_cpu_queue_postmerge
+          queue: small_cpu_queue_release
        commands:
          - "bash .buildkite/scripts/upload-release-wheels-pypi.sh"

@@ -323,7 +323,7 @@ steps:
      - step: input-rocm-config
        allow_failure: true  # Allow failure so non-UI builds can proceed (input step is skipped)
    agents:
-      queue: cpu_queue_postmerge
+      queue: cpu_queue_release
    commands:
      # Set configuration and check cache
      - |
@@ -465,7 +465,7 @@ steps:
      - step: build-rocm-base-wheels
        allow_failure: false
    agents:
-      queue: cpu_queue_postmerge
+      queue: cpu_queue_release
    timeout_in_minutes: 180
    commands:
      # Download artifacts and prepare Docker image
@@ -575,7 +575,7 @@ steps:
      - step: build-rocm-vllm-wheel
        allow_failure: false
    agents:
-      queue: cpu_queue_postmerge
+      queue: cpu_queue_release
    timeout_in_minutes: 60
    commands:
      # Download all wheel artifacts and run upload
@@ -624,7 +624,7 @@ steps:
      - step: input-release-version
        allow_failure: true
    agents:
-      queue: cpu_queue_postmerge
+      queue: cpu_queue_release
    commands:
      - "bash .buildkite/scripts/annotate-rocm-release.sh"
    env:
@@ -641,7 +641,7 @@ steps:
    depends_on: block-generate-root-index-rocm-wheels
    id: generate-root-index-rocm-wheels
    agents:
-      queue: cpu_queue_postmerge
+      queue: cpu_queue_release
    commands:
      - "bash tools/vllm-rocm/generate-rocm-wheels-root-index.sh"
    env:
@@ -655,7 +655,7 @@ steps:
      - step: build-rocm-base-wheels
        allow_failure: false
    agents:
-      queue: cpu_queue_postmerge
+      queue: cpu_queue_release
    timeout_in_minutes: 60
    commands:
      - |
@@ -16,6 +16,23 @@ RAY_BASE_URL="https://raw.githubusercontent.com/ray-project/ray/master/python"
 WORK_DIR=$(mktemp -d)
 trap 'rm -rf "$WORK_DIR"' EXIT

+# ── Detect PyTorch index URL ─────────────────────────────────────────────
+
+if python3 -c "import torch; assert torch.version.hip" 2>/dev/null; then
+    ROCM_VER=$(python3 -c "import torch; print(torch.version.hip.rsplit('.', 1)[0])")
+    CANDIDATE_URL="https://download.pytorch.org/whl/rocm${ROCM_VER}"
+    if curl -fsSL --head "${CANDIDATE_URL}/" >/dev/null 2>&1; then
+        TORCH_INDEX_URL="${CANDIDATE_URL}"
+    else
+        echo ">>> WARNING: ROCm ${ROCM_VER} wheel index not found at ${CANDIDATE_URL}"
+        echo ">>>          Falling back to default PyPI (resolution may be incomplete)"
+        TORCH_INDEX_URL=""
+    fi
+else
+    TORCH_INDEX_URL="https://download.pytorch.org/whl/cu129"
+fi
+echo ">>> Using PyTorch index: ${TORCH_INDEX_URL:-PyPI default}"
+
 # Fetch all Ray requirement files used in the LLM depset pipeline
 echo ">>> Fetching Ray requirement files"
 RAY_FILES=(
@@ -116,6 +133,11 @@ echo "============================================================"
 echo ">>> Resolving: Can Ray generate compatible lock files?"
 echo "============================================================"

+EXTRA_INDEX_ARGS=()
+if [[ -n "${TORCH_INDEX_URL}" ]]; then
+    EXTRA_INDEX_ARGS+=(--extra-index-url "${TORCH_INDEX_URL}")
+fi
+
 set +e
 uv pip compile \
    "${WORK_DIR}/requirements.txt" \
@@ -126,7 +148,7 @@ uv pip compile \
    -c "${WORK_DIR}/vllm-constraints.txt" \
    --python-version 3.12 \
    --python-platform x86_64-manylinux_2_31 \
-    --extra-index-url https://download.pytorch.org/whl/cu129 \
+    "${EXTRA_INDEX_ARGS[@]}" \
    --index-strategy unsafe-best-match \
    --unsafe-package setuptools \
    --unsafe-package ray \
@@ -326,8 +326,7 @@ apply_rocm_test_overrides() {
  if [[ $cmds == *" kernels/moe"* ]]; then
    cmds="${cmds} \
    --ignore=kernels/moe/test_moe.py \
-    --ignore=kernels/moe/test_cutlass_moe.py \
-    --ignore=kernels/moe/test_triton_moe_ptpc_fp8.py"
+    --ignore=kernels/moe/test_cutlass_moe.py"
  fi

  # --- Entrypoint ignores ---
@@ -336,14 +335,17 @@ apply_rocm_test_overrides() {
    --ignore=entrypoints/openai/chat_completion/test_audio.py \
    --ignore=entrypoints/openai/completion/test_shutdown.py \
    --ignore=entrypoints/openai/test_completion.py \
-    --ignore=entrypoints/openai/test_models.py \
-    --ignore=entrypoints/openai/test_lora_adapters.py \
+    --ignore=entrypoints/openai/models/test_models.py \
    --ignore=entrypoints/openai/test_return_tokens_as_ids.py \
    --ignore=entrypoints/openai/chat_completion/test_root_path.py \
-    --ignore=entrypoints/openai/test_tokenization.py \
    --ignore=entrypoints/openai/completion/test_prompt_validation.py "}
  fi

+  if [[ $cmds == *" entrypoints/serve"* ]]; then
+    cmds="${cmds} \
+    --ignore=entrypoints/serve/lora/test_lora_adapters.py"
+  fi
+
  if [[ $cmds == *" entrypoints/llm "* ]]; then
    cmds=${cmds//" entrypoints/llm "/" entrypoints/llm \
    --ignore=entrypoints/llm/test_chat.py \
@@ -127,7 +127,7 @@ run_and_track_test() {

 # --- Actual Test Execution ---
 run_and_track_test 1 "test_struct_output_generate.py" \
-    "python3 -m pytest -s -v /workspace/vllm/tests/v1/entrypoints/llm/test_struct_output_generate.py -k \"not test_structured_output_with_reasoning_matrices\""
+    "python3 -m pytest -s -v /workspace/vllm/tests/entrypoints/llm/test_struct_output_generate.py -k \"not test_structured_output_with_reasoning_matrices\""
 run_and_track_test 2 "test_moe_pallas.py" \
    "python3 -m pytest -s -v /workspace/vllm/tests/tpu/test_moe_pallas.py"
 run_and_track_test 3 "test_lora.py" \
@@ -1,11 +1,14 @@
 #!/usr/bin/env bash
 set -euxo pipefail
-
 # Nightly e2e test for prefetch offloading with a MoE model.
 # Runs DeepSeek-V2-Lite with prefetch offloading of MoE expert weights
 # and validates GSM8K accuracy matches baseline (no offloading).
 #
 # args: [THRESHOLD] [NUM_QUESTIONS] [START_PORT]
+#
+# Environment variables:
+#   ATTENTION_BACKEND   - attention backend to use (e.g., FLASH_ATTN,
+#                         ROCM_ATTN, FLASHINFER). If unset, uses vllm default.
 THRESHOLD=${1:-0.25}
 NUM_Q=${2:-1319}
 PORT=${3:-8030}
@@ -22,6 +25,14 @@ wait_for_server() {

 MODEL="deepseek-ai/DeepSeek-V2-Lite"

+# ── Build optional vllm serve flags ─────────────────────────────────────
+
+EXTRA_ARGS=()
+if [[ -n "${ATTENTION_BACKEND:-}" ]]; then
+  echo "Using attention backend: ${ATTENTION_BACKEND}"
+  EXTRA_ARGS+=(--attention-backend "${ATTENTION_BACKEND}")
+fi
+
 cleanup() {
  if [[ -n "${SERVER_PID:-}" ]] && kill -0 "${SERVER_PID}" 2>/dev/null; then
    kill "${SERVER_PID}" 2>/dev/null || true
@@ -40,7 +51,8 @@ vllm serve "$MODEL" \
  --offload-num-in-group 2 \
  --offload-prefetch-step 1 \
  --offload-params w13_weight w2_weight \
-  --port "$PORT" &
+  --port "$PORT" \
+  ${EXTRA_ARGS+"${EXTRA_ARGS[@]}"} &
 SERVER_PID=$!
 wait_for_server "$PORT"

@@ -59,7 +59,7 @@ steps:
  - export VLLM_TEST_CLEAN_GPU_MEMORY=1
  - pytest -s -v tests/compile/passes/distributed

- label: Fusion and Compile Unit Tests (B200)
+- label: Fusion and Compile Unit Tests (2xB200)
  timeout_in_minutes: 20
  working_dir: "/vllm-workspace/"
  device: b200
@@ -27,14 +27,14 @@ steps:
  - vllm/v1/engine/
  - vllm/v1/worker/
  - tests/v1/distributed
-  - tests/v1/entrypoints/openai/test_multi_api_servers.py
+  - tests/entrypoints/openai/test_multi_api_servers.py
  commands:
  # https://github.com/NVIDIA/nccl/issues/1838
  - export NCCL_CUMEM_HOST_ENABLE=0
  - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py
  - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
  - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py
-  - DP_SIZE=2 pytest -v -s v1/entrypoints/openai/test_multi_api_servers.py
+  - DP_SIZE=2 pytest -v -s entrypoints/openai/test_multi_api_servers.py

 - label: Distributed Compile + RPC Tests (2 GPUs)
  timeout_in_minutes: 20
@@ -88,7 +88,6 @@ steps:
  - vllm/distributed/
  - tests/distributed/test_torchrun_example.py
  - tests/distributed/test_torchrun_example_moe.py
-  - examples/offline_inference/rlhf.py
  - examples/offline_inference/rlhf_colocate.py
  - examples/rl/
  - tests/examples/offline_inference/data_parallel.py
@@ -194,7 +193,7 @@ steps:
  num_devices: 2
  commands:
    - pytest -v -s tests/distributed/test_context_parallel.py
-    - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/offline_inference/new_weight_syncing/rlhf_async_new_apis.py
+    - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_async_new_apis.py
    - VLLM_USE_DEEP_GEMM=1 VLLM_LOGGING_LEVEL=DEBUG python3 examples/offline_inference/data_parallel.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=deepep_high_throughput
    - pytest -v -s tests/v1/distributed/test_dbo.py

@@ -258,6 +257,17 @@ steps:
    - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt
    - CROSS_LAYERS_BLOCKS=True bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh

+- label: Hyrbid SSM NixlConnector PD accuracy tests (4 GPUs)
+  timeout_in_minutes: 20
+  working_dir: "/vllm-workspace/tests"
+  num_devices: 4
+  source_file_dependencies:
+    - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
+    - tests/v1/kv_connector/nixl_integration/
+  commands:
+    - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt
+    - HYBRID_SSM=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
+
 - label: NixlConnector PD + Spec Decode acceptance (2 GPUs)
  timeout_in_minutes: 30
  device: a100
@@ -70,3 +70,15 @@ steps:
      device: mi325_4
      depends_on:
      - image-build-amd
+
+- label: V1 e2e (4xH100)
+  timeout_in_minutes: 60
+  device: h100
+  num_devices: 4
+  optional: true
+  source_file_dependencies:
+    - vllm/v1/attention/backends/utils.py
+    - vllm/v1/worker/gpu_model_runner.py
+    - tests/v1/e2e/test_hybrid_chunked_prefill.py
+  commands:
+    - pytest -v -s v1/e2e/test_hybrid_chunked_prefill.py
@@ -10,7 +10,7 @@ steps:
  - tests/entrypoints/
  commands:
  - pytest -v -s entrypoints/openai/tool_parsers
-  - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/rpc --ignore=entrypoints/sleep --ignore=entrypoints/instrumentator --ignore=entrypoints/openai --ignore=entrypoints/offline_mode --ignore=entrypoints/test_chat_utils.py  --ignore=entrypoints/pooling
+  - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/rpc --ignore=entrypoints/sleep --ignore=entrypoints/serve/instrumentator --ignore=entrypoints/openai --ignore=entrypoints/offline_mode --ignore=entrypoints/test_chat_utils.py  --ignore=entrypoints/pooling

 - label: Entrypoints Integration (LLM)
  timeout_in_minutes: 40
@@ -25,8 +25,8 @@ steps:
  - pytest -v -s entrypoints/llm/test_generate.py # it needs a clean process
  - pytest -v -s entrypoints/offline_mode # Needs to avoid interference with other tests

- label: Entrypoints Integration (API Server 1)
-  timeout_in_minutes: 130
+- label: Entrypoints Integration (API Server openai - Part 1)
+  timeout_in_minutes: 50
  working_dir: "/vllm-workspace/tests"
  source_file_dependencies:
  - vllm/
@@ -34,7 +34,24 @@ steps:
  - tests/entrypoints/test_chat_utils
  commands:
  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-  - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/chat_completion/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/chat_completion/test_oot_registration.py --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py --ignore=entrypoints/openai/correctness/  --ignore=entrypoints/openai/tool_parsers/ --ignore=entrypoints/openai/responses
+  - pytest -v -s entrypoints/openai/chat_completion --ignore=entrypoints/openai/chat_completion/test_chat_with_tool_reasoning.py --ignore=entrypoints/openai/chat_completion/test_oot_registration.py
+  mirror:
+    amd:
+      device: mi325_1
+      depends_on:
+      - image-build-amd
+
+
+- label: Entrypoints Integration (API Server openai - Part 2)
+  timeout_in_minutes: 50
+  working_dir: "/vllm-workspace/tests"
+  source_file_dependencies:
+  - vllm/
+  - tests/entrypoints/openai
+  - tests/entrypoints/test_chat_utils
+  commands:
+  - pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py
+  - pytest -v -s entrypoints/openai/speech_to_text/
  - pytest -v -s entrypoints/test_chat_utils.py
  mirror:
    amd:
@@ -42,17 +59,28 @@ steps:
      depends_on:
      - image-build-amd

+- label: Entrypoints Integration (API Server openai - Part 3)
+  timeout_in_minutes: 50
+  working_dir: "/vllm-workspace/tests"
+  source_file_dependencies:
+  - vllm/
+  - tests/entrypoints/openai
+  - tests/entrypoints/test_chat_utils
+  commands:
+  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+  - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/speech_to_text/ --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/tool_parsers/ --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/test_multi_api_servers.py
+
 - label: Entrypoints Integration (API Server 2)
  timeout_in_minutes: 130
  working_dir: "/vllm-workspace/tests"
  source_file_dependencies:
  - vllm/
  - tests/entrypoints/rpc
-  - tests/entrypoints/instrumentator
+  - tests/entrypoints/serve/instrumentator
  - tests/tool_use
  commands:
  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-  - pytest -v -s entrypoints/instrumentator
+  - pytest -v -s entrypoints/serve/instrumentator
  - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/rpc
  - pytest -v -s tool_use

@@ -75,19 +103,6 @@ steps:
  commands:
  - pytest -v -s entrypoints/openai/responses

- label: Entrypoints V1
-  timeout_in_minutes: 50
-  source_file_dependencies:
-    - vllm/
-    - tests/v1
-  commands:
-    - pytest -v -s v1/entrypoints
-  mirror:
-    amd:
-      device: mi325_1
-      depends_on:
-      - image-build-amd
-
 - label: OpenAI API Correctness
  timeout_in_minutes: 30
  source_file_dependencies:
@@ -45,6 +45,22 @@ steps:
  commands:
  - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell.txt

+- label: LM Eval Qwen3.5 Models (B200)
+  timeout_in_minutes: 120
+  device: b200
+  optional: true
+  num_devices: 2
+  source_file_dependencies:
+  - vllm/model_executor/models/qwen3_5.py
+  - vllm/model_executor/models/qwen3_5_mtp.py
+  - vllm/transformers_utils/configs/qwen3_5.py
+  - vllm/transformers_utils/configs/qwen3_5_moe.py
+  - vllm/model_executor/models/qwen3_next.py
+  - vllm/model_executor/models/qwen3_next_mtp.py
+  - vllm/model_executor/layers/fla/ops/
+  commands:
+  - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-qwen35-blackwell.txt
+
 - label: LM Eval Large Models (H200)
  timeout_in_minutes: 60
  device: h200
@@ -74,6 +90,7 @@ steps:
  commands:
    - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor-dp-ep/config-b200.txt

+
 - label: GPQA Eval (GPT-OSS) (H100)
  timeout_in_minutes: 120
  device: h100
@@ -8,7 +8,7 @@ steps:
  - vllm/lora
  - tests/lora
  commands:
-    - pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --ignore=lora/test_chatglm3_tp.py --ignore=lora/test_llama_tp.py --ignore=lora/test_llm_with_multi_loras.py --ignore=lora/test_olmoe_tp.py --ignore=lora/test_deepseekv2_tp.py --ignore=lora/test_gptoss_tp.py --ignore=lora/test_qwen3moe_tp.py
+    - pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --ignore=lora/test_chatglm3_tp.py --ignore=lora/test_llama_tp.py --ignore=lora/test_llm_with_multi_loras.py --ignore=lora/test_olmoe_tp.py --ignore=lora/test_deepseekv2_tp.py --ignore=lora/test_gptoss_tp.py --ignore=lora/test_qwen3moe_tp.py --ignore=lora/test_qwen35_densemodel_lora.py 
  parallelism: 4


@@ -30,4 +30,5 @@ steps:
    - pytest -v -s -x lora/test_llama_tp.py
    - pytest -v -s -x lora/test_llm_with_multi_loras.py
    - pytest -v -s -x lora/test_olmoe_tp.py
-    - pytest -v -s -x lora/test_gptoss_tp.py
+    - pytest -v -s -x lora/test_gptoss_tp.py
+    - pytest -v -s -x lora/test_qwen35_densemodel_lora.py
@@ -2,11 +2,54 @@ group: Miscellaneous
 depends_on: 
  - image-build
 steps:
- label: V1 Others
-  timeout_in_minutes: 60
+- label: V1 Spec Decode
+  timeout_in_minutes: 30
  source_file_dependencies:
    - vllm/
-    - tests/v1
+    - tests/v1/spec_decode
+  commands:
+    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+    # TODO: create another `optional` test group for slow tests
+    - pytest -v -s -m 'not slow_test' v1/spec_decode
+  mirror:
+    amd:
+      device: mi325_1
+      depends_on:
+      - image-build-amd
+
+- label: V1 Sample + Logits
+  timeout_in_minutes: 30
+  source_file_dependencies:
+    - vllm/
+    - tests/v1/sample
+    - tests/v1/logits_processors
+    - tests/v1/test_oracle.py
+    - tests/v1/test_request.py
+    - tests/v1/test_outputs.py
+  commands:
+    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+    - pytest -v -s v1/sample
+    - pytest -v -s v1/logits_processors
+    - pytest -v -s v1/test_oracle.py
+    - pytest -v -s v1/test_request.py
+    - pytest -v -s v1/test_outputs.py
+  mirror:
+    amd:
+      device: mi325_1
+      depends_on:
+      - image-build-amd
+
+- label: V1 Core + KV + Metrics
+  timeout_in_minutes: 30
+  source_file_dependencies:
+    - vllm/
+    - tests/v1/core
+    - tests/v1/executor
+    - tests/v1/kv_offload
+    - tests/v1/worker
+    - tests/v1/kv_connector/unit
+    - tests/v1/metrics
+    - tests/entrypoints/openai/correctness/test_lmeval.py
  commands:
    - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt
    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
@@ -14,16 +57,9 @@ steps:
    - pytest -v -s -m 'not cpu_test' v1/core
    - pytest -v -s v1/executor
    - pytest -v -s v1/kv_offload
-    - pytest -v -s v1/sample
-    - pytest -v -s v1/logits_processors
    - pytest -v -s v1/worker
-    # TODO: create another `optional` test group for slow tests
-    - pytest -v -s -m 'not slow_test' v1/spec_decode
    - pytest -v -s -m 'not cpu_test' v1/kv_connector/unit
    - pytest -v -s -m 'not cpu_test' v1/metrics
-    - pytest -v -s v1/test_oracle.py
-    - pytest -v -s v1/test_request.py
-    - pytest -v -s v1/test_outputs.py
    # Integration test for streaming correctness (requires special branch).
    - pip install -U git+https://github.com/robertgshaw2-redhat/lm-evaluation-harness.git@streaming-api
    - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
@@ -39,7 +75,7 @@ steps:
  source_file_dependencies:
    - vllm/
    - tests/v1
-  device: cpu
+  device: cpu-small
  commands:
    # split the test to avoid interference
    - pytest -v -s -m 'cpu_test' v1/core
@@ -141,7 +177,7 @@ steps:
  - tests/tool_parsers
  - tests/transformers_utils
  - tests/config
-  device: cpu
+  device: cpu-small
  commands:
  - python3 standalone_tests/lazy_imports.py
  - pytest -v -s test_inputs.py
@@ -156,7 +192,7 @@ steps:
  - pytest -v -s config

 - label: Batch Invariance (H100)
-  timeout_in_minutes: 25
+  timeout_in_minutes: 30
  device: h100
  source_file_dependencies:
    - vllm/v1/attention
@@ -167,6 +203,8 @@ steps:
    - pip install pytest-timeout pytest-forked
    - pytest -v -s v1/determinism/test_batch_invariance.py
    - pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py
+    - VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle[TRITON_MLA]
+    - VLLM_TEST_MODEL=Qwen/Qwen3-30B-A3B-Thinking-2507-FP8 pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle[FLASH_ATTN]
  
 - label: Acceptance Length Test (Large Models) # optional
  timeout_in_minutes: 25
@@ -11,7 +11,7 @@ steps:
  - vllm/v1/attention/
  - tests/v1/engine/test_llm_engine.py
  - tests/v1/e2e/
-  - tests/v1/entrypoints/llm/test_struct_output_generate.py
+  - tests/entrypoints/llm/test_struct_output_generate.py
  commands:
  - set -x
  - export VLLM_USE_V2_MODEL_RUNNER=1
@@ -22,7 +22,7 @@ steps:
  - pytest -v -s v1/e2e/general/test_context_length.py
  - pytest -v -s v1/e2e/general/test_min_tokens.py
  # Temporary hack filter to exclude ngram spec decoding based tests.
-  - pytest -v -s v1/entrypoints/llm/test_struct_output_generate.py -k "xgrammar and not speculative_config6 and not speculative_config7 and not speculative_config8 and not speculative_config0"
+  - pytest -v -s entrypoints/llm/test_struct_output_generate.py -k "xgrammar and not speculative_config6 and not speculative_config7 and not speculative_config8 and not speculative_config0"

 - label: Model Runner V2 Examples
  timeout_in_minutes: 45
@@ -87,13 +87,12 @@ steps:
    - vllm/v1/worker/gpu/
    - vllm/v1/worker/gpu_worker.py
    - tests/distributed/test_pipeline_parallel.py
-    #- tests/distributed/test_pp_cudagraph.py
+    - tests/distributed/test_pp_cudagraph.py
  commands:
    - set -x
    - export VLLM_USE_V2_MODEL_RUNNER=1
    - pytest -v -s distributed/test_pipeline_parallel.py -k "not ray and not Jamba"
-    # TODO: Uncomment once https://github.com/vllm-project/vllm/pull/35162 is merged.
-    #- pytest -v -s distributed/test_pp_cudagraph.py -k "not ray"
+    - pytest -v -s distributed/test_pp_cudagraph.py -k "not ray"

 - label: Model Runner V2 Spec Decode
  timeout_in_minutes: 30
@@ -51,7 +51,7 @@ steps:
  - vllm/
  - tests/models/test_utils.py
  - tests/models/test_vision.py
-  device: cpu
+  device: cpu-small
  commands:
    - pytest -v -s models/test_utils.py models/test_vision.py

@@ -62,7 +62,7 @@ steps:
      depends_on:
      - image-build-amd

- label: Multi-Modal Processor Test (CPU)
+- label: Multi-Modal Processor (CPU)
  depends_on: 
  - image-build-cpu
  timeout_in_minutes: 60
@@ -70,7 +70,7 @@ steps:
  - vllm/
  - tests/models/multimodal
  - tests/models/registry.py
-  device: cpu
+  device: cpu-medium
  commands:
    - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
    - pytest -v -s models/multimodal/processing --ignore models/multimodal/processing/test_tensor_schema.py
@@ -95,34 +95,44 @@ steps:
  commands:
  - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-mm-small.txt --tp-size=1

- label: Multi-Modal Models (Extended) 1
+- label: Multi-Modal Models (Extended Generation 1)
  optional: true
  source_file_dependencies:
  - vllm/
-  - tests/models/multimodal
+  - tests/models/multimodal/generation
+  - tests/models/multimodal/test_mapping.py
  commands:
    - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
-    - pytest -v -s models/multimodal -m 'not core_model' --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/processing
+    - pytest -v -s models/multimodal/generation -m 'not core_model' --ignore models/multimodal/generation/test_common.py
+    - pytest -v -s models/multimodal/test_mapping.py
  mirror:
    amd:
      device: mi325_1
      depends_on:
      - image-build-amd

- label: Multi-Modal Models (Extended) 2
+- label: Multi-Modal Models (Extended Generation 2)
  optional: true
  source_file_dependencies:
  - vllm/
-  - tests/models/multimodal
+  - tests/models/multimodal/generation
  commands:
    - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
    - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=0) and not core_model'

- label: Multi-Modal Models (Extended) 3
+- label: Multi-Modal Models (Extended Generation 3)
  optional: true
  source_file_dependencies:
  - vllm/
-  - tests/models/multimodal
+  - tests/models/multimodal/generation
  commands:
    - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
    - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model'
+
+- label: Multi-Modal Models (Extended Pooling)
+  optional: true
+  source_file_dependencies:
+  - vllm/
+  - tests/models/multimodal/pooling
+  commands:
+    - pytest -v -s models/multimodal/pooling -m 'not core_model'
@@ -75,7 +75,7 @@ CMakeLists.txt @tlrmchlsmth @LucasWilkinson
 /tests/multimodal @DarkLight1337 @ywang96 @NickLucche
 /tests/quantization @mgoin @robertgshaw2-redhat @yewentao256 @pavanimajety
 /tests/test_inputs.py @DarkLight1337 @ywang96
-/tests/v1/entrypoints/llm/test_struct_output_generate.py @mgoin @russellb @aarnphm
+/tests/entrypoints/llm/test_struct_output_generate.py @mgoin @russellb @aarnphm
 /tests/v1/structured_output @mgoin @russellb @aarnphm
 /tests/v1/core @WoosukKwon @robertgshaw2-redhat @njhill @ywang96 @alexm-redhat @heheda12345 @ApostaC @orozery
 /tests/weight_loading @mgoin @youkaichao @yewentao256
@@ -171,6 +171,7 @@ mkdocs.yaml @hmellor

 # Pooling models
 /examples/pooling @noooop
+/docs/models/pooling_models @noooop
 /tests/models/*/pooling* @noooop
 /tests/entrypoints/pooling @noooop
 /vllm/config/pooler.py @noooop
@@ -260,7 +260,7 @@ pull_request_rules:
      - files=examples/offline_inference/structured_outputs.py
      - files=examples/online_serving/structured_outputs/structured_outputs.py
      - files~=^tests/v1/structured_output/
-      - files=tests/v1/entrypoints/llm/test_struct_output_generate.py
+      - files=tests/entrypoints/llm/test_struct_output_generate.py
      - files~=^vllm/v1/structured_output/
  actions:
    label:
@@ -333,9 +333,10 @@ pull_request_rules:
    - label != stale
    - or:
      - files~=^tests/tool_use/
-      - files~=^tests/entrypoints/openai/tool_parsers/
-      - files=tests/entrypoints/openai/chat_completion/test_chat_with_tool_reasoning.py
-      - files~=^vllm/entrypoints/openai/tool_parsers/
+      - files~=^tests/tool_parsers/
+      - files~=^tests/entrypoints/openai/.*tool.*
+      - files~=^tests/entrypoints/anthropic/.*tool.*
+      - files~=^vllm/tool_parsers/
      - files=docs/features/tool_calling.md
      - files~=^examples/tool_chat_*
      - files=examples/offline_inference/chat_with_tools.py
@@ -1,50 +0,0 @@
-#!/bin/bash
-
-set -eu
-
-# ensure 1 argument is passed
-if [ "$#" -ne 1 ]; then
-    echo "Usage: $0 <pr_number>"
-    exit 1
-fi
-
-PR_NUMBER=$1
-OLD=/tmp/orig_pr_body.txt
-NEW=/tmp/new_pr_body.txt
-
-gh pr view --json body --template "{{.body}}" "${PR_NUMBER}" > "${OLD}"
-cp "${OLD}" "${NEW}"
-
-# Remove markdown comments (like the <!-- markdownlint-disable --> at the start)
-sed -i '/<!--.*-->$/d' "${NEW}"
-
-# Remove "PLEASE FILL IN THE PR DESCRIPTION HERE ENSURING ALL CHECKLIST ITEMS (AT THE BOTTOM) HAVE BEEN CONSIDERED."
-sed -i '/PLEASE FILL IN THE PR DESCRIPTION HERE.*$/d' "${NEW}"
-
-# Remove all lines after and including "**BEFORE SUBMITTING, PLEASE READ THE CHECKLIST BELOW AND FILL IN THE DESCRIPTION ABOVE**"
-sed -i '/\*\*BEFORE SUBMITTING, PLEASE READ.*\*\*/,$d' "${NEW}"
-
-# Remove HTML <details> section that includes <summary> text of "PR Checklist (Click to Expand)"
-python3 - <<EOF
-import regex as re
-
-with open("${NEW}", "r") as file:
-    content = file.read()
-
-pattern = re.compile(r'(---\n\n)?<details>.*?<summary>.*?PR Checklist \(Click to Expand\).*?</summary>.*?</details>', re.DOTALL)
-content = re.sub(pattern, '', content)
-
-with open("${NEW}", "w") as file:
-    file.write(content)
-EOF
-
-# Run this only if ${NEW} is different than ${OLD}
-if ! cmp -s "${OLD}" "${NEW}"; then
-    gh pr edit --body-file "${NEW}" "${PR_NUMBER}"
-    echo
-    echo "Updated PR body:"
-    echo
-    cat "${NEW}"
-else
-    echo "No changes needed"
-fi
@@ -1,32 +0,0 @@
-name: Cleanup PR Body
-
-on:
-  pull_request_target:
-    types: [opened, reopened, edited]
-
-permissions:
-  pull-requests: write
-
-jobs:
-  update-description:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
-
-      - name: Set up Python
-        uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
-        with:
-          python-version: '3.12'
-          cache: 'pip'
-
-      - name: Install Python dependencies
-        run: |
-          python3 -m pip install --upgrade pip
-          python3 -m pip install regex
-
-      - name: Update PR description
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: bash .github/scripts/cleanup_pr_body.sh "${{ github.event.number }}"
@@ -383,4 +383,107 @@ jobs:
                  core.notice(`All users for label "${label}" already mentioned, skipping comment`);
                }
              }
-            }
+            }
+
+      - name: Request missing ROCm info from issue author
+        if: contains(steps.label-step.outputs.labels_added, 'rocm') && contains(toJSON(github.event.issue.labels.*.name), 'bug')
+        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd  # v8.0.0
+        with:
+          script: |
+            const body = (context.payload.issue.body || '').toLowerCase();
+
+            // Check for existing bot comments to avoid duplicate requests
+            const comments = await github.rest.issues.listComments({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+            });
+            const botAlreadyAsked = comments.data.some(
+              c => c.user.type === 'Bot' && c.body.includes('<!-- rocm-info-request -->')
+            );
+            if (botAlreadyAsked) {
+              core.notice('ROCm info request already posted, skipping');
+              return;
+            }
+
+            // Define required information and detection patterns
+            const requiredInfo = [
+              {
+                name: 'Reproducer',
+                patterns: [
+                  /reproduc/i, /minimal.?example/i, /repro\b/i, /steps to reproduce/i,
+                  /code.?snippet/i, /sample.?code/i,
+                  /```python[\s\S]*?```/, /```bash[\s\S]*?```/, /```sh[\s\S]*?```/,
+                ],
+                ask: 'A minimal reproducer (code snippet or script that triggers the issue)',
+              },
+              {
+                name: 'Error message',
+                patterns: [
+                  /error/i, /traceback/i, /exception/i, /fault/i, /crash/i,
+                  /failed/i, /abort/i, /panic/i,
+                ],
+                ask: 'The full error message or traceback',
+              },
+              {
+                name: 'Installation method',
+                patterns: [
+                  /docker/i, /rocm\/pytorch/i, /dockerfile/i, /from source/i,
+                  /pip install/i, /build.?from/i, /container/i, /image/i,
+                  /wheel/i, /\.whl/i, /nightly/i,
+                ],
+                ask: 'How you installed vLLM (Docker image name, pip install, or build from source steps)',
+              },
+              {
+                name: 'Command',
+                patterns: [
+                  /vllm serve/i, /python\s+\S+\.py/i, /```bash[\s\S]*?```/,
+                  /```sh[\s\S]*?```/, /command/i, /launch/i, /run\s/i,
+                  /--model/i, /--tensor-parallel/i, /--gpu-memory/i,
+                ],
+                ask: 'The command you used to launch vLLM (e.g., `vllm serve ...` or the Python script)',
+              },
+              {
+                name: 'GFX architecture',
+                patterns: [
+                  /gfx\d{3,4}/i, /mi\d{3}/i, /mi\d{2}\b/i, /radeon/i,
+                  /gpu.?arch/i, /rocm-smi/i, /rocminfo/i, /navi/i,
+                  /instinct/i,
+                ],
+                ask: 'Your GPU model and GFX architecture (e.g., MI300X / gfx942) — run `rocminfo | grep gfx`',
+              },
+            ];
+
+            const issueBody = context.payload.issue.body || '';
+            const missing = requiredInfo.filter(info =>
+              !info.patterns.some(p => p.test(issueBody))
+            );
+
+            if (missing.length === 0) {
+              core.notice('All required ROCm info appears to be present');
+              return;
+            }
+
+            const author = context.payload.issue.user.login;
+            const checklist = requiredInfo.map(info => {
+              const found = !missing.includes(info);
+              return `- [${found ? 'x' : ' '}] ${info.ask}`;
+            }).join('\n');
+            const message = [
+              '<!-- rocm-info-request -->',
+              `Hi @${author}, thanks for reporting this ROCm issue!`,
+              '',
+              'To help us investigate, please make sure the following information is included:',
+              '',
+              checklist,
+              '',
+              'Please provide any unchecked items above. This will help us reproduce and resolve the issue faster. Thank you!',
+            ].join('\n');
+
+            await github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+              body: message,
+            });
+            core.notice(`Requested missing ROCm info from @${author}: ${missing.map(m => m.name).join(', ')}`);
@@ -1,9 +1,9 @@
 name: macOS Apple Silicon Smoke Test

 on:
-  push:
-    branches:
-      - main
+  schedule:
+    # Daily at 2:30 AM UTC
+    - cron: '30 2 * * *'
  workflow_dispatch:  # Manual trigger

 permissions:
@@ -0,0 +1,96 @@
+name: New PR Bot
+
+on:
+  pull_request_target:
+    types: [opened]
+
+permissions:
+  pull-requests: write
+
+jobs:
+  update-description:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Update PR description
+        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
+        with:
+          script: |
+            const { owner, repo } = context.repo;
+            const pr_number = context.issue.number;
+
+            const { data: pr } = await github.rest.pulls.get({
+              owner,
+              repo,
+              pull_number: pr_number,
+            });
+
+            let body = pr.body || '';
+            const original = body;
+
+            // Remove markdown comments (<!-- ... -->)
+            body = body.replace(/^<!--.*-->$/gm, '');
+
+            // Remove "PLEASE FILL IN THE PR DESCRIPTION HERE ..."
+            body = body.replace(/^PLEASE FILL IN THE PR DESCRIPTION HERE.*$/gm, '');
+
+            // Remove all lines after and including "**BEFORE SUBMITTING, PLEASE READ ..."
+            body = body.replace(/\*\*BEFORE SUBMITTING, PLEASE READ.*\*\*[\s\S]*$/, '');
+
+            // Remove <details> section containing "PR Checklist (Click to Expand)"
+            body = body.replace(/(---\n\n)?<details>[\s\S]*?<summary>[\s\S]*?PR Checklist \(Click to Expand\)[\s\S]*?<\/summary>[\s\S]*?<\/details>/g, '');
+
+            if (body !== original) {
+              await github.rest.pulls.update({
+                owner,
+                repo,
+                pull_number: pr_number,
+                body,
+              });
+              console.log('Updated PR body');
+            } else {
+              console.log('No changes needed');
+            }
+
+  reminder-comment:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Post welcome comment for first-time contributors
+        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
+        with:
+          script: |
+            const { owner, repo } = context.repo;
+            const prAuthor = context.payload.pull_request.user.login;
+
+            const { data: searchResults } = await github.rest.search.issuesAndPullRequests({
+              q: `repo:${owner}/${repo} type:pr author:${prAuthor}`,
+              per_page: 1,
+            });
+
+            const authorPRCount = searchResults.total_count;
+            console.log(`Found ${authorPRCount} PRs by ${prAuthor}`);
+
+            if (authorPRCount === 1) {
+              console.log(`Posting welcome comment for first-time contributor: ${prAuthor}`);
+              await github.rest.issues.createComment({
+                owner,
+                repo,
+                issue_number: context.issue.number,
+                body: [
+                  '\u{1f44b} Hi! Thank you for contributing to the vLLM project.',
+                  '',
+                  '\u{1f4ac} Join our developer Slack at https://slack.vllm.ai to discuss your PR in #pr-reviews, coordinate on features in #feat- channels, or join special interest groups in #sig- channels.',
+                  '',
+                  'Just a reminder: PRs would not trigger full CI run by default.',
+                  '',
+                  'Once the PR is approved and ready to go, your PR reviewer(s) can run CI to test the changes comprehensively before merging.',
+                  '',
+                  'To run CI, PR reviewers can either: Add `ready` label to the PR or enable auto-merge.',
+                  '',
+                  'If you have any questions, please reach out to us on Slack at https://slack.vllm.ai.',
+                  '',
+                  '\u{1f680}',
+                ].join('\n'),
+              });
+            } else {
+              console.log(`Skipping comment for ${prAuthor} - not their first PR (${authorPRCount} PRs found)`);
+            }
@@ -11,9 +11,39 @@ concurrency:

 permissions:
  contents: read
+  pull-requests: read

 jobs:
+  pre-run-check:
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check PR label and author merge count
+      uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
+      with:
+        script: |
+          const { data: pr } = await github.rest.pulls.get({
+            ...context.repo,
+            pull_number: context.payload.pull_request.number,
+          });
+
+          const hasReadyLabel = pr.labels.some(l => l.name === 'ready');
+
+          const { data: mergedPRs } = await github.rest.search.issuesAndPullRequests({
+            q: `repo:${context.repo.owner}/${context.repo.repo} is:pr is:merged author:${pr.user.login}`,
+            per_page: 4,
+          });
+          const mergedCount = mergedPRs.total_count;
+
+          if (hasReadyLabel || mergedCount >= 4) {
+            core.info(`Check passed: ready label=${hasReadyLabel}, 4+ merged PRs=${mergedCount >= 4}`);
+          } else {
+            core.setFailed(`PR must have the 'ready' label or the author must have at least 4 merged PRs (found ${mergedCount}).`);
+          }
+
  pre-commit:
+    needs: pre-run-check
+    if: always() && (needs.pre-run-check.result == 'success' || needs.pre-run-check.result == 'skipped')
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
@@ -1,54 +0,0 @@
-name: PR Reminder Comment Bot
-permissions:
-  pull-requests: write
-on:
-  pull_request_target:
-    types: [opened]
-jobs:
-  pr_reminder:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Remind to run full CI on PR
-        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
-        with:
-          script: |
-            try {
-              // Get the PR author
-              const prAuthor = context.payload.pull_request.user.login;
-              
-              // Check if this is the author's first PR in this repository
-              // Use GitHub's search API to find all PRs by this author
-              const { data: searchResults } = await github.rest.search.issuesAndPullRequests({
-                q: `repo:${context.repo.owner}/${context.repo.repo} type:pr author:${prAuthor}`,
-                per_page: 100  
-              });
-              
-              const authorPRCount = searchResults.total_count;
-              
-              console.log(`Found ${authorPRCount} PRs by ${prAuthor}`);
-              
-              // Only post comment if this is the first PR (only one PR by this author)
-              if (authorPRCount === 1) {
-                console.log(`Posting welcome comment for first-time contributor: ${prAuthor}`);
-                await github.rest.issues.createComment({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                issue_number: context.issue.number,
-                body: '👋 Hi! Thank you for contributing to the vLLM project.\n\n' +
-                  '💬 Join our developer Slack at https://slack.vllm.ai to discuss your PR in #pr-reviews, coordinate on features in #feat- channels, or join special interest groups in #sig- channels.\n\n' +
-                  'Just a reminder: PRs would not trigger full CI run by default. Instead, it would only run `fastcheck` CI which starts running only a small and essential subset of CI tests to quickly catch errors. \n\n' +
-                  'You ask your reviewers to trigger select CI tests on top of `fastcheck` CI. \n\n' +
-                  'Once the PR is approved and ready to go, your PR reviewer(s) can run CI to test the changes comprehensively before merging.\n\n' +
-                  'To run CI, PR reviewers can either: Add `ready` label to the PR or enable auto-merge.\n\n' +
-                  'If you have any questions, please reach out to us on Slack at https://slack.vllm.ai.\n\n' +
-                  '🚀'
-                });
-              } else {
-                console.log(`Skipping comment for ${prAuthor} - not their first PR (${authorPRCount} PRs found)`);
-              }
-            } catch (error) {
-              console.error('Error checking PR history or posting comment:', error);
-              // Don't fail the workflow, just log the error
-            }
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -108,7 +108,7 @@ uv.lock
 # pyenv
 #   For a library or package, you might want to ignore these files since the code is
 #   intended to run in multiple environments; otherwise, check them in:
-# .python-version
+.python-version

 # pipenv
 #   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
@@ -340,11 +340,9 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")

  list(APPEND VLLM_EXT_SRC
    "csrc/quantization/awq/gemm_kernels.cu"
-    "csrc/permute_cols.cu"
    "csrc/quantization/w8a8/cutlass/scaled_mm_entry.cu"
    "csrc/quantization/fp4/nvfp4_quant_entry.cu"
    "csrc/quantization/fp4/nvfp4_scaled_mm_entry.cu"
-    "csrc/sparse/cutlass/sparse_scaled_mm_entry.cu"
    "csrc/cutlass_extensions/common.cpp"
    "csrc/quantization/w8a8/fp8/per_token_group_quant.cu"
    "csrc/quantization/w8a8/int8/per_token_group_quant.cu")
@@ -620,31 +618,6 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
    endif()
  endif()

-  #
-  # 2:4 Sparse Kernels
-
-  # The 2:4 sparse kernels cutlass_scaled_sparse_mm and cutlass_compressor
-  # require CUDA 12.2 or later (and only work on Hopper).
-  cuda_archs_loose_intersection(SCALED_MM_ARCHS "9.0a;" "${CUDA_ARCHS}")
-  if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.2 AND SCALED_MM_ARCHS)
-    set(SRCS "csrc/sparse/cutlass/sparse_scaled_mm_c3x.cu")
-    set_gencode_flags_for_srcs(
-      SRCS "${SRCS}"
-      CUDA_ARCHS "${SCALED_MM_ARCHS}")
-    list(APPEND VLLM_EXT_SRC "${SRCS}")
-    list(APPEND VLLM_GPU_FLAGS "-DENABLE_SPARSE_SCALED_MM_C3X=1")
-    message(STATUS "Building sparse_scaled_mm_c3x for archs: ${SCALED_MM_ARCHS}")
-  else()
-    if (NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.2 AND SCALED_MM_ARCHS)
-      message(STATUS "Not building sparse_scaled_mm_c3x kernels as CUDA Compiler version is "
-                     "not >= 12.2, we recommend upgrading to CUDA 12.2 or later "
-                     "if you intend on running FP8 sparse quantized models on Hopper.")
-    else()
-      message(STATUS "Not building sparse_scaled_mm_c3x as no compatible archs found "
-                     "in CUDA target architectures")
-    endif()
-  endif()
-
  # The nvfp4_scaled_mm_sm120 kernels for Geforce Blackwell SM120 require
  # CUDA 12.8 or later
  if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0)
@@ -691,6 +664,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
    list(APPEND VLLM_EXT_SRC "${SRCS}")
    list(APPEND VLLM_GPU_FLAGS "-DENABLE_NVFP4_SM100=1")
    list(APPEND VLLM_GPU_FLAGS "-DENABLE_CUTLASS_MOE_SM100=1")
+    set(VLLM_NVFP4_SM100_ENABLED TRUE)
    message(STATUS "Building NVFP4 for archs: ${FP4_ARCHS}")
  else()
    message(STATUS "Not building NVFP4 as no compatible archs were found.")
@@ -986,6 +960,54 @@ define_extension_target(
 # Setting this variable sidesteps the issue by calling the driver directly.
 target_compile_definitions(_C PRIVATE CUTLASS_ENABLE_DIRECT_CUDA_DRIVER_CALL=1)

+# Propagate ENABLE_NVFP4_SM100 to all languages (including C++ files such as
+# torch_bindings.cpp) so that per-SM op registrations are compiled in.
+if(VLLM_NVFP4_SM100_ENABLED)
+  target_compile_definitions(_C PRIVATE ENABLE_NVFP4_SM100=1)
+endif()
+
+# add OR VLLM_GPU_LANG STREQUAL "HIP" here once
+# https://github.com/vllm-project/vllm/issues/35163 is resolved
+if(VLLM_GPU_LANG STREQUAL "CUDA")
+  #
+  # _C_stable_libtorch extension (ops registered via STABLE_TORCH_LIBRARY)
+  #
+  set(VLLM_STABLE_EXT_SRC
+    "csrc/libtorch_stable/torch_bindings.cpp")
+
+  if(VLLM_GPU_LANG STREQUAL "CUDA")
+    list(APPEND VLLM_STABLE_EXT_SRC "csrc/libtorch_stable/permute_cols.cu")
+  endif()
+
+  if(VLLM_GPU_LANG STREQUAL "CUDA")
+    set_gencode_flags_for_srcs(
+      SRCS "${VLLM_STABLE_EXT_SRC}"
+      CUDA_ARCHS "${CUDA_ARCHS}")
+  endif()
+
+  message(STATUS "Enabling C_stable extension.")
+  define_extension_target(
+    _C_stable_libtorch
+    DESTINATION vllm
+    LANGUAGE ${VLLM_GPU_LANG}
+    SOURCES ${VLLM_STABLE_EXT_SRC}
+    COMPILE_FLAGS ${VLLM_GPU_FLAGS}
+    ARCHITECTURES ${VLLM_GPU_ARCHES}
+    USE_SABI 3
+    WITH_SOABI)
+
+  # Set TORCH_TARGET_VERSION for stable ABI compatibility.
+  # This ensures we only use C-shim APIs available in PyTorch 2.10.
+  # _C_stable_libtorch is abi compatible with PyTorch >= TORCH_TARGET_VERSION
+  # which is currently set to 2.10.
+  target_compile_definitions(_C_stable_libtorch PRIVATE
+    TORCH_TARGET_VERSION=0x020A000000000000ULL)
+
+  # Needed to use cuda APIs from C-shim
+  target_compile_definitions(_C_stable_libtorch PRIVATE
+    USE_CUDA)
+endif()
+
 #
 # _moe_C extension
 #
@@ -40,9 +40,9 @@ LLM engine. You can refer to the `vllm.engine.arg_utils.EngineArgs` for more
 details.
 """

-import dataclasses
 import random
 import time
+from dataclasses import fields

 from vllm import LLM, SamplingParams
 from vllm.engine.arg_utils import EngineArgs
@@ -124,7 +124,7 @@ def main(args):

    # Create the LLM engine
    engine_args = EngineArgs.from_cli_args(args)
-    llm = LLM(**dataclasses.asdict(engine_args))
+    llm = LLM(**{f.name: getattr(engine_args, f.name) for f in fields(engine_args)})
    sampling_params = SamplingParams(temperature=0, max_tokens=args.output_len)

    print("------warm up------")
@@ -32,6 +32,7 @@ import dataclasses
 import json
 import random
 import time
+from dataclasses import fields

 from transformers import PreTrainedTokenizerBase

@@ -196,7 +197,7 @@ def main(args):

    engine_args = EngineArgs.from_cli_args(args)

-    llm = LLM(**dataclasses.asdict(engine_args))
+    llm = LLM(**{f.name: getattr(engine_args, f.name) for f in fields(engine_args)})

    sampling_params = SamplingParams(
        temperature=0,
@@ -3,10 +3,10 @@
 """Benchmark offline prioritization."""

 import argparse
-import dataclasses
 import json
 import random
 import time
+from dataclasses import fields

 from transformers import AutoTokenizer, PreTrainedTokenizerBase

@@ -79,7 +79,7 @@ def run_vllm(
 ) -> float:
    from vllm import LLM, SamplingParams

-    llm = LLM(**dataclasses.asdict(engine_args))
+    llm = LLM(**{f.name: getattr(engine_args, f.name) for f in fields(engine_args)})

    assert all(
        llm.llm_engine.model_config.max_model_len >= (request[1] + request[2])
@@ -1,517 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-import argparse
-import copy
-import itertools
-import pickle as pkl
-import time
-from collections.abc import Callable, Iterable
-
-import torch
-import torch.utils.benchmark as TBenchmark
-from torch.utils.benchmark import Measurement as TMeasurement
-from utils import make_rand_sparse_tensors
-from weight_shapes import WEIGHT_SHAPES
-
-from vllm import _custom_ops as ops
-from vllm.utils.argparse_utils import FlexibleArgumentParser
-
-DEFAULT_MODELS = list(WEIGHT_SHAPES.keys())
-DEFAULT_BATCH_SIZES = [1, 16, 32, 64, 128, 256, 512]
-DEFAULT_TP_SIZES = [1]
-
-
-# bench
-def bench_fn(
-    label: str, sub_label: str, description: str, fn: Callable, *args, **kwargs
-) -> TMeasurement:
-    min_run_time = 1
-
-    globals = {
-        "args": args,
-        "kwargs": kwargs,
-        "fn": fn,
-    }
-    return TBenchmark.Timer(
-        stmt="fn(*args, **kwargs)",
-        globals=globals,
-        label=label,
-        sub_label=sub_label,
-        description=description,
-    ).blocked_autorange(min_run_time=min_run_time)
-
-
-def bench_int8(
-    dtype: torch.dtype, m: int, k: int, n: int, label: str, sub_label: str
-) -> Iterable[TMeasurement]:
-    assert dtype == torch.int8
-    b_compressed, e, a, b = make_rand_sparse_tensors(torch.int8, m, n, k)
-    scale_a = torch.tensor(1.0, device="cuda", dtype=torch.float32)
-    scale_b = torch.tensor(1.0, device="cuda", dtype=torch.float32)
-    bias = torch.zeros((n,), device="cuda", dtype=torch.bfloat16)
-
-    out = ops.cutlass_scaled_sparse_mm(
-        a, b_compressed, e, scale_a, scale_b, torch.bfloat16
-    )
-    out_ref = ops.cutlass_scaled_mm(a, b, scale_a, scale_b, torch.bfloat16)
-
-    if not torch.allclose(out, out_ref):
-        print("Incorrect results")
-        print(out)
-        print(out_ref)
-    else:
-        print("Correct results")
-
-    timers = []
-    # pytorch impl - bfloat16
-    timers.append(
-        bench_fn(
-            label,
-            sub_label,
-            "pytorch_bf16_bf16_bf16_matmul-no-scales",
-            torch.mm,
-            a.to(dtype=torch.bfloat16),
-            b.to(dtype=torch.bfloat16),
-        )
-    )
-
-    # pytorch impl - float16
-    timers.append(
-        bench_fn(
-            label,
-            sub_label,
-            "pytorch_fp16_fp16_fp16_matmul-no-scales",
-            torch.mm,
-            a.to(dtype=torch.float16),
-            b.to(dtype=torch.float16),
-        )
-    )
-
-    # cutlass impl
-    timers.append(
-        bench_fn(
-            label,
-            sub_label,
-            "cutlass_i8_i8_bf16_scaled_mm",
-            ops.cutlass_scaled_mm,
-            a,
-            b,
-            scale_a,
-            scale_b,
-            torch.bfloat16,
-        )
-    )
-
-    # cutlass with bias
-    timers.append(
-        bench_fn(
-            label,
-            sub_label,
-            "cutlass_i8_i8_bf16_scaled_mm_bias",
-            ops.cutlass_scaled_mm,
-            a,
-            b,
-            scale_a,
-            scale_b,
-            torch.bfloat16,
-            bias,
-        )
-    )
-
-    # cutlass sparse impl
-    timers.append(
-        bench_fn(
-            label,
-            sub_label,
-            "cutlass_i8_i8_bf16_scaled_sparse_mm",
-            ops.cutlass_scaled_sparse_mm,
-            a,
-            b_compressed,
-            e,
-            scale_a,
-            scale_b,
-            torch.bfloat16,
-        )
-    )
-
-    # cutlass sparse with bias
-    timers.append(
-        bench_fn(
-            label,
-            sub_label,
-            "cutlass_i8_i8_bf16_scaled_sparse_mm_bias",
-            ops.cutlass_scaled_sparse_mm,
-            a,
-            b_compressed,
-            e,
-            scale_a,
-            scale_b,
-            torch.bfloat16,
-            bias,
-        )
-    )
-
-    return timers
-
-
-def bench_fp8(
-    dtype: torch.dtype, m: int, k: int, n: int, label: str, sub_label: str
-) -> Iterable[TMeasurement]:
-    assert dtype == torch.float8_e4m3fn
-    b_compressed, e, a, b = make_rand_sparse_tensors(torch.float8_e4m3fn, m, n, k)
-    scale_a = torch.tensor(1.0, device="cuda", dtype=torch.float32)
-    scale_b = torch.tensor(1.0, device="cuda", dtype=torch.float32)
-    bias = torch.zeros((n,), device="cuda", dtype=torch.bfloat16)
-
-    out = ops.cutlass_scaled_sparse_mm(
-        a, b_compressed, e, scale_a, scale_b, torch.bfloat16
-    )
-    out_ref = ops.cutlass_scaled_mm(a, b, scale_a, scale_b, torch.bfloat16)
-
-    if not torch.allclose(out, out_ref):
-        print("Incorrect results")
-        print(out)
-        print(out_ref)
-    else:
-        print("Correct results")
-
-    timers = []
-
-    # pytorch impl w. bf16
-    timers.append(
-        bench_fn(
-            label,
-            sub_label,
-            "pytorch_bf16_bf16_bf16_matmul-no-scales",
-            torch.mm,
-            a.to(dtype=torch.bfloat16, device="cuda"),
-            b.to(dtype=torch.bfloat16, device="cuda"),
-        )
-    )
-
-    # pytorch impl: bf16 output, without fp8 fast accum
-    timers.append(
-        bench_fn(
-            label,
-            sub_label,
-            "pytorch_fp8_fp8_bf16_scaled_mm",
-            torch._scaled_mm,
-            a,
-            b,
-            scale_a=scale_a,
-            scale_b=scale_b,
-            out_dtype=torch.bfloat16,
-        )
-    )
-
-    # pytorch impl: bf16 output, with fp8 fast accum
-    timers.append(
-        bench_fn(
-            label,
-            sub_label,
-            "pytorch_fp8_fp8_bf16_scaled_mm_fast_accum",
-            torch._scaled_mm,
-            a,
-            b,
-            scale_a=scale_a,
-            scale_b=scale_b,
-            out_dtype=torch.bfloat16,
-            use_fast_accum=True,
-        )
-    )
-
-    # pytorch impl: fp16 output, without fp8 fast accum
-    timers.append(
-        bench_fn(
-            label,
-            sub_label,
-            "pytorch_fp8_fp8_fp16_scaled_mm",
-            torch._scaled_mm,
-            a,
-            b,
-            scale_a=scale_a,
-            scale_b=scale_b,
-            out_dtype=torch.float16,
-        )
-    )
-
-    # pytorch impl: fp16 output, with fp8 fast accum
-    timers.append(
-        bench_fn(
-            label,
-            sub_label,
-            "pytorch_fp8_fp8_fp16_scaled_mm_fast_accum",
-            torch._scaled_mm,
-            a,
-            b,
-            scale_a=scale_a,
-            scale_b=scale_b,
-            out_dtype=torch.float16,
-            use_fast_accum=True,
-        )
-    )
-
-    # cutlass impl: bf16 output
-    timers.append(
-        bench_fn(
-            label,
-            sub_label,
-            "cutlass_fp8_fp8_bf16_scaled_mm",
-            ops.cutlass_scaled_mm,
-            a,
-            b,
-            scale_a,
-            scale_b,
-            torch.bfloat16,
-        )
-    )
-
-    # cutlass impl: bf16 output
-    timers.append(
-        bench_fn(
-            label,
-            sub_label,
-            "cutlass_fp8_fp8_bf16_scaled_sparse_mm",
-            ops.cutlass_scaled_sparse_mm,
-            a,
-            b_compressed,
-            e,
-            scale_a,
-            scale_b,
-            torch.bfloat16,
-        )
-    )
-
-    # cutlass impl: fp16 output
-    timers.append(
-        bench_fn(
-            label,
-            sub_label,
-            "cutlass_fp8_fp8_fp16_scaled_sparse_mm",
-            ops.cutlass_scaled_sparse_mm,
-            a,
-            b_compressed,
-            e,
-            scale_a,
-            scale_b,
-            torch.float16,
-        )
-    )
-
-    # cutlass impl: bf16 output, with bias
-    timers.append(
-        bench_fn(
-            label,
-            sub_label,
-            "cutlass_fp8_fp8_bf16_scaled_sparse_mm_bias",
-            ops.cutlass_scaled_sparse_mm,
-            a,
-            b_compressed,
-            e,
-            scale_a,
-            scale_b,
-            torch.bfloat16,
-            bias,
-        )
-    )
-
-    # cutlass impl: fp16 output, with bias
-    timers.append(
-        bench_fn(
-            label,
-            sub_label,
-            "cutlass_fp8_fp8_fp16_scaled_sparse_mm_bias",
-            ops.cutlass_scaled_sparse_mm,
-            a,
-            b_compressed,
-            e,
-            scale_a,
-            scale_b,
-            torch.float16,
-            bias.to(dtype=torch.float16),
-        )
-    )
-
-    return timers
-
-
-def bench(
-    dtype: torch.dtype, m: int, k: int, n: int, label: str, sub_label: str
-) -> Iterable[TMeasurement]:
-    if dtype == torch.int8:
-        return bench_int8(dtype, m, k, n, label, sub_label)
-    if dtype == torch.float8_e4m3fn:
-        return bench_fp8(dtype, m, k, n, label, sub_label)
-    raise ValueError(
-        f"Unsupported dtype {dtype}: should be one of torch.int8, torch.float8_e4m3fn."
-    )
-
-
-# runner
-def print_timers(timers: Iterable[TMeasurement]):
-    compare = TBenchmark.Compare(timers)
-    compare.print()
-
-
-def run(
-    dtype: torch.dtype, MKNs: Iterable[tuple[int, int, int]]
-) -> Iterable[TMeasurement]:
-    results = []
-    for m, k, n in MKNs:
-        timers = bench(dtype, m, k, n, f"scaled-{dtype}-gemm", f"MKN=({m}x{k}x{n})")
-        print_timers(timers)
-        results.extend(timers)
-
-    return results
-
-
-# output makers
-def make_output(
-    data: Iterable[TMeasurement],
-    MKNs: Iterable[tuple[int, int, int]],
-    base_description: str,
-    timestamp=None,
-):
-    print(f"== All Results {base_description} ====")
-    print_timers(data)
-
-    # pickle all the results
-    timestamp = int(time.time()) if timestamp is None else timestamp
-    with open(f"{base_description}-{timestamp}.pkl", "wb") as f:
-        pkl.dump(data, f)
-
-
-# argparse runners
-
-
-def run_square_bench(args):
-    dim_sizes = list(range(args.dim_start, args.dim_end + 1, args.dim_increment))
-    MKNs = list(zip(dim_sizes, dim_sizes, dim_sizes))
-    data = run(args.dtype, MKNs)
-
-    make_output(data, MKNs, f"square_bench-{args.dtype}")
-
-
-def run_range_bench(args):
-    dim_sizes = list(range(args.dim_start, args.dim_end, args.dim_increment))
-    n = len(dim_sizes)
-    Ms = [args.m_constant] * n if args.m_constant is not None else dim_sizes
-    Ks = [args.k_constant] * n if args.k_constant is not None else dim_sizes
-    Ns = [args.n_constant] * n if args.n_constant is not None else dim_sizes
-    MKNs = list(zip(Ms, Ks, Ns))
-    data = run(args.dtype, MKNs)
-
-    make_output(data, MKNs, f"range_bench-{args.dtype}")
-
-
-def run_model_bench(args):
-    print("Benchmarking models:")
-    for i, model in enumerate(args.models):
-        print(f"[{i}]  {model}")
-
-    def model_shapes(model_name: str, tp_size: int) -> list[tuple[int, int]]:
-        KNs = []
-        for KN, tp_split_dim in copy.deepcopy(WEIGHT_SHAPES[model_name]):
-            KN[tp_split_dim] = KN[tp_split_dim] // tp_size
-            KNs.append(KN)
-        return KNs
-
-    model_bench_data = []
-    models_tps = list(itertools.product(args.models, args.tp_sizes))
-    for model, tp_size in models_tps:
-        Ms = args.batch_sizes
-        KNs = model_shapes(model, tp_size)
-        MKNs = []
-        for m in Ms:
-            for k, n in KNs:
-                MKNs.append((m, k, n))
-
-        data = run(args.dtype, MKNs)
-        model_bench_data.append(data)
-
-    # Print all results
-    for data, model_tp in zip(model_bench_data, models_tps):
-        model, tp_size = model_tp
-        print(f"== Results {args.dtype} {model}-TP{tp_size} ====")
-        print_timers(data)
-
-    timestamp = int(time.time())
-
-    all_data = []
-    for d in model_bench_data:
-        all_data.extend(d)
-    # pickle all data
-    with open(f"model_bench-{args.dtype}-{timestamp}.pkl", "wb") as f:
-        pkl.dump(all_data, f)
-
-
-if __name__ == "__main__":
-
-    def to_torch_dtype(dt):
-        if dt == "int8":
-            return torch.int8
-        if dt == "fp8":
-            return torch.float8_e4m3fn
-        raise ValueError("unsupported dtype")
-
-    parser = FlexibleArgumentParser(
-        description="""
-Benchmark Cutlass GEMM.
-
-    To run square GEMMs:
-        python3 ./benchmarks/cutlass_benchmarks/sparse_benchmarks.py --dtype fp8 square_bench --dim-start 128 --dim-end 512 --dim-increment 64
-    
-    To run constant N and K and sweep M:
-        python3 ./benchmarks/cutlass_benchmarks/sparse_benchmarks.py --dtype fp8 range_bench --dim-start 128 --dim-end 512 --dim-increment 64 --n-constant 16384 --k-constant 16384
-    
-    To run dimensions from a model:
-        python3 ./benchmarks/cutlass_benchmarks/sparse_benchmarks.py --dtype fp8 model_bench --models meta-llama/Llama-2-7b-hf --batch-sizes 16 --tp-sizes 1
-    
-    Output:
-        - a .pkl file, that is a list of raw torch.benchmark.utils.Measurements for the pytorch and cutlass implementations for the various GEMMs.
-            """,  # noqa: E501
-        formatter_class=argparse.RawTextHelpFormatter,
-    )
-
-    parser.add_argument(
-        "--dtype",
-        type=to_torch_dtype,
-        required=True,
-        help="Available options are ['int8', 'fp8']",
-    )
-    subparsers = parser.add_subparsers(dest="cmd")
-
-    square_parser = subparsers.add_parser("square_bench")
-    square_parser.add_argument("--dim-start", type=int, required=True)
-    square_parser.add_argument("--dim-end", type=int, required=True)
-    square_parser.add_argument("--dim-increment", type=int, required=True)
-    square_parser.set_defaults(func=run_square_bench)
-
-    range_parser = subparsers.add_parser("range_bench")
-    range_parser.add_argument("--dim-start", type=int, required=True)
-    range_parser.add_argument("--dim-end", type=int, required=True)
-    range_parser.add_argument("--dim-increment", type=int, required=True)
-    range_parser.add_argument("--m-constant", type=int, default=None)
-    range_parser.add_argument("--n-constant", type=int, default=None)
-    range_parser.add_argument("--k-constant", type=int, default=None)
-    range_parser.set_defaults(func=run_range_bench)
-
-    model_parser = subparsers.add_parser("model_bench")
-    model_parser.add_argument(
-        "--models",
-        nargs="+",
-        type=str,
-        default=DEFAULT_MODELS,
-        choices=WEIGHT_SHAPES.keys(),
-    )
-    model_parser.add_argument(
-        "--tp-sizes", nargs="+", type=int, default=DEFAULT_TP_SIZES
-    )
-    model_parser.add_argument(
-        "--batch-sizes", nargs="+", type=int, default=DEFAULT_BATCH_SIZES
-    )
-    model_parser.set_defaults(func=run_model_bench)
-
-    args = parser.parse_args()
-    args.func(args)
@@ -5,8 +5,6 @@

 import torch

-import vllm._custom_ops as ops
-

 def to_fp8(tensor: torch.Tensor) -> torch.Tensor:
    finfo = torch.finfo(torch.float8_e4m3fn)
@@ -39,49 +37,3 @@ def make_rand_tensors(
        return to_fp8(a), to_fp8(b)

    raise ValueError("unsupported dtype")
-
-
-def prune_to_2_4(tensor):
-    # Reshape tensor to [N, 4] where N is number of groups of 4
-    original_shape = tensor.shape
-    reshaped = tensor.reshape(-1, 4)
-
-    # Get indices of top 2 absolute values in each group of 4
-    _, indices = torch.topk(torch.abs(reshaped), k=2, dim=1)
-
-    # Create binary mask
-    mask = torch.zeros_like(reshaped)
-    mask.scatter_(dim=1, index=indices, src=torch.ones_like(indices, dtype=mask.dtype))
-
-    # Apply mask and reshape back
-    pruned = reshaped * mask
-
-    # Turn all -0.0 to 0.0
-    pruned[pruned == -0.0] = 0.0
-
-    return pruned.reshape(original_shape)
-
-
-def make_rand_sparse_tensors(
-    dtype: torch.dtype, m: int, n: int, k: int
-) -> tuple[torch.Tensor, torch.Tensor]:
-    a = torch.randn((m, k), device="cuda") * 5
-    b = torch.randn((n, k), device="cuda").t() * 5
-
-    b = prune_to_2_4(b.t()).t()
-
-    if dtype == torch.int8:
-        a, b = to_int8(a), to_int8(b)
-    elif dtype == torch.float8_e4m3fn:
-        a, b = to_fp8(a), to_fp8(b)
-    elif dtype == torch.float16:
-        a, b = to_fp16(a), to_fp16(b)
-    elif dtype == torch.bfloat16:
-        a, b = to_bf16(a), to_bf16(b)
-    else:
-        raise ValueError("unsupported dtype")
-
-    b_compressed, e = ops.cutlass_sparse_compress(b.t())
-
-    # Compressed B, Metadata, Original A, B
-    return b_compressed, e, a, b
@@ -0,0 +1,669 @@
+"""
+Benchmark: SM103 (B300) FP4 Ultra GEMM vs SM100 (B200) NVFP4 GEMM
+===================================================================
+
+This benchmark compares the performance of the SM103-optimized FP4 Ultra
+GEMM kernel against the SM100 NVFP4 GEMM kernel, both running on B300
+hardware.  It also benchmarks the effect of Programmatic Dependent Launch
+(PDL) on the quant->GEMM pipeline, where the GEMM consumer can begin
+before the quant producer finishes.
+
+SM103 kernels use:
+  - K=768 tile (vs K=256 on SM100)
+  - FP4 Ultra MMA (UltraVs16) schedule
+  - NoSmemWarpSpecialized epilogue
+  - Sm103BlockScaledConfig scale factor layout
+
+PDL kernels additionally set:
+  - cudaLaunchAttributeProgrammaticStreamSerialization on quant (producer)
+  - CUTLASS launch_with_pdl=true on GEMM (enables overlap with next kernel)
+
+Usage:
+    python benchmarks/kernels/benchmark_nvfp4_sm103.py [--mode gemm|quant|e2e|pdl|all]
+
+Requirements:
+    - B300 GPU (SM103 / compute capability 10.3)
+    - CUDA >= 12.9
+    - vLLM built with ENABLE_NVFP4_SM100=1 and SM103 support
+"""
+
+import argparse
+from typing import Optional
+
+import torch
+import vllm._C  # noqa: F401 - registers ops into torch.ops._C
+
+# ============================================================================
+# Helpers
+# ============================================================================
+
+
+def round_up(x: int, y: int) -> int:
+    return ((x + y - 1) // y) * y
+
+
+def get_sm_version() -> int:
+    """Return SM version as integer (e.g., 100, 103, 120)."""
+    cap = torch.cuda.get_device_capability()
+    return cap[0] * 10 + cap[1]
+
+
+def create_nvfp4_tensors(
+    m: int, n: int, k: int, dtype: torch.dtype = torch.bfloat16
+) -> dict:
+    """
+    Create synthetic NVFP4 GEMM input tensors (A, B, scales, alpha).
+
+    A: [m, k/2] uint8 (packed FP4)
+    B: [n, k/2] uint8 (packed FP4, column-major)
+    A_sf: [round_up(m,128), round_up(k/16,4)] float8_e4m3fn (SM100 swizzled)
+    B_sf: [round_up(n,128), round_up(k/16,4)] float8_e4m3fn (SM100 swizzled)
+    alpha: [1] float32
+    D: [m, n] output
+    """
+    # Packed FP4 data (random bytes -- content doesn't affect timing)
+    A = torch.randint(0, 256, (m, k // 2), dtype=torch.uint8, device="cuda")
+    B = torch.randint(0, 256, (n, k // 2), dtype=torch.uint8, device="cuda")
+
+    # Scale factors (SM100 swizzled layout)
+    sf_m = round_up(m, 128)
+    sf_n = round_up(n, 128)
+    sf_k = round_up(k // 16, 4)
+
+    A_sf_sm100 = torch.randint(
+        0, 256, (sf_m, sf_k), dtype=torch.uint8, device="cuda"
+    ).view(torch.float8_e4m3fn)
+    B_sf_sm100 = torch.randint(
+        0, 256, (sf_n, sf_k), dtype=torch.uint8, device="cuda"
+    ).view(torch.float8_e4m3fn)
+
+    # SM103 layout: convert from SM100 layout
+    A_sf_sm103 = torch.empty_like(A_sf_sm100)
+    B_sf_sm103 = torch.empty_like(B_sf_sm100)
+    torch.ops._C.convert_sf_layout_sm100_to_sm103(A_sf_sm103, A_sf_sm100)
+    torch.ops._C.convert_sf_layout_sm100_to_sm103(B_sf_sm103, B_sf_sm100)
+
+    # Global alpha
+    alpha = torch.tensor([1.0], dtype=torch.float32, device="cuda")
+
+    # Output
+    D = torch.empty(m, n, dtype=dtype, device="cuda")
+
+    return {
+        "A": A,
+        "B": B,
+        "A_sf_sm100": A_sf_sm100,
+        "B_sf_sm100": B_sf_sm100,
+        "A_sf_sm103": A_sf_sm103,
+        "B_sf_sm103": B_sf_sm103,
+        "alpha": alpha,
+        "D": D,
+    }
+
+
+def create_quant_tensors(
+    m: int, n: int, dtype: torch.dtype = torch.bfloat16
+) -> dict:
+    """Create inputs for activation quantization benchmark."""
+    input_tensor = torch.randn(m, n, dtype=dtype, device="cuda")
+    global_scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
+    return {"input": input_tensor, "global_scale": global_scale}
+
+
+def bench_fn(
+    fn,
+    warmup: int = 20,
+    iters: int = 100,
+    sync: bool = True,
+) -> float:
+    """Benchmark a function, returning median time in microseconds."""
+    # Warmup
+    for _ in range(warmup):
+        fn()
+    if sync:
+        torch.cuda.synchronize()
+
+    # Timed iterations using CUDA events
+    start_events = [torch.cuda.Event(enable_timing=True) for _ in range(iters)]
+    end_events = [torch.cuda.Event(enable_timing=True) for _ in range(iters)]
+
+    for i in range(iters):
+        start_events[i].record()
+        fn()
+        end_events[i].record()
+
+    torch.cuda.synchronize()
+
+    times = [s.elapsed_time(e) * 1000 for s, e in zip(start_events, end_events)]
+    times.sort()
+    # Return median in microseconds
+    return times[len(times) // 2]
+
+
+# ============================================================================
+# GEMM Benchmark
+# ============================================================================
+
+
+def benchmark_gemm(
+    m_sizes: list[int],
+    n: int = 7168,
+    k: int = 7168,
+    dtype: torch.dtype = torch.bfloat16,
+) -> list[dict]:
+    """
+    Benchmark SM100 vs SM103 vs SM103+PDL NVFP4 GEMM kernels side by side.
+
+    PDL on the GEMM sets ProgrammaticStreamSerialization, allowing the NEXT
+    kernel on the stream to overlap with the GEMM's tail.  For isolated GEMM
+    calls (no consumer kernel), the PDL overhead should be near-zero.
+    """
+    vllm_ops = torch.ops._C
+
+    has_sm100a = hasattr(vllm_ops, "cutlass_scaled_fp4_mm_sm100a")
+    has_sm103a = hasattr(vllm_ops, "cutlass_scaled_fp4_mm_sm103a")
+    has_sm103a_pdl = hasattr(vllm_ops, "cutlass_scaled_fp4_mm_sm103a_pdl")
+
+    if not has_sm100a and not has_sm103a:
+        print("WARNING: Neither sm100a nor sm103a ops are available. "
+              "Rebuild with ENABLE_NVFP4_SM100=1.")
+        return []
+
+    results = []
+
+    for m in m_sizes:
+        tensors = create_nvfp4_tensors(m, n, k, dtype)
+        D = tensors["D"]
+        A, B = tensors["A"], tensors["B"]
+        A_sf_sm100, B_sf_sm100 = tensors["A_sf_sm100"], tensors["B_sf_sm100"]
+        A_sf_sm103, B_sf_sm103 = tensors["A_sf_sm103"], tensors["B_sf_sm103"]
+        alpha = tensors["alpha"]
+
+        flops = 2.0 * m * n * k
+
+        time_sm100: Optional[float] = None
+        time_sm103: Optional[float] = None
+        time_sm103_pdl: Optional[float] = None
+
+        if has_sm100a:
+            def run_sm100():
+                vllm_ops.cutlass_scaled_fp4_mm_sm100a(
+                    D, A, B, A_sf_sm100, B_sf_sm100, alpha
+                )
+            time_sm100 = bench_fn(run_sm100, warmup=20, iters=100)
+
+        if has_sm103a:
+            def run_sm103():
+                vllm_ops.cutlass_scaled_fp4_mm_sm103a(
+                    D, A, B, A_sf_sm103, B_sf_sm103, alpha
+                )
+            time_sm103 = bench_fn(run_sm103, warmup=20, iters=100)
+
+        if has_sm103a_pdl:
+            def run_sm103_pdl():
+                vllm_ops.cutlass_scaled_fp4_mm_sm103a_pdl(
+                    D, A, B, A_sf_sm103, B_sf_sm103, alpha
+                )
+            time_sm103_pdl = bench_fn(run_sm103_pdl, warmup=20, iters=100)
+
+        row: dict = {"M": m, "N": n, "K": k}
+
+        if time_sm100 is not None:
+            row["sm100_us"] = time_sm100
+            row["sm100_tflops"] = flops / (time_sm100 * 1e-6) / 1e12
+
+        if time_sm103 is not None:
+            row["sm103_us"] = time_sm103
+            row["sm103_tflops"] = flops / (time_sm103 * 1e-6) / 1e12
+
+        if time_sm103_pdl is not None:
+            row["sm103pdl_us"] = time_sm103_pdl
+            row["sm103pdl_tflops"] = flops / (time_sm103_pdl * 1e-6) / 1e12
+
+        if time_sm100 is not None and time_sm103 is not None:
+            row["sm103_vs_100"] = time_sm100 / time_sm103
+
+        results.append(row)
+
+    return results
+
+
+# ============================================================================
+# Quantization Benchmark
+# ============================================================================
+
+
+def benchmark_quant(
+    m_sizes: list[int],
+    n: int = 7168,
+    dtype: torch.dtype = torch.bfloat16,
+) -> list[dict]:
+    """
+    Benchmark SM100 vs SM103 activation quantization (BF16 -> NVFP4).
+    """
+    vllm_ops = torch.ops._C
+    results = []
+
+    has_sm103_quant = hasattr(vllm_ops, "scaled_fp4_quant_sm103")
+
+    for m in m_sizes:
+        tensors = create_quant_tensors(m, n, dtype)
+        input_t = tensors["input"]
+        global_scale = tensors["global_scale"]
+
+        # SM100 quantization (swizzled layout)
+        def run_sm100_quant():
+            vllm_ops.scaled_fp4_quant(input_t, global_scale, True)
+
+        time_sm100 = bench_fn(run_sm100_quant, warmup=20, iters=100)
+
+        row: dict = {
+            "M": m,
+            "N": n,
+            "sm100_us": time_sm100,
+            "sm100_gb_s": (m * n * 2) / (time_sm100 * 1e-6) / 1e9,
+        }
+
+        if has_sm103_quant:
+            def run_sm103_quant():
+                vllm_ops.scaled_fp4_quant_sm103(input_t, global_scale)
+
+            time_sm103 = bench_fn(run_sm103_quant, warmup=20, iters=100)
+            row["sm103_us"] = time_sm103
+            row["sm103_gb_s"] = (m * n * 2) / (time_sm103 * 1e-6) / 1e9
+
+        results.append(row)
+
+    return results
+
+
+# ============================================================================
+# SF Layout Conversion Benchmark
+# ============================================================================
+
+
+def benchmark_sf_conversion(
+    m_sizes: list[int],
+    k: int = 7168,
+) -> list[dict]:
+    """
+    Benchmark the SM100 <-> SM103 scale factor layout conversion kernel.
+
+    This measures the overhead of converting scale factors between layouts,
+    which happens once at model load time for weights.
+    """
+    vllm_ops = torch.ops._C
+    results = []
+
+    for m in m_sizes:
+        sf_m = round_up(m, 128)
+        sf_k = round_up(k // 16, 4)
+
+        # Create source SF tensor (SM100 layout)
+        src = torch.randint(
+            0, 256, (sf_m, sf_k), dtype=torch.uint8, device="cuda"
+        ).view(torch.float8_e4m3fn)
+
+        # Allocate destination (same shape)
+        dst = torch.empty_like(src)
+
+        # Benchmark SM100 -> SM103 conversion
+        def run_convert():
+            vllm_ops.convert_sf_layout_sm100_to_sm103(dst, src)
+
+        time_us = bench_fn(run_convert, warmup=20, iters=200)
+
+        results.append({
+            "M": m,
+            "K": k,
+            "sf_shape": f"{sf_m}x{sf_k}",
+            "kernel": "SM100->SM103 SF convert",
+            "time_us": time_us,
+            "throughput_gb_s": (sf_m * sf_k) / (time_us * 1e-6) / 1e9,
+        })
+
+    return results
+
+
+# ============================================================================
+# End-to-End Benchmark (Quant + GEMM) with PDL comparison
+# ============================================================================
+
+
+def benchmark_e2e(
+    m_sizes: list[int],
+    n: int = 7168,
+    k: int = 7168,
+    dtype: torch.dtype = torch.bfloat16,
+) -> list[dict]:
+    """
+    Benchmark the full NVFP4 inference path: quantize activations + GEMM,
+    comparing SM100, SM103, and SM103+PDL.
+
+    This measures what a real transformer linear layer does:
+    1. Quantize BF16 activations to NVFP4 (with block scales)
+    2. NVFP4 x NVFP4 GEMM
+
+    SM103+PDL enables ProgrammaticStreamSerialization on the quant kernel
+    and launch_with_pdl on the GEMM, allowing the GEMM to begin executing
+    while the quant kernel is still completing its last thread blocks.
+    """
+    vllm_ops = torch.ops._C
+
+    has_sm100a = hasattr(vllm_ops, "cutlass_scaled_fp4_mm_sm100a")
+    has_sm103a = hasattr(vllm_ops, "cutlass_scaled_fp4_mm_sm103a")
+    has_sm103_quant = hasattr(vllm_ops, "scaled_fp4_quant_sm103")
+    has_sm103_pdl_quant = hasattr(vllm_ops, "scaled_fp4_quant_sm103_pdl")
+    has_sm103a_pdl = hasattr(vllm_ops, "cutlass_scaled_fp4_mm_sm103a_pdl")
+
+    if not has_sm100a and not has_sm103a:
+        print("WARNING: Neither sm100a nor sm103a ops are available. "
+              "Rebuild with ENABLE_NVFP4_SM100=1.")
+        return []
+
+    results = []
+
+    for m in m_sizes:
+        # Create activation input
+        activation = torch.randn(m, k, dtype=dtype, device="cuda")
+        global_scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
+
+        # Create weight (pre-quantized)
+        B = torch.randint(0, 256, (n, k // 2), dtype=torch.uint8, device="cuda")
+        sf_n = round_up(n, 128)
+        sf_k = round_up(k // 16, 4)
+
+        # Weight SFs in SM100 layout (for SM100 kernel)
+        B_sf_sm100 = torch.randint(
+            0, 256, (sf_n, sf_k), dtype=torch.uint8, device="cuda"
+        ).view(torch.float8_e4m3fn)
+        alpha = torch.tensor([1.0], dtype=torch.float32, device="cuda")
+
+        # Weight SFs in SM103 layout (pre-converted at load time)
+        B_sf_sm103 = torch.empty_like(B_sf_sm100)
+        vllm_ops.convert_sf_layout_sm100_to_sm103(B_sf_sm103, B_sf_sm100)
+
+        D = torch.empty(m, n, dtype=dtype, device="cuda")
+
+        flops = 2.0 * m * n * k
+        row: dict = {"M": m, "N": n, "K": k}
+
+        # --- SM100 baseline: SM100 quant + SM100 GEMM ---
+        if has_sm100a:
+            def run_e2e_sm100():
+                A_q, A_sf = vllm_ops.scaled_fp4_quant(
+                    activation, global_scale, True
+                )
+                A_sf = A_sf.view(torch.float8_e4m3fn)
+                vllm_ops.cutlass_scaled_fp4_mm_sm100a(
+                    D, A_q, B, A_sf, B_sf_sm100, alpha
+                )
+
+            time_sm100 = bench_fn(run_e2e_sm100, warmup=10, iters=50)
+            row["sm100_us"] = time_sm100
+            row["sm100_tflops"] = flops / (time_sm100 * 1e-6) / 1e12
+
+        # --- SM103 without PDL: SM103 quant + SM103 GEMM ---
+        if has_sm103a and has_sm103_quant:
+            def run_e2e_sm103():
+                A_q, A_sf = vllm_ops.scaled_fp4_quant_sm103(
+                    activation, global_scale
+                )
+                A_sf = A_sf.view(torch.float8_e4m3fn)
+                vllm_ops.cutlass_scaled_fp4_mm_sm103a(
+                    D, A_q, B, A_sf, B_sf_sm103, alpha
+                )
+
+            time_sm103 = bench_fn(run_e2e_sm103, warmup=10, iters=50)
+            row["sm103_us"] = time_sm103
+            row["sm103_tflops"] = flops / (time_sm103 * 1e-6) / 1e12
+
+        # --- SM103 with PDL: PDL quant + PDL GEMM ---
+        if has_sm103a_pdl and has_sm103_pdl_quant:
+            def run_e2e_sm103_pdl():
+                # PDL quant: ProgrammaticStreamSerialization allows GEMM to
+                # begin before quant finishes.
+                A_q, A_sf = vllm_ops.scaled_fp4_quant_sm103_pdl(
+                    activation, global_scale
+                )
+                A_sf = A_sf.view(torch.float8_e4m3fn)
+                # PDL GEMM: ProgrammaticStreamSerialization allows the next
+                # layer's kernel to begin before this GEMM finishes.
+                vllm_ops.cutlass_scaled_fp4_mm_sm103a_pdl(
+                    D, A_q, B, A_sf, B_sf_sm103, alpha
+                )
+
+            time_sm103_pdl = bench_fn(run_e2e_sm103_pdl, warmup=10, iters=50)
+            row["sm103pdl_us"] = time_sm103_pdl
+            row["sm103pdl_tflops"] = flops / (time_sm103_pdl * 1e-6) / 1e12
+
+        # Speedup columns
+        if "sm100_us" in row and "sm103_us" in row:
+            row["sm103_vs_100"] = row["sm100_us"] / row["sm103_us"]
+        if "sm103_us" in row and "sm103pdl_us" in row:
+            row["pdl_vs_nop"] = row["sm103_us"] / row["sm103pdl_us"]
+        if "sm100_us" in row and "sm103pdl_us" in row:
+            row["pdl_vs_100"] = row["sm100_us"] / row["sm103pdl_us"]
+
+        results.append(row)
+
+    return results
+
+
+# ============================================================================
+# PDL Pipeline Benchmark (back-to-back quant+GEMM pairs)
+# ============================================================================
+
+
+def benchmark_pdl_pipeline(
+    m_sizes: list[int],
+    n: int = 7168,
+    k: int = 7168,
+    num_layers: int = 4,
+    dtype: torch.dtype = torch.bfloat16,
+) -> list[dict]:
+    """
+    Benchmark the PDL pipeline benefit for back-to-back layers.
+
+    In a real transformer, the same quant->GEMM pattern repeats for each
+    linear layer.  With PDL enabled on both quant and GEMM, each kernel
+    launch overlaps with its predecessor's tail, creating a pipeline:
+
+        quant_1 -> GEMM_1 -> quant_2 -> GEMM_2 -> ...
+
+    This benchmark simulates `num_layers` consecutive quant+GEMM pairs
+    to measure the cumulative pipeline benefit.
+    """
+    vllm_ops = torch.ops._C
+
+    has_sm103_quant = hasattr(vllm_ops, "scaled_fp4_quant_sm103")
+    has_sm103_pdl_quant = hasattr(vllm_ops, "scaled_fp4_quant_sm103_pdl")
+    has_sm103a = hasattr(vllm_ops, "cutlass_scaled_fp4_mm_sm103a")
+    has_sm103a_pdl = hasattr(vllm_ops, "cutlass_scaled_fp4_mm_sm103a_pdl")
+
+    if not (has_sm103_quant and has_sm103a):
+        print("WARNING: SM103 ops not available.")
+        return []
+
+    results = []
+
+    for m in m_sizes:
+        activation = torch.randn(m, k, dtype=dtype, device="cuda")
+        global_scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
+        B = torch.randint(0, 256, (n, k // 2), dtype=torch.uint8, device="cuda")
+        sf_n = round_up(n, 128)
+        sf_k = round_up(k // 16, 4)
+        B_sf_sm100 = torch.randint(
+            0, 256, (sf_n, sf_k), dtype=torch.uint8, device="cuda"
+        ).view(torch.float8_e4m3fn)
+        B_sf_sm103 = torch.empty_like(B_sf_sm100)
+        vllm_ops.convert_sf_layout_sm100_to_sm103(B_sf_sm103, B_sf_sm100)
+        alpha = torch.tensor([1.0], dtype=torch.float32, device="cuda")
+        D = torch.empty(m, n, dtype=dtype, device="cuda")
+
+        total_flops = 2.0 * m * n * k * num_layers
+
+        # SM103 without PDL: num_layers sequential quant+GEMM
+        def run_pipeline_no_pdl():
+            for _ in range(num_layers):
+                A_q, A_sf = vllm_ops.scaled_fp4_quant_sm103(
+                    activation, global_scale
+                )
+                A_sf = A_sf.view(torch.float8_e4m3fn)
+                vllm_ops.cutlass_scaled_fp4_mm_sm103a(
+                    D, A_q, B, A_sf, B_sf_sm103, alpha
+                )
+
+        time_no_pdl = bench_fn(run_pipeline_no_pdl, warmup=5, iters=30)
+
+        row: dict = {
+            "M": m, "layers": num_layers,
+            "no_pdl_us": time_no_pdl,
+            "no_pdl_tflops": total_flops / (time_no_pdl * 1e-6) / 1e12,
+        }
+
+        # SM103 with PDL: num_layers pipelined quant+GEMM
+        if has_sm103_pdl_quant and has_sm103a_pdl:
+            def run_pipeline_pdl():
+                for _ in range(num_layers):
+                    A_q, A_sf = vllm_ops.scaled_fp4_quant_sm103_pdl(
+                        activation, global_scale
+                    )
+                    A_sf = A_sf.view(torch.float8_e4m3fn)
+                    vllm_ops.cutlass_scaled_fp4_mm_sm103a_pdl(
+                        D, A_q, B, A_sf, B_sf_sm103, alpha
+                    )
+
+            time_pdl = bench_fn(run_pipeline_pdl, warmup=5, iters=30)
+            row["pdl_us"] = time_pdl
+            row["pdl_tflops"] = total_flops / (time_pdl * 1e-6) / 1e12
+            row["pdl_speedup"] = time_no_pdl / time_pdl
+
+        results.append(row)
+
+    return results
+
+
+# ============================================================================
+# Main
+# ============================================================================
+
+
+def print_results(results: list[dict], title: str):
+    if not results:
+        return
+
+    print(f"\n{'=' * 80}")
+    print(f" {title}")
+    print(f"{'=' * 80}")
+
+    # Determine columns from first result
+    cols = list(results[0].keys())
+    # Header
+    header = " | ".join(f"{c:>15s}" for c in cols)
+    print(header)
+    print("-" * len(header))
+
+    for r in results:
+        row = []
+        for c in cols:
+            v = r.get(c, "")
+            if isinstance(v, float):
+                row.append(f"{v:>15.2f}")
+            elif isinstance(v, int):
+                row.append(f"{v:>15d}")
+            else:
+                row.append(f"{v:>15s}")
+        print(" | ".join(row))
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Benchmark NVFP4 SM103 vs SM100 kernels (with PDL)"
+    )
+    parser.add_argument(
+        "--mode",
+        choices=["gemm", "quant", "sf_convert", "e2e", "pdl", "all"],
+        default="all",
+        help="Which benchmark to run",
+    )
+    parser.add_argument(
+        "--n", type=int, default=7168,
+        help="N dimension (default: 7168, DeepSeek)",
+    )
+    parser.add_argument(
+        "--k", type=int, default=7168,
+        help="K dimension (default: 7168, DeepSeek)",
+    )
+    parser.add_argument(
+        "--layers", type=int, default=4,
+        help="Number of back-to-back layers for PDL pipeline benchmark",
+    )
+    args = parser.parse_args()
+
+    sm = get_sm_version()
+    print(f"GPU: {torch.cuda.get_device_name()}")
+    print(f"SM version: {sm}")
+    print(f"CUDA version: {torch.version.cuda}")
+
+    if sm < 100:
+        print("ERROR: This benchmark requires SM100+ (Blackwell) GPU.")
+        return
+
+    if sm == 103:
+        print("NOTE: Running on SM103 (B300) -- all kernel variants will run.")
+    else:
+        print(f"NOTE: Running on SM{sm} -- SM100 kernel is native; "
+              "SM103 kernel runs via forward compat (may be slower).")
+
+    # Problem sizes typical for LLM inference
+    # Small M = decode, large M = prefill
+    m_sizes = [1, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096]
+
+    if args.mode in ("gemm", "all"):
+        results = benchmark_gemm(m_sizes, n=args.n, k=args.k)
+        print_results(
+            results,
+            f"NVFP4 GEMM: SM100 vs SM103 vs SM103+PDL (N={args.n}, K={args.k})",
+        )
+
+    if args.mode in ("quant", "all"):
+        results = benchmark_quant(m_sizes, n=args.k)
+        print_results(results, f"NVFP4 Activation Quantization (N={args.k})")
+
+    if args.mode in ("sf_convert", "all"):
+        sf_m_sizes = [1024, 2048, 4096, 7168, 8192, 14336, 16384]
+        results = benchmark_sf_conversion(sf_m_sizes, k=args.k)
+        print_results(results, "SF Layout Conversion SM100 <-> SM103")
+
+    if args.mode in ("e2e", "all"):
+        results = benchmark_e2e(m_sizes, n=args.n, k=args.k)
+        print_results(
+            results,
+            f"E2E NVFP4 (Quant+GEMM): SM100 vs SM103 vs SM103+PDL "
+            f"(N={args.n}, K={args.k})",
+        )
+        print(
+            "\nNOTE: sm103_vs_100 = SM100_time / SM103_time (>1 means SM103 faster)\n"
+            "      pdl_vs_nop  = SM103_time / SM103+PDL_time (>1 means PDL faster)\n"
+            "      pdl_vs_100  = SM100_time / SM103+PDL_time (total speedup)"
+        )
+
+    if args.mode in ("pdl", "all"):
+        results = benchmark_pdl_pipeline(
+            m_sizes, n=args.n, k=args.k, num_layers=args.layers
+        )
+        print_results(
+            results,
+            f"PDL Pipeline ({args.layers} layers): SM103 vs SM103+PDL "
+            f"(N={args.n}, K={args.k})",
+        )
+        print(
+            "\nNOTE: pdl_speedup = no_pdl_time / pdl_time\n"
+            "      PDL overlaps quant tail with GEMM head across layer boundaries.\n"
+            "      Benefit is most visible with multiple back-to-back layers."
+        )
+
+
+if __name__ == "__main__":
+    main()
@@ -173,10 +173,13 @@ ScratchPadManager::ScratchPadManager() : size_(0), ptr_(nullptr) {
 void ScratchPadManager::realloc(size_t new_size) {
  new_size = round(new_size);
  if (new_size > size_) {
+    void* new_ptr = std::aligned_alloc(64, new_size);
+    TORCH_CHECK(new_ptr != nullptr,
+                "ScratchPadManager: aligned_alloc failed for size ", new_size);
    if (ptr_ != nullptr) {
      std::free(ptr_);
    }
-    ptr_ = std::aligned_alloc(64, new_size);
+    ptr_ = new_ptr;
    size_ = new_size;
  }
 }
@@ -232,6 +232,28 @@ void unmap_and_release(unsigned long long device, ssize_t size,
    }
  }

+  // ROCm workaround: hipMemRelease does not return physical VRAM to the
+  // free pool while the virtual-address reservation is still held.
+  // Cycling cuMemAddressFree → cuMemAddressReserve (at the same address)
+  // forces the driver to actually release the physical pages while keeping
+  // the same VA available for a later create_and_map.
+  if (first_error == no_error) {
+    first_error = cuMemAddressFree(d_mem, size);
+    if (first_error == no_error) {
+      CUdeviceptr d_mem_new = 0;
+      first_error = cuMemAddressReserve(&d_mem_new, size, 0, d_mem, 0);
+      if (first_error == no_error && d_mem_new != d_mem) {
+        cuMemAddressFree(d_mem_new, size);
+        snprintf(error_msg, sizeof(error_msg),
+                 "ROCm: VA re-reserve got %p instead of %p", (void*)d_mem_new,
+                 (void*)d_mem);
+        error_code = CUresult(1);
+        std::cerr << error_msg << std::endl;
+        return;
+      }
+    }
+  }
+
  if (first_error != no_error) {
    CUDA_CHECK(first_error);
  }
@@ -0,0 +1,9 @@
+#pragma once
+
+#include <torch/csrc/stable/library.h>
+#include <torch/csrc/stable/tensor.h>
+
+#ifndef USE_ROCM
+torch::stable::Tensor permute_cols(torch::stable::Tensor const& A,
+                                   torch::stable::Tensor const& perm);
+#endif
@@ -1,10 +1,13 @@
-#include <torch/all.h>
-
-#include <ATen/cuda/CUDAContext.h>
-#include <c10/cuda/CUDAGuard.h>
+#include <torch/csrc/stable/library.h>
+#include <torch/csrc/stable/tensor.h>
+#include <torch/csrc/stable/accelerator.h>
+#include <torch/csrc/stable/ops.h>
+#include <torch/headeronly/core/ScalarType.h>

 #include <cuda_fp16.h>

+#include "torch_utils.h"
+
 static constexpr int default_threads = 256;
 static constexpr int div_ceil(int a, int b) { return (a + b - 1) / b; }

@@ -64,19 +67,22 @@ __global__ void permute_cols_kernel(int4 const* __restrict__ a_int4_ptr,

 // More efficient version of A[..., perm]
 //  taken from gptq_marlin.cu
-torch::Tensor permute_cols(torch::Tensor const& A, torch::Tensor const& perm) {
-  const at::cuda::OptionalCUDAGuard device_guard(device_of(A));
-  auto dev = A.get_device();
-  auto stream = at::cuda::getCurrentCUDAStream(dev);
+torch::stable::Tensor permute_cols(torch::stable::Tensor const& A,
+                                   torch::stable::Tensor const& perm) {
+  const int32_t dev = A.get_device_index();
+  const torch::stable::accelerator::DeviceGuard device_guard(dev);
+  const auto stream = get_current_cuda_stream(dev);

-  TORCH_CHECK(A.scalar_type() == at::kHalf || A.scalar_type() == at::kBFloat16,
-              "Currently only 16bit types are supported");
-  TORCH_CHECK(A.is_contiguous(), "A must be contiguous");
-  TORCH_CHECK(A.size(-1) % 8 == 0,
-              "A columns must be a multiple of 8 (128bits)");
-  auto A_2d = A.view({-1, A.size(-1)});
+  STD_TORCH_CHECK(
+      A.scalar_type() == torch::headeronly::ScalarType::Half ||
+          A.scalar_type() == torch::headeronly::ScalarType::BFloat16,
+      "Currently only 16bit types are supported");
+  STD_TORCH_CHECK(A.is_contiguous(), "A must be contiguous");
+  STD_TORCH_CHECK(A.size(-1) % 8 == 0,
+                  "A columns must be a multiple of 8 (128bits)");
+  auto A_2d = torch::stable::view(A, {-1, A.size(-1)});

-  torch::Tensor D = torch::empty_like(A);
+  torch::stable::Tensor D = torch::stable::empty_like(A);
  int sms;
  cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, dev);
  int block_rows = div_ceil(A_2d.size(0), sms);
@@ -0,0 +1,21 @@
+#include "ops.h"
+#include "core/registration.h"
+
+#include <torch/csrc/stable/library.h>
+
+// Register ops with STABLE_TORCH_LIBRARY for libtorch stable ABI compatibility.
+// Note: We register under namespace "_C" so ops are accessible as
+// torch.ops._C.<op_name> for compatibility with existing code.
+STABLE_TORCH_LIBRARY_FRAGMENT(_C, m) {
+#ifndef USE_ROCM
+  m.def("permute_cols(Tensor A, Tensor perm) -> Tensor");
+#endif
+}
+
+STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, m) {
+#ifndef USE_ROCM
+  m.impl("permute_cols", TORCH_BOX(&permute_cols));
+#endif
+}
+
+REGISTER_EXTENSION(_C_stable_libtorch)
@@ -0,0 +1,13 @@
+#pragma once
+
+#include <torch/csrc/inductor/aoti_torch/c/shim.h>
+#include <cuda_runtime.h>
+
+// Utility to get the current CUDA stream for a given device using stable APIs.
+// Returns a cudaStream_t for use in kernel launches.
+inline cudaStream_t get_current_cuda_stream(int32_t device_index) {
+  void* stream_ptr = nullptr;
+  TORCH_ERROR_CODE_CHECK(
+      aoti_torch_get_current_cuda_stream(device_index, &stream_ptr));
+  return reinterpret_cast<cudaStream_t>(stream_ptr);
+}
@@ -201,7 +201,6 @@ torch::Tensor awq_dequantize(torch::Tensor _kernel,
                             torch::Tensor _zeros, int64_t split_k_iters,
                             int64_t thx, int64_t thy);

-torch::Tensor permute_cols(torch::Tensor const& A, torch::Tensor const& perm);
 #endif

 torch::Tensor ggml_dequantize(torch::Tensor W, int64_t type, int64_t m,
@@ -238,6 +237,7 @@ void cutlass_scaled_fp4_mm(torch::Tensor& D, torch::Tensor const& A,
                           torch::Tensor const& B_sf,
                           torch::Tensor const& alpha);

+
 void cutlass_scaled_mm(torch::Tensor& out, torch::Tensor const& a,
                       torch::Tensor const& b, torch::Tensor const& a_scales,
                       torch::Tensor const& b_scales,
@@ -286,16 +286,6 @@ void cutlass_scaled_mm_azp(torch::Tensor& out, torch::Tensor const& a,
                           std::optional<torch::Tensor> const& azp,
                           std::optional<torch::Tensor> const& bias);

-bool cutlass_sparse_scaled_mm_supported(int64_t cuda_device_capability);
-
-void cutlass_scaled_sparse_mm(torch::Tensor& out, torch::Tensor const& a,
-                              torch::Tensor const& b, torch::Tensor const& e,
-                              torch::Tensor const& a_scales,
-                              torch::Tensor const& b_scales,
-                              std::optional<torch::Tensor> const& bias);
-
-std::vector<torch::Tensor> cutlass_sparse_compress(torch::Tensor const& a);
-
 std::tuple<torch::Tensor, torch::Tensor> scaled_fp4_quant_func(
    torch::Tensor const& input, torch::Tensor const& input_scale,
    bool is_sf_swizzled_layout);
@@ -317,6 +307,11 @@ void silu_and_mul_scaled_fp4_experts_quant(
    torch::Tensor const& input_offset_by_experts,
    torch::Tensor const& output_scale_offset_by_experts);

+void convert_sf_layout_sm100_to_sm103(torch::Tensor& dst,
+                                      torch::Tensor const& src);
+void convert_sf_layout_sm103_to_sm100(torch::Tensor& dst,
+                                      torch::Tensor const& src);
+
 void per_token_group_quant_fp8(const torch::Tensor& input,
                               torch::Tensor& output_q, torch::Tensor& output_s,
                               int64_t group_size, double eps, double fp8_min,
@@ -27,6 +27,18 @@ void scaled_fp4_quant_sm1xxa(torch::Tensor const& output,
                             bool is_sf_swizzled_layout);
 #endif

+#if defined(ENABLE_NVFP4_SM100) && ENABLE_NVFP4_SM100
+void scaled_fp4_quant_sm103a(torch::Tensor const& output,
+                             torch::Tensor const& input,
+                             torch::Tensor const& output_sf,
+                             torch::Tensor const& input_sf);
+// PDL variant: launches quant with ProgrammaticStreamSerialization.
+void scaled_fp4_quant_sm103a_pdl(torch::Tensor const& output,
+                                 torch::Tensor const& input,
+                                 torch::Tensor const& output_sf,
+                                 torch::Tensor const& input_sf);
+#endif
+
 #if (defined(ENABLE_NVFP4_SM100) && ENABLE_NVFP4_SM100) || \
    (defined(ENABLE_NVFP4_SM120) && ENABLE_NVFP4_SM120)
 void scaled_fp4_experts_quant_sm1xxa(
@@ -132,3 +144,79 @@ void silu_and_mul_scaled_fp4_experts_quant(
  TORCH_CHECK_NOT_IMPLEMENTED(
      false, "No compiled silu_and_mul nvfp4 experts quantization kernel");
 }
+
+// SM103-native quantization: writes SM103-layout scale factors directly,
+// eliminating the SM100->SM103 conversion step on the critical path.
+std::tuple<torch::Tensor, torch::Tensor> scaled_fp4_quant_sm103a_func(
+    torch::Tensor const& input, torch::Tensor const& input_sf) {
+  int64_t n = input.size(-1);
+  int64_t m = input.numel() / n;
+  auto device = input.device();
+
+  auto output = torch::empty(
+      {m, n / 2}, torch::TensorOptions().device(device).dtype(torch::kUInt8));
+
+  auto [sf_m, sf_n] = vllm::computeSwizzledSFShape(m, n);
+  auto output_sf = torch::empty(
+      {sf_m, sf_n},
+      torch::TensorOptions().device(device).dtype(torch::kInt32));
+
+#if defined(ENABLE_NVFP4_SM100) && ENABLE_NVFP4_SM100
+  scaled_fp4_quant_sm103a(output, input, output_sf, input_sf);
+  return {output, output_sf};
+#endif
+  TORCH_CHECK_NOT_IMPLEMENTED(false,
+                              "No compiled SM103 nvfp4 quantization kernel");
+}
+
+void scaled_fp4_quant_sm103a_out(torch::Tensor const& input,
+                                 torch::Tensor const& input_sf,
+                                 torch::Tensor& output,
+                                 torch::Tensor& output_sf) {
+#if defined(ENABLE_NVFP4_SM100) && ENABLE_NVFP4_SM100
+  scaled_fp4_quant_sm103a(output, input, output_sf, input_sf);
+  return;
+#endif
+  TORCH_CHECK_NOT_IMPLEMENTED(false,
+                              "No compiled SM103 nvfp4 quantization kernel");
+}
+
+// ============================================================================
+// PDL-enabled SM103 quantization entry points.
+//
+// These launch the quant kernel with ProgrammaticStreamSerialization,
+// allowing the subsequent GEMM to begin before quantization completes.
+// ============================================================================
+std::tuple<torch::Tensor, torch::Tensor> scaled_fp4_quant_sm103a_pdl_func(
+    torch::Tensor const& input, torch::Tensor const& input_sf) {
+  int64_t n = input.size(-1);
+  int64_t m = input.numel() / n;
+  auto device = input.device();
+
+  auto output = torch::empty(
+      {m, n / 2}, torch::TensorOptions().device(device).dtype(torch::kUInt8));
+
+  auto [sf_m, sf_n] = vllm::computeSwizzledSFShape(m, n);
+  auto output_sf = torch::empty(
+      {sf_m, sf_n},
+      torch::TensorOptions().device(device).dtype(torch::kInt32));
+
+#if defined(ENABLE_NVFP4_SM100) && ENABLE_NVFP4_SM100
+  scaled_fp4_quant_sm103a_pdl(output, input, output_sf, input_sf);
+  return {output, output_sf};
+#endif
+  TORCH_CHECK_NOT_IMPLEMENTED(
+      false, "No compiled SM103 PDL nvfp4 quantization kernel");
+}
+
+void scaled_fp4_quant_sm103a_pdl_out(torch::Tensor const& input,
+                                     torch::Tensor const& input_sf,
+                                     torch::Tensor& output,
+                                     torch::Tensor& output_sf) {
+#if defined(ENABLE_NVFP4_SM100) && ENABLE_NVFP4_SM100
+  scaled_fp4_quant_sm103a_pdl(output, input, output_sf, input_sf);
+  return;
+#endif
+  TORCH_CHECK_NOT_IMPLEMENTED(
+      false, "No compiled SM103 PDL nvfp4 quantization kernel");
+}
@@ -171,8 +171,305 @@ __global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512))
  }
 }

+// ============================================================================
+// SM103 (B300) activation quantization kernel.
+//
+// Identical to the SM100 cvt_fp16_to_fp4 except it writes scale factors
+// in the SM103 swizzled layout (Sm103BlockScaledConfig).
+// ============================================================================
+template <class Type, bool UE8M0_SF = false>
+__global__ void __launch_bounds__(512, VLLM_BLOCKS_PER_SM(512))
+    cvt_fp16_to_fp4_sm103(int32_t numRows, int32_t numCols,
+                          int32_t num_padded_cols,
+                          Type const* __restrict__ in,
+                          float const* __restrict__ SFScale,
+                          uint32_t* __restrict__ out,
+                          uint32_t* __restrict__ SFout) {
+  using PackedVec = vllm::PackedVec<Type, CVT_FP4_PACK16>;
+
+  static constexpr int CVT_FP4_NUM_THREADS_PER_SF =
+      (CVT_FP4_SF_VEC_SIZE / CVT_FP4_ELTS_PER_THREAD);
+  static_assert(sizeof(PackedVec) == sizeof(Type) * CVT_FP4_ELTS_PER_THREAD,
+                "Vec size is not matched.");
+
+  int32_t const numKTiles = (numCols + 63) / 64;
+
+  int sf_m = round_up<int>(numRows, 128);
+  int32_t const colIdx = blockDim.x * blockIdx.y + threadIdx.x;
+  int elem_idx = colIdx * CVT_FP4_ELTS_PER_THREAD;
+
+  float const global_scale = (SFScale == nullptr) ? 1.0f : SFScale[0];
+
+  for (int rowIdx = blockIdx.x; rowIdx < sf_m; rowIdx += gridDim.x) {
+    if (colIdx < num_padded_cols) {
+      PackedVec in_vec;
+      int64_t inOffset = rowIdx * (numCols / CVT_FP4_ELTS_PER_THREAD) + colIdx;
+
+      bool valid = (rowIdx < numRows) && (elem_idx < numCols);
+      if constexpr (CVT_FP4_PACK16) {
+        ld256_cg_or_zero(reinterpret_cast<u32x8_t&>(in_vec),
+                         &reinterpret_cast<const uint32_t*>(in)[inOffset * 8],
+                         valid);
+      } else {
+        ld128_cg_or_zero(reinterpret_cast<uint4&>(in_vec),
+                         &reinterpret_cast<const uint32_t*>(in)[inOffset * 4],
+                         valid);
+      }
+
+      // SM103: Use SM103-specific SF offset function
+      auto sf_out =
+          cvt_quant_to_fp4_get_sf_out_offset_sm103<uint32_t,
+                                                   CVT_FP4_NUM_THREADS_PER_SF>(
+              rowIdx, colIdx, numKTiles, SFout);
+
+      auto out_val =
+          cvt_warp_fp16_to_fp4<Type, CVT_FP4_NUM_THREADS_PER_SF, UE8M0_SF>(
+              in_vec, global_scale, sf_out);
+
+      if (valid) {
+        if constexpr (CVT_FP4_PACK16) {
+          int64_t outOffset = rowIdx * (numCols / 8) + colIdx * 2;
+          uint64_t packed64 =
+              (uint64_t(out_val.hi) << 32) | uint64_t(out_val.lo);
+          reinterpret_cast<uint64_t*>(out)[outOffset >> 1] = packed64;
+        } else {
+          out[inOffset] = out_val;
+        }
+      }
+    }
+  }
+}
+
+// ============================================================================
+// Scale factor layout conversion: SM100 <-> SM103
+//
+// Converts an already-swizzled SF tensor between SM100 and SM103 layouts.
+// Both layouts use the same 512-byte tile structure (128 M-rows x 4 K-cols)
+// but arrange bytes differently within each tile.
+//
+// SM100 offset: outerM(=mIdx%32)*16 + innerM(=(mIdx/32)%4)*4 + innerK
+// SM103 offset: m8(=(mIdx/16)%8)*16 + m4a(=(mIdx/4)%4)*128 + m4b(=mIdx%4)*4
+//               + innerK
+// ============================================================================
+__global__ void convert_sf_sm100_to_sm103_kernel(
+    const uint8_t* __restrict__ src,
+    uint8_t* __restrict__ dst,
+    int32_t numMTiles,
+    int32_t numKTiles) {
+  // Each thread converts one byte (one SF value).
+  // Grid: numMTiles * numKTiles blocks, 512 threads per block.
+  int32_t tile_idx = blockIdx.x;
+  int32_t mTileIdx = tile_idx / numKTiles;
+  int32_t kTileIdx = tile_idx % numKTiles;
+
+  // Each tile is 512 bytes: 128 M-positions x 4 K-positions.
+  int32_t local_idx = threadIdx.x;  // 0..511
+  if (mTileIdx >= numMTiles) return;
+
+  int64_t tile_base = static_cast<int64_t>(tile_idx) << 9;
+
+  // Decode this thread's (mLocal, kLocal) from a simple linear index.
+  int32_t mLocal = local_idx >> 2;   // 0..127
+  int32_t kLocal = local_idx & 3;    // 0..3
+
+  // Compute SM100 source offset within tile.
+  int32_t outerMIdx = mLocal & 31;
+  int32_t innerMIdx = (mLocal >> 5) & 3;
+  int32_t sm100_off = (outerMIdx << 4) | (innerMIdx << 2) | kLocal;
+
+  // Compute SM103 destination offset within tile.
+  int32_t m4b = mLocal & 3;
+  int32_t m4a = (mLocal >> 2) & 3;
+  int32_t m8  = (mLocal >> 4) & 7;
+  int32_t sm103_off = (m8 << 4) | (m4a << 7) | (m4b << 2) | kLocal;
+
+  dst[tile_base + sm103_off] = src[tile_base + sm100_off];
+}
+
+__global__ void convert_sf_sm103_to_sm100_kernel(
+    const uint8_t* __restrict__ src,
+    uint8_t* __restrict__ dst,
+    int32_t numMTiles,
+    int32_t numKTiles) {
+  int32_t tile_idx = blockIdx.x;
+  int32_t mTileIdx = tile_idx / numKTiles;
+  if (mTileIdx >= numMTiles) return;
+
+  int32_t local_idx = threadIdx.x;
+  int64_t tile_base = static_cast<int64_t>(tile_idx) << 9;
+
+  int32_t mLocal = local_idx >> 2;
+  int32_t kLocal = local_idx & 3;
+
+  // SM103 source offset
+  int32_t m4b = mLocal & 3;
+  int32_t m4a = (mLocal >> 2) & 3;
+  int32_t m8  = (mLocal >> 4) & 7;
+  int32_t sm103_off = (m8 << 4) | (m4a << 7) | (m4b << 2) | kLocal;
+
+  // SM100 destination offset
+  int32_t outerMIdx = mLocal & 31;
+  int32_t innerMIdx = (mLocal >> 5) & 3;
+  int32_t sm100_off = (outerMIdx << 4) | (innerMIdx << 2) | kLocal;
+
+  dst[tile_base + sm100_off] = src[tile_base + sm103_off];
+}
+
 }  // namespace vllm

+// ============================================================================
+// Host entry: SM103 activation quantization
+//
+// When use_pdl=true, the kernel is launched with
+// cudaLaunchAttributeProgrammaticStreamSerialization, allowing the next
+// kernel on the same stream (typically the GEMM consumer) to begin
+// executing before this quantization kernel fully completes.  This
+// overlaps the tail of quantization with the head of the GEMM.
+// ============================================================================
+static void scaled_fp4_quant_sm103a_impl(torch::Tensor const& output,
+                                         torch::Tensor const& input,
+                                         torch::Tensor const& output_sf,
+                                         torch::Tensor const& input_sf,
+                                         bool use_pdl) {
+  int32_t m = input.size(0);
+  int32_t n = input.size(1);
+
+  TORCH_CHECK(n % 16 == 0, "The N dimension must be multiple of 16.");
+  TORCH_CHECK(input.scalar_type() == at::ScalarType::Half ||
+                  input.scalar_type() == at::ScalarType::BFloat16,
+              "Unsupported input data type for quantize_to_fp4.");
+
+  int multiProcessorCount =
+      get_device_attribute(cudaDevAttrMultiProcessorCount, -1);
+
+  auto input_sf_ptr = static_cast<float const*>(input_sf.data_ptr());
+  auto sf_out = static_cast<int32_t*>(output_sf.data_ptr());
+  auto output_ptr = static_cast<int64_t*>(output.data_ptr());
+  const at::cuda::OptionalCUDAGuard device_guard(device_of(input));
+  auto stream = at::cuda::getCurrentCUDAStream(input.get_device());
+
+  int sf_n_unpadded = int(n / CVT_FP4_SF_VEC_SIZE);
+
+  dim3 block(std::min(int(n / ELTS_PER_THREAD), 512));
+  int const numBlocksPerSM =
+      vllm_runtime_blocks_per_sm(static_cast<int>(block.x));
+
+  // SM103 always uses swizzled layout (the SM103 variant)
+  int sf_n_int = int(vllm::round_up(sf_n_unpadded, 4) / 4);
+  int32_t num_padded_cols =
+      sf_n_int * 4 * CVT_FP4_SF_VEC_SIZE / CVT_FP4_ELTS_PER_THREAD;
+
+  int grid_y = vllm::div_round_up(num_padded_cols, static_cast<int>(block.x));
+  int grid_x =
+      std::min(vllm::computeEffectiveRows(m),
+               std::max(1, (multiProcessorCount * numBlocksPerSM) / grid_y));
+  dim3 grid(grid_x, grid_y);
+
+  VLLM_DISPATCH_HALF_TYPES(input.scalar_type(), "nvfp4_quant_sm103", [&] {
+    using cuda_type = vllm::CUDATypeConverter<scalar_t>::Type;
+    auto input_ptr = static_cast<cuda_type const*>(input.data_ptr());
+    auto output_u32 = reinterpret_cast<uint32_t*>(output_ptr);
+    auto sf_out_u32 = reinterpret_cast<uint32_t*>(sf_out);
+
+    if (use_pdl) {
+      // PDL launch: set ProgrammaticStreamSerialization so the next kernel
+      // (GEMM) can begin before this quant kernel fully completes.
+      cudaLaunchConfig_t launch_config = {};
+      launch_config.gridDim = grid;
+      launch_config.blockDim = block;
+      launch_config.dynamicSmemBytes = 0;
+      launch_config.stream = stream;
+
+      cudaLaunchAttribute pdl_attr;
+      pdl_attr.id = cudaLaunchAttributeProgrammaticStreamSerialization;
+      pdl_attr.val.programmaticStreamSerializationAllowed = 1;
+      launch_config.numAttrs = 1;
+      launch_config.attrs = &pdl_attr;
+
+      CUDA_CHECK(cudaLaunchKernelEx(
+          &launch_config,
+          vllm::cvt_fp16_to_fp4_sm103<cuda_type, false>,
+          m, n, num_padded_cols, input_ptr, input_sf_ptr,
+          output_u32, sf_out_u32));
+    } else {
+      vllm::cvt_fp16_to_fp4_sm103<cuda_type, false>
+          <<<grid, block, 0, stream>>>(
+              m, n, num_padded_cols, input_ptr, input_sf_ptr,
+              output_u32, sf_out_u32);
+    }
+  });
+}
+
+// Original entry point (no PDL).
+void scaled_fp4_quant_sm103a(torch::Tensor const& output,
+                             torch::Tensor const& input,
+                             torch::Tensor const& output_sf,
+                             torch::Tensor const& input_sf) {
+  scaled_fp4_quant_sm103a_impl(output, input, output_sf, input_sf,
+                                /*use_pdl=*/false);
+}
+
+// PDL-enabled entry point: launches quant kernel with
+// ProgrammaticStreamSerialization to overlap with a subsequent GEMM.
+void scaled_fp4_quant_sm103a_pdl(torch::Tensor const& output,
+                                 torch::Tensor const& input,
+                                 torch::Tensor const& output_sf,
+                                 torch::Tensor const& input_sf) {
+  scaled_fp4_quant_sm103a_impl(output, input, output_sf, input_sf,
+                                /*use_pdl=*/true);
+}
+
+// ============================================================================
+// Host entry: SF layout conversion SM100 <-> SM103
+// ============================================================================
+void convert_sf_layout_sm100_to_sm103(torch::Tensor& dst,
+                                      torch::Tensor const& src) {
+  TORCH_CHECK(src.is_contiguous(), "Source SF tensor must be contiguous");
+  TORCH_CHECK(dst.is_contiguous(), "Destination SF tensor must be contiguous");
+  TORCH_CHECK(src.numel() == dst.numel(),
+              "Source and destination must have the same number of elements");
+
+  // SF tensors are stored as int32 with shape (rounded_m, rounded_k / 4)
+  // Total bytes = rounded_m * (rounded_k / 4) * 4 = rounded_m * rounded_k
+  int64_t total_bytes = src.numel() * src.element_size();
+  int32_t numMTiles = src.size(0) / 128;
+  int32_t numKTiles = total_bytes / (numMTiles * 512);
+
+  const at::cuda::OptionalCUDAGuard device_guard(device_of(src));
+  auto stream = at::cuda::getCurrentCUDAStream(src.get_device());
+
+  int32_t num_tiles = numMTiles * numKTiles;
+  dim3 grid(num_tiles);
+  dim3 block(512);
+
+  vllm::convert_sf_sm100_to_sm103_kernel<<<grid, block, 0, stream>>>(
+      static_cast<const uint8_t*>(src.data_ptr()),
+      static_cast<uint8_t*>(dst.data_ptr()),
+      numMTiles, numKTiles);
+}
+
+void convert_sf_layout_sm103_to_sm100(torch::Tensor& dst,
+                                      torch::Tensor const& src) {
+  TORCH_CHECK(src.is_contiguous() && dst.is_contiguous());
+  TORCH_CHECK(src.numel() == dst.numel());
+
+  int64_t total_bytes = src.numel() * src.element_size();
+  int32_t numMTiles = src.size(0) / 128;
+  int32_t numKTiles = total_bytes / (numMTiles * 512);
+
+  const at::cuda::OptionalCUDAGuard device_guard(device_of(src));
+  auto stream = at::cuda::getCurrentCUDAStream(src.get_device());
+
+  int32_t num_tiles = numMTiles * numKTiles;
+  vllm::convert_sf_sm103_to_sm100_kernel<<<dim3(num_tiles), dim3(512), 0, stream>>>(
+      static_cast<const uint8_t*>(src.data_ptr()),
+      static_cast<uint8_t*>(dst.data_ptr()),
+      numMTiles, numKTiles);
+}
+
+// ============================================================================
+// Original SM100 host entry
+// ============================================================================
 void scaled_fp4_quant_sm1xxa(torch::Tensor const& output,
                             torch::Tensor const& input,
                             torch::Tensor const& output_sf,
@@ -24,6 +24,20 @@ void cutlass_scaled_fp4_mm_sm100a(torch::Tensor& D, torch::Tensor const& A,
                                  torch::Tensor const& A_sf,
                                  torch::Tensor const& B_sf,
                                  torch::Tensor const& alpha);
+// SM103 (B300) uses FP4 Ultra MMA -- separate entry point compiled from
+// the same source file, guarded by CUTLASS_ARCH_MMA_SM103_SUPPORTED.
+void cutlass_scaled_fp4_mm_sm103a(torch::Tensor& D, torch::Tensor const& A,
+                                  torch::Tensor const& B,
+                                  torch::Tensor const& A_sf,
+                                  torch::Tensor const& B_sf,
+                                  torch::Tensor const& alpha);
+// PDL variant: GEMM launched with ProgrammaticStreamSerialization.
+void cutlass_scaled_fp4_mm_sm103a_pdl(torch::Tensor& D,
+                                      torch::Tensor const& A,
+                                      torch::Tensor const& B,
+                                      torch::Tensor const& A_sf,
+                                      torch::Tensor const& B_sf,
+                                      torch::Tensor const& alpha);
 #endif

 #if defined ENABLE_NVFP4_SM120 && ENABLE_NVFP4_SM120
@@ -43,6 +57,14 @@ void cutlass_scaled_fp4_mm(torch::Tensor& D, const torch::Tensor& A,
  const int32_t sm = get_sm_version_num();

 #if defined(ENABLE_NVFP4_SM100) && ENABLE_NVFP4_SM100
+  // SM103 (B300): Use FP4 Ultra kernels with K=768 tiles for higher
+  // throughput. Falls through to SM100 path if SM103 kernels weren’t compiled
+  // (e.g., CUDA < 12.9).
+  if (sm == 103) {
+    cutlass_scaled_fp4_mm_sm103a(D, A, B, A_sf, B_sf, alpha);
+    return;
+  }
+
  if (sm >= 100 && sm < 120) {
    cutlass_scaled_fp4_mm_sm100a(D, A, B, A_sf, B_sf, alpha);
    return;
@@ -36,6 +36,10 @@ using namespace cute;

 #if defined(CUTLASS_ARCH_MMA_SM100_SUPPORTED)

+// ============================================================================
+// SM100 (B200) Tile Configurations
+// ============================================================================
+
 // Configuration for M in (256, inf)
 struct sm100_fp4_config_default {
  using KernelSchedule = cutlass::gemm::collective::KernelScheduleAuto;
@@ -63,6 +67,51 @@ struct sm100_fp4_config_M16 {
  using PerSmTileShape_MNK = Shape<_128, _128, _256>;
 };

+// ============================================================================
+// SM103 (B300 / Blackwell Ultra) Tile Configurations
+//
+// Key differences from SM100:
+//   - Tile K = 768 is MANDATORY (CUTLASS static_assert)
+//   - Uses FP4 Ultra MMA instructions (UltraVs16) for higher throughput
+//   - Uses NoSmem epilogue (saves shared memory for mainloop)
+//   - 1SM for small M, 2SM for large M (cooperative SM pairs)
+// ============================================================================
+#if defined(CUTLASS_ARCH_MMA_SM103_SUPPORTED)
+
+// SM103 configuration for M in (256, inf) -- 2SM cooperative execution
+struct sm103_fp4_config_default {
+  // 2SM schedule: two SMs cooperate on one tile for higher throughput
+  using KernelSchedule = cutlass::gemm::
+      KernelTmaWarpSpecialized2SmBlockScaledMxNvf4UltraVs16Sm103;
+  using EpilogueSchedule = cutlass::epilogue::NoSmemWarpSpecialized2Sm;
+  using TileShape = Shape<_256, _256, Int<768>>;
+  using ClusterShape = Shape<_2, _2, _1>;
+  using PerSmTileShape_MNK = Shape<_128, _256, Int<768>>;
+};
+
+// SM103 configuration for M in (16, 256] -- 2SM with smaller N tile
+struct sm103_fp4_config_M256 {
+  using KernelSchedule = cutlass::gemm::
+      KernelTmaWarpSpecialized2SmBlockScaledMxNvf4UltraVs16Sm103;
+  using EpilogueSchedule = cutlass::epilogue::NoSmemWarpSpecialized2Sm;
+  using TileShape = Shape<_256, _128, Int<768>>;
+  using ClusterShape = Shape<_2, _1, _1>;
+  using PerSmTileShape_MNK = Shape<_128, _128, Int<768>>;
+};
+
+// SM103 configuration for M in [1, 16] -- 1SM (decode / small batch)
+struct sm103_fp4_config_M16 {
+  // 1SM schedule: single SM per tile, lower latency for small problems
+  using KernelSchedule = cutlass::gemm::
+      KernelTmaWarpSpecialized1SmBlockScaledMxNvf4UltraVs16Sm103;
+  using EpilogueSchedule = cutlass::epilogue::NoSmemWarpSpecialized1Sm;
+  using TileShape = Shape<_128, _128, Int<768>>;
+  using ClusterShape = Shape<_1, _1, _1>;
+  using PerSmTileShape_MNK = Shape<_128, _128, Int<768>>;
+};
+
+#endif  // CUTLASS_ARCH_MMA_SM103_SUPPORTED
+
 template <typename Config, typename OutType>
 struct Fp4GemmSm100 {
  // A matrix configuration
@@ -125,6 +174,99 @@ struct Fp4GemmSm100 {
  using LayoutD = decltype(cute::make_layout(make_shape(0, 0, 0), StrideD{}));
 };

+// ============================================================================
+// SM103 GEMM Definition (FP4 Ultra)
+//
+// SM103 differs from SM100 in several fundamental ways:
+//   1. Uses cutlass::arch::Sm103 (separate CollectiveBuilder specialization)
+//   2. Element types passed as cute::tuple<DataType, ScaleFactorType>
+//      (SM100 uses nv_float4_t<float_e2m1_t> wrapper instead)
+//   3. Tile K = 768 (SM100 uses K = 256)
+//   4. Epilogue uses NoSmemWarpSpecialized (SM100 uses TmaWarpSpecialized)
+//   5. Scale factor memory layout uses Sm103BlockScaledConfig
+//      (different swizzle pattern from SM100's Sm1xxBlockScaledConfig)
+//
+// IMPORTANT: Scale factor layout compatibility
+//   SM103 and SM100 use DIFFERENT physical scale factor layouts in memory.
+//   The activation quantization kernel (scaled_fp4_quant) and the weight
+//   scale factors in NVFP4 checkpoints must produce/store data in the
+//   SM103-expected layout when using these kernels. Passing SM100-format
+//   scale factors to SM103 kernels will produce incorrect results.
+//   See Sm103BlockScaledConfig::tile_atom_to_shape_SFA for the expected
+//   layout.
+// ============================================================================
+#if defined(CUTLASS_ARCH_MMA_SM103_SUPPORTED)
+
+template <typename Config, typename OutType>
+struct Fp4GemmSm103 {
+  // A matrix configuration -- bare float_e2m1_t (not nv_float4_t wrapper)
+  using ElementA = cutlass::float_e2m1_t;
+  using ElementSFA = cutlass::float_ue4m3_t;
+  using LayoutATag = cutlass::layout::RowMajor;
+  static constexpr int AlignmentA = 32;
+
+  // B matrix configuration
+  using ElementB = cutlass::float_e2m1_t;
+  using ElementSFB = cutlass::float_ue4m3_t;
+  using LayoutBTag = cutlass::layout::ColumnMajor;
+  static constexpr int AlignmentB = 32;
+
+  // C/D matrix configuration
+  using ElementD = OutType;
+  using ElementC = OutType;
+  using LayoutCTag = cutlass::layout::RowMajor;
+  using LayoutDTag = cutlass::layout::RowMajor;
+  static constexpr int AlignmentD = 128 / cutlass::sizeof_bits<ElementD>::value;
+  static constexpr int AlignmentC = 128 / cutlass::sizeof_bits<ElementC>::value;
+
+  // Kernel functional config
+  using ElementAccumulator = float;
+  using ArchTag = cutlass::arch::Sm103;
+  using OperatorClass = cutlass::arch::OpClassBlockScaledTensorOp;
+
+  // Use config's tile shapes (K=768 mandatory for SM103)
+  using MmaTileShape = typename Config::TileShape;
+  using ClusterShape = typename Config::ClusterShape;
+  using PerSmTileShape_MNK = typename Config::PerSmTileShape_MNK;
+
+  // Epilogue: SM103 uses NoSmem variant with OpClassTensorOp
+  // Note: epilogue builder uses Sm100 arch tag (shared epilogue HW)
+  using CollectiveEpilogue =
+      typename cutlass::epilogue::collective::CollectiveBuilder<
+          cutlass::arch::Sm100, cutlass::arch::OpClassTensorOp,
+          PerSmTileShape_MNK, ClusterShape,
+          cutlass::epilogue::collective::EpilogueTileAuto, ElementAccumulator,
+          ElementAccumulator, ElementC, LayoutCTag, AlignmentC, ElementD,
+          LayoutDTag, AlignmentD,
+          typename Config::EpilogueSchedule>::CollectiveOp;
+
+  // Mainloop: SM103 passes element+SF types as tuples to the builder
+  using CollectiveMainloop =
+      typename cutlass::gemm::collective::CollectiveBuilder<
+          ArchTag, OperatorClass, cute::tuple<ElementA, ElementSFA>, LayoutATag,
+          AlignmentA, cute::tuple<ElementB, ElementSFB>, LayoutBTag, AlignmentB,
+          ElementAccumulator, MmaTileShape, ClusterShape,
+          cutlass::gemm::collective::StageCountAutoCarveout<static_cast<int>(
+              sizeof(typename CollectiveEpilogue::SharedStorage))>,
+          typename Config::KernelSchedule>::CollectiveOp;
+
+  using GemmKernel = cutlass::gemm::kernel::GemmUniversal<
+      Shape<int, int, int, int>, CollectiveMainloop, CollectiveEpilogue, void>;
+  using Gemm = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
+  using StrideA = typename Gemm::GemmKernel::StrideA;
+  using LayoutA = decltype(cute::make_layout(make_shape(0, 0, 0), StrideA{}));
+  using LayoutSFA = typename Gemm::GemmKernel::CollectiveMainloop::LayoutSFA;
+  using StrideB = typename Gemm::GemmKernel::StrideB;
+  using LayoutB = decltype(cute::make_layout(make_shape(0, 0, 0), StrideB{}));
+  using LayoutSFB = typename Gemm::GemmKernel::CollectiveMainloop::LayoutSFB;
+  using StrideC = typename Gemm::GemmKernel::StrideC;
+  using LayoutC = decltype(cute::make_layout(make_shape(0, 0, 0), StrideC{}));
+  using StrideD = typename Gemm::GemmKernel::StrideD;
+  using LayoutD = decltype(cute::make_layout(make_shape(0, 0, 0), StrideD{}));
+};
+
+#endif  // CUTLASS_ARCH_MMA_SM103_SUPPORTED
+
 template <typename Config>
 typename Config::Gemm::Arguments args_from_options(
    at::Tensor& D, at::Tensor const& A, at::Tensor const& B,
@@ -177,7 +319,7 @@ template <typename Config>
 void runGemm(at::Tensor& D, at::Tensor const& A, at::Tensor const& B,
             at::Tensor const& A_sf, at::Tensor const& B_sf,
             at::Tensor const& alpha, int64_t m, int64_t n, int64_t k,
-             cudaStream_t stream) {
+             cudaStream_t stream, bool launch_with_pdl = false) {
  typename Config::Gemm gemm;

  auto arguments =
@@ -192,7 +334,12 @@ void runGemm(at::Tensor& D, at::Tensor const& A, at::Tensor const& B,

  CUTLASS_CHECK(gemm.initialize(arguments, workspace.data_ptr(), stream));

-  CUTLASS_CHECK(gemm.run(arguments, workspace.data_ptr(), stream));
+  // When launch_with_pdl=true, CUTLASS sets
+  // cudaLaunchAttributeProgrammaticStreamSerialization on the GEMM kernel,
+  // allowing the next kernel on the stream to begin before this GEMM
+  // fully completes.
+  CUTLASS_CHECK(gemm.run(arguments, workspace.data_ptr(), stream,
+                         /*cuda_adapter=*/nullptr, launch_with_pdl));
 }

 // Dispatch function to select appropriate config based on M
@@ -220,6 +367,39 @@ void cutlass_fp4_gemm_dispatch(torch::Tensor& D, torch::Tensor const& A,
  }
 }

+// ============================================================================
+// SM103 Dispatch
+// ============================================================================
+#if defined(CUTLASS_ARCH_MMA_SM103_SUPPORTED)
+
+template <typename OutType>
+void cutlass_fp4_gemm_sm103_dispatch(torch::Tensor& D, torch::Tensor const& A,
+                                     torch::Tensor const& B,
+                                     torch::Tensor const& A_sf,
+                                     torch::Tensor const& B_sf,
+                                     torch::Tensor const& alpha, int64_t m,
+                                     int64_t n, int64_t k,
+                                     cudaStream_t stream,
+                                     bool launch_with_pdl = false) {
+  uint32_t const mp2 = std::max(static_cast<uint32_t>(16), next_pow_2(m));
+
+  if (mp2 <= 16) {
+    // m in [1, 16] -- 1SM, low-latency decode
+    runGemm<Fp4GemmSm103<sm103_fp4_config_M16, OutType>>(
+        D, A, B, A_sf, B_sf, alpha, m, n, k, stream, launch_with_pdl);
+  } else if (mp2 <= 256) {
+    // m in (16, 256] -- 2SM, small tile
+    runGemm<Fp4GemmSm103<sm103_fp4_config_M256, OutType>>(
+        D, A, B, A_sf, B_sf, alpha, m, n, k, stream, launch_with_pdl);
+  } else {
+    // m in (256, inf) -- 2SM, large tile
+    runGemm<Fp4GemmSm103<sm103_fp4_config_default, OutType>>(
+        D, A, B, A_sf, B_sf, alpha, m, n, k, stream, launch_with_pdl);
+  }
+}
+
+#endif  // CUTLASS_ARCH_MMA_SM103_SUPPORTED
+
 #else
 template <typename OutType>
 void cutlass_fp4_gemm_dispatch(torch::Tensor& D, torch::Tensor const& A,
@@ -315,3 +495,107 @@ void cutlass_scaled_fp4_mm_sm100a(torch::Tensor& D, torch::Tensor const& A,
                ")");
  }
 }
+
+// ============================================================================
+// SM103 Entry Point (B300 / Blackwell Ultra)
+//
+// Uses FP4 Ultra MMA instructions with K=768 tiles for higher throughput.
+// Scale factors must be in Sm103BlockScaledConfig layout (different from SM100).
+//
+// When launch_with_pdl=true, the CUTLASS GEMM is launched with
+// ProgrammaticStreamSerialization, allowing the next kernel on the stream
+// to begin before this GEMM completes.  Combined with a PDL-enabled
+// quantization producer, this creates a pipelined quant->GEMM overlap.
+// ============================================================================
+#if defined(CUTLASS_ARCH_MMA_SM103_SUPPORTED)
+
+static void cutlass_scaled_fp4_mm_sm103a_impl(
+    torch::Tensor& D, torch::Tensor const& A, torch::Tensor const& B,
+    torch::Tensor const& A_sf, torch::Tensor const& B_sf,
+    torch::Tensor const& alpha, bool launch_with_pdl) {
+  CHECK_INPUT(A, FLOAT4_E2M1X2, "a");
+  CHECK_INPUT(B, FLOAT4_E2M1X2, "b");
+
+  CHECK_INPUT(A_sf, SF_DTYPE, "scale_a");
+  CHECK_INPUT(B_sf, SF_DTYPE, "scale_b");
+
+  CHECK_INPUT(alpha, at::ScalarType::Float, "alpha");
+
+  TORCH_CHECK(A.dim() == 2, "a must be a matrix");
+  TORCH_CHECK(B.dim() == 2, "b must be a matrix");
+  TORCH_CHECK(A.sizes()[1] == B.sizes()[1],
+              "a and b shapes cannot be multiplied (", A.sizes()[0], "x",
+              A.sizes()[1], " and ", B.sizes()[0], "x", B.sizes()[1], ")");
+
+  auto const m = A.sizes()[0];
+  auto const n = B.sizes()[0];
+  auto const k = A.sizes()[1] * 2;
+
+  constexpr int alignment = 32;
+  TORCH_CHECK(k % alignment == 0, "Expected k to be divisible by ", alignment,
+              ", but got a shape: (", A.sizes()[0], "x", A.sizes()[1],
+              "), k: ", k, ".");
+  TORCH_CHECK(n % alignment == 0, "Expected n to be divisible by ", alignment,
+              ", but got b shape: (", B.sizes()[0], "x", B.sizes()[1], ").");
+
+  // SM103 scale factor shape validation.
+  // Physical dimensions are the same as SM100 (padded to 128 x ceil(k/16,4)),
+  // but the internal swizzle pattern (Sm103BlockScaledConfig) differs.
+  auto round_up = [](int x, int y) { return (x + y - 1) / y * y; };
+  int rounded_m = round_up(m, 128);
+  int rounded_n = round_up(n, 128);
+  int rounded_k = round_up(k / 16, 4);
+
+  TORCH_CHECK(A_sf.dim() == 2, "scale_a must be a matrix");
+  TORCH_CHECK(B_sf.dim() == 2, "scale_b must be a matrix");
+  TORCH_CHECK(A_sf.sizes()[1] == B_sf.sizes()[1],
+              "scale_a and scale_b shapes cannot be multiplied (",
+              A_sf.sizes()[0], "x", A_sf.sizes()[1], " and ", B_sf.sizes()[0],
+              "x", B_sf.sizes()[1], ")");
+  TORCH_CHECK(A_sf.sizes()[0] == rounded_m && A_sf.sizes()[1] == rounded_k,
+              "scale_a must be padded and swizzled to a shape (", rounded_m,
+              "x", rounded_k, "), but got a shape (", A_sf.sizes()[0], "x",
+              A_sf.sizes()[1], ")");
+  TORCH_CHECK(B_sf.sizes()[0] == rounded_n && B_sf.sizes()[1] == rounded_k,
+              "scale_b must be padded and swizzled to a shape (", rounded_n,
+              "x", rounded_k, "), but got a shape (", B_sf.sizes()[0], "x",
+              B_sf.sizes()[1], ")");
+
+  auto out_dtype = D.dtype();
+  const at::cuda::OptionalCUDAGuard device_guard(device_of(A));
+  const cudaStream_t stream = at::cuda::getCurrentCUDAStream(A.get_device());
+
+  if (out_dtype == at::ScalarType::Half) {
+    cutlass_fp4_gemm_sm103_dispatch<cutlass::half_t>(
+        D, A, B, A_sf, B_sf, alpha, m, n, k, stream, launch_with_pdl);
+  } else if (out_dtype == at::ScalarType::BFloat16) {
+    cutlass_fp4_gemm_sm103_dispatch<cutlass::bfloat16_t>(
+        D, A, B, A_sf, B_sf, alpha, m, n, k, stream, launch_with_pdl);
+  } else {
+    TORCH_CHECK(false, "Unsupported output data type of nvfp4 mm (", out_dtype,
+                ")");
+  }
+}
+
+// Original entry point (no PDL).
+void cutlass_scaled_fp4_mm_sm103a(torch::Tensor& D, torch::Tensor const& A,
+                                  torch::Tensor const& B,
+                                  torch::Tensor const& A_sf,
+                                  torch::Tensor const& B_sf,
+                                  torch::Tensor const& alpha) {
+  cutlass_scaled_fp4_mm_sm103a_impl(D, A, B, A_sf, B_sf, alpha,
+                                     /*launch_with_pdl=*/false);
+}
+
+// PDL-enabled entry point: GEMM launched with ProgrammaticStreamSerialization
+// so the next kernel on the stream can overlap with this GEMM's tail.
+void cutlass_scaled_fp4_mm_sm103a_pdl(torch::Tensor& D, torch::Tensor const& A,
+                                      torch::Tensor const& B,
+                                      torch::Tensor const& A_sf,
+                                      torch::Tensor const& B_sf,
+                                      torch::Tensor const& alpha) {
+  cutlass_scaled_fp4_mm_sm103a_impl(D, A, B, A_sf, B_sf, alpha,
+                                     /*launch_with_pdl=*/true);
+}
+
+#endif  // CUTLASS_ARCH_MMA_SM103_SUPPORTED
@@ -199,6 +199,55 @@ __device__ __forceinline__ uint8_t* cvt_quant_to_fp4_get_sf_out_offset(
  return reinterpret_cast<uint8_t*>(SFout) + SFOffset;
 }

+// ============================================================================
+// SM103 (Blackwell Ultra / B300) swizzled SF offset.
+//
+// SM103 uses Sm103BlockScaledConfig with a 3-level M decomposition:
+//   M -> (m8, m4a, m4b)  where mIdx = m4b + m4a*4 + m8*16
+//   K -> (sfv16_broadcast, k4)
+//
+// Atom layout:
+//   Shape:  <Shape<_8, _4, _4>, Shape<SFVecSize=16, _4>>
+//   Stride: <Stride<_16, _128, _4>, Stride<_0, _1>>
+//
+// Physical offset = m8*16 + m4a*128 + m4b*4 + k4
+// Each 128-row x 4-col tile occupies 512 bytes (same as SM100).
+// ============================================================================
+template <class SFType, int CVT_FP4_NUM_THREADS_PER_SF>
+__device__ __forceinline__ uint8_t* cvt_quant_to_fp4_get_sf_out_offset_sm103(
+    int rowIdx, int colIdx, int32_t numKTiles, SFType* SFout) {
+  static_assert(CVT_FP4_NUM_THREADS_PER_SF == 1 ||
+                CVT_FP4_NUM_THREADS_PER_SF == 2);
+
+  if (threadIdx.x % CVT_FP4_NUM_THREADS_PER_SF != 0) {
+    return nullptr;
+  }
+
+  int32_t kIdx = colIdx / CVT_FP4_NUM_THREADS_PER_SF;
+  int32_t mIdx = rowIdx;
+
+  // SM103 tile decomposition (128 rows per M-tile, 4 K-positions per K-tile).
+  int32_t mTileIdx = mIdx >> 7;         // mIdx / 128
+  int32_t mLocal = mIdx & 127;          // mIdx % 128
+
+  // SM103 3-level M decomposition: mLocal = m4b + m4a*4 + m8*16
+  int32_t m4b = mLocal & 3;             // mLocal % 4
+  int32_t m4a = (mLocal >> 2) & 3;      // (mLocal / 4) % 4
+  int32_t m8  = (mLocal >> 4) & 7;      // (mLocal / 16) % 8
+
+  int32_t kTileIdx = kIdx >> 2;         // kIdx / 4
+  int32_t innerKIdx = kIdx & 3;         // kIdx % 4
+
+  // Physical offset within the 512-byte tile:
+  //   m8 * 16 + m4a * 128 + m4b * 4 + innerKIdx
+  // Tile base: (mTileIdx * numKTiles + kTileIdx) * 512
+  int64_t SFOffset = (static_cast<int64_t>(mTileIdx) * numKTiles + kTileIdx)
+                         << 9 |
+                     (m8 << 4) | (m4a << 7) | (m4b << 2) | innerKIdx;
+
+  return reinterpret_cast<uint8_t*>(SFout) + SFOffset;
+}
+
 template <class SFType>
 __device__ __forceinline__ uint8_t* sf_out_rowmajor_u8(int row, int pack,
                                                       int packs_per_row_sf,
@@ -26,6 +26,16 @@
  #define __HIP__GFX9__
 #endif

+#if defined(__HIPCC__) &&                                                    \
+    (defined(__gfx1100__) || defined(__gfx1101__) || defined(__gfx1150__) || \
+     defined(__gfx1151__) || defined(__gfx1200__) || defined(__gfx1201__))
+  #define __HIP__GFX1X__
+#endif
+
+#if defined(__HIPCC__) && (defined(__gfx1200__) || defined(__gfx1201__))
+  #define __HIP__GFX12__
+#endif
+
 #if defined(__HIPCC__) && (defined(__gfx942__) || defined(__gfx950__))
  #define __HIP__MI3XX__
 #endif
@@ -37,15 +47,31 @@
 #endif

 int get_lds_size() {
-  static bool is_cached = false;
-  static int result;
-  if (is_cached == false) {
-    auto dprops = at::cuda::getCurrentDeviceProperties();
-    std::string device_arch = dprops->gcnArchName;
-    size_t substring = device_arch.find("gfx95");
-    result = (substring == std::string::npos ? 64 * 1024 : 160 * 1024);
-    is_cached = true;
-  }
+  static const int result = [] {
+    const auto* dprops = at::cuda::getCurrentDeviceProperties();
+    const std::string device_arch = dprops->gcnArchName;
+    return device_arch.find("gfx95") == std::string::npos ? 64 * 1024
+                                                          : 160 * 1024;
+  }();
+  return result;
+}
+
+bool on_gfx1x() {
+  static const bool result = [] {
+    const auto* dprops = at::cuda::getCurrentDeviceProperties();
+    const std::string device_arch = dprops->gcnArchName;
+    return device_arch.find("gfx11") != std::string::npos ||
+           device_arch.find("gfx12") != std::string::npos;
+  }();
+  return result;
+}
+
+bool on_gfx12() {
+  static const bool result = [] {
+    const auto* dprops = at::cuda::getCurrentDeviceProperties();
+    const std::string device_arch = dprops->gcnArchName;
+    return device_arch.find("gfx12") != std::string::npos;
+  }();
  return result;
 }

@@ -286,21 +312,35 @@ torch::Tensor LLMM1(at::Tensor& in_a, at::Tensor& in_b,
  return out_c;
 }

-#define DOT2C(V0, V2, V3)                                                     \
-  if constexpr (std::is_same_v<scalar_t, half>) {                             \
-    asm("v_dot2c_f32_f16 %0, %2, %3" : "=v"(V0) : "0"(V0), "v"(V2), "v"(V3)); \
-  } else if constexpr (std::is_same_v<scalar_t, __hip_bfloat16>) {            \
-    float2 s = __bfloat1622float2(*((__hip_bfloat162*)(&(V2)))) *             \
-               __bfloat1622float2(*((__hip_bfloat162*)(&(V3))));              \
-    V0 += (s.x + s.y);                                                        \
-  }
+#if defined(__HIP__GFX9__) && !defined(__HIP__GFX1X__)
+  #define DOT2C(V0, V2, V3)                                          \
+    if constexpr (std::is_same_v<scalar_t, half>) {                  \
+      asm("v_dot2c_f32_f16 %0, %2, %3"                               \
+          : "=v"(V0)                                                 \
+          : "0"(V0), "v"(V2), "v"(V3));                              \
+    } else if constexpr (std::is_same_v<scalar_t, __hip_bfloat16>) { \
+      float2 s = __bfloat1622float2(*((__hip_bfloat162*)(&(V2)))) *  \
+                 __bfloat1622float2(*((__hip_bfloat162*)(&(V3))));   \
+      V0 += (s.x + s.y);                                             \
+    }
+#elif defined(__HIP__GFX1X__)
+  // gfx1x: v_dot2_f32_f16 (VOP3-P, dot10-insts, available on gfx11+gfx12)
+  #define DOT2C(V0, V2, V3)                                               \
+    if constexpr (std::is_same_v<scalar_t, half>) {                       \
+      asm("v_dot2_f32_f16 %0, %1, %2, %0" : "+v"(V0) : "v"(V2), "v"(V3)); \
+    } else if constexpr (std::is_same_v<scalar_t, __hip_bfloat16>) {      \
+      float2 s = __bfloat1622float2(*((__hip_bfloat162*)(&(V2)))) *       \
+                 __bfloat1622float2(*((__hip_bfloat162*)(&(V3))));        \
+      V0 += (s.x + s.y);                                                  \
+    }
+#endif

 // To avoid LLVM silently upcasting to double
 __device__ inline unsigned int min__(uint32_t a, uint32_t b) {
  return min(a, b);
 }

-#if defined(__HIP__GFX9__)  // TODO: Add NAVI support
+#if defined(__HIP__GFX9__) || defined(__HIP__GFX1X__)
 // This version targets cases where A[] fits LDS capacity
 template <typename scalar_t, int THRDS, int YTILE, int WvPrGrp, int A_CHUNK,
          int UNRL, int N>
@@ -442,14 +482,18 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
                                                1);  // row_shr2
          sum[n][y] += __builtin_amdgcn_mov_dpp(sum[n][y], 0x111, 0xf, 0xf,
                                                1);  // row_shr1
+  #if defined(__HIP__GFX9__)
          sum[n][y] += __builtin_amdgcn_mov_dpp(sum[n][y], 0x142, 0xf, 0xf,
                                                1);  // ROW_BCAST15
          sum[n][y] += __builtin_amdgcn_mov_dpp(sum[n][y], 0x143, 0xf, 0xf,
                                                1);  // ROW_BCAST31
+  #else
+          sum[n][y] += __shfl_xor(sum[n][y], 16);
+  #endif
        }
      }

-      if (threadIdx.x == 63) {
+      if (threadIdx.x == (THRDS - 1)) {
        scalar_t biases[N][YTILE] = {};
        if (BIAS)
          for (int n = 0; n < N; n++) {
@@ -469,9 +513,10 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
        }
      }
    } else {
-  #pragma unroll
+  #ifdef __HIP__GFX9__
+    #pragma unroll
      for (int n = 0; n < N; n++) {
-  #pragma unroll
+    #pragma unroll
        for (int y = 0; y < YTILE; y++) {
          /*float accm1 = 0;
           for (int i=0; i<64; i++)
@@ -498,7 +543,7 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
          sum4[n][y][0] = accm;
        }
      }
-      if (threadIdx.x == 63) {
+      if (threadIdx.x == (THRDS - 1)) {
        scalar_t biases[N][YTILE] = {};
        if (BIAS)
          for (int n = 0; n < N; n++) {
@@ -513,11 +558,12 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
          }
        }
      }
+  #endif  // __HIP__GFX9__ (MFMA path)
    }
    m += CuCount * _WvPrGrp * YTILE;
  }
 }
-#else   // !defined(__HIP__GFX9__) TODO: Add NAVI support
+#else
 template <typename scalar_t, int THRDS, int YTILE, int WvPrGrp, int A_CHUNK,
          int UNRL, int N>
 __global__ void wvSplitK_hf_sml_(const int K, const int Kbp, const int Kap,
@@ -528,9 +574,9 @@ __global__ void wvSplitK_hf_sml_(const int K, const int Kbp, const int Kap,
                                 const int _WvPrGrp, const int CuCount) {
  UNREACHABLE_CODE
 }
-#endif  // defined(__HIP__GFX9__) TODO: Add NAVI support
+#endif

-#if defined(__HIP__GFX9__)  // TODO: Add NAVI support
+#if defined(__HIP__GFX9__) || defined(__HIP__GFX1X__)
 // This version targets cases where A[] marginally exceeds LDS capacity
 template <typename scalar_t, int THRDS, int YTILE, int WvPrGrp, int A_CHUNK,
          int UNRL, int N>
@@ -657,14 +703,18 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
                                                1);  // row_shr2
          sum[n][y] += __builtin_amdgcn_mov_dpp(sum[n][y], 0x111, 0xf, 0xf,
                                                1);  // row_shr1
+  #if defined(__HIP__GFX9__)
          sum[n][y] += __builtin_amdgcn_mov_dpp(sum[n][y], 0x142, 0xf, 0xf,
                                                1);  // ROW_BCAST15
          sum[n][y] += __builtin_amdgcn_mov_dpp(sum[n][y], 0x143, 0xf, 0xf,
                                                1);  // ROW_BCAST31
+  #else
+          sum[n][y] += __shfl_xor(sum[n][y], 16);
+  #endif
        }
      }

-      if (threadIdx.x == 63) {
+      if (threadIdx.x == (THRDS - 1)) {
        scalar_t biases[N][YTILE] = {};
        if (BIAS)
          for (int n = 0; n < N; n++) {
@@ -686,9 +736,10 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
        }
      }
    } else {
-  #pragma unroll
+  #ifdef __HIP__GFX9__
+    #pragma unroll
      for (int n = 0; n < N; n++) {
-  #pragma unroll
+    #pragma unroll
        for (int y = 0; y < YTILE; y++) {
          // float accm1 = 0;
          // for (int i=0; i<64; i++)
@@ -713,7 +764,7 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
          sum4[n][y][0] = accm;
        }
      }
-      if (threadIdx.x == 63) {
+      if (threadIdx.x == (THRDS - 1)) {
        scalar_t biases[N][YTILE] = {};
        if (BIAS)
          for (int n = 0; n < N; n++) {
@@ -730,6 +781,7 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
          }
        }
      }
+  #endif  // __HIP__GFX9__ (MFMA path)
    }

    m += CuCount * _WvPrGrp * YTILE;
@@ -746,7 +798,7 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
  }
 }

-#else   // !defined(__HIP__GFX9__) TODO: Add NAVI support
+#else
 template <typename scalar_t, int THRDS, int YTILE, int WvPrGrp, int A_CHUNK,
          int UNRL, int N>
 __global__ void wvSplitK_hf_(const int K, const int Kbp, const int Kap,
@@ -756,9 +808,9 @@ __global__ void wvSplitK_hf_(const int K, const int Kbp, const int Kap,
                             const int _WvPrGrp, const int CuCount) {
  UNREACHABLE_CODE
 }
-#endif  // defined(__HIP__GFX9__) TODO: Add NAVI support
+#endif

-#if defined(__HIP__GFX9__)  // TODO: Add NAVI support
+#if defined(__HIP__GFX9__) || defined(__HIP__GFX1X__)
 // This version targets big A[] cases, where it is much larger than LDS capacity
 template <typename scalar_t, int THRDS, int YTILE, int WvPrGrp, int A_CHUNK,
          int UNRL, int N>
@@ -1004,14 +1056,18 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
                                                1);  // row_shr2
          sum[n][y] += __builtin_amdgcn_mov_dpp(sum[n][y], 0x111, 0xf, 0xf,
                                                1);  // row_shr1
+  #if defined(__HIP__GFX9__)
          sum[n][y] += __builtin_amdgcn_mov_dpp(sum[n][y], 0x142, 0xf, 0xf,
                                                1);  // ROW_BCAST15
          sum[n][y] += __builtin_amdgcn_mov_dpp(sum[n][y], 0x143, 0xf, 0xf,
                                                1);  // ROW_BCAST31
+  #else
+          sum[n][y] += __shfl_xor(sum[n][y], 16);
+  #endif
        }
      }

-      if (threadIdx.x == 63) {
+      if (threadIdx.x == (THRDS - 1)) {
        scalar_t biases[N][YTILE] = {};
        if (BIAS)
          for (int n = 0; n < N; n++) {
@@ -1033,9 +1089,10 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
        }
      }
    } else {
-  #pragma unroll
+  #ifdef __HIP__GFX9__
+    #pragma unroll
      for (int n = 0; n < N; n++) {
-  #pragma unroll
+    #pragma unroll
        for (int y = 0; y < YTILE; y++) {
          float accm = sum4[n][y][0];
          accm += __builtin_amdgcn_mov_dpp(sum4[n][y][1], 0x101, 0xf, 0xf,
@@ -1057,7 +1114,7 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
          sum4[n][y][0] = accm;
        }
      }
-      if (threadIdx.x == 63) {
+      if (threadIdx.x == (THRDS - 1)) {
        scalar_t biases[N][YTILE] = {};
        if (BIAS)
          for (int n = 0; n < N; n++) {
@@ -1074,6 +1131,7 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
          }
        }
      }
+  #endif  // __HIP__GFX9__ (MFMA path)
    }

    m += CuCount * _WvPrGrp * YTILE;
@@ -1090,7 +1148,7 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
    }
  }
 }
-#else   // !defined(__HIP__GFX9__) TODO: Add NAVI support
+#else
 template <typename scalar_t, int THRDS, int YTILE, int WvPrGrp, int A_CHUNK,
          int UNRL, int N>
 __global__ void wvSplitK_hf_big_(const int K, const int Kbp, const int Kap,
@@ -1101,7 +1159,7 @@ __global__ void wvSplitK_hf_big_(const int K, const int Kbp, const int Kap,
                                 const int _WvPrGrp, const int CuCount) {
  UNREACHABLE_CODE
 }
-#endif  // defined(__HIP__GFX9__) TODO: Add NAVI support
+#endif

 // Find the min val of div2 that doesn't increase N/(div1*div2)
 int mindiv(int N, int div1, int div2) {
@@ -1148,40 +1206,40 @@ torch::Tensor wvSplitK(const at::Tensor& in_a, const at::Tensor& in_b,
  const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
  const int max_lds_len = get_lds_size() / 2;

-#define WVSPLITK(_YTILE, _UNRL, _N)                                           \
+#define WVSPLITK_CFG(_THRDS, _WVPRGRP, _YTILE, _UNRL, _N)                     \
  {                                                                           \
-    dim3 block(64, 16);                                                       \
-    int __wvPrGrp = mindiv(M_in, CuCount * _YTILE, 16);                       \
+    dim3 block(_THRDS, _WVPRGRP);                                             \
+    int __wvPrGrp = mindiv(M_in, CuCount * _YTILE, _WVPRGRP);                 \
    if ((Kbp_in * N_in <= max_lds_len) && (M_in % _YTILE == 0))               \
-      wvSplitK_hf_sml_<fptype, 64, _YTILE, 16, 8, _UNRL, _N>                  \
+      wvSplitK_hf_sml_<fptype, _THRDS, _YTILE, _WVPRGRP, 8, _UNRL, _N>        \
          <<<grid, block, 0, stream>>>(K_in, Kap_in, Kbp_in, M_in, Bx_in,     \
                                       By_in, af4, bf4, biasf4, c, __wvPrGrp, \
                                       CuCount);                              \
    else if (Kbp_in * N_in <= max_lds_len * 1.2)                              \
-      wvSplitK_hf_<fptype, 64, _YTILE, 16, 8, _UNRL, _N>                      \
+      wvSplitK_hf_<fptype, _THRDS, _YTILE, _WVPRGRP, 8, _UNRL, _N>            \
          <<<grid, block, 0, stream>>>(K_in, Kap_in, Kbp_in, M_in, Bx_in,     \
                                       By_in, af4, bf4, biasf4, c, __wvPrGrp, \
                                       CuCount);                              \
    else                                                                      \
-      wvSplitK_hf_big_<fptype, 64, _YTILE, 16, 8, _UNRL, _N>                  \
+      wvSplitK_hf_big_<fptype, _THRDS, _YTILE, _WVPRGRP, 8, _UNRL, _N>        \
          <<<grid, block, 0, stream>>>(K_in, Kap_in, Kbp_in, M_in, Bx_in,     \
                                       By_in, af4, bf4, biasf4, c, __wvPrGrp, \
                                       CuCount);                              \
  }

-#define WVSPLIT_TILE(_sYT, __N)                           \
+#define WVSPLIT_TILE_CFG(_THRDS, _WVPRGRP, _sYT, __N)     \
  {                                                       \
    bool fit_lds = (Kbp_in * N_in <= max_lds_len);        \
    if (_sYT <= 1)                                        \
-      WVSPLITK(1, 4, __N)                                 \
+      WVSPLITK_CFG(_THRDS, _WVPRGRP, 1, 4, __N)           \
    else if ((__N == 1) || (!fit_lds) || (_sYT <= 4 * 2)) \
-      WVSPLITK(2, 2, __N)                                 \
+      WVSPLITK_CFG(_THRDS, _WVPRGRP, 2, 2, __N)           \
    else if (_sYT <= 4 * 3)                               \
-      WVSPLITK(3, 2, __N)                                 \
+      WVSPLITK_CFG(_THRDS, _WVPRGRP, 3, 2, __N)           \
    else if (__N == 4)                                    \
-      WVSPLITK(4, 1, __N)                                 \
+      WVSPLITK_CFG(_THRDS, _WVPRGRP, 4, 1, __N)           \
    else                                                  \
-      WVSPLITK(4, 2, __N)                                 \
+      WVSPLITK_CFG(_THRDS, _WVPRGRP, 4, 2, __N)           \
  }

  AT_DISPATCH_REDUCED_FLOATING_TYPES(in_b.scalar_type(), "wvSplitK", [&] {
@@ -1198,18 +1256,31 @@ torch::Tensor wvSplitK(const at::Tensor& in_a, const at::Tensor& in_b,
    // then cut the active waves to balance their distribution...
    int sYT = (M_in + CuCount * 4 - 1) / (CuCount * 4);

+    const bool use_wave32 = on_gfx1x();
    switch (N_in) {
      case 1:
-        WVSPLIT_TILE(sYT, 1)
+        if (use_wave32)
+          WVSPLIT_TILE_CFG(32, 16, sYT, 1)
+        else
+          WVSPLIT_TILE_CFG(64, 16, sYT, 1)
        break;
      case 2:
-        WVSPLIT_TILE(sYT, 2)
+        if (use_wave32)
+          WVSPLIT_TILE_CFG(32, 16, sYT, 2)
+        else
+          WVSPLIT_TILE_CFG(64, 16, sYT, 2)
        break;
      case 3:
-        WVSPLIT_TILE(sYT, 3)
+        if (use_wave32)
+          WVSPLIT_TILE_CFG(32, 16, sYT, 3)
+        else
+          WVSPLIT_TILE_CFG(64, 16, sYT, 3)
        break;
      case 4:
-        WVSPLIT_TILE(sYT, 4)
+        if (use_wave32)
+          WVSPLIT_TILE_CFG(32, 16, sYT, 4)
+        else
+          WVSPLIT_TILE_CFG(64, 16, sYT, 4)
        break;
      default:
        throw std::runtime_error(
@@ -1653,7 +1724,7 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
  #endif
  }
 }
-#else   // !defined(__HIP__GFX9__) TODO: Add NAVI support
+#else
 template <typename scalar_t, int THRDS, int YTILE, int WvPrGrp, int A_CHUNK,
          int UNRL, int N, int GrpsShrB, int CHUNKK, int DTRMNSTC>
 __global__ void wvSplitKrc_(const int actlN, const int K, const int Kap,
@@ -1688,6 +1759,8 @@ torch::Tensor wvSplitKrc(const at::Tensor& in_a, const at::Tensor& in_b,
  TORCH_CHECK(in_a.dtype() == torch::kFloat16 ||
              in_a.dtype() == torch::kBFloat16);

+  const at::cuda::OptionalCUDAGuard device_guard(device_of(in_a));
+
  auto out_c = torch::empty(
      {N_in, M_in},
      torch::TensorOptions().dtype(in_a.dtype()).device(in_a.device()));
@@ -1696,7 +1769,6 @@ torch::Tensor wvSplitKrc(const at::Tensor& in_a, const at::Tensor& in_b,

  dim3 grid(CuCount);

-  const at::cuda::OptionalCUDAGuard device_guard(device_of(in_a));
  const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
  // const int max_lds_len = get_lds_size() / 2;

@@ -1773,7 +1845,7 @@ torch::Tensor wvSplitKrc(const at::Tensor& in_a, const at::Tensor& in_b,
  return out_c;
 }

-#if defined(__HIP__MI3XX__)  // TODO: Add NAVI support
+#if defined(__HIP__MI3XX__) || defined(__HIP__GFX12__)
 template <typename scalar_t, typename fp8_t, int THRDS, int YTILE, int WvPrGrp,
          int A_CHUNK, int UNRL, int N>
 __global__ void __launch_bounds__(WvPrGrp* THRDS)
@@ -1817,12 +1889,17 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)

  uint32_t m = (blockIdx.x * _WvPrGrp + (threadIdx.y % _WvPrGrp)) * YTILE;

-  using floatx16 = __attribute__((__vector_size__(16 * sizeof(float)))) float;
  float sA = *s_A;
  float sB = *s_B;

  while (m < M) {
+  #ifdef __HIP__GFX12__
+    // gfx12: per-lane scalar accumulation via v_dot4_f32_fp8_fp8
+    float sum[N][YTILE] = {};
+  #else
+    // gfx9: MFMA accumulation
    scalar8 sum[N][YTILE] = {};
+  #endif
    for (uint32_t k1 = 0; k1 < K; k1 += THRDS * A_CHUNK * UNRL) {
      bigType bigA[N][UNRL] = {};
      bigType bigB[YTILE][UNRL];
@@ -1854,6 +1931,17 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
  #pragma unroll
      for (uint32_t k2 = 0; k2 < UNRL; k2++) {
        for (uint32_t n = 0; n < N; n++) {
+  #ifdef __HIP__GFX12__
+          // gfx12: 4 x dot4 per A_CHUNK=16 bytes (4 FP8 per dot4)
+          for (int y = 0; y < YTILE; ++y) {
+    #pragma unroll
+            for (int i = 0; i < A_CHUNK / 4; i++) {
+              sum[n][y] = __builtin_amdgcn_dot4_f32_fp8_fp8(
+                  bigA[n][k2].i[i], bigB[y][k2].i[i], sum[n][y]);
+            }
+          }
+  #else
+          // gfx9: MFMA path
          for (int i = 0; i < A_CHUNK; i += 8) {
            for (int y = 0; y < YTILE; ++y) {
              sum[n][y] = __builtin_amdgcn_mfma_f32_16x16x32_fp8_fp8(
@@ -1861,11 +1949,33 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
                  0);
            }
          }
+  #endif
        }
      }
    }

    // Final reduction
+  #ifdef __HIP__GFX12__
+    // gfx12 wave32: DPP row_shr within 16-lane rows + cross-row shuffle
+    for (int n = 0; n < N; n++) {
+      for (int y = 0; y < YTILE; y++) {
+        asm("s_nop 0\n\tv_add_f32 %0, %2, %3 row_shr:8 bound_ctrl:0 "
+            : "=v"(sum[n][y])
+            : "0"(sum[n][y]), "v"(sum[n][y]), "v"(sum[n][y]));
+        asm("s_nop 0\n\tv_add_f32 %0, %2, %3 row_shr:4 bound_ctrl:0 "
+            : "=v"(sum[n][y])
+            : "0"(sum[n][y]), "v"(sum[n][y]), "v"(sum[n][y]));
+        asm("s_nop 0\n\tv_add_f32 %0, %2, %3 row_shr:2 bound_ctrl:0 "
+            : "=v"(sum[n][y])
+            : "0"(sum[n][y]), "v"(sum[n][y]), "v"(sum[n][y]));
+        asm("s_nop 0\n\tv_add_f32 %0, %2, %3 row_shr:1 bound_ctrl:0 "
+            : "=v"(sum[n][y])
+            : "0"(sum[n][y]), "v"(sum[n][y]), "v"(sum[n][y]));
+        sum[n][y] += __shfl_xor(sum[n][y], 16);
+      }
+    }
+  #else
+    // gfx9 MFMA reduction
    for (int n = 0; n < N; n++) {
      for (int y = 0; y < YTILE; y++) {
        float accm0 = sum[n][y][0];
@@ -1880,8 +1990,15 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
        sum[n][y][0] = accm0;
      }
    }
+  #endif

-    if (threadIdx.x == 0) {
+    const bool writeback_lane =
+  #ifdef __HIP__GFX12__
+        threadIdx.x == (THRDS - 1);
+  #else
+        threadIdx.x == 0;
+  #endif
+    if (writeback_lane) {
      scalar_t biases[N][YTILE] = {};
      if (BIAS)
        for (int n = 0; n < N; n++) {
@@ -1892,13 +2009,17 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
      for (int n = 0; n < N; n++) {
        for (int y = 0; y < YTILE; y++) {
          if (y + m >= M) break;  // To avoid mem access fault.
-          sum[n][y][0] *= sA * sB;
+  #ifdef __HIP__GFX12__
+          float result = sum[n][y] * sA * sB;
+  #else
+          float result = sum[n][y][0] * sA * sB;
+  #endif
          if constexpr (std::is_same_v<scalar_t, half>) {
-            sum[n][y][0] += __half2float(biases[n][y]);
+            result += __half2float(biases[n][y]);
          } else if constexpr (std::is_same_v<scalar_t, __hip_bfloat16>) {
-            sum[n][y][0] += __bfloat162float(biases[n][y]);
+            result += __bfloat162float(biases[n][y]);
          }
-          C[m + y + n * M] = __float2s<scalar_t>(sum[n][y][0]);
+          C[m + y + n * M] = __float2s<scalar_t>(result);
        }
      }
    }
@@ -1906,7 +2027,7 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
    m += CuCount * _WvPrGrp * YTILE;
  }
 }
-#else   // !defined(__HIP__MI3XX__) TODO: Add NAVI support
+#else   // !defined(__HIP__MI3XX__) && !defined(__HIP__GFX12__)
 template <typename scalar_t, typename fp8_t, int THRDS, int YTILE, int WvPrGrp,
          int A_CHUNK, int UNRL, int N>
 __global__ void wvSplitKQ_hf_sml_(const int K, const int Kap, const int Kbp,
@@ -1918,9 +2039,9 @@ __global__ void wvSplitKQ_hf_sml_(const int K, const int Kap, const int Kbp,
                                  const int _WvPrGrp, const int CuCount) {
  UNREACHABLE_CODE
 }
-#endif  // defined(__HIP__MI3XX__) TODO: Add NAVI support
+#endif  // defined(__HIP__MI3XX__) || defined(__HIP__GFX12__)

-#if defined(__HIP__MI3XX__)  // TODO: Add NAVI support
+#if defined(__HIP__MI3XX__) || defined(__HIP__GFX12__)
 template <typename scalar_t, typename fp8_t, int THRDS, int YTILE, int WvPrGrp,
          int A_CHUNK, int UNRL, int N>
 __global__ void __launch_bounds__(WvPrGrp* THRDS)
@@ -1963,12 +2084,17 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)

  uint32_t m = (blockIdx.x * _WvPrGrp + (threadIdx.y % _WvPrGrp)) * YTILE;

-  using floatx16 = __attribute__((__vector_size__(16 * sizeof(float)))) float;
  float sA = *s_A;
  float sB = *s_B;

  while (m < M) {
+  #ifdef __HIP__GFX12__
+    // gfx12: per-lane scalar accumulation via v_dot4_f32_fp8_fp8
+    float sum[N][YTILE] = {};
+  #else
+    // gfx9: MFMA accumulation
    scalar8 sum[N][YTILE] = {};
+  #endif
    for (uint32_t k1 = 0; k1 < K; k1 += THRDS * A_CHUNK * UNRL) {
      bigType bigA[N][UNRL] = {};
      bigType bigB[YTILE][UNRL];
@@ -2002,6 +2128,17 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
  #pragma unroll
      for (uint32_t k2 = 0; k2 < UNRL; k2++) {
        for (uint32_t n = 0; n < N; n++) {
+  #ifdef __HIP__GFX12__
+          // gfx12: 4 x dot4 per A_CHUNK=16 bytes (4 FP8 per dot4)
+          for (int y = 0; y < YTILE; ++y) {
+    #pragma unroll
+            for (int i = 0; i < A_CHUNK / 4; i++) {
+              sum[n][y] = __builtin_amdgcn_dot4_f32_fp8_fp8(
+                  bigA[n][k2].i[i], bigB[y][k2].i[i], sum[n][y]);
+            }
+          }
+  #else
+          // gfx9: MFMA path
          for (int i = 0; i < A_CHUNK; i += 8) {
            for (int y = 0; y < YTILE; ++y) {
              sum[n][y] = __builtin_amdgcn_mfma_f32_16x16x32_fp8_fp8(
@@ -2009,11 +2146,33 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
                  0);
            }
          }
+  #endif
        }
      }
    }

    // Final reduction
+  #ifdef __HIP__GFX12__
+    // gfx12 wave32: DPP row_shr within 16-lane rows + cross-row shuffle
+    for (int n = 0; n < N; n++) {
+      for (int y = 0; y < YTILE; y++) {
+        asm("s_nop 0\n\tv_add_f32 %0, %2, %3 row_shr:8 bound_ctrl:0 "
+            : "=v"(sum[n][y])
+            : "0"(sum[n][y]), "v"(sum[n][y]), "v"(sum[n][y]));
+        asm("s_nop 0\n\tv_add_f32 %0, %2, %3 row_shr:4 bound_ctrl:0 "
+            : "=v"(sum[n][y])
+            : "0"(sum[n][y]), "v"(sum[n][y]), "v"(sum[n][y]));
+        asm("s_nop 0\n\tv_add_f32 %0, %2, %3 row_shr:2 bound_ctrl:0 "
+            : "=v"(sum[n][y])
+            : "0"(sum[n][y]), "v"(sum[n][y]), "v"(sum[n][y]));
+        asm("s_nop 0\n\tv_add_f32 %0, %2, %3 row_shr:1 bound_ctrl:0 "
+            : "=v"(sum[n][y])
+            : "0"(sum[n][y]), "v"(sum[n][y]), "v"(sum[n][y]));
+        sum[n][y] += __shfl_xor(sum[n][y], 16);
+      }
+    }
+  #else
+    // gfx9 MFMA reduction
    for (int n = 0; n < N; n++) {
      for (int y = 0; y < YTILE; y++) {
        float accm0 = sum[n][y][0];
@@ -2028,8 +2187,15 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
        sum[n][y][0] = accm0;
      }
    }
+  #endif

-    if (threadIdx.x == 0) {
+    const bool writeback_lane =
+  #ifdef __HIP__GFX12__
+        threadIdx.x == (THRDS - 1);
+  #else
+        threadIdx.x == 0;
+  #endif
+    if (writeback_lane) {
      scalar_t biases[N][YTILE] = {};
      if (BIAS)
        for (int n = 0; n < N; n++) {
@@ -2040,13 +2206,17 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
      for (int n = 0; n < N; n++) {
        for (int y = 0; y < YTILE; y++) {
          if (y + m >= M) break;  // To avoid mem access fault.
-          sum[n][y][0] *= sA * sB;
+  #ifdef __HIP__GFX12__
+          float result = sum[n][y] * sA * sB;
+  #else
+          float result = sum[n][y][0] * sA * sB;
+  #endif
          if constexpr (std::is_same_v<scalar_t, half>) {
-            sum[n][y][0] += __half2float(biases[n][y]);
+            result += __half2float(biases[n][y]);
          } else if constexpr (std::is_same_v<scalar_t, __hip_bfloat16>) {
-            sum[n][y][0] += __bfloat162float(biases[n][y]);
+            result += __bfloat162float(biases[n][y]);
          }
-          C[m + y + n * M] = __float2s<scalar_t>(sum[n][y][0]);
+          C[m + y + n * M] = __float2s<scalar_t>(result);
        }
      }
    }
@@ -2054,7 +2224,7 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS)
    m += CuCount * _WvPrGrp * YTILE;
  }
 }
-#else   // !defined(__HIP__MI3XX__) TODO: Add NAVI support
+#else   // !defined(__HIP__MI3XX__) && !defined(__HIP__GFX12__)
 template <typename scalar_t, typename fp8_t, int THRDS, int YTILE, int WvPrGrp,
          int A_CHUNK, int UNRL, int N>
 __global__ void wvSplitKQ_hf_(const int K, const int Kap, const int Kbp,
@@ -2066,7 +2236,7 @@ __global__ void wvSplitKQ_hf_(const int K, const int Kap, const int Kbp,
                              const int CuCount) {
  UNREACHABLE_CODE
 }
-#endif  // defined(__HIP__MI3XX__) TODO: Add NAVI support
+#endif  // defined(__HIP__MI3XX__) || defined(__HIP__GFX12__)

 void wvSplitKQ(const at::Tensor& in_b, const at::Tensor& in_a,
               const std::optional<at::Tensor>& in_bias, at::Tensor& out_c,
@@ -2099,24 +2269,30 @@ void wvSplitKQ(const at::Tensor& in_b, const at::Tensor& in_a,
  const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
  const int max_lds_len = get_lds_size();

-#define WVSPLITKQ(_WvPrGrp, _YTILEs, _YTILEm, _UNRLs, _UNRLm, _N)             \
-  {                                                                           \
-    dim3 block(64, _WvPrGrp);                                                 \
-    if ((Kap_in * N_in <= max_lds_len) && (M_in % _YTILEs == 0)) {            \
-      int __wvPrGrp = min(_WvPrGrp, mindiv(M_in, CuCount * _YTILEs, 16));     \
-      wvSplitKQ_hf_sml_<fptype, fp8_t, 64, _YTILEs, _WvPrGrp, 16, _UNRLs, _N> \
-          <<<grid, block, 0, stream>>>(K_in, Kap_in, Kbp_in, M_in, Bx_in,     \
-                                       By_in, b_ptr, a_ptr, bias_ptr, c_ptr,  \
-                                       s_a, s_b, __wvPrGrp, CuCount);         \
-    } else {                                                                  \
-      int __wvPrGrp = min(_WvPrGrp, mindiv(M_in, CuCount * _YTILEm, 16));     \
-      wvSplitKQ_hf_<fptype, fp8_t, 64, _YTILEm, _WvPrGrp, 16, _UNRLm, _N>     \
-          <<<grid, block, 0, stream>>>(K_in, Kap_in, Kbp_in, M_in, Bx_in,     \
-                                       By_in, b_ptr, a_ptr, bias_ptr, c_ptr,  \
-                                       s_a, s_b, __wvPrGrp, CuCount);         \
-    }                                                                         \
+#define WVSPLITKQ_IMPL(_THRDS, _WvPrGrp, _YTILEs, _YTILEm, _UNRLs, _UNRLm, _N) \
+  {                                                                            \
+    dim3 block(_THRDS, _WvPrGrp);                                              \
+    if ((Kap_in * N_in <= max_lds_len) && (M_in % _YTILEs == 0)) {             \
+      int __wvPrGrp = min(_WvPrGrp, mindiv(M_in, CuCount * _YTILEs, 16));      \
+      wvSplitKQ_hf_sml_<fptype, fp8_t, _THRDS, _YTILEs, _WvPrGrp, 16, _UNRLs,  \
+                        _N><<<grid, block, 0, stream>>>(                       \
+          K_in, Kap_in, Kbp_in, M_in, Bx_in, By_in, b_ptr, a_ptr, bias_ptr,    \
+          c_ptr, s_a, s_b, __wvPrGrp, CuCount);                                \
+    } else {                                                                   \
+      int __wvPrGrp = min(_WvPrGrp, mindiv(M_in, CuCount * _YTILEm, 16));      \
+      wvSplitKQ_hf_<fptype, fp8_t, _THRDS, _YTILEm, _WvPrGrp, 16, _UNRLm, _N>  \
+          <<<grid, block, 0, stream>>>(K_in, Kap_in, Kbp_in, M_in, Bx_in,      \
+                                       By_in, b_ptr, a_ptr, bias_ptr, c_ptr,   \
+                                       s_a, s_b, __wvPrGrp, CuCount);          \
+    }                                                                          \
  }

+#define WVSPLITKQ(_WvPrGrp, _YTILEs, _YTILEm, _UNRLs, _UNRLm, _N)      \
+  if (on_gfx12())                                                      \
+    WVSPLITKQ_IMPL(32, _WvPrGrp, _YTILEs, _YTILEm, _UNRLs, _UNRLm, _N) \
+  else                                                                 \
+    WVSPLITKQ_IMPL(64, _WvPrGrp, _YTILEs, _YTILEm, _UNRLs, _UNRLm, _N)
+
  AT_DISPATCH_REDUCED_FLOATING_TYPES(out_c.scalar_type(), "wvSplitKQ", [&] {
    using fptype = typename scalar<scalar_t>::type;
    auto c_ptr = reinterpret_cast<fptype*>(out_c.data_ptr());
@@ -2136,10 +2312,10 @@ void wvSplitKQ(const at::Tensor& in_b, const at::Tensor& in_a,
          WVSPLITKQ(16, 2, 2, 2, 2, 2)
          break;
        case 3:
-          WVSPLITKQ(16, 2, 2, 2, 2, 3)
+          WVSPLITKQ(16, 2, 2, 1, 1, 3)
          break;
        case 4:
-          WVSPLITKQ(16, 2, 2, 2, 2, 4)
+          WVSPLITKQ(16, 2, 2, 1, 1, 4)
          break;
        default:
          throw std::runtime_error(
@@ -1,90 +0,0 @@
-#pragma once
-
-// clang-format will break include orders
-// clang-format off
-#include <cudaTypedefs.h>
-
-#if defined CUDA_VERSION && CUDA_VERSION >= 12020
-#include "sparse_scaled_mm_c3x.cuh"
-
-#include "cutlass/numeric_conversion.h"
-#include "cutlass/transform/device/transform_universal_adapter.hpp"
-#include "cutlass/transform/kernel/sparse_gemm_compressor.hpp"
-#include "cutlass/epilogue/collective/default_epilogue.hpp"
-
-// clang-format on
-
-using namespace cute;
-using namespace vllm;
-
-using CompressorResult = std::tuple<torch::Tensor, torch::Tensor>;
-/// Make A structured sparse by replacing elements with 0 and compress it
-template <typename Gemm>
-CompressorResult cutlass_sparse_compress(torch::Tensor const& a) {
-  // Checks for conformality
-  TORCH_CHECK(a.dtype() == torch::kInt8 || a.dtype() == torch::kFloat8_e4m3fn ||
-              a.dtype() == torch::kFloat16 || a.dtype() == torch::kBFloat16);
-  TORCH_CHECK(a.dim() == 2)
-  // Check for strides and alignment
-  TORCH_CHECK(a.stride(0) % 4 == 0)  // Required for semi-structured sparsity
-  TORCH_CHECK(a.stride(1) == 1)
-
-  using GemmKernel = typename Gemm::KernelType;
-  using ElementA = typename Gemm::ElementAB;
-  using ElementE = typename GemmKernel::CollectiveMainloop::ElementE;
-
-  int m = a.size(0);
-  int k = a.size(1);
-  using ProblemShape = typename GemmKernel::ProblemShape;
-  ProblemShape prob_shape{m, 1, k, 1};
-
-  int64_t lda = a.stride(0);
-  using StrideA = Stride<int64_t, Int<1>, int64_t>;
-  StrideA a_stride{lda, Int<1>{}, 0};
-
-  using CompressorUtility = typename Gemm::CompressorUtility;
-  CompressorUtility compressor_utility(prob_shape, a_stride);
-
-  // Allocate buffers for the metadata E and the compressed matrix A
-  int ME = compressor_utility.get_metadata_m_physical();
-  int KE = compressor_utility.get_metadata_k_physical();
-  int MC = compressor_utility.get_tensorA_m_physical();
-  int KC = compressor_utility.get_tensorA_k_physical();
-
-  auto const a_meta_options =
-      torch::TensorOptions().dtype(torch::kUInt8).device(a.device());
-  auto const a_nzs_options =
-      torch::TensorOptions().dtype(a.dtype()).device(a.device());
-
-  auto a_meta = torch::zeros({ME, KE}, a_meta_options);
-  auto a_nzs = torch::zeros({MC, KC}, a_nzs_options);
-
-  auto a_ptr = static_cast<ElementA*>(a.data_ptr());
-  auto a_nzs_ptr = static_cast<ElementA*>(a_nzs.data_ptr());
-  auto a_meta_ptr = static_cast<ElementE*>(a_meta.data_ptr());
-
-  cutlass::KernelHardwareInfo hw_info;
-  hw_info.device_id = a.device().index();
-  hw_info.sm_count =
-      cutlass::KernelHardwareInfo::query_device_multiprocessor_count(
-          hw_info.device_id);
-
-  using Compressor = typename Gemm::Compressor;
-  typename Compressor::Arguments arguments{
-      prob_shape, {a_ptr, a_stride, a_nzs_ptr, a_meta_ptr}, {hw_info}};
-
-  Compressor compressor_op;
-  size_t workspace_size = Compressor::get_workspace_size(arguments);
-  auto const workspace_options =
-      torch::TensorOptions().dtype(torch::kUInt8).device(a.device());
-  auto workspace = torch::empty(workspace_size, workspace_options);
-
-  CUTLASS_CHECK(compressor_op.can_implement(arguments));
-  CUTLASS_CHECK(compressor_op.initialize(arguments, workspace.data_ptr()));
-  CUTLASS_CHECK(compressor_op.run());
-  CUDA_CHECK(cudaDeviceSynchronize());
-
-  return {a_meta, a_nzs};
-}
-
-#endif
@@ -1,307 +0,0 @@
-// clang-format will break include orders
-// clang-format off
-#include <cudaTypedefs.h>
-
-#if defined CUDA_VERSION && CUDA_VERSION >= 12020
-#include "sparse_scaled_mm_c3x.cuh"
-// clang-format on
-
-using namespace cute;
-using namespace vllm;
-
-struct GemmCallerTraits {
-  using return_type = void;
-
-  template <typename GemmConfig, typename... Args>
-  static return_type invoke(Args&&... args) {
-    return cutlass_sparse_gemm_caller<GemmConfig>(std::forward<Args>(args)...);
-  }
-};
-
-struct GemmCompressorTraits {
-  using return_type = CompressorResult;
-
-  template <typename GemmConfig, typename... Args>
-  static return_type invoke(Args&&... args) {
-    return cutlass_sparse_compress<GemmConfig>(std::forward<Args>(args)...);
-  }
-};
-
-template <typename InType, typename OutType,
-          template <typename, typename, typename> typename Epilogue,
-          typename DispatchFunc, typename... Args>
-typename DispatchFunc::return_type cutlass_gemm_sm90_fp8_dispatch(
-    uint32_t m, uint32_t n, Args&&... args) {
-  static_assert(std::is_same_v<InType, cutlass::float_e4m3_t>);
-
-  using Cutlass3xGemmDefault =
-      typename sm90_config_default<InType, OutType, Epilogue>::Cutlass3xGemm;
-  using Cutlass3xGemmM64 =
-      typename sm90_fp8_config_M64<InType, OutType, Epilogue>::Cutlass3xGemm;
-  using Cutlass3xGemmM128 =
-      typename sm90_fp8_config_M128<InType, OutType, Epilogue>::Cutlass3xGemm;
-  using Cutlass3xGemmM256 =
-      typename sm90_fp8_config_M256<InType, OutType, Epilogue>::Cutlass3xGemm;
-  using Cutlass3xGemmM512 =
-      typename sm90_fp8_config_M512<InType, OutType, Epilogue>::Cutlass3xGemm;
-
-  using Cutlass3xGemm1 =
-      typename sm90_fp8_config_1<InType, OutType, Epilogue>::Cutlass3xGemm;
-  using Cutlass3xGemm2 =
-      typename sm90_fp8_config_2<InType, OutType, Epilogue>::Cutlass3xGemm;
-  using Cutlass3xGemm3 =
-      typename sm90_fp8_config_3<InType, OutType, Epilogue>::Cutlass3xGemm;
-  using Cutlass3xGemm4 =
-      typename sm90_fp8_config_4<InType, OutType, Epilogue>::Cutlass3xGemm;
-  using Cutlass3xGemm5 =
-      typename sm90_fp8_config_5<InType, OutType, Epilogue>::Cutlass3xGemm;
-  using Cutlass3xGemm6 =
-      typename sm90_fp8_config_6<InType, OutType, Epilogue>::Cutlass3xGemm;
-  using Cutlass3xGemm7 =
-      typename sm90_fp8_config_7<InType, OutType, Epilogue>::Cutlass3xGemm;
-  using Cutlass3xGemm8 =
-      typename sm90_fp8_config_8<InType, OutType, Epilogue>::Cutlass3xGemm;
-
-  uint32_t const mp2 =
-      std::max(static_cast<uint32_t>(64), next_pow_2(m));  // next power of 2
-
-  if (mp2 <= 64) {
-    if (n == 28672) {
-      return DispatchFunc::template invoke<Cutlass3xGemm2>(
-          std::forward<Args>(args)...);
-    } else if (n == 4096 || n == 6144) {
-      return DispatchFunc::template invoke<Cutlass3xGemm1>(
-          std::forward<Args>(args)...);
-    }
-  } else if (mp2 <= 128) {
-    if (n == 4096) {
-      return DispatchFunc::template invoke<Cutlass3xGemm3>(
-          std::forward<Args>(args)...);
-    } else if (n == 28672) {
-      return DispatchFunc::template invoke<Cutlass3xGemm5>(
-          std::forward<Args>(args)...);
-    } else if (n == 6144) {
-      return DispatchFunc::template invoke<Cutlass3xGemm4>(
-          std::forward<Args>(args)...);
-    }
-  } else if (mp2 <= 256) {
-    if (n == 4096) {
-      return DispatchFunc::template invoke<Cutlass3xGemm6>(
-          std::forward<Args>(args)...);
-    } else if (n == 28672) {
-      return DispatchFunc::template invoke<Cutlass3xGemm8>(
-          std::forward<Args>(args)...);
-    } else if (n == 6144) {
-      return DispatchFunc::template invoke<Cutlass3xGemm7>(
-          std::forward<Args>(args)...);
-    }
-  } else {
-    if (n == 6144 || n == 28672) {
-      return DispatchFunc::template invoke<Cutlass3xGemm8>(
-          std::forward<Args>(args)...);
-    } else if (n == 4096) {
-      return DispatchFunc::template invoke<Cutlass3xGemm7>(
-          std::forward<Args>(args)...);
-    }
-  }
-
-  // Otherwise the default heuristic
-  if (mp2 <= 64) {
-    // n in [1, 64]
-    return DispatchFunc::template invoke<Cutlass3xGemmM64>(
-        std::forward<Args>(args)...);
-  } else if (mp2 <= 128) {
-    // n in (64, 128]
-    return DispatchFunc::template invoke<Cutlass3xGemmM128>(
-        std::forward<Args>(args)...);
-  } else if (mp2 <= 256) {
-    // n in (128, 256]
-    return DispatchFunc::template invoke<Cutlass3xGemmM256>(
-        std::forward<Args>(args)...);
-  } else {
-    // n in (256, inf)
-    return DispatchFunc::template invoke<Cutlass3xGemmM512>(
-        std::forward<Args>(args)...);
-  }
-}
-
-template <typename InType, typename OutType,
-          template <typename, typename, typename> typename Epilogue,
-          typename DispatchFunc, typename... Args>
-typename DispatchFunc::return_type cutlass_gemm_sm90_16bit_dispatch(
-    uint32_t m, uint32_t n, Args&&... args) {
-  using Cutlass3xGemmDefault =
-      typename sm90_config_default<InType, OutType, Epilogue>::Cutlass3xGemm;
-
-  return DispatchFunc::template invoke<Cutlass3xGemmDefault>(
-      std::forward<Args>(args)...);
-}
-
-template <typename InType, typename OutType,
-          template <typename, typename, typename> typename Epilogue,
-          typename DispatchFunc, typename... Args>
-typename DispatchFunc::return_type cutlass_gemm_sm90_int8_dispatch(
-    uint32_t m, uint32_t n, Args&&... args) {
-  static_assert(std::is_same_v<InType, int8_t>);
-
-  using Cutlass3xGemmDefault =
-      typename sm90_config_default<InType, OutType, Epilogue>::Cutlass3xGemm;
-  using Cutlass3xGemmM128 =
-      typename sm90_int8_config_M128<InType, OutType, Epilogue>::Cutlass3xGemm;
-  using Cutlass3xGemmM64 =
-      typename sm90_int8_config_M64<InType, OutType, Epilogue>::Cutlass3xGemm;
-  using Cutlass3xGemmM32NBig =
-      typename sm90_int8_config_M32_NBig<InType, OutType,
-                                         Epilogue>::Cutlass3xGemm;
-  using Cutlass3xGemmM32NSmall =
-      typename sm90_int8_config_M32_NSmall<InType, OutType,
-                                           Epilogue>::Cutlass3xGemm;
-
-  bool const is_small_n = n < 8192;
-  uint32_t const mp2 =
-      std::max(static_cast<uint32_t>(32), next_pow_2(m));  // next power of 2
-
-  if (mp2 <= 32) {
-    // m in [1, 32]
-    if (is_small_n) {
-      return DispatchFunc::template invoke<Cutlass3xGemmM32NSmall>(
-          std::forward<Args>(args)...);
-    } else {
-      return DispatchFunc::template invoke<Cutlass3xGemmM32NBig>(
-          std::forward<Args>(args)...);
-    }
-  } else if (mp2 <= 64) {
-    // m in (32, 64]
-    return DispatchFunc::template invoke<Cutlass3xGemmM64>(
-        std::forward<Args>(args)...);
-  } else if (mp2 <= 128) {
-    // m in (64, 128]
-    return DispatchFunc::template invoke<Cutlass3xGemmM128>(
-        std::forward<Args>(args)...);
-  } else {
-    // m in (128, inf)
-    return DispatchFunc::template invoke<Cutlass3xGemmDefault>(
-        std::forward<Args>(args)...);
-  }
-}
-
-// Dispatch to GEMM implementations based on element types
-template <template <typename, typename, typename> typename Epilogue,
-          typename... EpilogueArgs>
-void cutlass_scaled_sparse_mm_sm90_epilogue(torch::Tensor& out,
-                                            torch::Tensor const& a,
-                                            torch::Tensor const& bt_nzs,
-                                            torch::Tensor const& bt_meta,
-                                            EpilogueArgs&&... epilogue_args) {
-  uint32_t const m = out.size(0);
-  uint32_t const n = out.size(1);
-
-  // TODO: add dispatch functions to all of these
-  TORCH_CHECK(bt_meta.dtype() == torch::kUInt8);
-  if (a.dtype() == torch::kInt8) {
-    TORCH_CHECK(bt_nzs.dtype() == torch::kInt8);
-
-    if (out.dtype() == torch::kBFloat16) {
-      return cutlass_gemm_sm90_int8_dispatch<int8_t, cutlass::bfloat16_t,
-                                             Epilogue, GemmCallerTraits>(
-          m, n, out, a, bt_nzs, bt_meta,
-          std::forward<EpilogueArgs>(epilogue_args)...);
-    } else {
-      TORCH_CHECK(out.dtype() == torch::kFloat16);
-      return cutlass_gemm_sm90_int8_dispatch<int8_t, cutlass::half_t, Epilogue,
-                                             GemmCallerTraits>(
-          m, n, out, a, bt_nzs, bt_meta,
-          std::forward<EpilogueArgs>(epilogue_args)...);
-    }
-  } else if (a.dtype() == torch::kFloat8_e4m3fn) {
-    TORCH_CHECK(bt_nzs.dtype() == torch::kFloat8_e4m3fn);
-
-    if (out.dtype() == torch::kBFloat16) {
-      return cutlass_gemm_sm90_fp8_dispatch<cutlass::float_e4m3_t,
-                                            cutlass::bfloat16_t, Epilogue,
-                                            GemmCallerTraits>(
-          m, n, out, a, bt_nzs, bt_meta,
-          std::forward<EpilogueArgs>(epilogue_args)...);
-    } else {
-      TORCH_CHECK(out.dtype() == torch::kFloat16);
-      return cutlass_gemm_sm90_fp8_dispatch<
-          cutlass::float_e4m3_t, cutlass::half_t, Epilogue, GemmCallerTraits>(
-          m, n, out, a, bt_nzs, bt_meta,
-          std::forward<EpilogueArgs>(epilogue_args)...);
-    }
-  } else if (a.dtype() == torch::kFloat16) {
-    TORCH_CHECK(bt_nzs.dtype() == torch::kFloat16);
-    TORCH_CHECK(out.dtype() == torch::kFloat16);
-
-    return cutlass_gemm_sm90_16bit_dispatch<cutlass::half_t, cutlass::half_t,
-                                            Epilogue, GemmCallerTraits>(
-        m, n, out, a, bt_nzs, bt_meta,
-        std::forward<EpilogueArgs>(epilogue_args)...);
-  } else {  // a.dtype() == torch::kBFloat16
-    TORCH_CHECK(a.dtype() == torch::kBFloat16);
-    TORCH_CHECK(bt_nzs.dtype() == torch::kBFloat16);
-    TORCH_CHECK(out.dtype() == torch::kBFloat16);
-
-    return cutlass_gemm_sm90_16bit_dispatch<
-        cutlass::bfloat16_t, cutlass::bfloat16_t, Epilogue, GemmCallerTraits>(
-        m, n, out, a, bt_nzs, bt_meta,
-        std::forward<EpilogueArgs>(epilogue_args)...);
-  }
-}
-
-void cutlass_scaled_sparse_mm_sm90(torch::Tensor& out, torch::Tensor const& a,
-                                   torch::Tensor const& bt_nzs,
-                                   torch::Tensor const& bt_meta,
-                                   torch::Tensor const& a_scales,
-                                   torch::Tensor const& b_scales,
-                                   std::optional<torch::Tensor> const& bias) {
-  TORCH_CHECK(bt_meta.dtype() == torch::kUInt8);
-  TORCH_CHECK(a_scales.dtype() == torch::kFloat32);
-  TORCH_CHECK(b_scales.dtype() == torch::kFloat32);
-
-  if (bias) {
-    TORCH_CHECK(bias->dtype() == out.dtype(),
-                "CUTLASS scaled_mm bias dtype must match output dtype ",
-                out.dtype());
-    return cutlass_scaled_sparse_mm_sm90_epilogue<
-        c3x::ScaledEpilogueColumnBias>(out, a, bt_nzs, bt_meta, b_scales,
-                                       a_scales, *bias);
-  } else {
-    return cutlass_scaled_sparse_mm_sm90_epilogue<c3x::ScaledEpilogue>(
-        out, a, bt_nzs, bt_meta, b_scales, a_scales);
-  }
-}
-
-CompressorResult cutlass_sparse_compress_sm90(torch::Tensor const& a) {
-  // These m and n variables are fordispatching to different GEMM algorithms.
-  uint32_t const m = 1;  // Set M to 1 for compression
-  uint32_t const n = a.size(1);
-
-  // Note: For correctness, the compressed format must be invariant in:
-  //  - M, the flattened number of tokens
-  //  - Whether output dtype is fp16 or bf16
-  //  - CUTLASS epilogues
-
-  if (a.dtype() == torch::kInt8) {
-    return cutlass_gemm_sm90_int8_dispatch<int8_t, cutlass::bfloat16_t,
-                                           c3x::TrivialEpilogue,
-                                           GemmCompressorTraits>(m, n, a);
-  } else if (a.dtype() == torch::kFloat8_e4m3fn) {
-    return cutlass_gemm_sm90_fp8_dispatch<
-        cutlass::float_e4m3_t, cutlass::bfloat16_t, c3x::TrivialEpilogue,
-        GemmCompressorTraits>(m, n, a);
-  } else if (a.dtype() == torch::kFloat16) {
-    return cutlass_gemm_sm90_16bit_dispatch<
-        cutlass::bfloat16_t, cutlass::bfloat16_t, c3x::TrivialEpilogue,
-        GemmCompressorTraits>(m, n, a);
-  } else {
-    TORCH_CHECK(a.dtype() == torch::kBFloat16,
-                "cutlass_sparse_compress only supports int8, fp8_e4m3, fp16, "
-                "and bf16 datatypes");
-    return cutlass_gemm_sm90_16bit_dispatch<cutlass::half_t, cutlass::half_t,
-                                            c3x::TrivialEpilogue,
-                                            GemmCompressorTraits>(m, n, a);
-  }
-}
-
-#endif
@@ -1,570 +0,0 @@
-#pragma once
-
-// clang-format will break include orders
-// clang-format off
-#include <cudaTypedefs.h>
-
-#include <torch/all.h>
-
-#include <ATen/cuda/CUDAContext.h>
-
-#include "cuda_utils.h"
-
-#include "cutlass/cutlass.h"
-
-#include "cutlass/gemm/device/gemm_universal_adapter.h"
-#include "cutlass/epilogue/collective/collective_builder.hpp"
-#include "cutlass/gemm/collective/collective_builder.hpp"
-
-#include "cutlass/transform/device/transform_universal_adapter.hpp"
-#include "cutlass/transform/kernel/sparse_gemm_compressor.hpp"
-
-#include "core/math.hpp"
-#include "cutlass_extensions/cute_utils.cuh"
-#include "cutlass_extensions/epilogue/scaled_mm_epilogues_c3x.hpp"
-#include "cutlass_extensions/common.hpp"
-#include "cutlass_extensions/torch_utils.hpp"
-// clang-format on
-
-using namespace cute;
-
-/*
-   This file defines 2:4 sparse GEMM operations using the CUTLASS 3.x API,
-   for NVIDIA GPUs with sm90a (Hopper) or later.
-*/
-
-namespace {
-
-// A wrapper for the GEMM kernel that is used to guard against compilation on
-// architectures that will never use the kernel. The purpose of this is to
-// reduce the size of the compiled binary.
-// __CUDA_ARCH__ is not defined in host code, so this lets us smuggle the ifdef
-// into code that will be executed on the device where it is defined.
-template <typename Kernel>
-struct enable_sm90_or_later : Kernel {
-  template <typename... Args>
-  CUTLASS_DEVICE void operator()(Args&&... args) {
-#if defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 900
-    Kernel::operator()(std::forward<Args>(args)...);
-#endif
-  }
-};
-
-using GemmUniversalMode = cutlass::gemm::GemmUniversalMode;
-
-/*
- * cutlass_sparse_3x_gemm defines a 2:4 sparse GEMM kernel via CUTLASS
- * for SM90 Hopper systems.
- */
-template <typename ElementAB_, typename ElementD_,
-          template <typename, typename, typename> typename Epilogue_,
-          typename TileShape, typename ClusterShape, typename KernelSchedule,
-          typename EpilogueSchedule>
-struct cutlass_sparse_3x_gemm {
-  using ElementAB = ElementAB_;
-  using ElementD = ElementD_;
-  using ElementAcc =
-      typename std::conditional<std::is_same_v<ElementAB, int8_t>, int32_t,
-                                float>::type;
-
-  using Epilogue = Epilogue_<ElementAcc, ElementD, TileShape>;
-
-  using ElementC = void;
-  using LayoutC = cutlass::layout::RowMajor;
-  using LayoutC_Transpose =
-      typename cutlass::layout::LayoutTranspose<LayoutC>::type;
-
-  using EVTCompute = typename Epilogue::EVTCompute;
-
-  // These are the minimum alignments needed for the kernels to compile
-  static constexpr int AlignmentAB =
-      128 / cutlass::sizeof_bits<ElementAB>::value;
-  static constexpr int AlignmentCD =
-      128 / cutlass::sizeof_bits<ElementD>::value;
-
-  using CollectiveEpilogue =
-      typename cutlass::epilogue::collective::CollectiveBuilder<
-          cutlass::arch::Sm90, cutlass::arch::OpClassTensorOp, TileShape,
-          ClusterShape, cutlass::epilogue::collective::EpilogueTileAuto,
-          ElementAcc, float, ElementC, LayoutC_Transpose, AlignmentCD, ElementD,
-          LayoutC_Transpose, AlignmentCD, EpilogueSchedule,
-          EVTCompute>::CollectiveOp;
-
-  static constexpr size_t CEStorageSize =
-      sizeof(typename CollectiveEpilogue::SharedStorage);
-  using Stages = typename cutlass::gemm::collective::StageCountAutoCarveout<
-      static_cast<int>(CEStorageSize)>;
-
-  // clang-format off
-  using CollectiveMainloop =
-      typename cutlass::gemm::collective::CollectiveBuilder<
-          cutlass::arch::Sm90, cutlass::arch::OpClassSparseTensorOp,
-          ElementAB, cutlass::layout::RowMajor, AlignmentAB,
-          ElementAB, cutlass::layout::ColumnMajor, AlignmentAB,
-          ElementAcc, TileShape, ClusterShape,
-          Stages,
-          KernelSchedule>::CollectiveOp;
-  // clang-format on
-
-  using KernelType = enable_sm90_or_later<cutlass::gemm::kernel::GemmUniversal<
-      cute::Shape<int, int, int, int>, CollectiveMainloop, CollectiveEpilogue,
-      cutlass::gemm::PersistentScheduler>>;
-
-  struct GemmKernel : public KernelType {};
-
-  // Sparse compressor definitions
-  using SparseConfig = typename GemmKernel::CollectiveMainloop::SparseConfig;
-  using LayoutTagA = cutlass::layout::RowMajor;
-  using CompressorUtility =
-      cutlass::transform::kernel::StructuredSparseCompressorUtility<
-          typename GemmKernel::ProblemShape, ElementAB, LayoutTagA,
-          SparseConfig>;
-  using CompressorKernel =
-      cutlass::transform::kernel::StructuredSparseCompressor<
-          typename GemmKernel::ProblemShape, ElementAB, LayoutTagA,
-          SparseConfig, cutlass::arch::Sm90>;
-  using Compressor =
-      cutlass::transform::device::TransformUniversalAdapter<CompressorKernel>;
-};
-
-/*
- * This class defines kernel to compress a 2:4 sparse matrix.
- * The particular format is defined by the Gemm template parameter,
- * which is a cutlass_sparse_3x_gemm.
- */
-using CompressorResult = std::tuple<torch::Tensor, torch::Tensor>;
-/// Make A structured sparse by replacing elements with 0 and compress it
-template <typename Gemm>
-CompressorResult cutlass_sparse_compress(torch::Tensor const& a) {
-  // Checks for conformality
-  TORCH_CHECK(a.dtype() == torch::kInt8 || a.dtype() == torch::kFloat8_e4m3fn ||
-              a.dtype() == torch::kFloat16 || a.dtype() == torch::kBFloat16);
-  TORCH_CHECK(a.dim() == 2)
-  // Check for strides and alignment
-  TORCH_CHECK(a.stride(0) % 4 == 0)  // Required for semi-structured sparsity
-  TORCH_CHECK(a.stride(1) == 1)
-
-  using GemmKernel = typename Gemm::KernelType;
-  using ElementA = typename Gemm::ElementAB;
-  using ElementE = typename GemmKernel::CollectiveMainloop::ElementE;
-
-  int m = a.size(0);
-  int k = a.size(1);
-  using ProblemShape = typename GemmKernel::ProblemShape;
-  ProblemShape prob_shape{m, 1, k, 1};
-
-  int64_t lda = a.stride(0);
-  using StrideA = Stride<int64_t, Int<1>, int64_t>;
-  StrideA a_stride{lda, Int<1>{}, 0};
-
-  using CompressorUtility = typename Gemm::CompressorUtility;
-  CompressorUtility compressor_utility(prob_shape, a_stride);
-
-  // Allocate buffers for the metadata E and the compressed matrix A
-  int ME = compressor_utility.get_metadata_m_physical();
-  int KE = compressor_utility.get_metadata_k_physical();
-  int MC = compressor_utility.get_tensorA_m_physical();
-  int KC = compressor_utility.get_tensorA_k_physical();
-
-  auto const a_meta_options =
-      torch::TensorOptions().dtype(torch::kUInt8).device(a.device());
-  auto const a_nzs_options =
-      torch::TensorOptions().dtype(a.dtype()).device(a.device());
-
-  auto a_meta = torch::zeros({ME, KE}, a_meta_options);
-  auto a_nzs = torch::zeros({MC, KC}, a_nzs_options);
-
-  auto a_ptr = static_cast<ElementA*>(a.data_ptr());
-  auto a_nzs_ptr = static_cast<ElementA*>(a_nzs.data_ptr());
-  auto a_meta_ptr = static_cast<ElementE*>(a_meta.data_ptr());
-
-  cutlass::KernelHardwareInfo hw_info;
-  hw_info.device_id = a.device().index();
-  hw_info.sm_count =
-      cutlass::KernelHardwareInfo::query_device_multiprocessor_count(
-          hw_info.device_id);
-
-  using Compressor = typename Gemm::Compressor;
-  typename Compressor::Arguments arguments{
-      prob_shape, {a_ptr, a_stride, a_nzs_ptr, a_meta_ptr}, {hw_info}};
-
-  Compressor compressor_op;
-  size_t workspace_size = Compressor::get_workspace_size(arguments);
-  auto const workspace_options =
-      torch::TensorOptions().dtype(torch::kUInt8).device(a.device());
-  auto workspace = torch::empty(workspace_size, workspace_options);
-
-  CUTLASS_CHECK(compressor_op.can_implement(arguments));
-  CUTLASS_CHECK(compressor_op.initialize(arguments, workspace.data_ptr()));
-  CUTLASS_CHECK(compressor_op.run());
-  CUDA_CHECK(cudaDeviceSynchronize());
-
-  return {a_meta, a_nzs};
-}
-
-template <typename Gemm, typename... EpilogueArgs>
-void cutlass_sparse_gemm_caller(torch::Tensor& out, torch::Tensor const& a,
-                                torch::Tensor const& bt_nzs,
-                                torch::Tensor const& bt_meta,
-                                EpilogueArgs&&... epilogue_params) {
-  using ElementAB = typename Gemm::ElementAB;
-  using ElementD = typename Gemm::ElementD;
-
-  // Interface stride expected from the argument a (will get transposed)
-  // We compute C^T = B^T * A^T, but we assume B is transposed before
-  // compression and hence the bt_* naming
-  using LayoutB = typename Gemm::GemmKernel::CollectiveMainloop::LayoutA;
-  using LayoutE = typename Gemm::GemmKernel::CollectiveMainloop::LayoutE;
-
-  // M, N, K after transposition
-  int32_t m = out.size(1);
-  int32_t n = out.size(0);
-  int32_t k = a.size(1);
-
-  int64_t lda = a.stride(0);
-  int64_t ldc = out.stride(0);
-
-  using StrideA = Stride<int64_t, Int<1>, int64_t>;
-  using StrideC = Stride<Int<1>, int64_t, int64_t>;
-
-  StrideA a_stride{lda, Int<1>{}, Int<0>{}};
-  StrideC c_stride{Int<1>{}, ldc, Int<0>{}};
-
-  using GemmKernel = typename Gemm::GemmKernel;
-  typename GemmKernel::ProblemShape prob_shape{m, n, k, 1};
-
-  using ElementE = typename GemmKernel::CollectiveMainloop::ElementE;
-  using SparseConfig = typename GemmKernel::CollectiveMainloop::SparseConfig;
-
-  LayoutB b_layout = SparseConfig::fill_layoutA(prob_shape);
-  LayoutE e_layout = SparseConfig::fill_layoutE(prob_shape);
-
-  auto a_ptr = static_cast<ElementAB*>(a.data_ptr());
-  auto b_ptr = static_cast<ElementAB*>(bt_nzs.data_ptr());
-  auto e_ptr = static_cast<ElementE*>(bt_meta.data_ptr());
-  typename GemmKernel::MainloopArguments mainloop_args{
-      b_ptr, b_layout, a_ptr, a_stride, e_ptr, e_layout};
-
-  auto c_ptr = static_cast<ElementD*>(out.data_ptr());
-  typename GemmKernel::EpilogueArguments epilogue_args{
-      Gemm::Epilogue::prepare_args(
-          std::forward<EpilogueArgs>(epilogue_params)...),
-      c_ptr, c_stride, c_ptr, c_stride};
-
-  typename GemmKernel::Arguments args{cutlass::gemm::GemmUniversalMode::kGemm,
-                                      prob_shape, mainloop_args, epilogue_args};
-
-  // Launch the CUTLASS GEMM kernel.
-  using GemmOp = cutlass::gemm::device::GemmUniversalAdapter<GemmKernel>;
-  GemmOp gemm_op;
-  CUTLASS_CHECK(gemm_op.can_implement(args));
-
-  size_t workspace_size = gemm_op.get_workspace_size(args);
-  auto const workspace_options =
-      torch::TensorOptions().dtype(torch::kUInt8).device(a.device());
-  auto workspace = torch::empty(workspace_size, workspace_options);
-
-  auto stream = at::cuda::getCurrentCUDAStream(a.get_device());
-
-  cutlass::Status status = gemm_op.run(args, workspace.data_ptr(), stream);
-  CUTLASS_CHECK(status);
-}
-
-//////////////////////////////////////////////////
-// Gemm Configs are defined below
-//////////////////////////////////////////////////
-
-template <typename InType, typename OutType,
-          template <typename, typename, typename> typename Epilogue>
-struct sm90_config_default {};
-
-template <typename OutType,
-          template <typename, typename, typename> typename Epilogue>
-struct sm90_config_default<half_t, OutType, Epilogue> {
-  using KernelSchedule = cutlass::gemm::KernelTmaWarpSpecialized;
-  using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
-  using TileShape = Shape<_128, _128, _128>;
-  using ClusterShape = Shape<_1, _1, _1>;
-  using Cutlass3xGemm =
-      cutlass_sparse_3x_gemm<half_t, OutType, Epilogue, TileShape, ClusterShape,
-                             KernelSchedule, EpilogueSchedule>;
-};
-
-template <typename OutType,
-          template <typename, typename, typename> typename Epilogue>
-struct sm90_config_default<cutlass::bfloat16_t, OutType, Epilogue> {
-  using KernelSchedule = cutlass::gemm::KernelTmaWarpSpecialized;
-  using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
-  using TileShape = Shape<_128, _128, _128>;
-  using ClusterShape = Shape<_1, _1, _1>;
-  using Cutlass3xGemm =
-      cutlass_sparse_3x_gemm<cutlass::bfloat16_t, OutType, Epilogue, TileShape,
-                             ClusterShape, KernelSchedule, EpilogueSchedule>;
-};
-
-//////////////////////// Cherry-Picking Kernels ////////////////////////
-template <typename InType, typename OutType,
-          template <typename, typename, typename> typename Epilogue>
-struct sm90_fp8_config_1 {
-  static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
-  using KernelSchedule = cutlass::gemm::KernelTmaWarpSpecializedFP8FastAccum;
-  using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
-  using TileShape = Shape<_64, _64, _256>;
-  using ClusterShape = Shape<_8, _1, _1>;
-  using Cutlass3xGemm =
-      cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
-                             KernelSchedule, EpilogueSchedule>;
-};
-
-template <typename InType, typename OutType,
-          template <typename, typename, typename> typename Epilogue>
-struct sm90_fp8_config_2 {
-  static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
-  using KernelSchedule =
-      cutlass::gemm::KernelTmaWarpSpecializedCooperativeFP8FastAccum;
-  using EpilogueSchedule =
-      typename cutlass::epilogue::TmaWarpSpecializedCooperative;
-  using TileShape = Shape<_128, _64, _256>;
-  using ClusterShape = Shape<_8, _1, _1>;
-  using Cutlass3xGemm =
-      cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
-                             KernelSchedule, EpilogueSchedule>;
-};
-
-template <typename InType, typename OutType,
-          template <typename, typename, typename> typename Epilogue>
-struct sm90_fp8_config_3 {
-  static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
-  using KernelSchedule = cutlass::gemm::KernelTmaWarpSpecializedFP8FastAccum;
-  using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
-  using TileShape = Shape<_64, _64, _256>;
-  using ClusterShape = Shape<_1, _2, _1>;
-  using Cutlass3xGemm =
-      cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
-                             KernelSchedule, EpilogueSchedule>;
-};
-
-template <typename InType, typename OutType,
-          template <typename, typename, typename> typename Epilogue>
-struct sm90_fp8_config_4 {
-  static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
-  using KernelSchedule = cutlass::gemm::KernelTmaWarpSpecializedFP8FastAccum;
-  using EpilogueSchedule =
-      typename cutlass::epilogue::TmaWarpSpecializedCooperative;
-  using TileShape = Shape<_64, _128, _256>;
-  using ClusterShape = Shape<_8, _1, _1>;
-  using Cutlass3xGemm =
-      cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
-                             KernelSchedule, EpilogueSchedule>;
-};
-
-template <typename InType, typename OutType,
-          template <typename, typename, typename> typename Epilogue>
-struct sm90_fp8_config_5 {
-  static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
-  using KernelSchedule =
-      cutlass::gemm::KernelTmaWarpSpecializedPingpongFP8FastAccum;
-  using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
-  using TileShape = Shape<_128, _128, _256>;
-  using ClusterShape = Shape<_8, _1, _1>;
-  using Cutlass3xGemm =
-      cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
-                             KernelSchedule, EpilogueSchedule>;
-};
-
-template <typename InType, typename OutType,
-          template <typename, typename, typename> typename Epilogue>
-struct sm90_fp8_config_6 {
-  static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
-  using KernelSchedule = cutlass::gemm::KernelTmaWarpSpecializedFP8FastAccum;
-  using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
-  using TileShape = Shape<_64, _128, _256>;
-  using ClusterShape = Shape<_1, _2, _1>;
-  using Cutlass3xGemm =
-      cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
-                             KernelSchedule, EpilogueSchedule>;
-};
-
-template <typename InType, typename OutType,
-          template <typename, typename, typename> typename Epilogue>
-struct sm90_fp8_config_7 {
-  static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
-  using KernelSchedule =
-      cutlass::gemm::KernelTmaWarpSpecializedCooperativeFP8FastAccum;
-  using EpilogueSchedule =
-      typename cutlass::epilogue::TmaWarpSpecializedCooperative;
-  using TileShape = Shape<_128, _128, _256>;
-  using ClusterShape = Shape<_1, _1, _1>;
-  using Cutlass3xGemm =
-      cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
-                             KernelSchedule, EpilogueSchedule>;
-};
-
-template <typename InType, typename OutType,
-          template <typename, typename, typename> typename Epilogue>
-struct sm90_fp8_config_8 {
-  static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
-  using KernelSchedule =
-      cutlass::gemm::KernelTmaWarpSpecializedCooperativeFP8FastAccum;
-  using EpilogueSchedule =
-      typename cutlass::epilogue::TmaWarpSpecializedCooperative;
-  using TileShape = Shape<_128, _256, _128>;
-  using ClusterShape = Shape<_8, _1, _1>;
-  using Cutlass3xGemm =
-      cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
-                             KernelSchedule, EpilogueSchedule>;
-};
-////////////////////////////////////////////////////////////////////////
-
-template <typename OutType,
-          template <typename, typename, typename> typename Epilogue>
-struct sm90_config_default<cutlass::float_e4m3_t, OutType, Epilogue> {
-  // M in (128, inf)
-  using KernelSchedule = cutlass::gemm::KernelTmaWarpSpecializedFP8FastAccum;
-  using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
-  using TileShape = Shape<_128, _128, _128>;
-  using ClusterShape = Shape<_1, _2, _1>;
-  using Cutlass3xGemm =
-      cutlass_sparse_3x_gemm<cutlass::float_e4m3_t, OutType, Epilogue,
-                             TileShape, ClusterShape, KernelSchedule,
-                             EpilogueSchedule>;
-};
-
-template <typename InType, typename OutType,
-          template <typename, typename, typename> typename Epilogue>
-struct sm90_fp8_config_M64 {
-  // M in [1, 64]
-  static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
-  using KernelSchedule = cutlass::gemm::KernelTmaWarpSpecializedFP8FastAccum;
-  using EpilogueSchedule =
-      typename cutlass::epilogue::TmaWarpSpecializedCooperative;
-  using TileShape = Shape<_64, _64, _256>;
-  using ClusterShape = Shape<_1, _1, _1>;
-
-  using Cutlass3xGemm =
-      cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
-                             KernelSchedule, EpilogueSchedule>;
-};
-
-template <typename InType, typename OutType,
-          template <typename, typename, typename> typename Epilogue>
-struct sm90_fp8_config_M128 {
-  // M in (64, 128]
-  static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
-  using KernelSchedule =
-      cutlass::gemm::KernelTmaWarpSpecializedPingpongFP8FastAccum;
-  using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
-  using TileShape = Shape<_64, _128, _256>;
-  using ClusterShape = Shape<_1, _1, _1>;
-
-  using Cutlass3xGemm =
-      cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
-                             KernelSchedule, EpilogueSchedule>;
-};
-
-template <typename InType, typename OutType,
-          template <typename, typename, typename> typename Epilogue>
-struct sm90_fp8_config_M256 {
-  // M in (128, 256]
-  static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
-  using KernelSchedule =
-      cutlass::gemm::KernelTmaWarpSpecializedCooperativeFP8FastAccum;
-  using EpilogueSchedule =
-      typename cutlass::epilogue::TmaWarpSpecializedCooperative;
-  using TileShape = Shape<_128, _128, _256>;
-  using ClusterShape = Shape<_1, _1, _1>;
-
-  using Cutlass3xGemm =
-      cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
-                             KernelSchedule, EpilogueSchedule>;
-};
-
-template <typename InType, typename OutType,
-          template <typename, typename, typename> typename Epilogue>
-struct sm90_fp8_config_M512 {
-  // M in (256, ]
-  static_assert(std::is_same<InType, cutlass::float_e4m3_t>());
-  using KernelSchedule =
-      cutlass::gemm::KernelTmaWarpSpecializedCooperativeFP8FastAccum;
-  using EpilogueSchedule =
-      typename cutlass::epilogue::TmaWarpSpecializedCooperative;
-  using TileShape = Shape<_128, _128, _256>;
-  using ClusterShape = Shape<_1, _1, _1>;
-
-  using Cutlass3xGemm =
-      cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
-                             KernelSchedule, EpilogueSchedule>;
-};
-
-template <typename OutType,
-          template <typename, typename, typename> typename Epilogue>
-struct sm90_config_default<int8_t, OutType, Epilogue> {
-  // For M > 128 and any N
-  using KernelSchedule =
-      typename cutlass::gemm::KernelTmaWarpSpecializedPingpong;
-  using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
-  using TileShape = Shape<_128, _128, _128>;
-  using ClusterShape = Shape<_2, _1, _1>;
-  using Cutlass3xGemm =
-      cutlass_sparse_3x_gemm<int8_t, OutType, Epilogue, TileShape, ClusterShape,
-                             KernelSchedule, EpilogueSchedule>;
-};
-
-template <typename InType, typename OutType,
-          template <typename, typename, typename> typename Epilogue>
-struct sm90_int8_config_M128 {
-  // For M in (64, 128] and any N
-  static_assert(std::is_same<InType, int8_t>());
-  using KernelSchedule =
-      typename cutlass::gemm::KernelTmaWarpSpecializedPingpong;
-  using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
-  using TileShape = Shape<_64, _128, _128>;
-  using ClusterShape = Shape<_2, _1, _1>;
-  using Cutlass3xGemm =
-      cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
-                             KernelSchedule, EpilogueSchedule>;
-};
-
-template <typename InType, typename OutType,
-          template <typename, typename, typename> typename Epilogue>
-struct sm90_int8_config_M64 {
-  // For M in (32, 64] and any N
-  static_assert(std::is_same<InType, int8_t>());
-  using KernelSchedule = typename cutlass::gemm::KernelTmaWarpSpecialized;
-  using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
-  using TileShape = Shape<_64, _64, _256>;
-  using ClusterShape = Shape<_1, _1, _1>;
-  using Cutlass3xGemm =
-      cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
-                             KernelSchedule, EpilogueSchedule>;
-};
-
-template <typename InType, typename OutType,
-          template <typename, typename, typename> typename Epilogue>
-struct sm90_int8_config_M32_NBig {
-  // For M in [1, 32] and N >= 8192
-  static_assert(std::is_same<InType, int8_t>());
-  using KernelSchedule = typename cutlass::gemm::KernelTmaWarpSpecialized;
-  using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
-  using TileShape = Shape<_64, _128, _256>;
-  using ClusterShape = Shape<_1, _4, _1>;
-  using Cutlass3xGemm =
-      cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
-                             KernelSchedule, EpilogueSchedule>;
-};
-
-template <typename InType, typename OutType,
-          template <typename, typename, typename> typename Epilogue>
-struct sm90_int8_config_M32_NSmall {
-  // For M in [1, 32] and N < 8192
-  static_assert(std::is_same<InType, int8_t>());
-  using KernelSchedule = typename cutlass::gemm::KernelTmaWarpSpecialized;
-  using EpilogueSchedule = typename cutlass::epilogue::TmaWarpSpecialized;
-  using TileShape = Shape<_64, _64, _256>;
-  using ClusterShape = Shape<_1, _8, _1>;
-  using Cutlass3xGemm =
-      cutlass_sparse_3x_gemm<InType, OutType, Epilogue, TileShape, ClusterShape,
-                             KernelSchedule, EpilogueSchedule>;
-};
-
-}  // namespace
@@ -1,104 +0,0 @@
-#include <cudaTypedefs.h>
-
-#include <c10/cuda/CUDAGuard.h>
-#include <torch/all.h>
-
-#include "cutlass_extensions/common.hpp"
-
-bool cutlass_sparse_scaled_mm_supported(int64_t cuda_device_capability) {
-  // sparse CUTLASS kernels need exactly hopper and are not forward compatible
-  //   CUDA 12.2 and SM90 (Hopper)
-
-#if defined CUDA_VERSION
-  return CUDA_VERSION >= 12020 && cuda_device_capability == 90;
-#endif
-
-  return false;
-}
-
-#if defined ENABLE_SPARSE_SCALED_MM_C3X && ENABLE_SPARSE_SCALED_MM_C3X
-void cutlass_scaled_sparse_mm_sm90(torch::Tensor& c, torch::Tensor const& a,
-                                   torch::Tensor const& b,
-                                   torch::Tensor const& e,
-                                   torch::Tensor const& a_scales,
-                                   torch::Tensor const& b_scales,
-                                   std::optional<torch::Tensor> const& bias);
-
-using CompressorResult = std::tuple<torch::Tensor, torch::Tensor>;
-CompressorResult cutlass_sparse_compress_sm90(torch::Tensor const& a);
-#endif
-
-void cutlass_scaled_sparse_mm(torch::Tensor& c, torch::Tensor const& a,
-                              torch::Tensor const& bt_nzs,
-                              torch::Tensor const& bt_meta,
-                              torch::Tensor const& a_scales,
-                              torch::Tensor const& b_scales,
-                              std::optional<torch::Tensor> const& bias) {
-  // Checks for conformality
-  TORCH_CHECK(a.dim() == 2 && bt_nzs.dim() == 2 && c.dim() == 2);
-  TORCH_CHECK(c.size(1) == bt_nzs.size(0) && bt_nzs.size(1) * 2 == a.size(1) &&
-              a.size(0) == c.size(0));
-  TORCH_CHECK(a_scales.numel() == 1 || a_scales.numel() == a.size(0));
-  TORCH_CHECK(b_scales.numel() == 1 || b_scales.numel() == bt_nzs.size(0));
-
-  // Check for strides and alignment
-  TORCH_CHECK(a.stride(1) == 1 && bt_nzs.stride(1) == 1 &&
-              c.stride(1) == 1);            // Row-major
-  TORCH_CHECK(c.stride(0) % 16 == 0);       // 16 Byte Alignment
-  TORCH_CHECK(bt_nzs.stride(0) % 16 == 0);  // 16 Byte Alignment
-  TORCH_CHECK(a_scales.is_contiguous() && b_scales.is_contiguous());
-
-  if (bias) {
-    TORCH_CHECK(bias->numel() == bt_nzs.size(0) && bias->is_contiguous() &&
-                bias->dim() == 1);
-  }
-
-  at::cuda::OptionalCUDAGuard const device_guard(device_of(a));
-  int32_t version_num = get_sm_version_num();
-
-  // Guard against compilation issues for sm90 kernels
-#if defined ENABLE_SPARSE_SCALED_MM_C3X && ENABLE_SPARSE_SCALED_MM_C3X
-  // We build for 9.0a which is not forward compatible, so restrict this to
-  // Hopper only
-  if (version_num == 90) {
-    cutlass_scaled_sparse_mm_sm90(c, a, bt_nzs, bt_meta, a_scales, b_scales,
-                                  bias);
-    return;
-  }
-#endif
-
-  TORCH_CHECK_NOT_IMPLEMENTED(
-      false,
-      "No compiled cutlass_scaled_sparse_mm for a compute capability less than "
-      "CUDA device capability: ",
-      version_num);
-}
-
-std::vector<torch::Tensor> cutlass_sparse_compress(torch::Tensor const& a) {
-  // Check for strides and alignment
-  TORCH_CHECK(a.stride(1) == 1);      // Row-major
-  TORCH_CHECK(a.stride(0) % 8 == 0);  // 8 Byte Alignment for Compression
-
-  at::cuda::OptionalCUDAGuard const device_guard(device_of(a));
-  int32_t version_num = get_sm_version_num();
-
-  // Guard against compilation issues for sm90 kernels
-#if defined ENABLE_SPARSE_SCALED_MM_C3X && ENABLE_SPARSE_SCALED_MM_C3X
-  // We build for 9.0a which is not forward compatible, so restrict this to
-  // Hopper only
-  if (version_num == 90) {
-    std::vector<torch::Tensor> result_tensors;
-
-    auto [a_meta, a_nzs] = cutlass_sparse_compress_sm90(a);
-    result_tensors.push_back(std::move(a_nzs));
-    result_tensors.push_back(std::move(a_meta));
-    return result_tensors;
-  }
-#endif
-
-  TORCH_CHECK_NOT_IMPLEMENTED(
-      false,
-      "No compiled cutlass_sparse_compress for a compute capability equal to "
-      "CUDA device capability: ",
-      version_num);
-}
@@ -6,6 +6,41 @@
 #include <torch/library.h>
 #include <torch/version.h>

+// Forward declarations for per-SM NVFP4 GEMM and quantization entry points.
+// Defined in nvfp4_scaled_mm_kernels.cu / nvfp4_quant_entry.cu and only
+// compiled when ENABLE_NVFP4_SM100 is set.
+#if defined(ENABLE_NVFP4_SM100) && ENABLE_NVFP4_SM100
+void cutlass_scaled_fp4_mm_sm100a(torch::Tensor& D, torch::Tensor const& A,
+                                  torch::Tensor const& B,
+                                  torch::Tensor const& A_sf,
+                                  torch::Tensor const& B_sf,
+                                  torch::Tensor const& alpha);
+void cutlass_scaled_fp4_mm_sm103a(torch::Tensor& D, torch::Tensor const& A,
+                                  torch::Tensor const& B,
+                                  torch::Tensor const& A_sf,
+                                  torch::Tensor const& B_sf,
+                                  torch::Tensor const& alpha);
+std::tuple<torch::Tensor, torch::Tensor> scaled_fp4_quant_sm103a_func(
+    torch::Tensor const& input, torch::Tensor const& input_sf);
+void scaled_fp4_quant_sm103a_out(torch::Tensor const& input,
+                                 torch::Tensor const& input_sf,
+                                 torch::Tensor& output,
+                                 torch::Tensor& output_sf);
+// PDL-enabled variants (ProgrammaticStreamSerialization).
+void cutlass_scaled_fp4_mm_sm103a_pdl(torch::Tensor& D,
+                                      torch::Tensor const& A,
+                                      torch::Tensor const& B,
+                                      torch::Tensor const& A_sf,
+                                      torch::Tensor const& B_sf,
+                                      torch::Tensor const& alpha);
+std::tuple<torch::Tensor, torch::Tensor> scaled_fp4_quant_sm103a_pdl_func(
+    torch::Tensor const& input, torch::Tensor const& input_sf);
+void scaled_fp4_quant_sm103a_pdl_out(torch::Tensor const& input,
+                                     torch::Tensor const& input_sf,
+                                     torch::Tensor& output,
+                                     torch::Tensor& output_sf);
+#endif
+
 // Note on op signatures:
 // The X_meta signatures are for the meta functions corresponding to op X.
 // They must be kept in sync with the signature for X. Generally, only
@@ -303,9 +338,6 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
      ") -> Tensor");
  // conditionally compiled so impl registration is in source file

-  ops.def("permute_cols(Tensor A, Tensor perm) -> Tensor");
-  ops.impl("permute_cols", torch::kCUDA, &permute_cols);
-
  // Marlin Optimized Quantized GEMM (supports GPTQ, AWQ, FP8, NVFP4, MXFP4).
  ops.def(
      "marlin_gemm(Tensor a, Tensor? c_or_none, Tensor b_q_weight, "
@@ -419,6 +451,65 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
      "                      Tensor alpha) -> ()");
  ops.impl("cutlass_scaled_fp4_mm", torch::kCUDA, &cutlass_scaled_fp4_mm);

+#if defined(ENABLE_NVFP4_SM100) && ENABLE_NVFP4_SM100
+  // SM100-specific entry point (B200 / Blackwell, SM100 SF layout)
+  ops.def(
+      "cutlass_scaled_fp4_mm_sm100a(Tensor! out, Tensor a, Tensor b,"
+      "                             Tensor block_scale_a, Tensor block_scale_b,"
+      "                             Tensor alpha) -> ()");
+  ops.impl("cutlass_scaled_fp4_mm_sm100a", torch::kCUDA,
+           &cutlass_scaled_fp4_mm_sm100a);
+
+  // SM103-specific entry point (B300 / Blackwell Ultra, SM103 SF layout)
+  ops.def(
+      "cutlass_scaled_fp4_mm_sm103a(Tensor! out, Tensor a, Tensor b,"
+      "                             Tensor block_scale_a, Tensor block_scale_b,"
+      "                             Tensor alpha) -> ()");
+  ops.impl("cutlass_scaled_fp4_mm_sm103a", torch::kCUDA,
+           &cutlass_scaled_fp4_mm_sm103a);
+
+  // SM103-native quantization: produces SM103-layout scale factors directly.
+  ops.def(
+      "scaled_fp4_quant_sm103(Tensor input,"
+      "                       Tensor input_scale) -> (Tensor, Tensor)");
+  ops.impl("scaled_fp4_quant_sm103", torch::kCUDA, &scaled_fp4_quant_sm103a_func);
+
+  ops.def(
+      "scaled_fp4_quant_sm103.out(Tensor input,"
+      "                           Tensor input_scale,"
+      "                           *, Tensor(a!) output, Tensor(b!) output_scale)"
+      " -> ()");
+  ops.impl("scaled_fp4_quant_sm103.out", torch::kCUDA,
+           &scaled_fp4_quant_sm103a_out);
+
+  // PDL-enabled SM103 GEMM: launched with ProgrammaticStreamSerialization
+  // so the next kernel on the stream can overlap with this GEMM's tail.
+  ops.def(
+      "cutlass_scaled_fp4_mm_sm103a_pdl(Tensor! out, Tensor a, Tensor b,"
+      "                                 Tensor block_scale_a,"
+      "                                 Tensor block_scale_b,"
+      "                                 Tensor alpha) -> ()");
+  ops.impl("cutlass_scaled_fp4_mm_sm103a_pdl", torch::kCUDA,
+           &cutlass_scaled_fp4_mm_sm103a_pdl);
+
+  // PDL-enabled SM103 quantization: launched with
+  // ProgrammaticStreamSerialization so the subsequent GEMM can begin
+  // before this quant kernel completes.
+  ops.def(
+      "scaled_fp4_quant_sm103_pdl(Tensor input,"
+      "                           Tensor input_scale) -> (Tensor, Tensor)");
+  ops.impl("scaled_fp4_quant_sm103_pdl", torch::kCUDA,
+           &scaled_fp4_quant_sm103a_pdl_func);
+
+  ops.def(
+      "scaled_fp4_quant_sm103_pdl.out(Tensor input,"
+      "                               Tensor input_scale,"
+      "                               *, Tensor(a!) output,"
+      "                               Tensor(b!) output_scale) -> ()");
+  ops.impl("scaled_fp4_quant_sm103_pdl.out", torch::kCUDA,
+           &scaled_fp4_quant_sm103a_pdl_out);
+#endif
+
  // cutlass nvfp4 block scaled group GEMM
  ops.def(
      "cutlass_fp4_group_mm(Tensor! out, Tensor a, Tensor b,"
@@ -526,26 +617,6 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
  ops.impl("cutlass_scaled_mm_supports_block_fp8",
           &cutlass_scaled_mm_supports_block_fp8);

-  // Check if cutlass sparse scaled_mm is supported for CUDA devices of the
-  // given capability
-  ops.def(
-      "cutlass_sparse_scaled_mm_supported(int cuda_device_capability) -> bool");
-  ops.impl("cutlass_sparse_scaled_mm_supported",
-           &cutlass_sparse_scaled_mm_supported);
-
-  // CUTLASS sparse GEMM, supporting symmetric per-tensor or per-row/column
-  // quantization, as well as bias
-  ops.def(
-      "cutlass_scaled_sparse_mm(Tensor! out, Tensor a,"
-      "                         Tensor bt_nzs,"
-      "                         Tensor bt_meta, Tensor a_scales,"
-      "                         Tensor b_scales, Tensor? bias) -> ()");
-  ops.impl("cutlass_scaled_sparse_mm", torch::kCUDA, &cutlass_scaled_sparse_mm);
-
-  // CUTLASS sparse matrix compressor
-  ops.def("cutlass_sparse_compress(Tensor a) -> Tensor[]");
-  ops.impl("cutlass_sparse_compress", &cutlass_sparse_compress);
-
  // SM100 CUTLASS MLA decode
  ops.def(
      "sm100_cutlass_mla_decode(Tensor! out, Tensor! lse, Tensor q_nope,"
@@ -596,6 +667,16 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
  ops.impl("silu_and_mul_scaled_fp4_experts_quant", torch::kCUDA,
           &silu_and_mul_scaled_fp4_experts_quant);

+  // SM100 <-> SM103 scale factor layout conversion (B300 / Blackwell Ultra)
+  ops.def(
+      "convert_sf_layout_sm100_to_sm103(Tensor(a!) dst, Tensor src) -> ()");
+  ops.impl("convert_sf_layout_sm100_to_sm103", torch::kCUDA,
+           &convert_sf_layout_sm100_to_sm103);
+  ops.def(
+      "convert_sf_layout_sm103_to_sm100(Tensor(a!) dst, Tensor src) -> ()");
+  ops.impl("convert_sf_layout_sm103_to_sm100", torch::kCUDA,
+           &convert_sf_layout_sm103_to_sm100);
+
  // Check if cutlass_scaled_mm_fp4 is supported for CUDA devices
  // of the given capability
  ops.def("cutlass_scaled_mm_supports_fp4(int cuda_device_capability) -> bool");
@@ -44,7 +44,7 @@ ENV DEBIAN_FRONTEND=noninteractive

 # Install Python and other dependencies
 RUN apt-get update -y \
-    && apt-get install -y software-properties-common git curl sudo vim less libgfortran5 libopenmpi-dev libpci-dev \
+    && apt-get install -y software-properties-common git curl sudo vim less libgfortran5 libopenmpi-dev libpci-dev liblzma-dev pkg-config \
    && for i in 1 2 3; do \
        add-apt-repository -y ppa:deadsnakes/ppa && break || \
        { echo "Attempt $i failed, retrying in 5s..."; sleep 5; }; \
@@ -25,7 +25,7 @@ nav:
    - Models:
      - models/supported_models.md
      - models/generative_models.md
-      - models/pooling_models.md
+      - Pooling Models: models/pooling_models
      - models/extensions
      - Hardware Supported Models:
        - models/hardware_supported_models/*
@@ -37,7 +37,7 @@ For [generative models](../../models/generative_models.md), there are two levels

 #### Pooling models

-For [pooling models](../../models/pooling_models.md), we simply check the cosine similarity, as defined in [tests/models/utils.py](../../../tests/models/utils.py).
+For [pooling models](../../models/pooling_models/README.md), we simply check the cosine similarity, as defined in [tests/models/utils.py](../../../tests/models/utils.py).

 ### Multi-modal processing

@@ -3,6 +3,10 @@
 !!! warning
    Profiling is only intended for vLLM developers and maintainers to understand the proportion of time spent in different parts of the codebase. **vLLM end-users should never turn on profiling** as it will significantly slow down the inference.

+!!! tip "Choosing a profiler"
+    - Use **Nsight Systems** for low-overhead, performance-critical profiling.
+    - Use **PyTorch Profiler** for medium-overhead profiling with richer debugging information (e.g., stack traces, memory, shapes). Note that enabling these features adds overhead and is not recommended for benchmarking.
+
 ## Profile with PyTorch Profiler

 We support tracing vLLM workers using different profilers. You can enable profiling by setting the `--profiler-config` flag when launching the server.
@@ -51,11 +51,8 @@ For example:
 **1. Attention:**

 ```python
--8<-- "vllm/model_executor/layers/attention/mm_encoder_attention.py:mm_encoder_attn"
-
 --8<-- "vllm/model_executor/layers/mla.py:multi_head_latent_attention"

--8<-- "vllm/model_executor/models/deepencoder.py:rel_pos_attention"
 ```

 **2. Activation:**
@@ -170,6 +167,16 @@ For example:
 --8<-- "vllm/model_executor/layers/rotary_embedding/common.py:apply_rotary_emb"
 ```

+**12. Encoder:**
+
+```python
+--8<-- "vllm/model_executor/models/deepencoder2.py:qwen2_decoder"
+
+--8<-- "vllm/model_executor/layers/attention/mm_encoder_attention.py:mm_encoder_attn"
+
+--8<-- "vllm/model_executor/models/deepencoder.py:rel_pos_attention"
+```
+
 ## Guidelines for Implementing a New CustomOp

 ### Implement a New CustomOp in vLLM
@@ -244,6 +244,7 @@ statistics relating to that iteration:
  prefill in this iteration. However, we calculate this interval
  relative to when the request was first received by the frontend
  (`arrival_time`) in order to account for input processing time.
+  Currently `arrival_time` starts when tokenization begins.

 For any requests that were completed in a given iteration, we also
 record:
@@ -587,7 +588,7 @@ see:
 - [Benchmarking LLM Workloads for Performance Evaluation and Autoscaling in Kubernetes](https://docs.google.com/document/d/1k4Q4X14hW4vftElIuYGDu5KDe2LtV1XammoG-Xi3bbQ)
 - [Inference Perf](https://github.com/kubernetes-sigs/wg-serving/tree/main/proposals/013-inference-perf)
 - <https://github.com/vllm-project/vllm/issues/5041> and <https://github.com/vllm-project/vllm/pull/12726>.
-  
+
 This is a non-trivial topic. Consider this comment from Rob:

 > I think this metric should focus on trying to estimate what the max
@@ -33,10 +33,10 @@ th {
 | Backend | Output act. format | Quant. types | Quant. format | Async | Apply Weight On Input | Subclass |
 | ------- | ------------------ | ------------ | ------------- | ----- | --------------------- | --------- |
 | naive | standard | all<sup>1</sup> | G,A,T | N | <sup>6</sup> | [layer.py][vllm.model_executor.layers.fused_moe.layer.FusedMoE] |
-| deepep_high_throughput | standard | fp8 | G(128),A,T<sup>2</sup> | Y | Y | [`DeepEPHTPrepareAndFinalize`][vllm.model_executor.layers.fused_moe.deepep_ht_prepare_finalize.DeepEPHTPrepareAndFinalize] |
-| deepep_low_latency | batched | fp8 | G(128),A,T<sup>3</sup> | Y | Y | [`DeepEPLLPrepareAndFinalize`][vllm.model_executor.layers.fused_moe.deepep_ll_prepare_finalize.DeepEPLLPrepareAndFinalize] |
-| flashinfer_nvlink_two_sided | standard | nvfp4,fp8 | G,A,T | N | N | [`FlashInferNVLinkTwoSidedPrepareAndFinalize`][vllm.model_executor.layers.fused_moe.flashinfer_nvlink_two_sided_prepare_finalize.FlashInferNVLinkTwoSidedPrepareAndFinalize] |
-| flashinfer_nvlink_one_sided | standard | nvfp4 | G,A,T | N | N | [`FlashInferNVLinkOneSidedPrepareAndFinalize`][vllm.model_executor.layers.fused_moe.flashinfer_nvlink_one_sided_prepare_finalize.FlashInferNVLinkOneSidedPrepareAndFinalize] |
+| deepep_high_throughput | standard | fp8 | G(128),A,T<sup>2</sup> | Y | Y | [`DeepEPHTPrepareAndFinalize`][vllm.model_executor.layers.fused_moe.prepare_finalize.deepep_ht.DeepEPHTPrepareAndFinalize] |
+| deepep_low_latency | batched | fp8 | G(128),A,T<sup>3</sup> | Y | Y | [`DeepEPLLPrepareAndFinalize`][vllm.model_executor.layers.fused_moe.prepare_finalize.deepep_ll.DeepEPLLPrepareAndFinalize] |
+| flashinfer_nvlink_two_sided | standard | nvfp4,fp8 | G,A,T | N | N | [`FlashInferNVLinkTwoSidedPrepareAndFinalize`][vllm.model_executor.layers.fused_moe.prepare_finalize.flashinfer_nvlink_two_sided.FlashInferNVLinkTwoSidedPrepareAndFinalize] |
+| flashinfer_nvlink_one_sided | standard | nvfp4 | G,A,T | N | N | [`FlashInferNVLinkOneSidedPrepareAndFinalize`][vllm.model_executor.layers.fused_moe.prepare_finalize.flashinfer_nvlink_one_sided.FlashInferNVLinkOneSidedPrepareAndFinalize] |

 !!! info "Table key"
    1. All types: mxfp4, nvfp4, int4, int8, fp8
@@ -88,8 +88,8 @@ To be used with a particular `FusedMoEPrepareAndFinalizeModular` subclass, MoE k
 | flashinfer | standard | nvfp4,</br>fp8 | T | <sup>5</sup> | N | Y | [`FlashInferExperts`][vllm.model_executor.layers.fused_moe.flashinfer_cutlass_moe.FlashInferExperts] |
 | gpt oss triton | standard | N/A | N/A | <sup>5</sup> | Y | Y | [`triton_kernel_fused_experts`][vllm.model_executor.layers.fused_moe.gpt_oss_triton_kernels_moe.triton_kernel_fused_experts],</br>[`OAITritonExperts`][vllm.model_executor.layers.fused_moe.gpt_oss_triton_kernels_moe.OAITritonExperts] |
 | marlin | standard,</br>batched | <sup>3</sup> / N/A | <sup>3</sup> / N/A | silu,</br>swigluoai | Y | Y | [`fused_marlin_moe`][vllm.model_executor.layers.fused_moe.fused_marlin_moe.fused_marlin_moe],</br>[`MarlinExperts`][vllm.model_executor.layers.fused_moe.fused_marlin_moe.MarlinExperts],</br>[`BatchedMarlinExperts`][vllm.model_executor.layers.fused_moe.fused_marlin_moe.BatchedMarlinExperts] |
-| trtllm | standard | mxfp4,</br>nvfp4 | G(16),G(32) | <sup>5</sup> | N | Y | [`TrtLlmGenExperts`][vllm.model_executor.layers.fused_moe.trtllm_moe.TrtLlmGenExperts] |
-| rocm aiter moe | standard | fp8 | G(128),A,T | silu, gelu | Y | N | [`rocm_aiter_fused_experts`][vllm.model_executor.layers.fused_moe.rocm_aiter_fused_moe.rocm_aiter_fused_experts] |
+| trtllm | standard | mxfp4,</br>nvfp4 | G(16),G(32) | <sup>5</sup> | N | Y | [`TrtLlmMxfp4ExpertsMonolithic`][vllm.model_executor.layers.fused_moe.experts.trtllm_mxfp4_moe.TrtLlmMxfp4ExpertsMonolithic],</br>[`TrtLlmMxfp4ExpertsModular`][vllm.model_executor.layers.fused_moe.experts.trtllm_mxfp4_moe.TrtLlmMxfp4ExpertsModular],</br>[`TrtLlmNvFp4ExpertsMonolithic`][vllm.model_executor.layers.fused_moe.experts.trtllm_nvfp4_moe.TrtLlmNvFp4ExpertsMonolithic],</br>[`TrtLlmNvfp4ExpertsModular`][vllm.model_executor.layers.fused_moe.experts.trtllm_nvfp4_moe.TrtLlmNvFp4ExpertsModular] |
+| rocm aiter moe | standard | mxfp4,</br>fp8 | G(32),G(128),A,T | silu, gelu,</br>swigluoai | Y | N | `rocm_aiter_fused_experts`,</br>`AiterExperts` |
 | cpu_fused_moe | standard | N/A | N/A | silu | N | N | [`CPUFusedMOE`][vllm.model_executor.layers.fused_moe.cpu_fused_moe.CPUFusedMOE] |
 | naive batched<sup>4</sup> | batched | int8,</br>fp8 | G,A,T | silu, gelu | <sup>6</sup> | Y | [`NaiveBatchedExperts`][vllm.model_executor.layers.fused_moe.fused_batched_moe.NaiveBatchedExperts] |

@@ -103,7 +103,7 @@ To be used with a particular `FusedMoEPrepareAndFinalizeModular` subclass, MoE k

 ## Modular Kernel "families"

-The following table shows "families" of modular kernels that are intended to work together. There are some combinations which may work but have not yet been tested, e.g. flashinfer with other fp8 experts. Note that the "naive" backend will work with any non-modular experts.
+The following table shows "families" of modular kernels that are intended to work together. There are some combinations which may work but have not yet been tested, e.g. flashinfer with other fp8 experts.

 | backend | `FusedMoEPrepareAndFinalizeModular` subclasses | `FusedMoEExpertsModular` subclasses |
 | ------- | ---------------------------------------------- | ----------------------------------- |
@@ -29,10 +29,9 @@ To compile a multimodal component such as an encoder, we follow the same mechani
 1. The `@support_torch_compile` decorator should include `enable_if=should_torch_compile_mm_encoder`. This will gate the compilation behind our
 `compile_mm_encoder` configuration

-2. `with set_model_tag("<component_name>", is_encoder=True)` context manager should be used around the nn.Module's instantiation. Since torch.compile
-relies on caching artifacts to reduce start time, we must properly propagate the `<component_name>` information to the cache in order to avoid collisions
-with the LLM text-backbone, or other instances of the same artifact (as is the case with vision block). `is_encoder=True` is also needed for encoder
-components (see Compile Range Integration).
+2. The `@support_torch_compile` decorator should include `is_encoder=True` for encoder components. This is needed for compile range integration
+(see Compile Range Integration). The decorator automatically uses the class name as the cache directory prefix, avoiding collisions between
+independently compiled sub-modules (e.g. vision encoder components vs the text backbone).

 ### CompilationConfig

@@ -57,8 +56,8 @@ tradeoff
 ### Compile ranges

 The torch.compile integration will try to rely on max_batch_size to infer compilation ranges for dynamic shapes; however, for modules used in the encoder, this
-shape can be difficult to infer due to the unspecified range of shapes the encoder may see as input. Therefore, we rely on `is_encoder=True` in the `set_model_tag`
-to alert torch.compile to the fact that this range cannot be inferred, and we default to the range (1, MAX_INT).
+shape can be difficult to infer due to the unspecified range of shapes the encoder may see as input. Therefore, we rely on `is_encoder=True` in the
+`@support_torch_compile` decorator to alert torch.compile to the fact that this range cannot be inferred, and we default to the range (1, MAX_INT).

 !!! note
    We may seek to tighten this range for better performance in the future
@@ -36,14 +36,14 @@ th:not(:first-child) {
 }
 </style>

-| Feature | [CP](../configuration/optimization.md#chunked-prefill) | [APC](automatic_prefix_caching.md) | [LoRA](lora.md) | [SD](speculative_decoding/README.md) | CUDA graph | [pooling](../models/pooling_models.md) | <abbr title="Encoder-Decoder Models">enc-dec</abbr> | <abbr title="Logprobs">logP</abbr> | <abbr title="Prompt Logprobs">prmpt logP</abbr> | <abbr title="Async Output Processing">async output</abbr> | multi-step | <abbr title="Multimodal Inputs">mm</abbr> | best-of | beam-search | [prompt-embeds](prompt_embeds.md) |
+| Feature | [CP](../configuration/optimization.md#chunked-prefill) | [APC](automatic_prefix_caching.md) | [LoRA](lora.md) | [SD](speculative_decoding/README.md) | CUDA graph | [pooling](../models/pooling_models/README.md) | <abbr title="Encoder-Decoder Models">enc-dec</abbr> | <abbr title="Logprobs">logP</abbr> | <abbr title="Prompt Logprobs">prmpt logP</abbr> | <abbr title="Async Output Processing">async output</abbr> | multi-step | <abbr title="Multimodal Inputs">mm</abbr> | best-of | beam-search | [prompt-embeds](prompt_embeds.md) |
 | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | - |
 | [CP](../configuration/optimization.md#chunked-prefill) | ✅ | | | | | | | | | | | | | | |
 | [APC](automatic_prefix_caching.md) | ✅ | ✅ | | | | | | | | | | | | | |
 | [LoRA](lora.md) | ✅ | ✅ | ✅ | | | | | | | | | | | | |
 | [SD](speculative_decoding/README.md) | ✅ | ✅ | ❌ | ✅ | | | | | | | | | | | |
 | CUDA graph | ✅ | ✅ | ✅ | ✅ | ✅ | | | | | | | | | | |
-| [pooling](../models/pooling_models.md) | 🟠\* | 🟠\* | ✅ | ❌ | ✅ | ✅ | | | | | | | | | |
+| [pooling](../models/pooling_models/README.md) | 🟠\* | 🟠\* | ✅ | ❌ | ✅ | ✅ | | | | | | | | | |
 | <abbr title="Encoder-Decoder Models">enc-dec</abbr> | ❌ | [❌](https://github.com/vllm-project/vllm/issues/7366) | ❌ | [❌](https://github.com/vllm-project/vllm/issues/7366) | ✅ | ✅ | ✅ | | | | | | | | |
 | <abbr title="Logprobs">logP</abbr> | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | | | | | | | |
 | <abbr title="Prompt Logprobs">prmpt logP</abbr> | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | | | | | | |
@@ -66,7 +66,7 @@ th:not(:first-child) {
 | [LoRA](lora.md) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
 | [SD](speculative_decoding/README.md) | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ |
 | CUDA graph | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | [❌](https://github.com/vllm-project/vllm/issues/26970) |
-| [pooling](../models/pooling_models.md) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
+| [pooling](../models/pooling_models/README.md) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
 | <abbr title="Encoder-Decoder Models">enc-dec</abbr> | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ |
 | [mm](multimodal_inputs.md) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
 | [prompt-embeds](prompt_embeds.md) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❔ | ✅ |
@@ -137,6 +137,3 @@ llm = LLM("facebook/opt-125m", quantization="fp8")
 result = llm.generate("Hello, my name is")
 print(result[0].outputs[0].text)
 ```
-
-!!! warning
-    Currently, we load the model at original precision before quantizing down to 8-bits, so you need enough memory to load the whole model.
@@ -5,7 +5,7 @@ vLLM offers support for reasoning models like [DeepSeek R1](https://huggingface.
 Reasoning models return an additional `reasoning` field in their outputs, which contains the reasoning steps that led to the final conclusion. This field is not present in the outputs of other models.

 !!! warning
-    `reasoning` used to be called `reasoning_content`. For now, `reasoning_content` will continue to work. However, we encourage you to migrate to `reasoning` in case `reasoning_content` is removed in future.
+    `reasoning` used to be called `reasoning_content`. To migrate, directly replace `reasoning_content` with `reasoning`.

 ## Supported Models

@@ -1,7 +1,8 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 """
-MkDocs hook to enable the following links to render correctly:
+MkDocs hook + markdown extension to enable the following links to render correctly,
+including inside content included via pymdownx.snippets:

 - Relative file links outside of the `docs/` directory, e.g.:
    - [Text](../some_file.py)
@@ -12,13 +13,17 @@ MkDocs hook to enable the following links to render correctly:
        e.g. <...pull/123> -> [Pull Request #123](.../pull/123)
    - Works for external repos too by including the `owner/repo` in the link title

-The goal is to simplify cross-referencing common GitHub resources
-in project docs.
+The link replacement runs as a markdown preprocessor (priority 25) so that it executes
+after pymdownx.snippets (priority 32) has expanded all included content.
+The on_page_markdown hook passes the current page context to the preprocessor before
+each page is converted.
 """

 from pathlib import Path

 import regex as re
+from markdown import Extension
+from markdown.preprocessors import Preprocessor
 from mkdocs.config.defaults import MkDocsConfig
 from mkdocs.structure.files import Files
 from mkdocs.structure.pages import Page
@@ -26,7 +31,6 @@ from mkdocs.structure.pages import Page
 ROOT_DIR = Path(__file__).parent.parent.parent.parent.resolve()
 DOC_DIR = ROOT_DIR / "docs"

-
 gh_icon = ":octicons-mark-github-16:"

 # Regex pieces
@@ -48,46 +52,90 @@ github_link = re.compile(rf"(\[{TITLE}\]\(|<){URL}(\)|>)")
 relative_link = re.compile(rf"\[{TITLE}\]\({RELATIVE}\)")


+class UrlSchemesPreprocessor(Preprocessor):
+    """Preprocessor that runs after pymdownx.snippets to process all links."""
+
+    def __init__(self, md, ext):
+        super().__init__(md)
+        self.ext = ext
+
+    def run(self, lines):
+        page = self.ext.page
+        if page is None or getattr(page.file, "abs_src_path", None) is None:
+            return lines
+
+        def replace_relative_link(match: re.Match) -> str:
+            """
+            Replace relative file links with URLs if they point outside the docs dir.
+            """
+            title = match.group("title")
+            path = match.group("path")
+            path = (Path(page.file.abs_src_path).parent / path).resolve()
+            fragment = match.group("fragment") or ""
+
+            # Check if the path exists and is outside the docs dir
+            if not path.exists() or path.is_relative_to(DOC_DIR):
+                return match.group(0)
+
+            # Files and directories have different URL schemes on GitHub
+            slug = "tree/main" if path.is_dir() else "blob/main"
+
+            path = path.relative_to(ROOT_DIR)
+            url = f"https://github.com/vllm-project/vllm/{slug}/{path}{fragment}"
+            return f"[{gh_icon} {title}]({url})"
+
+        def replace_github_link(match: re.Match) -> str:
+            """
+            Replace GitHub issue, PR, and project links with enhanced Markdown links.
+            """
+            repo = match.group("repo")
+            type = match.group("type")
+            number = match.group("number")
+            # Title and fragment could be None
+            title = match.group("title") or ""
+            fragment = match.group("fragment") or ""
+
+            # Use default titles for raw links
+            if not title:
+                title = TITLES[type]
+                if "vllm-project" not in repo:
+                    title += repo
+                title += f"#{number}"
+
+            url = f"https://github.com/{repo}/{type}/{number}{fragment}"
+            return f"[{gh_icon} {title}]({url})"
+
+        markdown = "\n".join(lines)
+        markdown = relative_link.sub(replace_relative_link, markdown)
+        markdown = github_link.sub(replace_github_link, markdown)
+        return markdown.split("\n")
+
+
+class UrlSchemesExtension(Extension):
+    """Markdown extension that registers the URL schemes preprocessor."""
+
+    def __init__(self, **kwargs):
+        self.page = None
+        super().__init__(**kwargs)
+
+    def extendMarkdown(self, md):
+        # Priority 25 runs after pymdownx.snippets (priority 32)
+        md.preprocessors.register(UrlSchemesPreprocessor(md, self), "url_schemes", 25)
+
+
+# Singleton extension instance shared between the hook and the preprocessor.
+_ext = UrlSchemesExtension()
+
+
+def on_config(config: MkDocsConfig) -> MkDocsConfig:
+    """Register the URL schemes markdown extension."""
+    config["markdown_extensions"].append(_ext)
+    return config
+
+
 def on_page_markdown(
    markdown: str, *, page: Page, config: MkDocsConfig, files: Files
 ) -> str:
-    def replace_relative_link(match: re.Match) -> str:
-        """Replace relative file links with URLs if they point outside the docs dir."""
-        title = match.group("title")
-        path = match.group("path")
-        path = (Path(page.file.abs_src_path).parent / path).resolve()
-        fragment = match.group("fragment") or ""
-
-        # Check if the path exists and is outside the docs dir
-        if not path.exists() or path.is_relative_to(DOC_DIR):
-            return match.group(0)
-
-        # Files and directories have different URL schemes on GitHub
-        slug = "tree/main" if path.is_dir() else "blob/main"
-
-        path = path.relative_to(ROOT_DIR)
-        url = f"https://github.com/vllm-project/vllm/{slug}/{path}{fragment}"
-        return f"[{gh_icon} {title}]({url})"
-
-    def replace_github_link(match: re.Match) -> str:
-        """Replace GitHub issue, PR, and project links with enhanced Markdown links."""
-        repo = match.group("repo")
-        type = match.group("type")
-        number = match.group("number")
-        # Title and fragment could be None
-        title = match.group("title") or ""
-        fragment = match.group("fragment") or ""
-
-        # Use default titles for raw links
-        if not title:
-            title = TITLES[type]
-            if "vllm-project" not in repo:
-                title += repo
-            title += f"#{number}"
-
-        url = f"https://github.com/{repo}/{type}/{number}{fragment}"
-        return f"[{gh_icon} {title}]({url})"
-
-    markdown = relative_link.sub(replace_relative_link, markdown)
-    markdown = github_link.sub(replace_github_link, markdown)
+    """Pass the current page context to the preprocessor."""
+    _ext.page = page
    return markdown
@@ -1,716 +0,0 @@
-# Pooling Models
-
-vLLM also supports pooling models, such as embedding, classification, and reward models.
-
-In vLLM, pooling models implement the [VllmModelForPooling][vllm.model_executor.models.VllmModelForPooling] interface.
-These models use a [Pooler][vllm.model_executor.layers.pooler.Pooler] to extract the final hidden states of the input
-before returning them.
-
-!!! note
-    We currently support pooling models primarily for convenience. This is not guaranteed to provide any performance improvements over using Hugging Face Transformers or Sentence Transformers directly.
-
-    We plan to optimize pooling models in vLLM. Please comment on <https://github.com/vllm-project/vllm/issues/21796> if you have any suggestions!
-
-## Configuration
-
-### Model Runner
-
-Run a model in pooling mode via the option `--runner pooling`.
-
-!!! tip
-    There is no need to set this option in the vast majority of cases as vLLM can automatically
-    detect the appropriate model runner via `--runner auto`.
-
-### Model Conversion
-
-vLLM can adapt models for various pooling tasks via the option `--convert <type>`.
-
-If `--runner pooling` has been set (manually or automatically) but the model does not implement the
-[VllmModelForPooling][vllm.model_executor.models.VllmModelForPooling] interface,
-vLLM will attempt to automatically convert the model according to the architecture names
-shown in the table below.
-
-| Architecture                                    | `--convert` | Supported pooling tasks               |
-| ----------------------------------------------- | ----------- | ------------------------------------- |
-| `*ForTextEncoding`, `*EmbeddingModel`, `*Model` | `embed`     | `token_embed`, `embed`                |
-| `*ForRewardModeling`, `*RewardModel`            | `embed`     | `token_embed`, `embed`                |
-| `*For*Classification`, `*ClassificationModel`   | `classify`  | `token_classify`, `classify`, `score` |
-
-!!! tip
-    You can explicitly set `--convert <type>` to specify how to convert the model.
-
-### Pooling Tasks
-
-Each pooling model in vLLM supports one or more of these tasks according to
-[Pooler.get_supported_tasks][vllm.model_executor.layers.pooler.Pooler.get_supported_tasks],
-enabling the corresponding APIs:
-
-| Task             | APIs                                                                          |
-| ---------------- | ----------------------------------------------------------------------------- |
-| `embed`          | `LLM.embed(...)`, `LLM.score(...)`\*, `LLM.encode(..., pooling_task="embed")` |
-| `classify`       | `LLM.classify(...)`, `LLM.encode(..., pooling_task="classify")`               |
-| `score`          | `LLM.score(...)`                                                              |
-| `token_classify` | `LLM.reward(...)`, `LLM.encode(..., pooling_task="token_classify")`           |
-| `token_embed`    | `LLM.encode(..., pooling_task="token_embed")`                                 |
-| `plugin`         | `LLM.encode(..., pooling_task="plugin")`                                      |
-
-\* The `LLM.score(...)` API falls back to `embed` task if the model does not support `score` task.
-
-### Pooler Configuration
-
-#### Predefined models
-
-If the [Pooler][vllm.model_executor.layers.pooler.Pooler] defined by the model accepts `pooler_config`,
-you can override some of its attributes via the `--pooler-config` option.
-
-#### Converted models
-
-If the model has been converted via `--convert` (see above),
-the pooler assigned to each task has the following attributes by default:
-
-| Task       | Pooling Type | Normalization | Softmax |
-| ---------- | ------------ | ------------- | ------- |
-| `embed`    | `LAST`       | ✅︎            | ❌      |
-| `classify` | `LAST`       | ❌            | ✅︎      |
-
-When loading [Sentence Transformers](https://huggingface.co/sentence-transformers) models,
-its Sentence Transformers configuration file (`modules.json`) takes priority over the model's defaults.
-
-You can further customize this via the `--pooler-config` option,
-which takes priority over both the model's and Sentence Transformers' defaults.
-
-## Offline Inference
-
-The [LLM][vllm.LLM] class provides various methods for offline inference.
-See [configuration](../api/README.md#configuration) for a list of options when initializing the model.
-
-### `LLM.embed`
-
-The [embed][vllm.LLM.embed] method outputs an embedding vector for each prompt.
-It is primarily designed for embedding models.
-
-```python
-from vllm import LLM
-
-llm = LLM(model="intfloat/e5-small", runner="pooling")
-(output,) = llm.embed("Hello, my name is")
-
-embeds = output.outputs.embedding
-print(f"Embeddings: {embeds!r} (size={len(embeds)})")
-```
-
-A code example can be found here: [examples/basic/offline_inference/embed.py](../../examples/basic/offline_inference/embed.py)
-
-### `LLM.classify`
-
-The [classify][vllm.LLM.classify] method outputs a probability vector for each prompt.
-It is primarily designed for classification models.
-
-```python
-from vllm import LLM
-
-llm = LLM(model="jason9693/Qwen2.5-1.5B-apeach", runner="pooling")
-(output,) = llm.classify("Hello, my name is")
-
-probs = output.outputs.probs
-print(f"Class Probabilities: {probs!r} (size={len(probs)})")
-```
-
-A code example can be found here: [examples/basic/offline_inference/classify.py](../../examples/basic/offline_inference/classify.py)
-
-### `LLM.score`
-
-The [score][vllm.LLM.score] method outputs similarity scores between sentence pairs.
-It is designed for embedding models and cross-encoder models. Embedding models use cosine similarity, and [cross-encoder models](https://www.sbert.net/examples/applications/cross-encoder/README.html) serve as rerankers between candidate query-document pairs in RAG systems.
-
-!!! note
-    vLLM can only perform the model inference component (e.g. embedding, reranking) of RAG.
-    To handle RAG at a higher level, you should use integration frameworks such as [LangChain](https://github.com/langchain-ai/langchain).
-
-```python
-from vllm import LLM
-
-llm = LLM(model="BAAI/bge-reranker-v2-m3", runner="pooling")
-(output,) = llm.score(
-    "What is the capital of France?",
-    "The capital of Brazil is Brasilia.",
-)
-
-score = output.outputs.score
-print(f"Score: {score}")
-```
-
-A code example can be found here: [examples/basic/offline_inference/score.py](../../examples/basic/offline_inference/score.py)
-
-### `LLM.reward`
-
-The [reward][vllm.LLM.reward] method is available to all reward models in vLLM.
-
-```python
-from vllm import LLM
-
-llm = LLM(model="internlm/internlm2-1_8b-reward", runner="pooling", trust_remote_code=True)
-(output,) = llm.reward("Hello, my name is")
-
-data = output.outputs.data
-print(f"Data: {data!r}")
-```
-
-A code example can be found here: [examples/basic/offline_inference/reward.py](../../examples/basic/offline_inference/reward.py)
-
-### `LLM.encode`
-
-The [encode][vllm.LLM.encode] method is available to all pooling models in vLLM.
-
-!!! note
-    Please use one of the more specific methods or set the task directly when using `LLM.encode`:
-
-    - For embeddings, use `LLM.embed(...)` or `pooling_task="embed"`.
-    - For classification logits, use `LLM.classify(...)` or `pooling_task="classify"`.
-    - For similarity scores, use `LLM.score(...)`.
-    - For rewards, use `LLM.reward(...)` or `pooling_task="token_classify"`.
-    - For token classification, use `pooling_task="token_classify"`.
-    - For multi-vector retrieval, use `pooling_task="token_embed"`.
-    - For IO Processor Plugins, use `pooling_task="plugin"`.
-
-```python
-from vllm import LLM
-
-llm = LLM(model="intfloat/e5-small", runner="pooling")
-(output,) = llm.encode("Hello, my name is", pooling_task="embed")
-
-data = output.outputs.data
-print(f"Data: {data!r}")
-```
-
-## Online Serving
-
-Our [OpenAI-Compatible Server](../serving/openai_compatible_server.md) provides endpoints that correspond to the offline APIs:
-
- [Embeddings API](../serving/openai_compatible_server.md#embeddings-api) is similar to `LLM.embed`, accepting both text and [multi-modal inputs](../features/multimodal_inputs.md) for embedding models.
- [Classification API](../serving/openai_compatible_server.md#classification-api) is similar to `LLM.classify` and is applicable to sequence classification models.
- [Score API](../serving/openai_compatible_server.md#score-api) is similar to `LLM.score` for cross-encoder models.
- [Pooling API](../serving/openai_compatible_server.md#pooling-api) is similar to `LLM.encode`, being applicable to all types of pooling models.
-
-!!! note
-    Please use one of the more specific endpoints or set the task directly when using the [Pooling API](../serving/openai_compatible_server.md#pooling-api):
-
-    - For embeddings, use [Embeddings API](../serving/openai_compatible_server.md#embeddings-api) or `"task":"embed"`.
-    - For classification logits, use [Classification API](../serving/openai_compatible_server.md#classification-api) or `"task":"classify"`.
-    - For similarity scores, use [Score API](../serving/openai_compatible_server.md#score-api).
-    - For rewards, use `"task":"token_classify"`.
-    - For token classification, use `"task":"token_classify"`.
-    - For multi-vector retrieval, use `"task":"token_embed"`.
-    - For IO Processor Plugins, use `"task":"plugin"`.
-
-```python
-# start a supported embeddings model server with `vllm serve`, e.g.
-# vllm serve intfloat/e5-small
-import requests
-
-host = "localhost"
-port = "8000"
-model_name = "intfloat/e5-small"
-
-api_url = f"http://{host}:{port}/pooling"
-
-prompts = [
-    "Hello, my name is",
-    "The president of the United States is",
-    "The capital of France is",
-    "The future of AI is",
-]
-prompt = {"model": model_name, "input": prompts, "task": "embed"}
-
-response = requests.post(api_url, json=prompt)
-
-for output in response.json()["data"]:
-    data = output["data"]
-    print(f"Data: {data!r} (size={len(data)})")
-```
-
-## Matryoshka Embeddings
-
-[Matryoshka Embeddings](https://sbert.net/examples/sentence_transformer/training/matryoshka/README.html#matryoshka-embeddings) or [Matryoshka Representation Learning (MRL)](https://arxiv.org/abs/2205.13147) is a technique used in training embedding models. It allows users to trade off between performance and cost.
-
-!!! warning
-    Not all embedding models are trained using Matryoshka Representation Learning. To avoid misuse of the `dimensions` parameter, vLLM returns an error for requests that attempt to change the output dimension of models that do not support Matryoshka Embeddings.
-
-    For example, setting `dimensions` parameter while using the `BAAI/bge-m3` model will result in the following error.
-
-    ```json
-    {"object":"error","message":"Model \"BAAI/bge-m3\" does not support matryoshka representation, changing output dimensions will lead to poor results.","type":"BadRequestError","param":null,"code":400}
-    ```
-
-### Manually enable Matryoshka Embeddings
-
-There is currently no official interface for specifying support for Matryoshka Embeddings. In vLLM, if `is_matryoshka` is `True` in `config.json`, you can change the output dimension to arbitrary values. Use `matryoshka_dimensions` to control the allowed output dimensions.
-
-For models that support Matryoshka Embeddings but are not recognized by vLLM, manually override the config using `hf_overrides={"is_matryoshka": True}` or `hf_overrides={"matryoshka_dimensions": [<allowed output dimensions>]}` (offline), or `--hf-overrides '{"is_matryoshka": true}'` or `--hf-overrides '{"matryoshka_dimensions": [<allowed output dimensions>]}'` (online).
-
-Here is an example to serve a model with Matryoshka Embeddings enabled.
-
-```bash
-vllm serve Snowflake/snowflake-arctic-embed-m-v1.5 --hf-overrides '{"matryoshka_dimensions":[256]}'
-```
-
-### Offline Inference
-
-You can change the output dimensions of embedding models that support Matryoshka Embeddings by using the dimensions parameter in [PoolingParams][vllm.PoolingParams].
-
-```python
-from vllm import LLM, PoolingParams
-
-llm = LLM(
-    model="jinaai/jina-embeddings-v3",
-    runner="pooling",
-    trust_remote_code=True,
-)
-outputs = llm.embed(
-    ["Follow the white rabbit."],
-    pooling_params=PoolingParams(dimensions=32),
-)
-print(outputs[0].outputs)
-```
-
-A code example can be found here: [examples/pooling/embed/embed_matryoshka_fy_offline.py](../../examples/pooling/embed/embed_matryoshka_fy_offline.py)
-
-### Online Inference
-
-Use the following command to start the vLLM server.
-
-```bash
-vllm serve jinaai/jina-embeddings-v3 --trust-remote-code
-```
-
-You can change the output dimensions of embedding models that support Matryoshka Embeddings by using the dimensions parameter.
-
-```bash
-curl http://127.0.0.1:8000/v1/embeddings \
-  -H 'accept: application/json' \
-  -H 'Content-Type: application/json' \
-  -d '{
-    "input": "Follow the white rabbit.",
-    "model": "jinaai/jina-embeddings-v3",
-    "encoding_format": "float",
-    "dimensions": 32
-  }'
-```
-
-Expected output:
-
-```json
-{"id":"embd-5c21fc9a5c9d4384a1b021daccaf9f64","object":"list","created":1745476417,"model":"jinaai/jina-embeddings-v3","data":[{"index":0,"object":"embedding","embedding":[-0.3828125,-0.1357421875,0.03759765625,0.125,0.21875,0.09521484375,-0.003662109375,0.1591796875,-0.130859375,-0.0869140625,-0.1982421875,0.1689453125,-0.220703125,0.1728515625,-0.2275390625,-0.0712890625,-0.162109375,-0.283203125,-0.055419921875,-0.0693359375,0.031982421875,-0.04052734375,-0.2734375,0.1826171875,-0.091796875,0.220703125,0.37890625,-0.0888671875,-0.12890625,-0.021484375,-0.0091552734375,0.23046875]}],"usage":{"prompt_tokens":8,"total_tokens":8,"completion_tokens":0,"prompt_tokens_details":null}}
-```
-
-An OpenAI client example can be found here: [examples/pooling/embed/openai_embedding_matryoshka_fy_client.py](../../examples/pooling/embed/openai_embedding_matryoshka_fy_client.py)
-
-## Specific models
-
-### ColBERT Late Interaction Models
-
-[ColBERT](https://arxiv.org/abs/2004.12832) (Contextualized Late Interaction over BERT) is a retrieval model that uses per-token embeddings and MaxSim scoring for document ranking. Unlike single-vector embedding models, ColBERT retains token-level representations and computes relevance scores through late interaction, providing better accuracy while being more efficient than cross-encoders.
-
-vLLM supports ColBERT models with multiple encoder backbones:
-
-| Architecture | Backbone | Example HF Models |
-| - | - | - |
-| `HF_ColBERT` | BERT | `answerdotai/answerai-colbert-small-v1`, `colbert-ir/colbertv2.0` |
-| `ColBERTModernBertModel` | ModernBERT | `lightonai/GTE-ModernColBERT-v1` |
-| `ColBERTJinaRobertaModel` | Jina XLM-RoBERTa | `jinaai/jina-colbert-v2` |
-
-**BERT-based ColBERT** models work out of the box:
-
-```shell
-vllm serve answerdotai/answerai-colbert-small-v1
-```
-
-For **non-BERT backbones**, use `--hf-overrides` to set the correct architecture:
-
-```shell
-# ModernBERT backbone
-vllm serve lightonai/GTE-ModernColBERT-v1 \
-    --hf-overrides '{"architectures": ["ColBERTModernBertModel"]}'
-
-# Jina XLM-RoBERTa backbone
-vllm serve jinaai/jina-colbert-v2 \
-    --hf-overrides '{"architectures": ["ColBERTJinaRobertaModel"]}' \
-    --trust-remote-code
-```
-
-Then you can use the rerank endpoint:
-
-```shell
-curl -s http://localhost:8000/rerank -H "Content-Type: application/json" -d '{
-    "model": "answerdotai/answerai-colbert-small-v1",
-    "query": "What is machine learning?",
-    "documents": [
-        "Machine learning is a subset of artificial intelligence.",
-        "Python is a programming language.",
-        "Deep learning uses neural networks."
-    ]
-}'
-```
-
-Or the score endpoint:
-
-```shell
-curl -s http://localhost:8000/score -H "Content-Type: application/json" -d '{
-    "model": "answerdotai/answerai-colbert-small-v1",
-    "text_1": "What is machine learning?",
-    "text_2": ["Machine learning is a subset of AI.", "The weather is sunny."]
-}'
-```
-
-You can also get the raw token embeddings using the pooling endpoint with `token_embed` task:
-
-```shell
-curl -s http://localhost:8000/pooling -H "Content-Type: application/json" -d '{
-    "model": "answerdotai/answerai-colbert-small-v1",
-    "input": "What is machine learning?",
-    "task": "token_embed"
-}'
-```
-
-An example can be found here: [examples/pooling/score/colbert_rerank_online.py](../../examples/pooling/score/colbert_rerank_online.py)
-
-### ColQwen3 Multi-Modal Late Interaction Models
-
-ColQwen3 is based on [ColPali](https://arxiv.org/abs/2407.01449), which extends ColBERT's late interaction approach to **multi-modal** inputs. While ColBERT operates on text-only token embeddings, ColPali/ColQwen3 can embed both **text and images** (e.g. PDF pages, screenshots, diagrams) into per-token L2-normalized vectors and compute relevance via MaxSim scoring. ColQwen3 specifically uses Qwen3-VL as its vision-language backbone.
-
-| Architecture | Backbone | Example HF Models |
-| - | - | - |
-| `ColQwen3` | Qwen3-VL | `TomoroAI/tomoro-colqwen3-embed-4b`, `TomoroAI/tomoro-colqwen3-embed-8b` |
-| `OpsColQwen3Model` | Qwen3-VL | `OpenSearch-AI/Ops-Colqwen3-4B`, `OpenSearch-AI/Ops-Colqwen3-8B` |
-| `Qwen3VLNemotronEmbedModel` | Qwen3-VL | `nvidia/nemotron-colembed-vl-4b-v2`, `nvidia/nemotron-colembed-vl-8b-v2` |
-
-Start the server:
-
-```shell
-vllm serve TomoroAI/tomoro-colqwen3-embed-4b --max-model-len 4096
-```
-
-#### Text-only scoring and reranking
-
-Use the `/rerank` endpoint:
-
-```shell
-curl -s http://localhost:8000/rerank -H "Content-Type: application/json" -d '{
-    "model": "TomoroAI/tomoro-colqwen3-embed-4b",
-    "query": "What is machine learning?",
-    "documents": [
-        "Machine learning is a subset of artificial intelligence.",
-        "Python is a programming language.",
-        "Deep learning uses neural networks."
-    ]
-}'
-```
-
-Or the `/score` endpoint:
-
-```shell
-curl -s http://localhost:8000/score -H "Content-Type: application/json" -d '{
-    "model": "TomoroAI/tomoro-colqwen3-embed-4b",
-    "text_1": "What is the capital of France?",
-    "text_2": ["The capital of France is Paris.", "Python is a programming language."]
-}'
-```
-
-#### Multi-modal scoring and reranking (text query × image documents)
-
-The `/score` and `/rerank` endpoints also accept multi-modal inputs directly.
-Pass image documents using the `data_1`/`data_2` (for `/score`) or `documents` (for `/rerank`) fields
-with a `content` list containing `image_url` and `text` parts — the same format used by the
-OpenAI chat completion API:
-
-Score a text query against image documents:
-
-```shell
-curl -s http://localhost:8000/score -H "Content-Type: application/json" -d '{
-    "model": "TomoroAI/tomoro-colqwen3-embed-4b",
-    "data_1": "Retrieve the city of Beijing",
-    "data_2": [
-        {
-            "content": [
-                {"type": "image_url", "image_url": {"url": "data:image/png;base64,<BASE64>"}},
-                {"type": "text", "text": "Describe the image."}
-            ]
-        }
-    ]
-}'
-```
-
-Rerank image documents by a text query:
-
-```shell
-curl -s http://localhost:8000/rerank -H "Content-Type: application/json" -d '{
-    "model": "TomoroAI/tomoro-colqwen3-embed-4b",
-    "query": "Retrieve the city of Beijing",
-    "documents": [
-        {
-            "content": [
-                {"type": "image_url", "image_url": {"url": "data:image/png;base64,<BASE64_1>"}},
-                {"type": "text", "text": "Describe the image."}
-            ]
-        },
-        {
-            "content": [
-                {"type": "image_url", "image_url": {"url": "data:image/png;base64,<BASE64_2>"}},
-                {"type": "text", "text": "Describe the image."}
-            ]
-        }
-    ],
-    "top_n": 2
-}'
-```
-
-#### Raw token embeddings
-
-You can also get the raw token embeddings using the `/pooling` endpoint with `token_embed` task:
-
-```shell
-curl -s http://localhost:8000/pooling -H "Content-Type: application/json" -d '{
-    "model": "TomoroAI/tomoro-colqwen3-embed-4b",
-    "input": "What is machine learning?",
-    "task": "token_embed"
-}'
-```
-
-For **image inputs** via the pooling endpoint, use the chat-style `messages` field:
-
-```shell
-curl -s http://localhost:8000/pooling -H "Content-Type: application/json" -d '{
-    "model": "TomoroAI/tomoro-colqwen3-embed-4b",
-    "messages": [
-        {
-            "role": "user",
-            "content": [
-                {"type": "image_url", "image_url": {"url": "data:image/png;base64,<BASE64>"}},
-                {"type": "text", "text": "Describe the image."}
-            ]
-        }
-    ]
-}'
-```
-
-#### Examples
-
- Multi-vector retrieval: [examples/pooling/token_embed/colqwen3_token_embed_online.py](../../examples/pooling/token_embed/colqwen3_token_embed_online.py)
- Reranking (text + multi-modal): [examples/pooling/score/colqwen3_rerank_online.py](../../examples/pooling/score/colqwen3_rerank_online.py)
-
-### Llama Nemotron Multimodal
-
-#### Embedding Model
-
-Llama Nemotron VL Embedding models combine the bidirectional Llama embedding backbone
-(from `nvidia/llama-nemotron-embed-1b-v2`) with SigLIP as the vision encoder to produce
-single-vector embeddings from text and/or images.
-
-| Architecture | Backbone | Example HF Models |
-| - | - | - |
-| `LlamaNemotronVLModel` | Bidirectional Llama + SigLIP | `nvidia/llama-nemotron-embed-vl-1b-v2` |
-
-Start the server:
-
-```shell
-vllm serve nvidia/llama-nemotron-embed-vl-1b-v2 \
-    --trust-remote-code \
-    --chat-template examples/pooling/embed/template/nemotron_embed_vl.jinja
-```
-
-!!! note
-    The chat template bundled with this model's tokenizer is not suitable for
-    the embeddings API. Use the provided override template above when serving
-    with the `messages`-based (chat-style) embeddings endpoint.
-
-    The override template uses the message `role` to automatically prepend the
-    appropriate prefix: set `role` to `"query"` for queries (prepends `query: `)
-    or `"document"` for passages (prepends `passage: `). Any other role omits
-    the prefix.
-
-Embed text queries:
-
-```shell
-curl -s http://localhost:8000/v1/embeddings -H "Content-Type: application/json" -d '{
-    "model": "nvidia/llama-nemotron-embed-vl-1b-v2",
-    "messages": [
-        {
-            "role": "query",
-            "content": [
-                {"type": "text", "text": "What is machine learning?"}
-            ]
-        }
-    ]
-}'
-```
-
-Embed images via the chat-style `messages` field:
-
-```shell
-curl -s http://localhost:8000/v1/embeddings -H "Content-Type: application/json" -d '{
-    "model": "nvidia/llama-nemotron-embed-vl-1b-v2",
-    "messages": [
-        {
-            "role": "document",
-            "content": [
-                {"type": "image_url", "image_url": {"url": "data:image/png;base64,<BASE64>"}},
-                {"type": "text", "text": "Describe the image."}
-            ]
-        }
-    ]
-}'
-```
-
-#### Reranker Model
-
-Llama Nemotron VL reranker models combine the same bidirectional Llama + SigLIP
-backbone with a sequence-classification head for cross-encoder scoring and reranking.
-
-| Architecture | Backbone | Example HF Models |
-| - | - | - |
-| `LlamaNemotronVLForSequenceClassification` | Bidirectional Llama + SigLIP | `nvidia/llama-nemotron-rerank-vl-1b-v2` |
-
-Start the server:
-
-```shell
-vllm serve nvidia/llama-nemotron-rerank-vl-1b-v2 \
-    --runner pooling \
-    --trust-remote-code \
-    --chat-template examples/pooling/score/template/nemotron-vl-rerank.jinja
-```
-
-!!! note
-    The chat template bundled with this checkpoint's tokenizer is not suitable
-    for the Score/Rerank APIs. Use the provided override template when serving:
-    `examples/pooling/score/template/nemotron-vl-rerank.jinja`.
-
-Score a text query against an image document:
-
-```shell
-curl -s http://localhost:8000/score -H "Content-Type: application/json" -d '{
-    "model": "nvidia/llama-nemotron-rerank-vl-1b-v2",
-    "data_1": "Find diagrams about autonomous robots",
-    "data_2": [
-        {
-            "content": [
-                {"type": "image_url", "image_url": {"url": "data:image/png;base64,<BASE64>"}},
-                {"type": "text", "text": "Robotics workflow diagram."}
-            ]
-        }
-    ]
-}'
-```
-
-Rerank image documents by a text query:
-
-```shell
-curl -s http://localhost:8000/rerank -H "Content-Type: application/json" -d '{
-    "model": "nvidia/llama-nemotron-rerank-vl-1b-v2",
-    "query": "Find diagrams about autonomous robots",
-    "documents": [
-        {
-            "content": [
-                {"type": "image_url", "image_url": {"url": "data:image/png;base64,<BASE64_1>"}},
-                {"type": "text", "text": "Robotics workflow diagram."}
-            ]
-        },
-        {
-            "content": [
-                {"type": "image_url", "image_url": {"url": "data:image/png;base64,<BASE64_2>"}},
-                {"type": "text", "text": "General skyline photo."}
-            ]
-        }
-    ],
-    "top_n": 2
-}'
-```
-
-### ColQwen3.5 Multi-Modal Late Interaction Models
-
-ColQwen3.5 is based on [ColPali](https://arxiv.org/abs/2407.01449), extending ColBERT's late interaction approach to **multi-modal** inputs. It uses the Qwen3.5 hybrid backbone (linear + full attention) and produces per-token L2-normalized vectors for MaxSim scoring.
-
-| Architecture | Backbone | Example HF Models |
-| - | - | - |
-| `ColQwen3_5` | Qwen3.5 | `athrael-soju/colqwen3.5-4.5B` |
-
-Start the server:
-
-```shell
-vllm serve athrael-soju/colqwen3.5-4.5B --max-model-len 4096
-```
-
-Then you can use the rerank endpoint:
-
-```shell
-curl -s http://localhost:8000/rerank -H "Content-Type: application/json" -d '{
-    "model": "athrael-soju/colqwen3.5-4.5B",
-    "query": "What is machine learning?",
-    "documents": [
-        "Machine learning is a subset of artificial intelligence.",
-        "Python is a programming language.",
-        "Deep learning uses neural networks."
-    ]
-}'
-```
-
-Or the score endpoint:
-
-```shell
-curl -s http://localhost:8000/score -H "Content-Type: application/json" -d '{
-    "model": "athrael-soju/colqwen3.5-4.5B",
-    "text_1": "What is the capital of France?",
-    "text_2": ["The capital of France is Paris.", "Python is a programming language."]
-}'
-```
-
-An example can be found here: [examples/pooling/score/colqwen3_5_rerank_online.py](../../examples/pooling/score/colqwen3_5_rerank_online.py)
-
-### BAAI/bge-m3
-
-The `BAAI/bge-m3` model comes with extra weights for sparse and colbert embeddings but unfortunately in its `config.json`
-the architecture is declared as `XLMRobertaModel`, which makes `vLLM` load it as a vanilla ROBERTA model without the
-extra weights. To load the full model weights, override its architecture like this:
-
-```shell
-vllm serve BAAI/bge-m3 --hf-overrides '{"architectures": ["BgeM3EmbeddingModel"]}'
-```
-
-Then you obtain the sparse embeddings like this:
-
-```shell
-curl -s http://localhost:8000/pooling -H "Content-Type: application/json" -d '{
-     "model": "BAAI/bge-m3",
-     "task": "token_classify",
-     "input": ["What is BGE M3?", "Definition of BM25"]
-}'
-```
-
-Due to limitations in the output schema, the output consists of a list of
-token scores for each token for each input. This means that you'll have to call
-`/tokenize` as well to be able to pair tokens with scores.
-Refer to the tests in  `tests/models/language/pooling/test_bge_m3.py` to see how
-to do that.
-
-You can obtain the colbert embeddings like this:
-
-```shell
-curl -s http://localhost:8000/pooling -H "Content-Type: application/json" -d '{
-     "model": "BAAI/bge-m3",
-     "task": "token_embed",
-     "input": ["What is BGE M3?", "Definition of BM25"]
-}'
-```
-
-## Deprecated Features
-
-### Encode task
-
-We have split the `encode` task into two more specific token-wise tasks: `token_embed` and `token_classify`:
-
- `token_embed` is the same as `embed`, using normalization as the activation.
- `token_classify` is the same as `classify`, by default using softmax as the activation.
-
-Pooling models now default support all pooling, you can use it without any settings.
-
- Extracting hidden states prefers using `token_embed` task.
- Reward models prefers using `token_classify` task.
@@ -0,0 +1,260 @@
+# Pooling Models
+
+!!! note
+    We currently support pooling models primarily for convenience. This is not guaranteed to provide any performance improvements over using Hugging Face Transformers or Sentence Transformers directly.
+
+    We plan to optimize pooling models in vLLM. Please comment on <https://github.com/vllm-project/vllm/issues/21796> if you have any suggestions!
+
+## What are pooling models?
+
+Natural Language Processing (NLP) can be primarily divided into the following two types of tasks:
+
+- Natural Language Understanding (NLU)
+- Natural Language Generation (NLG)
+
+The generative models supported by vLLM cover a variety of task types, such as the large language models (LLMs) we are familiar with, multimodal models (VLM) that handle multimodal inputs like images, videos, and audio, speech-to-text transcription models, and real-time models that support streaming input. Their common feature is the ability to generate text. Taking it a step further, vLLM-Omni supports the generation of multimodal content, including images, videos, and audio.
+
+As the capabilities of generative models continue to improve, the boundaries of these models are also constantly expanding. However, certain application scenarios still require specialized small language models to efficiently complete specific tasks. These models typically have the following characteristics:
+
+- They do not require content generation.
+- They only need to perform very limited functions, without requiring strong generalization, creativity, or high intelligence.
+- They demand extremely low latency and may operate on cost-constrained hardware.
+- Text-only models typically have fewer than 1 billion parameters, while multimodal models generally have fewer than 10 billion parameters.
+
+Although these models are relatively small in scale, they are still based on the Transformer architecture, similar or even identical to the most advanced large language models today. Many recently released pooling models are also fine-tuned from large language models, allowing them to benefit from the continuous improvements in large models. This architecture similarity enables them to reuse much of vLLM’s infrastructure. If compatible, we would be happy to help them leverage the latest features of vLLM as well.
+
+### Sequence-wise Task and Token-wise Task
+
+The key distinction between sequence-wise task and token-wise task lies in their output granularity: sequence-wise task produces a single result for an entire input sequence, whereas token-wise task yields a result for each individual token within the sequence.
+
+Of course, we also have "plugin" tasks that allow users to customize input and output processors. For more information, please refer to [IO Processor Plugins](../../design/io_processor_plugins.md).
+
+### Pooling Tasks
+
+| Pooling Tasks         | Granularity   | Outputs                                         |
+|-----------------------|---------------|-------------------------------------------------|
+| `classify` (see note) | Sequence-wise | probability vector of classes for each sequence |
+| `embed`               | Sequence-wise | vector representations for each sequence        |
+| `token_classify`      | Token-wise    | probability vector of classes for each token    |
+| `token_embed`         | Token-wise    | vector representations for each token           |
+
+!!! note
+    Within classification tasks, there is a specialized subcategory: Cross-encoder (aka reranker) models. These models are a subset of classification models that accept two prompts as input and output num_labels equal to 1.
+
+### Score Types
+
+The scoring models is designed to compute similarity scores between two input prompts. It supports three model types (aka `score_type`): `cross-encoder`, `late-interaction`, and `bi-encoder`.
+
+| Pooling Tasks         | Granularity   | Outputs                                      | Score Types        | scoring function         |
+|-----------------------|---------------|----------------------------------------------|--------------------|--------------------------|
+| `classify` (see note) | Sequence-wise | reranker score for each sequence             | `cross-encoder`    | linear classifier        |
+| `embed`               | Sequence-wise | vector representations for each sequence     | `bi-encoder`       | cosine similarity        |
+| `token_classify`      | Token-wise    | probability vector of classes for each token | nan                | nan                      |
+| `token_embed`         | Token-wise    | vector representations for each token        | `late-interaction` | late interaction(MaxSim) |
+
+!!! note
+    Only when a classification model outputs num_labels equal to 1 can it be used as a scoring model and have its scoring API enabled.
+
+### Pooling Usages
+
+| Pooling Usages              | Description                                                                                                                                             |
+|-----------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------|
+| Classification Usages       | Predicting which predefined category, class, or label best corresponds to a given input.                                                                |
+| Embedding Usages            | Converts unstructured data (text, images, audio, etc.) into structured numerical vectors (embeddings).                                                  |
+| Token Classification Usages | Token-wise classification                                                                                                                               |
+| Token Embedding Usages      | Token-wise embedding                                                                                                                                    |
+| Scoring Usages              | Computes similarity scores between two inputs. It supports three model types (aka `score_type`): `cross-encoder`, `late-interaction`, and `bi-encoder`. |
+| Reward Usages               | Evaluates the quality of outputs generated by a language model, acting as a proxy for human preferences.                                                |
+
+We also have some special models that support multiple pooling tasks, or have specific usage scenarios, or support special inputs and outputs.
+
+For more detailed information, please refer to the link below.
+
+- [Classification Usages](classify.md)
+- [Embedding Usages](embed.md)
+- [Reward Usages](reward.md)
+- [Token Classification Usages](token_classify.md)
+- [Token Embedding Usages](token_embed.md)
+- [Scoring Usages](scoring.md)
+- [Specific Model Examples](specific_models.md)
+
+## Offline Inference
+
+Each pooling model in vLLM supports one or more of these tasks according to
+[Pooler.get_supported_tasks][vllm.model_executor.layers.pooler.Pooler.get_supported_tasks],
+enabling the corresponding APIs.
+
+### Offline APIs corresponding to pooling tasks
+
+| Task             | APIs                                                                                  |
+|------------------|---------------------------------------------------------------------------------------|
+| `embed`          | `LLM.embed(...)`, `LLM.encode(..., pooling_task="embed")`, `LLM.score(...)`(see note) |
+| `classify`       | `LLM.classify(...)`, `LLM.encode(..., pooling_task="classify")`, `LLM.score(...)`     |
+| `token_classify` | `LLM.reward(...)`, `LLM.encode(..., pooling_task="token_classify")`                   |
+| `token_embed`    | `LLM.encode(..., pooling_task="token_embed")`, `LLM.score(...)`                       |
+| `plugin`         | `LLM.encode(..., pooling_task="plugin")`                                              |
+
+!!! note
+    Only when a classification model outputs num_labels equal to 1 can it be used as a scoring model and have its scoring API enabled.
+
+### `LLM.classify`
+
+The [classify][vllm.LLM.classify] method outputs a probability vector for each prompt.
+It is primarily designed for [classification models](classify.md).
+For more information about `LLM.embed`, see [this page](classify.md#offline-inference).
+
+### `LLM.embed`
+
+The [embed][vllm.LLM.embed] method outputs an embedding vector for each prompt.
+It is primarily designed for [embedding models](embed.md).
+For more information about `LLM.embed`, see [this page](embed.md#offline-inference).
+
+### `LLM.score`
+
+The [score][vllm.LLM.score] method outputs similarity scores between sentence pairs.
+It is primarily designed for [score models](scoring.md).
+
+### `LLM.encode`
+
+The [encode][vllm.LLM.encode] method is available to all pooling models in vLLM.
+
+Please use one of the more specific methods or set the task directly when using `LLM.encode`, refer to the [table above](#offline-apis-corresponding-to-pooling-tasks).
+
+### Examples
+
+```python
+from vllm import LLM
+
+llm = LLM(model="intfloat/e5-small", runner="pooling")
+(output,) = llm.encode("Hello, my name is", pooling_task="embed")
+
+data = output.outputs.data
+print(f"Data: {data!r}")
+```
+
+## Online Serving
+
+Our online Server provides endpoints that correspond to the offline APIs:
+
+- Corresponding to `LLM.embed`:
+    - [Cohere Embed API](embed.md#cohere-embed-api) (`/v2/embed`)
+    - [Openai-compatible Embeddings API](embed.md#openai-compatible-embeddings-api) (`/v1/embeddings`)
+- Corresponding to `LLM.classify`:
+    - [Classification API](classify.md#online-serving)(`/classify`)
+- Corresponding to `LLM.score`:
+    - [Score API](scoring.md#score-api)(`/score`)
+    - [Rerank API](scoring.md#rerank-api) (`/rerank`, `/v1/rerank`, `/v2/rerank`)
+- Pooling API (`/pooling`) is similar to `LLM.encode`, being applicable to all types of pooling models.
+
+The following introduces the Pooling API. For other APIs, please refer to the link above.
+
+### Pooling API
+
+Our Pooling API (`/pooling`) is similar to `LLM.encode`, being applicable to all types of pooling models.
+
+The input format is the same as [Embeddings API](embed.md#openai-compatible-embeddings-api), but the output data can contain an arbitrary nested list, not just a 1-D list of floats.
+
+Please use one of the more specific APIs or set the task directly when using the Pooling API, refer to the [table above](#offline-apis-corresponding-to-pooling-tasks).
+
+Code example: [examples/pooling/pooling/pooling_online.py](../../../examples/pooling/pooling/pooling_online.py)
+
+### Examples
+
+```python
+# start a supported embeddings model server with `vllm serve`, e.g.
+# vllm serve intfloat/e5-small
+import requests
+
+host = "localhost"
+port = "8000"
+model_name = "intfloat/e5-small"
+
+api_url = f"http://{host}:{port}/pooling"
+
+prompts = [
+    "Hello, my name is",
+    "The president of the United States is",
+    "The capital of France is",
+    "The future of AI is",
+]
+prompt = {"model": model_name, "input": prompts, "task": "embed"}
+
+response = requests.post(api_url, json=prompt)
+
+for output in response.json()["data"]:
+    data = output["data"]
+    print(f"Data: {data!r} (size={len(data)})")
+```
+
+## Configuration
+
+In vLLM, pooling models implement the [VllmModelForPooling][vllm.model_executor.models.VllmModelForPooling] interface.
+These models use a [Pooler][vllm.model_executor.layers.pooler.Pooler] to extract the final hidden states of the input
+before returning them.
+
+### Model Runner
+
+Run a model in pooling mode via the option `--runner pooling`.
+
+!!! tip
+    There is no need to set this option in the vast majority of cases as vLLM can automatically
+    detect the appropriate model runner via `--runner auto`.
+
+### Model Conversion
+
+vLLM can adapt models for various pooling tasks via the option `--convert <type>`.
+
+If `--runner pooling` has been set (manually or automatically) but the model does not implement the
+[VllmModelForPooling][vllm.model_executor.models.VllmModelForPooling] interface,
+vLLM will attempt to automatically convert the model according to the architecture names
+shown in the table below.
+
+| Architecture                                    | `--convert` | Supported pooling tasks      |
+|-------------------------------------------------|-------------|------------------------------|
+| `*ForTextEncoding`, `*EmbeddingModel`, `*Model` | `embed`     | `token_embed`, `embed`       |
+| `*ForRewardModeling`, `*RewardModel`            | `embed`     | `token_embed`, `embed`       |
+| `*For*Classification`, `*ClassificationModel`   | `classify`  | `token_classify`, `classify` |
+
+!!! tip
+    You can explicitly set `--convert <type>` to specify how to convert the model.
+
+### Pooler Configuration
+
+#### Predefined models
+
+If the [Pooler][vllm.model_executor.layers.pooler.Pooler] defined by the model accepts `pooler_config`,
+you can override some of its attributes via the `--pooler-config` option.
+
+#### Converted models
+
+If the model has been converted via `--convert` (see above),
+the pooler assigned to each task has the following attributes by default:
+
+| Task       | Pooling Type | Normalization | Softmax |
+| ---------- | ------------ | ------------- | ------- |
+| `embed`    | `LAST`       | ✅︎            | ❌      |
+| `classify` | `LAST`       | ❌            | ✅︎      |
+
+When loading [Sentence Transformers](https://huggingface.co/sentence-transformers) models,
+its Sentence Transformers configuration file (`modules.json`) takes priority over the model's defaults.
+
+You can further customize this via the `--pooler-config` option,
+which takes priority over both the model's and Sentence Transformers' defaults.
+
+## Removed Features
+
+### Encode task
+
+We have split the `encode` task into two more specific token-wise tasks: `token_embed` and `token_classify`:
+
+- `token_embed` is the same as `embed`, using normalization as the activation.
+- `token_classify` is the same as `classify`, by default using softmax as the activation.
+
+Pooling models now default support all pooling, you can use it without any settings.
+
+- Extracting hidden states prefers using `token_embed` task.
+- Named Entity Recognition (NER) and reward models prefers using `token_classify` task.
+
+### Score task
+
+`score` task is deprecated and will be removed in v0.20. Please use `classify` instead. Only when a classification model outputs num_labels equal to 1 can it be used as a scoring model and have its scoring API enabled.
@@ -0,0 +1,278 @@
+# Classification Usages
+
+Classification involves predicting which predefined category, class, or label best corresponds to a given input.
+
+## Summary
+
+- Model Usage: (sequence) classification
+- Pooling Task: `classify`
+- Offline APIs:
+    - `LLM.classify(...)`
+    - `LLM.encode(..., pooling_task="classify")`
+- Online APIs:
+    - [Classification API](classify.md#online-serving) (`/classify`)
+    - Pooling API (`/pooling`)
+
+The key distinction between (sequence) classification and token classification lies in their output granularity: (sequence) classification produces a single result for an entire input sequence, whereas token classification yields a result for each individual token within the sequence.
+
+Many classification models support both (sequence) classification and token classification. For further details on token classification, please refer to [this page](token_classify.md).
+
+Only when a classification model outputs num_labels equal to 1 can it be used as a scoring model and have its scoring API enabled, please refer to [this page](scoring.md).
+
+## Typical Use Cases
+
+### Classification
+
+The most fundamental application of classification models is to categorize input data into predefined classes.
+
+## Supported Models
+
+### Text-only Models
+
+| Architecture | Models | Example HF Models | [LoRA](../../features/lora.md) | [PP](../../serving/parallelism_scaling.md) |
+| ------------ | ------ | ----------------- | ------------------------------ | ------------------------------------------ |
+| `ErnieForSequenceClassification` | BERT-like Chinese ERNIE | `Forrest20231206/ernie-3.0-base-zh-cls` | | |
+| `GPT2ForSequenceClassification` | GPT2 | `nie3e/sentiment-polish-gpt2-small` | | |
+| `Qwen2ForSequenceClassification`<sup>C</sup> | Qwen2-based | `jason9693/Qwen2.5-1.5B-apeach` | | |
+| `*Model`<sup>C</sup>, `*ForCausalLM`<sup>C</sup>, etc. | Generative models | N/A | \* | \* |
+
+### Multimodal Models
+
+!!! note
+    For more information about multimodal models inputs, see [this page](../supported_models.md#list-of-multimodal-language-models).
+
+| Architecture | Models | Inputs | Example HF Models | [LoRA](../../features/lora.md) | [PP](../../serving/parallelism_scaling.md) |
+| ------------ | ------ | ------ | ----------------- | ------------------------------ | ------------------------------------------ |
+| `Qwen2_5_VLForSequenceClassification`<sup>C</sup> | Qwen2_5_VL-based | T + I<sup>E+</sup> + V<sup>E+</sup> | `muziyongshixin/Qwen2.5-VL-7B-for-VideoCls` | | |
+| `*ForConditionalGeneration`<sup>C</sup>, `*ForCausalLM`<sup>C</sup>, etc. | Generative models | \* | N/A | \* | \* |
+
+<sup>C</sup> Automatically converted into a classification model via `--convert classify`. ([details](./README.md#model-conversion))  
+\* Feature support is the same as that of the original model.
+
+If your model is not in the above list, we will try to automatically convert the model using
+[as_seq_cls_model][vllm.model_executor.models.adapters.as_seq_cls_model]. By default, the class probabilities are extracted from the softmaxed hidden state corresponding to the last token.
+
+### Cross-encoder Models
+
+Cross-encoder (aka reranker) models are a subset of classification models that accept two prompts as input and output num_labels equal to 1. Most classification models can also be used as [cross-encoder models](scoring.md#cross-encoder-models). For more information on cross-encoder models, please refer to [this page](scoring.md).
+
+--8<-- "docs/models/pooling_models/scoring.md:supported-cross-encoder-models"
+
+### Reward Models
+
+Using (sequence) classification models as reward models. For more information, see [Reward Models](reward.md).
+
+--8<-- "docs/models/pooling_models/reward.md:supported-sequence-reward-models"
+
+## Offline Inference
+
+### Pooling Parameters
+
+The following [pooling parameters][vllm.PoolingParams] are supported.
+
+```python
+--8<-- "vllm/pooling_params.py:common-pooling-params"
+--8<-- "vllm/pooling_params.py:classify-pooling-params"
+```
+
+### `LLM.classify`
+
+The [classify][vllm.LLM.classify] method outputs a probability vector for each prompt.
+
+```python
+from vllm import LLM
+
+llm = LLM(model="jason9693/Qwen2.5-1.5B-apeach", runner="pooling")
+(output,) = llm.classify("Hello, my name is")
+
+probs = output.outputs.probs
+print(f"Class Probabilities: {probs!r} (size={len(probs)})")
+```
+
+A code example can be found here: [examples/offline_inference/basic/classify.py](../../../examples/basic/offline_inference/classify.py)
+
+### `LLM.encode`
+
+The [encode][vllm.LLM.encode] method is available to all pooling models in vLLM.
+
+Set `pooling_task="classify"` when using `LLM.encode` for classification Models:
+
+```python
+from vllm import LLM
+
+llm = LLM(model="jason9693/Qwen2.5-1.5B-apeach", runner="pooling")
+(output,) = llm.encode("Hello, my name is", pooling_task="classify")
+
+data = output.outputs.data
+print(f"Data: {data!r}")
+```
+
+## Online Serving
+
+### Classification API
+
+Online `/classify` API is similar to `LLM.classify`.
+
+#### Completion Parameters
+
+The following Classification API parameters are supported:
+
+??? code
+
+    ```python
+    --8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-params"
+    --8<-- "vllm/entrypoints/pooling/base/protocol.py:completion-params"
+    --8<-- "vllm/entrypoints/pooling/base/protocol.py:classify-params"
+    ```
+
+The following extra parameters are supported:
+
+??? code
+
+    ```python
+    --8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-extra-params"
+    --8<-- "vllm/entrypoints/pooling/base/protocol.py:completion-extra-params"
+    --8<-- "vllm/entrypoints/pooling/base/protocol.py:classify-extra-params"
+    ```
+
+#### Chat Parameters
+
+For chat-like input (i.e. if `messages` is passed), the following parameters are supported:
+
+??? code
+
+    ```python
+    --8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-params"
+    --8<-- "vllm/entrypoints/pooling/base/protocol.py:chat-params"
+    --8<-- "vllm/entrypoints/pooling/base/protocol.py:classify-params"
+    ```
+
+these extra parameters are supported instead:
+
+??? code
+
+    ```python
+    --8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-extra-params"
+    --8<-- "vllm/entrypoints/pooling/base/protocol.py:chat-extra-params"
+    --8<-- "vllm/entrypoints/pooling/base/protocol.py:classify-extra-params"
+    ```
+
+#### Example Requests
+
+Code example: [examples/pooling/classify/classification_online.py](../../../examples/pooling/classify/classification_online.py)
+
+You can classify multiple texts by passing an array of strings:
+
+```bash
+curl -v "http://127.0.0.1:8000/classify" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "jason9693/Qwen2.5-1.5B-apeach",
+    "input": [
+      "Loved the new café—coffee was great.",
+      "This update broke everything. Frustrating."
+    ]
+  }'
+```
+
+??? console "Response"
+
+    ```json
+    {
+      "id": "classify-7c87cac407b749a6935d8c7ce2a8fba2",
+      "object": "list",
+      "created": 1745383065,
+      "model": "jason9693/Qwen2.5-1.5B-apeach",
+      "data": [
+        {
+          "index": 0,
+          "label": "Default",
+          "probs": [
+            0.565970778465271,
+            0.4340292513370514
+          ],
+          "num_classes": 2
+        },
+        {
+          "index": 1,
+          "label": "Spoiled",
+          "probs": [
+            0.26448777318000793,
+            0.7355121970176697
+          ],
+          "num_classes": 2
+        }
+      ],
+      "usage": {
+        "prompt_tokens": 20,
+        "total_tokens": 20,
+        "completion_tokens": 0,
+        "prompt_tokens_details": null
+      }
+    }
+    ```
+
+You can also pass a string directly to the `input` field:
+
+```bash
+curl -v "http://127.0.0.1:8000/classify" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "jason9693/Qwen2.5-1.5B-apeach",
+    "input": "Loved the new café—coffee was great."
+  }'
+```
+
+??? console "Response"
+
+    ```json
+    {
+      "id": "classify-9bf17f2847b046c7b2d5495f4b4f9682",
+      "object": "list",
+      "created": 1745383213,
+      "model": "jason9693/Qwen2.5-1.5B-apeach",
+      "data": [
+        {
+          "index": 0,
+          "label": "Default",
+          "probs": [
+            0.565970778465271,
+            0.4340292513370514
+          ],
+          "num_classes": 2
+        }
+      ],
+      "usage": {
+        "prompt_tokens": 10,
+        "total_tokens": 10,
+        "completion_tokens": 0,
+        "prompt_tokens_details": null
+      }
+    }
+    ```
+
+## More examples
+
+More examples can be found here: [examples/pooling/classify](../../../examples/pooling/classify)
+
+## Supported Features
+
+### Enable/disable activation
+
+You can enable or disable activation via `use_activation`.
+
+### Problem type (e.g. `multi_label_classification`)
+
+You can modify the `problem_type` via problem_type in the Hugging Face config. The supported problem types are: `single_label_classification`, `multi_label_classification`, and `regression`.
+
+Implement alignment with transformers [ForSequenceClassificationLoss](https://github.com/huggingface/transformers/blob/57bb6db6ee4cfaccc45b8d474dfad5a17811ca60/src/transformers/loss/loss_utils.py#L92).
+
+### Logit bias
+
+You can modify the `logit_bias` (aka `sigmoid_normalize`) through the logit_bias parameter in `vllm.config.PoolerConfig`.
+
+## Removed Features
+
+### Remove softmax from PoolingParams
+
+We have already removed `softmax` and `activation` from PoolingParams. Instead, use `use_activation`, since we allow `classify` and `token_classify` to use any activation function.
@@ -0,0 +1,546 @@
+# Embedding Usages
+
+Embedding models are a class of machine learning models designed to transform unstructured data—such as text, images, or audio—into a structured numerical representation known as an embedding.
+
+## Summary
+
+- Model Usage: (sequence) embedding
+- Pooling Task: `embed`
+- Offline APIs:
+    - `LLM.embed(...)`
+    - `LLM.encode(..., pooling_task="embed")`
+    - `LLM.score(...)`
+- Online APIs:
+    - [Cohere Embed API](embed.md#cohere-embed-api) (`/v2/embed`)
+    - [Openai-compatible Embeddings API](embed.md#openai-compatible-embeddings-api) (`/v1/embeddings`)
+    - Pooling API (`/pooling`)
+
+The primary distinction between (sequence) embedding and token embedding lies in their output granularity: (sequence) embedding produces a single embedding vector for an entire input sequence, whereas token embedding generates an embedding for each individual token within the sequence.
+
+Many embedding models support both (sequence) embedding and token embedding. For further details on token embedding, please refer to [this page](token_embed.md).
+
+## Typical Use Cases
+
+### Embedding
+
+The most basic use case of embedding models is to embed the inputs, e.g. for RAG.
+
+### Pairwise Similarity
+
+You can compute pairwise similarity scores to build a similarity matrix using the [Score API](scoring.md).
+
+## Supported Models
+
+--8<-- [start:supported-embed-models]
+
+### Text-only Models
+
+| Architecture | Models | Example HF Models | [LoRA](../../features/lora.md) | [PP](../../serving/parallelism_scaling.md) |
+| ------------ | ------ | ----------------- | ------------------------------ | ------------------------------------------ |
+| `BertModel` | BERT-based | `BAAI/bge-base-en-v1.5`, `Snowflake/snowflake-arctic-embed-xs`, etc. | | |
+| `BertSpladeSparseEmbeddingModel` | SPLADE | `naver/splade-v3` | | |
+| `ErnieModel` | BERT-like Chinese ERNIE | `shibing624/text2vec-base-chinese-sentence` | | |
+| `Gemma2Model`<sup>C</sup> | Gemma 2-based | `BAAI/bge-multilingual-gemma2`, etc. | ✅︎ | ✅︎ |
+| `Gemma3TextModel`<sup>C</sup> | Gemma 3-based | `google/embeddinggemma-300m`, etc. | ✅︎ | ✅︎ |
+| `GritLM` | GritLM | `parasail-ai/GritLM-7B-vllm`. | ✅︎ | ✅︎ |
+| `GteModel` | Arctic-Embed-2.0-M | `Snowflake/snowflake-arctic-embed-m-v2.0`. | | |
+| `GteNewModel` | mGTE-TRM (see note) | `Alibaba-NLP/gte-multilingual-base`, etc. | | |
+| `LlamaBidirectionalModel`<sup>C</sup> | Llama-based with bidirectional attention | `nvidia/llama-nemotron-embed-1b-v2`, etc. | ✅︎ | ✅︎ |
+| `LlamaModel`<sup>C</sup>, `LlamaForCausalLM`<sup>C</sup>, `MistralModel`<sup>C</sup>, etc. | Llama-based | `intfloat/e5-mistral-7b-instruct`, etc. | ✅︎ | ✅︎ |
+| `ModernBertModel` | ModernBERT-based | `Alibaba-NLP/gte-modernbert-base`, etc. | | |
+| `NomicBertModel` | Nomic BERT | `nomic-ai/nomic-embed-text-v1`, `nomic-ai/nomic-embed-text-v2-moe`, `Snowflake/snowflake-arctic-embed-m-long`, etc. | | |
+| `Qwen2Model`<sup>C</sup>, `Qwen2ForCausalLM`<sup>C</sup> | Qwen2-based | `ssmits/Qwen2-7B-Instruct-embed-base` (see note), `Alibaba-NLP/gte-Qwen2-7B-instruct` (see note), etc. | ✅︎ | ✅︎ |
+| `Qwen3Model`<sup>C</sup>, `Qwen3ForCausalLM`<sup>C</sup> | Qwen3-based | `Qwen/Qwen3-Embedding-0.6B`, etc. | ✅︎ | ✅︎ |
+| `RobertaModel`, `RobertaForMaskedLM` | RoBERTa-based | `sentence-transformers/all-roberta-large-v1`, etc. | | |
+| `VoyageQwen3BidirectionalEmbedModel`<sup>C</sup> | Voyage Qwen3-based with bidirectional attention | `voyageai/voyage-4-nano`, etc. | ✅︎ | ✅︎ |
+| `XLMRobertaModel` | XLMRobertaModel-based | `BAAI/bge-m3` (see note), `intfloat/multilingual-e5-base`, `jinaai/jina-embeddings-v3` (see note), etc. | | |
+| `*Model`<sup>C</sup>, `*ForCausalLM`<sup>C</sup>, etc. | Generative models | N/A | \* | \* |
+
+!!! note
+    The second-generation GTE model (mGTE-TRM) is named `NewModel`. The name `NewModel` is too generic, you should set `--hf-overrides '{"architectures": ["GteNewModel"]}'` to specify the use of the `GteNewModel` architecture.
+
+!!! note
+    `ssmits/Qwen2-7B-Instruct-embed-base` has an improperly defined Sentence Transformers config.
+    You need to manually set mean pooling by passing `--pooler-config '{"pooling_type": "MEAN"}'`.
+
+!!! note
+    For `Alibaba-NLP/gte-Qwen2-*`, you need to enable `--trust-remote-code` for the correct tokenizer to be loaded.
+    See [relevant issue on HF Transformers](https://github.com/huggingface/transformers/issues/34882).
+
+!!! note
+    The `BAAI/bge-m3` model comes with extra weights for sparse and colbert embeddings, See [this page](specific_models.md#baaibge-m3) for more information.
+
+!!! note
+    `jinaai/jina-embeddings-v3` supports multiple tasks through LoRA, while vllm temporarily only supports text-matching tasks by merging LoRA weights.
+
+### Multimodal Models
+
+!!! note
+    For more information about multimodal models inputs, see [this page](../supported_models.md#list-of-multimodal-language-models).
+
+| Architecture | Models | Inputs | Example HF Models | [LoRA](../../features/lora.md) | [PP](../../serving/parallelism_scaling.md) |
+| ------------ | ------ | ------ | ----------------- | ------------------------------ | ------------------------------------------ |
+| `CLIPModel` | CLIP | T / I | `openai/clip-vit-base-patch32`, `openai/clip-vit-large-patch14`, etc. | | |
+| `LlamaNemotronVLModel` | Llama Nemotron Embedding + SigLIP | T + I | `nvidia/llama-nemotron-embed-vl-1b-v2` | | |
+| `LlavaNextForConditionalGeneration`<sup>C</sup> | LLaVA-NeXT-based | T / I | `royokong/e5-v` | | ✅︎ |
+| `Phi3VForCausalLM`<sup>C</sup> | Phi-3-Vision-based | T + I | `TIGER-Lab/VLM2Vec-Full` | | ✅︎ |
+| `Qwen3VLForConditionalGeneration`<sup>C</sup> | Qwen3-VL | T + I + V | `Qwen/Qwen3-VL-Embedding-2B`, etc. | ✅︎ | ✅︎ |
+| `SiglipModel` | SigLIP, SigLIP2 | T / I | `google/siglip-base-patch16-224`, `google/siglip2-base-patch16-224` | | |
+| `*ForConditionalGeneration`<sup>C</sup>, `*ForCausalLM`<sup>C</sup>, etc. | Generative models | \* | N/A | \* | \* |
+
+<sup>C</sup> Automatically converted into an embedding model via `--convert embed`. ([details](./README.md#model-conversion))  
+\* Feature support is the same as that of the original model.
+
+If your model is not in the above list, we will try to automatically convert the model using
+[as_embedding_model][vllm.model_executor.models.adapters.as_embedding_model]. By default, the embeddings
+of the whole prompt are extracted from the normalized hidden state corresponding to the last token.
+
+!!! note
+    Although vLLM supports automatically converting models of any architecture into embedding models via --convert embed, to get the best results, you should use pooling models that are specifically trained as such.
+
+--8<-- [end:supported-embed-models]
+
+## Offline Inference
+
+### Pooling Parameters
+
+The following [pooling parameters][vllm.PoolingParams] are supported.
+
+```python
+--8<-- "vllm/pooling_params.py:common-pooling-params"
+--8<-- "vllm/pooling_params.py:embed-pooling-params"
+```
+
+### `LLM.embed`
+
+The [embed][vllm.LLM.embed] method outputs an embedding vector for each prompt.
+
+```python
+from vllm import LLM
+
+llm = LLM(model="intfloat/e5-small", runner="pooling")
+(output,) = llm.embed("Hello, my name is")
+
+embeds = output.outputs.embedding
+print(f"Embeddings: {embeds!r} (size={len(embeds)})")
+```
+
+A code example can be found here: [examples/offline_inference/basic/embed.py](../../../examples/basic/offline_inference/embed.py)
+
+### `LLM.encode`
+
+The [encode][vllm.LLM.encode] method is available to all pooling models in vLLM.
+
+Set `pooling_task="embed"` when using `LLM.encode` for embedding Models:
+
+```python
+from vllm import LLM
+
+llm = LLM(model="intfloat/e5-small", runner="pooling")
+(output,) = llm.encode("Hello, my name is", pooling_task="embed")
+
+data = output.outputs.data
+print(f"Data: {data!r}")
+```
+
+### `LLM.score`
+
+The [score][vllm.LLM.score] method outputs similarity scores between sentence pairs.
+
+All models that support embedding task also support using the score API to compute similarity scores by calculating the cosine similarity of two input prompt's embeddings.
+
+```python
+from vllm import LLM
+
+llm = LLM(model="intfloat/e5-small", runner="pooling")
+(output,) = llm.score(
+    "What is the capital of France?",
+    "The capital of Brazil is Brasilia.",
+)
+
+score = output.outputs.score
+print(f"Score: {score}")
+```
+
+## Online Serving
+
+### OpenAI-Compatible Embeddings API
+
+Our Embeddings API is compatible with [OpenAI's Embeddings API](https://platform.openai.com/docs/api-reference/embeddings);
+you can use the [official OpenAI Python client](https://github.com/openai/openai-python) to interact with it.
+
+Code example: [examples/pooling/embed/openai_embedding_client.py](../../../examples/pooling/embed/openai_embedding_client.py)
+
+#### Completion Parameters
+
+The following Classification API parameters are supported:
+
+??? code
+
+    ```python
+    --8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-params"
+    --8<-- "vllm/entrypoints/pooling/base/protocol.py:completion-params"
+    --8<-- "vllm/entrypoints/pooling/base/protocol.py:encoding-params"
+    --8<-- "vllm/entrypoints/pooling/base/protocol.py:embed-params"
+    ```
+
+The following extra parameters are supported:
+
+??? code
+
+    ```python
+    --8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-extra-params"
+    --8<-- "vllm/entrypoints/pooling/base/protocol.py:completion-extra-params"
+    --8<-- "vllm/entrypoints/pooling/base/protocol.py:encoding-extra-params"
+    --8<-- "vllm/entrypoints/pooling/base/protocol.py:embed-extra-params"
+    ```
+
+#### Chat Parameters
+
+For chat-like input (i.e. if `messages` is passed), the following parameters are supported:
+
+??? code
+
+    ```python
+    --8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-params"
+    --8<-- "vllm/entrypoints/pooling/base/protocol.py:chat-params"
+    --8<-- "vllm/entrypoints/pooling/base/protocol.py:encoding-params"
+    --8<-- "vllm/entrypoints/pooling/base/protocol.py:embed-params"
+    ```
+
+these extra parameters are supported instead:
+
+??? code
+
+    ```python
+    --8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-extra-params"
+    --8<-- "vllm/entrypoints/pooling/base/protocol.py:chat-extra-params"
+    --8<-- "vllm/entrypoints/pooling/base/protocol.py:encoding-extra-params"
+    --8<-- "vllm/entrypoints/pooling/base/protocol.py:embed-extra-params"
+    ```
+
+#### Examples
+
+If the model has a [chat template](../../serving/openai_compatible_server.md#chat-template), you can replace `inputs` with a list of `messages` (same schema as [Chat API](../../serving/openai_compatible_server.md#chat-api))
+which will be treated as a single prompt to the model. Here is a convenience function for calling the API while retaining OpenAI's type annotations:
+
+??? code
+
+    ```python
+    from openai import OpenAI
+    from openai._types import NOT_GIVEN, NotGiven
+    from openai.types.chat import ChatCompletionMessageParam
+    from openai.types.create_embedding_response import CreateEmbeddingResponse
+
+    def create_chat_embeddings(
+        client: OpenAI,
+        *,
+        messages: list[ChatCompletionMessageParam],
+        model: str,
+        encoding_format: Union[Literal["base64", "float"], NotGiven] = NOT_GIVEN,
+    ) -> CreateEmbeddingResponse:
+        return client.post(
+            "/embeddings",
+            cast_to=CreateEmbeddingResponse,
+            body={"messages": messages, "model": model, "encoding_format": encoding_format},
+        )
+    ```
+
+##### Multi-modal inputs
+
+You can pass multi-modal inputs to embedding models by defining a custom chat template for the server
+and passing a list of `messages` in the request. Refer to the examples below for illustration.
+
+=== "VLM2Vec"
+
+    To serve the model:
+
+    ```bash
+    vllm serve TIGER-Lab/VLM2Vec-Full --runner pooling \
+      --trust-remote-code \
+      --max-model-len 4096 \
+      --chat-template examples/pooling/embed/template/vlm2vec_phi3v.jinja
+    ```
+
+    !!! important
+        Since VLM2Vec has the same model architecture as Phi-3.5-Vision, we have to explicitly pass `--runner pooling`
+        to run this model in embedding mode instead of text generation mode.
+
+        The custom chat template is completely different from the original one for this model,
+        and can be found here: [examples/pooling/embed/template/vlm2vec_phi3v.jinja](../../../examples/pooling/embed/template/vlm2vec_phi3v.jinja)
+
+    Since the request schema is not defined by OpenAI client, we post a request to the server using the lower-level `requests` library:
+
+    ??? code
+
+        ```python
+        from openai import OpenAI
+        client = OpenAI(
+            base_url="http://localhost:8000/v1",
+            api_key="EMPTY",
+        )
+        image_url = "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
+
+        response = create_chat_embeddings(
+            client,
+            model="TIGER-Lab/VLM2Vec-Full",
+            messages=[
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "image_url", "image_url": {"url": image_url}},
+                        {"type": "text", "text": "Represent the given image."},
+                    ],
+                }
+            ],
+            encoding_format="float",
+        )
+
+        print("Image embedding output:", response.data[0].embedding)
+        ```
+
+=== "DSE-Qwen2-MRL"
+
+    To serve the model:
+
+    ```bash
+    vllm serve MrLight/dse-qwen2-2b-mrl-v1 --runner pooling \
+      --trust-remote-code \
+      --max-model-len 8192 \
+      --chat-template examples/pooling/embed/template/dse_qwen2_vl.jinja
+    ```
+
+    !!! important
+        Like with VLM2Vec, we have to explicitly pass `--runner pooling`.
+
+        Additionally, `MrLight/dse-qwen2-2b-mrl-v1` requires an EOS token for embeddings, which is handled
+        by a custom chat template: [examples/pooling/embed/template/dse_qwen2_vl.jinja](../../../examples/pooling/embed/template/dse_qwen2_vl.jinja)
+
+    !!! important
+        `MrLight/dse-qwen2-2b-mrl-v1` requires a placeholder image of the minimum image size for text query embeddings. See the full code
+        example below for details.
+
+Full example: [examples/pooling/embed/vision_embedding_online.py](../../../examples/pooling/embed/vision_embedding_online.py)
+
+### Cohere Embed API
+
+Our API is also compatible with [Cohere's Embed v2 API](https://docs.cohere.com/reference/embed) which adds support for some modern embedding feature such as truncation, output dimensions, embedding types, and input types. This endpoint works with any embedding model (including multimodal models).
+
+#### Cohere Embed API request parameters
+
+| Parameter | Type | Required | Description |
+| --------- | ---- | -------- | ----------- |
+| `model` | string | Yes | Model name |
+| `input_type` | string | No | Prompt prefix key (model-dependent, see below) |
+| `texts` | list[string] | No | Text inputs (use one of `texts`, `images`, or `inputs`) |
+| `images` | list[string] | No | Base64 data URI images |
+| `inputs` | list[object] | No | Mixed text and image content objects |
+| `embedding_types` | list[string] | No | Output types (default: `["float"]`) |
+| `output_dimension` | int | No | Truncate embeddings to this dimension (Matryoshka) |
+| `truncate` | string | No | `END`, `START`, or `NONE` (default: `END`) |
+
+#### Text embedding
+
+```bash
+curl -X POST "http://localhost:8000/v2/embed" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "Snowflake/snowflake-arctic-embed-m-v1.5",
+    "input_type": "query",
+    "texts": ["Hello world", "How are you?"],
+    "embedding_types": ["float"]
+  }'
+```
+
+??? console "Response"
+
+    ```json
+    {
+      "id": "embd-...",
+      "embeddings": {
+        "float": [
+          [0.012, -0.034, ...],
+          [0.056, 0.078, ...]
+        ]
+      },
+      "texts": ["Hello world", "How are you?"],
+      "meta": {
+        "api_version": {"version": "2"},
+        "billed_units": {"input_tokens": 12}
+      }
+    }
+    ```
+
+#### Mixed text and image inputs
+
+For multimodal models, you can embed images by passing base64 data URIs. The `inputs` field accepts a list of objects with mixed text and image content:
+
+```bash
+curl -X POST "http://localhost:8000/v2/embed" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "google/siglip-so400m-patch14-384",
+    "inputs": [
+      {
+        "content": [
+          {"type": "text", "text": "A photo of a cat"},
+          {"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBOR..."}}
+        ]
+      }
+    ],
+    "embedding_types": ["float"]
+  }'
+```
+
+#### Embedding types
+
+The `embedding_types` parameter controls the output format. Multiple types can be requested in a single call:
+
+| Type | Description |
+| ---- | ----------- |
+| `float` | Raw float32 embeddings (default) |
+| `binary` | Bit-packed signed binary |
+| `ubinary` | Bit-packed unsigned binary |
+| `base64` | Little-endian float32 encoded as base64 |
+
+```bash
+curl -X POST "http://localhost:8000/v2/embed" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "Snowflake/snowflake-arctic-embed-m-v1.5",
+    "input_type": "query",
+    "texts": ["What is machine learning?"],
+    "embedding_types": ["float", "binary"]
+  }'
+```
+
+??? console "Response"
+
+    ```json
+    {
+      "id": "embd-...",
+      "embeddings": {
+        "float": [[0.012, -0.034, ...]],
+        "binary": [[42, -117, ...]]
+      },
+      "texts": ["What is machine learning?"],
+      "meta": {
+        "api_version": {"version": "2"},
+        "billed_units": {"input_tokens": 8}
+      }
+    }
+    ```
+
+#### Truncation
+
+The `truncate` parameter controls how inputs exceeding the model's maximum sequence length are handled:
+
+| Value | Behavior |
+| ----- | --------- |
+| `END` (default) | Keep the first tokens, drop the end |
+| `START` | Keep the last tokens, drop the beginning |
+| `NONE` | Return an error if the input is too long |
+
+#### Input type and prompt prefixes
+
+The `input_type` field selects a prompt prefix to prepend to each text input. The available values
+depend on the model:
+
+- **Models with `task_instructions` in `config.json`**: The keys from the `task_instructions` dict are
+  the valid `input_type` values and the corresponding value is prepended to each text.
+- **Models with `config_sentence_transformers.json` prompts**: The keys from the `prompts` dict are
+  the valid `input_type` values. For example, `Snowflake/snowflake-arctic-embed-xs` defines `"query"`,
+  so setting `input_type: "query"` prepends `"Represent this sentence for searching relevant passages: "`.
+- **Other models**: `input_type` is not accepted and will raise a validation error if passed.
+
+## More examples
+
+More examples can be found here: [examples/pooling/embed](../../../examples/pooling/embed)
+
+## Supported Features
+
+### Enable/disable normalize
+
+You can enable or disable normalize via `use_activation`.
+
+### Matryoshka Embeddings
+
+[Matryoshka Embeddings](https://sbert.net/examples/sentence_transformer/training/matryoshka/README.html#matryoshka-embeddings) or [Matryoshka Representation Learning (MRL)](https://arxiv.org/abs/2205.13147) is a technique used in training embedding models. It allows users to trade off between performance and cost.
+
+!!! warning
+    Not all embedding models are trained using Matryoshka Representation Learning. To avoid misuse of the `dimensions` parameter, vLLM returns an error for requests that attempt to change the output dimension of models that do not support Matryoshka Embeddings.
+
+    For example, setting `dimensions` parameter while using the `BAAI/bge-m3` model will result in the following error.
+
+    ```json
+    {"object":"error","message":"Model \"BAAI/bge-m3\" does not support matryoshka representation, changing output dimensions will lead to poor results.","type":"BadRequestError","param":null,"code":400}
+    ```
+
+#### Manually enable Matryoshka Embeddings
+
+There is currently no official interface for specifying support for Matryoshka Embeddings. In vLLM, if `is_matryoshka` is `True` in `config.json`, you can change the output dimension to arbitrary values. Use `matryoshka_dimensions` to control the allowed output dimensions.
+
+For models that support Matryoshka Embeddings but are not recognized by vLLM, manually override the config using `hf_overrides={"is_matryoshka": True}` or `hf_overrides={"matryoshka_dimensions": [<allowed output dimensions>]}` (offline), or `--hf-overrides '{"is_matryoshka": true}'` or `--hf-overrides '{"matryoshka_dimensions": [<allowed output dimensions>]}'` (online).
+
+Here is an example to serve a model with Matryoshka Embeddings enabled.
+
+```bash
+vllm serve Snowflake/snowflake-arctic-embed-m-v1.5 --hf-overrides '{"matryoshka_dimensions":[256]}'
+```
+
+#### Offline Inference
+
+You can change the output dimensions of embedding models that support Matryoshka Embeddings by using the dimensions parameter in [PoolingParams][vllm.PoolingParams].
+
+```python
+from vllm import LLM, PoolingParams
+
+llm = LLM(
+    model="jinaai/jina-embeddings-v3",
+    runner="pooling",
+    trust_remote_code=True,
+)
+outputs = llm.embed(
+    ["Follow the white rabbit."],
+    pooling_params=PoolingParams(dimensions=32),
+)
+print(outputs[0].outputs)
+```
+
+A code example can be found here: [examples/pooling/embed/embed_matryoshka_fy_offline.py](../../../examples/pooling/embed/embed_matryoshka_fy_offline.py)
+
+#### Online Inference
+
+Use the following command to start the vLLM server.
+
+```bash
+vllm serve jinaai/jina-embeddings-v3 --trust-remote-code
+```
+
+You can change the output dimensions of embedding models that support Matryoshka Embeddings by using the dimensions parameter.
+
+```bash
+curl http://127.0.0.1:8000/v1/embeddings \
+  -H 'accept: application/json' \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "input": "Follow the white rabbit.",
+    "model": "jinaai/jina-embeddings-v3",
+    "encoding_format": "float",
+    "dimensions": 32
+  }'
+```
+
+Expected output:
+
+```json
+{"id":"embd-5c21fc9a5c9d4384a1b021daccaf9f64","object":"list","created":1745476417,"model":"jinaai/jina-embeddings-v3","data":[{"index":0,"object":"embedding","embedding":[-0.3828125,-0.1357421875,0.03759765625,0.125,0.21875,0.09521484375,-0.003662109375,0.1591796875,-0.130859375,-0.0869140625,-0.1982421875,0.1689453125,-0.220703125,0.1728515625,-0.2275390625,-0.0712890625,-0.162109375,-0.283203125,-0.055419921875,-0.0693359375,0.031982421875,-0.04052734375,-0.2734375,0.1826171875,-0.091796875,0.220703125,0.37890625,-0.0888671875,-0.12890625,-0.021484375,-0.0091552734375,0.23046875]}],"usage":{"prompt_tokens":8,"total_tokens":8,"completion_tokens":0,"prompt_tokens_details":null}}
+```
+
+An OpenAI client example can be found here: [examples/pooling/embed/openai_embedding_matryoshka_fy_client.py](../../../examples/pooling/embed/openai_embedding_matryoshka_fy_client.py)
+
+## Removed Features
+
+### Remove `normalize` from PoolingParams
+
+We have already removed `normalize` from PoolingParams, use `use_activation` instead.
@@ -0,0 +1,136 @@
+# Reward Usages
+
+A reward model (RM) is designed to evaluate and score the quality of outputs generated by a language model, acting as a proxy for human preferences.
+
+## Summary
+
+- Model Usage: reward
+- Pooling Task:
+
+| Model Types                        | Pooling Tasks  |
+|------------------------------------|----------------|
+| (sequence) (outcome) reward models | classify       |
+| token (outcome) reward models      | token_classify |
+| process reward models              | token_classify |
+
+- Offline APIs:
+    - `LLM.encode(..., pooling_task="...")`
+- Online APIs:
+    - Pooling API (`/pooling`)
+
+## Supported Models
+
+### Reward Models
+
+Using sequence classification models as (sequence) (outcome) reward models, the usage and supported features are the same as for normal [classification models](classify.md).
+
+--8<-- [start:supported-sequence-reward-models]
+
+| Architecture | Models | Example HF Models | [LoRA](../../features/lora.md) | [PP](../../serving/parallelism_scaling.md) |
+| ------------ | ------ | ----------------- | -------------------- | ------------------------- |
+| `JambaForSequenceClassification` | Jamba | `ai21labs/Jamba-tiny-reward-dev`, etc. | ✅︎ | ✅︎ |
+| `Qwen3ForSequenceClassification`<sup>C</sup> | Qwen3-based | `Skywork/Skywork-Reward-V2-Qwen3-0.6B`, etc. | ✅︎ | ✅︎ |
+| `LlamaForSequenceClassification`<sup>C</sup> | Llama-based | `Skywork/Skywork-Reward-V2-Llama-3.2-1B`, etc. | ✅︎ | ✅︎ |
+| `*Model`<sup>C</sup>, `*ForCausalLM`<sup>C</sup>, etc. | Generative models | N/A | \* | \* |
+
+<sup>C</sup> Automatically converted into a classification model via `--convert classify`. ([details](./README.md#model-conversion))  
+
+If your model is not in the above list, we will try to automatically convert the model using
+[as_seq_cls_model][vllm.model_executor.models.adapters.as_seq_cls_model]. By default, the class probabilities are extracted from the softmaxed hidden state corresponding to the last token.
+
+--8<-- [end:supported-sequence-reward-models]
+
+### Token Reward Models
+
+The key distinction between (sequence) classification and token classification lies in their output granularity: (sequence) classification produces a single result for an entire input sequence, whereas token classification yields a result for each individual token within the sequence.
+
+Using token classification models as token (outcome) reward models, the usage and supported features are the same as for normal [token classification models](token_classify.md).
+
+--8<-- [start:supported-token-reward-models]
+
+| Architecture | Models | Example HF Models | [LoRA](../../features/lora.md) | [PP](../../serving/parallelism_scaling.md) |
+| ------------ | ------ | ----------------- | -------------------- | ------------------------- |
+| `InternLM2ForRewardModel` | InternLM2-based | `internlm/internlm2-1_8b-reward`, `internlm/internlm2-7b-reward`, etc. | ✅︎ | ✅︎ |
+| `Qwen2ForRewardModel` | Qwen2-based | `Qwen/Qwen2.5-Math-RM-72B`, etc. | ✅︎ | ✅︎ |
+| `*Model`<sup>C</sup>, `*ForCausalLM`<sup>C</sup>, etc. | Generative models | N/A | \* | \* |
+
+<sup>C</sup> Automatically converted into a classification model via `--convert classify`. ([details](./README.md#model-conversion))  
+
+If your model is not in the above list, we will try to automatically convert the model using
+[as_seq_cls_model][vllm.model_executor.models.adapters.as_seq_cls_model].
+
+--8<-- [end:supported-token-reward-models]
+
+### Process Reward Models
+
+The process reward models used for evaluating intermediate steps are crucial to achieving the desired outcome.
+
+| Architecture | Models | Example HF Models | [LoRA](../../features/lora.md) | [PP](../../serving/parallelism_scaling.md) |
+| ------------ | ------ | ----------------- | -------------------- | ------------------------- |
+| `LlamaForCausalLM` | Llama-based | `peiyi9979/math-shepherd-mistral-7b-prm`, etc. | ✅︎ | ✅︎ |
+| `Qwen2ForProcessRewardModel` | Qwen2-based | `Qwen/Qwen2.5-Math-PRM-7B`, etc. | ✅︎ | ✅︎ |
+
+!!! important
+    For process-supervised reward models such as `peiyi9979/math-shepherd-mistral-7b-prm`, the pooling config should be set explicitly,
+    e.g.: `--pooler-config '{"pooling_type": "STEP", "step_tag_id": 123, "returned_token_ids": [456, 789]}'`.
+
+## Offline Inference
+
+### Pooling Parameters
+
+The following [pooling parameters][vllm.PoolingParams] are supported.
+
+```python
+--8<-- "vllm/pooling_params.py:common-pooling-params"
+--8<-- "vllm/pooling_params.py:classify-pooling-params"
+```
+
+### `LLM.encode`
+
+The [encode][vllm.LLM.encode] method is available to all pooling models in vLLM.
+
+- Reward Models
+
+Set `pooling_task="classify"` when using `LLM.encode` for (sequence) (outcome) reward models:
+
+```python
+from vllm import LLM
+
+llm = LLM(model="Skywork/Skywork-Reward-V2-Qwen3-0.6B", runner="pooling")
+(output,) = llm.encode("Hello, my name is", pooling_task="classify")
+
+data = output.outputs.data
+print(f"Data: {data!r}")
+```
+
+- Token Reward Models
+
+Set `pooling_task="token_classify"` when using `LLM.encode` for token (outcome) reward models:
+
+```python
+from vllm import LLM
+
+llm = LLM(model="internlm/internlm2-1_8b-reward", runner="pooling", trust_remote_code=True)
+(output,) = llm.encode("Hello, my name is", pooling_task="token_classify")
+
+data = output.outputs.data
+print(f"Data: {data!r}")
+```
+
+- Process Reward Models
+
+Set `pooling_task="token_classify"` when using `LLM.encode` for token (outcome) reward models:
+
+```python
+from vllm import LLM
+
+llm = LLM(model="Qwen/Qwen2.5-Math-PRM-7B", runner="pooling")
+(output,) = llm.encode("Hello, my name is<extra_0><extra_0><extra_0>", pooling_task="token_classify")
+
+data = output.outputs.data
+print(f"Data: {data!r}")
+```
+
+## Online Serving
+
+Please refer to the [pooling API](README.md#pooling-api). Pooling task corresponding to reward model types refer to the [table above](#summary).
@@ -0,0 +1,451 @@
+# Scoring Usages
+
+The score models is designed to compute similarity scores between two input prompts. It supports three model types (aka `score_type`): `cross-encoder`, `late-interaction`, and `bi-encoder`.
+
+!!! note
+    vLLM handles only the model inference component of RAG pipelines (such as embedding generation and reranking). For higher-level RAG orchestration, you should leverage integration frameworks like [LangChain](https://github.com/langchain-ai/langchain).
+
+## Summary
+
+- Model Usage: Scoring
+- Pooling Task:
+
+| Score Types        | Pooling Tasks         | scoring function         |
+|--------------------|-----------------------|--------------------------|
+| `cross-encoder`    | `classify` (see note) | linear classifier        |
+| `late-interaction` | `token_embed`         | late interaction(MaxSim) |
+| `bi-encoder`       | `embed`               | cosine similarity        |
+
+- Offline APIs:
+    - `LLM.score`
+- Online APIs:
+    - [Score API](scoring.md#score-api) (`/score`)
+    - [Rerank API](scoring.md#rerank-api) (`/rerank`, `/v1/rerank`, `/v2/rerank`)
+
+!!! note
+    Only when a classification model outputs num_labels equal to 1 can it be used as a scoring model and have its scoring API enabled.
+
+## Supported Models
+
+### Cross-encoder models
+
+[Cross-encoder](https://www.sbert.net/examples/applications/cross-encoder/README.html) (aka reranker) models are a subset of classification models that accept two prompts as input and output num_labels equal to 1.
+
+--8<-- [start:supported-cross-encoder-models]
+
+#### Text-only Models
+
+| Architecture | Models | Example HF Models | Score template (see note) | [LoRA](../../features/lora.md) | [PP](../../serving/parallelism_scaling.md) |
+| ------------ | ------ | ----------------- | ------------------------- | --------------------------- | --------------------------------------- |
+| `BertForSequenceClassification` | BERT-based | `cross-encoder/ms-marco-MiniLM-L-6-v2`, etc. | N/A | | |
+| `GemmaForSequenceClassification` | Gemma-based | `BAAI/bge-reranker-v2-gemma`(see note), etc. | [bge-reranker-v2-gemma.jinja](../../../examples/pooling/score/template/bge-reranker-v2-gemma.jinja) | ✅︎ | ✅︎ |
+| `GteNewForSequenceClassification` | mGTE-TRM (see note) | `Alibaba-NLP/gte-multilingual-reranker-base`, etc. | N/A | | |
+| `LlamaBidirectionalForSequenceClassification`<sup>C</sup> | Llama-based with bidirectional attention | `nvidia/llama-nemotron-rerank-1b-v2`, etc. | [nemotron-rerank.jinja](../../../examples/pooling/score/template/nemotron-rerank.jinja) | ✅︎ | ✅︎ |
+| `Qwen2ForSequenceClassification`<sup>C</sup> | Qwen2-based | `mixedbread-ai/mxbai-rerank-base-v2`(see note), etc. | [mxbai_rerank_v2.jinja](../../../examples/pooling/score/template/mxbai_rerank_v2.jinja) | ✅︎ | ✅︎ |
+| `Qwen3ForSequenceClassification`<sup>C</sup> | Qwen3-based | `tomaarsen/Qwen3-Reranker-0.6B-seq-cls`, `Qwen/Qwen3-Reranker-0.6B`(see note), etc. | [qwen3_reranker.jinja](../../../examples/pooling/score/template/qwen3_reranker.jinja) | ✅︎ | ✅︎ |
+| `RobertaForSequenceClassification` | RoBERTa-based | `cross-encoder/quora-roberta-base`, etc. | N/A | | |
+| `XLMRobertaForSequenceClassification` | XLM-RoBERTa-based | `BAAI/bge-reranker-v2-m3`, etc. | N/A | | |
+| `*Model`<sup>C</sup>, `*ForCausalLM`<sup>C</sup>, etc. | Generative models | N/A | N/A | \* | \* |
+
+<sup>C</sup> Automatically converted into a classification model via `--convert classify`. ([details](./README.md#model-conversion))  
+\* Feature support is the same as that of the original model.
+
+!!! note
+    Some models require a specific prompt format to work correctly.
+
+    You can find Example HF Models's corresponding score template in [examples/pooling/score/template/](../../../examples/pooling/score/template)
+
+    Examples : [examples/pooling/score/using_template_offline.py](../../../examples/pooling/score/using_template_offline.py) [examples/pooling/score/using_template_online.py](../../../examples/pooling/score/using_template_online.py)
+
+!!! note
+    Load the official original `BAAI/bge-reranker-v2-gemma` by using the following command.
+
+    ```bash
+    vllm serve BAAI/bge-reranker-v2-gemma --hf_overrides '{"architectures": ["GemmaForSequenceClassification"],"classifier_from_token": ["Yes"],"method": "no_post_processing"}'
+    ```
+
+!!! note
+    The second-generation GTE model (mGTE-TRM) is named `NewForSequenceClassification`. The name `NewForSequenceClassification` is too generic, you should set `--hf-overrides '{"architectures": ["GteNewForSequenceClassification"]}'` to specify the use of the `GteNewForSequenceClassification` architecture.
+
+!!! note
+    Load the official original `mxbai-rerank-v2` by using the following command.
+
+    ```bash
+    vllm serve mixedbread-ai/mxbai-rerank-base-v2 --hf_overrides '{"architectures": ["Qwen2ForSequenceClassification"],"classifier_from_token": ["0", "1"], "method": "from_2_way_softmax"}'
+    ```
+
+!!! note
+    Load the official original `Qwen3 Reranker` by using the following command. More information can be found at: [examples/pooling/score/qwen3_reranker_offline.py](../../../examples/pooling/score/qwen3_reranker_offline.py) [examples/pooling/score/qwen3_reranker_online.py](../../../examples/pooling/score/qwen3_reranker_online.py).
+
+    ```bash
+    vllm serve Qwen/Qwen3-Reranker-0.6B --hf_overrides '{"architectures": ["Qwen3ForSequenceClassification"],"classifier_from_token": ["no", "yes"],"is_original_qwen3_reranker": true}'
+    ```
+
+#### Multimodal Models
+
+!!! note
+    For more information about multimodal models inputs, see [this page](../supported_models.md#list-of-multimodal-language-models).
+
+| Architecture | Models | Inputs | Example HF Models | [LoRA](../../features/lora.md) | [PP](../../serving/parallelism_scaling.md) |
+| ------------ | ------ | ------ | ----------------- | ------------------------------ | ------------------------------------------ |
+| `JinaVLForSequenceClassification` | JinaVL-based | T + I<sup>E+</sup> | `jinaai/jina-reranker-m0`, etc. | ✅︎ | ✅︎ |
+| `LlamaNemotronVLForSequenceClassification` | Llama Nemotron Reranker + SigLIP | T + I<sup>E+</sup> | `nvidia/llama-nemotron-rerank-vl-1b-v2` | | |
+| `Qwen3VLForSequenceClassification` | Qwen3-VL-Reranker | T + I<sup>E+</sup> + V<sup>E+</sup> | `Qwen/Qwen3-VL-Reranker-2B`(see note), etc. | ✅︎ | ✅︎ |
+
+<sup>C</sup> Automatically converted into a classification model via `--convert classify`. ([details](README.md#model-conversion))  
+\* Feature support is the same as that of the original model.
+
+!!! note
+    Similar to Qwen3-Reranker, you need to use the following `--hf_overrides` to load the official original `Qwen3-VL-Reranker`.
+
+    ```bash
+    vllm serve Qwen/Qwen3-VL-Reranker-2B --hf_overrides '{"architectures": ["Qwen3VLForSequenceClassification"],"classifier_from_token": ["no", "yes"],"is_original_qwen3_reranker": true}'
+    ```
+
+--8<-- [end:supported-cross-encoder-models]
+
+### Late-interaction models
+
+All models that support token embedding task also support using the score API to compute similarity scores by calculating the late interaction of two input prompts. See [this page](token_embed.md) for more information about token embedding models.
+
+--8<-- "docs/models/pooling_models/token_embed.md:supported-token-embed-models"
+
+### Bi-encoder
+
+All models that support embedding task also support using the score API to compute similarity scores by calculating the cosine similarity of two input prompt's embeddings. See [this page](embed.md) for more information about embedding models.
+
+--8<-- "docs/models/pooling_models/embed.md:supported-embed-models"
+
+## Offline Inference
+
+### Pooling Parameters
+
+The following [pooling parameters][vllm.PoolingParams] are only supported by cross-encoder models and do not work for late-interaction and bi-encoder models.
+
+```python
+--8<-- "vllm/pooling_params.py:common-pooling-params"
+--8<-- "vllm/pooling_params.py:classify-pooling-params"
+```
+
+### `LLM.score`
+
+The [score][vllm.LLM.score] method outputs similarity scores between sentence pairs.
+
+```python
+from vllm import LLM
+
+llm = LLM(model="BAAI/bge-reranker-v2-m3", runner="pooling")
+(output,) = llm.score(
+    "What is the capital of France?",
+    "The capital of Brazil is Brasilia.",
+)
+
+score = output.outputs.score
+print(f"Score: {score}")
+```
+
+A code example can be found here: [examples/basic/offline_inference/score.py](../../../examples/basic/offline_inference/score.py)
+
+## Online Serving
+
+### Score API
+
+Our Score API (`/score`) is similar to `LLM.score`, compute similarity scores between two input prompts.
+
+#### Parameters
+
+The following Score API parameters are supported:
+
+```python
+--8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-params"
+--8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-extra-params"
+--8<-- "vllm/entrypoints/pooling/base/protocol.py:classify-extra-params"
+```
+
+#### Examples
+
+##### Single inference
+
+You can pass a string to both `queries` and `documents`, forming a single sentence pair.
+
+```bash
+curl -X 'POST' \
+  'http://127.0.0.1:8000/score' \
+  -H 'accept: application/json' \
+  -H 'Content-Type: application/json' \
+  -d '{
+  "model": "BAAI/bge-reranker-v2-m3",
+  "encoding_format": "float",
+  "queries": "What is the capital of France?",
+  "documents": "The capital of France is Paris."
+}'
+```
+
+??? console "Response"
+
+    ```json
+    {
+      "id": "score-request-id",
+      "object": "list",
+      "created": 693447,
+      "model": "BAAI/bge-reranker-v2-m3",
+      "data": [
+        {
+          "index": 0,
+          "object": "score",
+          "score": 1
+        }
+      ],
+      "usage": {}
+    }
+    ```
+
+##### Batch inference
+
+You can pass a string to `queries` and a list to `documents`, forming multiple sentence pairs
+where each pair is built from `queries` and a string in `documents`.
+The total number of pairs is `len(documents)`.
+
+??? console "Request"
+
+    ```bash
+    curl -X 'POST' \
+      'http://127.0.0.1:8000/score' \
+      -H 'accept: application/json' \
+      -H 'Content-Type: application/json' \
+      -d '{
+      "model": "BAAI/bge-reranker-v2-m3",
+      "queries": "What is the capital of France?",
+      "documents": [
+        "The capital of Brazil is Brasilia.",
+        "The capital of France is Paris."
+      ]
+    }'
+    ```
+
+??? console "Response"
+
+    ```json
+    {
+      "id": "score-request-id",
+      "object": "list",
+      "created": 693570,
+      "model": "BAAI/bge-reranker-v2-m3",
+      "data": [
+        {
+          "index": 0,
+          "object": "score",
+          "score": 0.001094818115234375
+        },
+        {
+          "index": 1,
+          "object": "score",
+          "score": 1
+        }
+      ],
+      "usage": {}
+    }
+    ```
+
+You can pass a list to both `queries` and `documents`, forming multiple sentence pairs
+where each pair is built from a string in `queries` and the corresponding string in `documents` (similar to `zip()`).
+The total number of pairs is `len(documents)`.
+
+??? console "Request"
+
+    ```bash
+    curl -X 'POST' \
+      'http://127.0.0.1:8000/score' \
+      -H 'accept: application/json' \
+      -H 'Content-Type: application/json' \
+      -d '{
+      "model": "BAAI/bge-reranker-v2-m3",
+      "encoding_format": "float",
+      "queries": [
+        "What is the capital of Brazil?",
+        "What is the capital of France?"
+      ],
+      "documents": [
+        "The capital of Brazil is Brasilia.",
+        "The capital of France is Paris."
+      ]
+    }'
+    ```
+
+??? console "Response"
+
+    ```json
+    {
+      "id": "score-request-id",
+      "object": "list",
+      "created": 693447,
+      "model": "BAAI/bge-reranker-v2-m3",
+      "data": [
+        {
+          "index": 0,
+          "object": "score",
+          "score": 1
+        },
+        {
+          "index": 1,
+          "object": "score",
+          "score": 1
+        }
+      ],
+      "usage": {}
+    }
+    ```
+
+##### Multi-modal inputs
+
+You can pass multi-modal inputs to scoring models by passing `content` including a list of multi-modal input (image, etc.) in the request. Refer to the examples below for illustration.
+
+=== "JinaVL-Reranker"
+
+    To serve the model:
+
+    ```bash
+    vllm serve jinaai/jina-reranker-m0
+    ```
+
+    Since the request schema is not defined by OpenAI client, we post a request to the server using the lower-level `requests` library:
+
+    ??? Code
+
+        ```python
+        import requests
+        
+        response = requests.post(
+            "http://localhost:8000/v1/score",
+            json={
+                "model": "jinaai/jina-reranker-m0",
+                "queries": "slm markdown",
+                "documents": [
+                    {
+                        "content": [
+                            {
+                                "type": "image_url",
+                                "image_url": {
+                                    "url": "https://raw.githubusercontent.com/jina-ai/multimodal-reranker-test/main/handelsblatt-preview.png"
+                                },
+                            }
+                        ],
+                    },
+                    {
+                        "content": [
+                            {
+                                "type": "image_url",
+                                "image_url": {
+                                    "url": "https://raw.githubusercontent.com/jina-ai/multimodal-reranker-test/main/handelsblatt-preview.png"
+                                },
+                            }
+                        ]
+                    },
+                ],
+            },
+        )
+        response.raise_for_status()
+        response_json = response.json()
+        print("Scoring output:", response_json["data"][0]["score"])
+        print("Scoring output:", response_json["data"][1]["score"])
+        ```
+Full example:
+
+- [examples/pooling/score/vision_score_api_online.py](../../../examples/pooling/score/vision_score_api_online.py)
+- [examples/pooling/score/vision_rerank_api_online.py](../../../examples/pooling/score/vision_rerank_api_online.py)
+
+### Rerank API
+
+`/rerank`, `/v1/rerank`, and `/v2/rerank` APIs are compatible with both [Jina AI's rerank API interface](https://jina.ai/reranker/) and
+[Cohere's rerank API interface](https://docs.cohere.com/v2/reference/rerank) to ensure compatibility with
+popular open-source tools.
+
+Code example: [examples/pooling/score/rerank_api_online.py](../../../examples/pooling/score/rerank_api_online.py)
+
+#### Parameters
+
+The following rerank api parameters are supported:
+
+```python
+--8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-params"
+--8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-extra-params"
+--8<-- "vllm/entrypoints/pooling/base/protocol.py:classify-extra-params"
+```
+
+#### Examples
+
+Note that the `top_n` request parameter is optional and will default to the length of the `documents` field.
+Result documents will be sorted by relevance, and the `index` property can be used to determine original order.
+
+??? console "Request"
+
+    ```bash
+    curl -X 'POST' \
+      'http://127.0.0.1:8000/v1/rerank' \
+      -H 'accept: application/json' \
+      -H 'Content-Type: application/json' \
+      -d '{
+      "model": "BAAI/bge-reranker-base",
+      "query": "What is the capital of France?",
+      "documents": [
+        "The capital of Brazil is Brasilia.",
+        "The capital of France is Paris.",
+        "Horses and cows are both animals"
+      ]
+    }'
+    ```
+
+??? console "Response"
+
+    ```json
+    {
+      "id": "rerank-fae51b2b664d4ed38f5969b612edff77",
+      "model": "BAAI/bge-reranker-base",
+      "usage": {
+        "total_tokens": 56
+      },
+      "results": [
+        {
+          "index": 1,
+          "document": {
+            "text": "The capital of France is Paris."
+          },
+          "relevance_score": 0.99853515625
+        },
+        {
+          "index": 0,
+          "document": {
+            "text": "The capital of Brazil is Brasilia."
+          },
+          "relevance_score": 0.0005860328674316406
+        }
+      ]
+    }
+    ```
+
+## More examples
+
+More examples can be found here: [examples/pooling/score](../../../examples/pooling/score)
+
+## Supported Features
+
+AS cross-encoder models are a subset of classification models that accept two prompts as input and output num_labels equal to 1, cross-encoder features should be consistent with (sequence) classification. For more information, see [this page](classify.md#supported-features).
+
+### Score Template
+
+Score templates are supported for **cross-encoder** models only. If you are using an **embedding** model for scoring, vLLM does not apply a score template.
+
+Some scoring models require a specific prompt format to work correctly. You can specify a custom score template using the `--chat-template` parameter (see [Chat Template](../../serving/openai_compatible_server.md#chat-template)).
+
+Like chat templates, the score template receives a `messages` list. For scoring, each message has a `role` attribute—either `"query"` or `"document"`. For the usual kind of point-wise cross-encoder, you can expect exactly two messages: one query and one document. To access the query and document content, use Jinja's `selectattr` filter:
+
+- **Query**: `{{ (messages | selectattr("role", "eq", "query") | first).content }}`
+- **Document**: `{{ (messages | selectattr("role", "eq", "document") | first).content }}`
+
+This approach is more robust than index-based access (`messages[0]`, `messages[1]`) because it selects messages by their semantic role. It also avoids assumptions about message ordering if additional message types are added to `messages` in the future.
+
+Example template file: [examples/pooling/score/template/nemotron-rerank.jinja](../../../examples/pooling/score/template/nemotron-rerank.jinja)
+
+### Enable/disable activation
+
+You can enable or disable activation via `use_activation` only works for cross-encoder models.
@@ -0,0 +1,400 @@
+# Specific Model Examples
+
+## ColBERT Late Interaction Models
+
+[ColBERT](https://arxiv.org/abs/2004.12832) (Contextualized Late Interaction over BERT) is a retrieval model that uses per-token embeddings and MaxSim scoring for document ranking. Unlike single-vector embedding models, ColBERT retains token-level representations and computes relevance scores through late interaction, providing better accuracy while being more efficient than cross-encoders.
+
+vLLM supports ColBERT models with multiple encoder backbones:
+
+| Architecture | Backbone | Example HF Models |
+| - | - | - |
+| `HF_ColBERT` | BERT | `answerdotai/answerai-colbert-small-v1`, `colbert-ir/colbertv2.0` |
+| `ColBERTModernBertModel` | ModernBERT | `lightonai/GTE-ModernColBERT-v1` |
+| `ColBERTJinaRobertaModel` | Jina XLM-RoBERTa | `jinaai/jina-colbert-v2` |
+| `ColBERTLfm2Model` | LFM2 | `LiquidAI/LFM2-ColBERT-350M` |
+
+**BERT-based ColBERT** models work out of the box:
+
+```shell
+vllm serve answerdotai/answerai-colbert-small-v1
+```
+
+For **non-BERT backbones**, use `--hf-overrides` to set the correct architecture:
+
+```shell
+# ModernBERT backbone
+vllm serve lightonai/GTE-ModernColBERT-v1 \
+    --hf-overrides '{"architectures": ["ColBERTModernBertModel"]}'
+
+# Jina XLM-RoBERTa backbone
+vllm serve jinaai/jina-colbert-v2 \
+    --hf-overrides '{"architectures": ["ColBERTJinaRobertaModel"]}' \
+    --trust-remote-code
+
+# LFM2 backbone
+vllm serve LiquidAI/LFM2-ColBERT-350M \
+    --hf-overrides '{"architectures": ["ColBERTLfm2Model"]}'
+```
+
+Then you can use the rerank API:
+
+```shell
+curl -s http://localhost:8000/rerank -H "Content-Type: application/json" -d '{
+    "model": "answerdotai/answerai-colbert-small-v1",
+    "query": "What is machine learning?",
+    "documents": [
+        "Machine learning is a subset of artificial intelligence.",
+        "Python is a programming language.",
+        "Deep learning uses neural networks."
+    ]
+}'
+```
+
+Or the score API:
+
+```shell
+curl -s http://localhost:8000/score -H "Content-Type: application/json" -d '{
+    "model": "answerdotai/answerai-colbert-small-v1",
+    "text_1": "What is machine learning?",
+    "text_2": ["Machine learning is a subset of AI.", "The weather is sunny."]
+}'
+```
+
+You can also get the raw token embeddings using the pooling API with `token_embed` task:
+
+```shell
+curl -s http://localhost:8000/pooling -H "Content-Type: application/json" -d '{
+    "model": "answerdotai/answerai-colbert-small-v1",
+    "input": "What is machine learning?",
+    "task": "token_embed"
+}'
+```
+
+An example can be found here: [examples/pooling/score/colbert_rerank_online.py](../../../examples/pooling/score/colbert_rerank_online.py)
+
+## ColQwen3 Multi-Modal Late Interaction Models
+
+ColQwen3 is based on [ColPali](https://arxiv.org/abs/2407.01449), which extends ColBERT's late interaction approach to **multi-modal** inputs. While ColBERT operates on text-only token embeddings, ColPali/ColQwen3 can embed both **text and images** (e.g. PDF pages, screenshots, diagrams) into per-token L2-normalized vectors and compute relevance via MaxSim scoring. ColQwen3 specifically uses Qwen3-VL as its vision-language backbone.
+
+| Architecture | Backbone | Example HF Models |
+| - | - | - |
+| `ColQwen3` | Qwen3-VL | `TomoroAI/tomoro-colqwen3-embed-4b`, `TomoroAI/tomoro-colqwen3-embed-8b` |
+| `OpsColQwen3Model` | Qwen3-VL | `OpenSearch-AI/Ops-Colqwen3-4B`, `OpenSearch-AI/Ops-Colqwen3-8B` |
+| `Qwen3VLNemotronEmbedModel` | Qwen3-VL | `nvidia/nemotron-colembed-vl-4b-v2`, `nvidia/nemotron-colembed-vl-8b-v2` |
+
+Start the server:
+
+```shell
+vllm serve TomoroAI/tomoro-colqwen3-embed-4b --max-model-len 4096
+```
+
+### Text-only scoring and reranking
+
+Use the `/rerank` API:
+
+```shell
+curl -s http://localhost:8000/rerank -H "Content-Type: application/json" -d '{
+    "model": "TomoroAI/tomoro-colqwen3-embed-4b",
+    "query": "What is machine learning?",
+    "documents": [
+        "Machine learning is a subset of artificial intelligence.",
+        "Python is a programming language.",
+        "Deep learning uses neural networks."
+    ]
+}'
+```
+
+Or the `/score` API:
+
+```shell
+curl -s http://localhost:8000/score -H "Content-Type: application/json" -d '{
+    "model": "TomoroAI/tomoro-colqwen3-embed-4b",
+    "text_1": "What is the capital of France?",
+    "text_2": ["The capital of France is Paris.", "Python is a programming language."]
+}'
+```
+
+### Multi-modal scoring and reranking (text query × image documents)
+
+The `/score` and `/rerank` APIs also accept multi-modal inputs directly.
+Pass image documents using the `data_1`/`data_2` (for `/score`) or `documents` (for `/rerank`) fields
+with a `content` list containing `image_url` and `text` parts — the same format used by the
+OpenAI chat completion API:
+
+Score a text query against image documents:
+
+```shell
+curl -s http://localhost:8000/score -H "Content-Type: application/json" -d '{
+    "model": "TomoroAI/tomoro-colqwen3-embed-4b",
+    "data_1": "Retrieve the city of Beijing",
+    "data_2": [
+        {
+            "content": [
+                {"type": "image_url", "image_url": {"url": "data:image/png;base64,<BASE64>"}},
+                {"type": "text", "text": "Describe the image."}
+            ]
+        }
+    ]
+}'
+```
+
+Rerank image documents by a text query:
+
+```shell
+curl -s http://localhost:8000/rerank -H "Content-Type: application/json" -d '{
+    "model": "TomoroAI/tomoro-colqwen3-embed-4b",
+    "query": "Retrieve the city of Beijing",
+    "documents": [
+        {
+            "content": [
+                {"type": "image_url", "image_url": {"url": "data:image/png;base64,<BASE64_1>"}},
+                {"type": "text", "text": "Describe the image."}
+            ]
+        },
+        {
+            "content": [
+                {"type": "image_url", "image_url": {"url": "data:image/png;base64,<BASE64_2>"}},
+                {"type": "text", "text": "Describe the image."}
+            ]
+        }
+    ],
+    "top_n": 2
+}'
+```
+
+### Raw token embeddings
+
+You can also get the raw token embeddings using the `/pooling` API with `token_embed` task:
+
+```shell
+curl -s http://localhost:8000/pooling -H "Content-Type: application/json" -d '{
+    "model": "TomoroAI/tomoro-colqwen3-embed-4b",
+    "input": "What is machine learning?",
+    "task": "token_embed"
+}'
+```
+
+For **image inputs** via the pooling API, use the chat-style `messages` field:
+
+```shell
+curl -s http://localhost:8000/pooling -H "Content-Type: application/json" -d '{
+    "model": "TomoroAI/tomoro-colqwen3-embed-4b",
+    "messages": [
+        {
+            "role": "user",
+            "content": [
+                {"type": "image_url", "image_url": {"url": "data:image/png;base64,<BASE64>"}},
+                {"type": "text", "text": "Describe the image."}
+            ]
+        }
+    ]
+}'
+```
+
+### Examples
+
+- Multi-vector retrieval: [examples/pooling/token_embed/colqwen3_token_embed_online.py](../../../examples/pooling/token_embed/colqwen3_token_embed_online.py)
+- Reranking (text + multi-modal): [examples/pooling/score/colqwen3_rerank_online.py](../../../examples/pooling/score/colqwen3_rerank_online.py)
+
+## ColQwen3.5 Multi-Modal Late Interaction Models
+
+ColQwen3.5 is based on [ColPali](https://arxiv.org/abs/2407.01449), extending ColBERT's late interaction approach to **multi-modal** inputs. It uses the Qwen3.5 hybrid backbone (linear + full attention) and produces per-token L2-normalized vectors for MaxSim scoring.
+
+| Architecture | Backbone | Example HF Models |
+| - | - | - |
+| `ColQwen3_5` | Qwen3.5 | `athrael-soju/colqwen3.5-4.5B` |
+
+Start the server:
+
+```shell
+vllm serve athrael-soju/colqwen3.5-4.5B --max-model-len 4096
+```
+
+Then you can use the rerank endpoint:
+
+```shell
+curl -s http://localhost:8000/rerank -H "Content-Type: application/json" -d '{
+    "model": "athrael-soju/colqwen3.5-4.5B",
+    "query": "What is machine learning?",
+    "documents": [
+        "Machine learning is a subset of artificial intelligence.",
+        "Python is a programming language.",
+        "Deep learning uses neural networks."
+    ]
+}'
+```
+
+Or the score endpoint:
+
+```shell
+curl -s http://localhost:8000/score -H "Content-Type: application/json" -d '{
+    "model": "athrael-soju/colqwen3.5-4.5B",
+    "text_1": "What is the capital of France?",
+    "text_2": ["The capital of France is Paris.", "Python is a programming language."]
+}'
+```
+
+An example can be found here: [examples/pooling/score/colqwen3_5_rerank_online.py](../../../examples/pooling/score/colqwen3_5_rerank_online.py)
+
+## Llama Nemotron Multimodal
+
+### Embedding Model
+
+Llama Nemotron VL Embedding models combine the bidirectional Llama embedding backbone
+(from `nvidia/llama-nemotron-embed-1b-v2`) with SigLIP as the vision encoder to produce
+single-vector embeddings from text and/or images.
+
+| Architecture | Backbone | Example HF Models |
+| - | - | - |
+| `LlamaNemotronVLModel` | Bidirectional Llama + SigLIP | `nvidia/llama-nemotron-embed-vl-1b-v2` |
+
+Start the server:
+
+```shell
+vllm serve nvidia/llama-nemotron-embed-vl-1b-v2 \
+    --trust-remote-code \
+    --chat-template examples/pooling/embed/template/nemotron_embed_vl.jinja
+```
+
+!!! note
+    The chat template bundled with this model's tokenizer is not suitable for
+    the embeddings API. Use the provided override template above when serving
+    with the `messages`-based (chat-style) embeddings API.
+
+    The override template uses the message `role` to automatically prepend the
+    appropriate prefix: set `role` to `"query"` for queries (prepends `query: `)
+    or `"document"` for passages (prepends `passage: `). Any other role omits
+    the prefix.
+
+Embed text queries:
+
+```shell
+curl -s http://localhost:8000/v1/embeddings -H "Content-Type: application/json" -d '{
+    "model": "nvidia/llama-nemotron-embed-vl-1b-v2",
+    "messages": [
+        {
+            "role": "query",
+            "content": [
+                {"type": "text", "text": "What is machine learning?"}
+            ]
+        }
+    ]
+}'
+```
+
+Embed images via the chat-style `messages` field:
+
+```shell
+curl -s http://localhost:8000/v1/embeddings -H "Content-Type: application/json" -d '{
+    "model": "nvidia/llama-nemotron-embed-vl-1b-v2",
+    "messages": [
+        {
+            "role": "document",
+            "content": [
+                {"type": "image_url", "image_url": {"url": "data:image/png;base64,<BASE64>"}},
+                {"type": "text", "text": "Describe the image."}
+            ]
+        }
+    ]
+}'
+```
+
+### Reranker Model
+
+Llama Nemotron VL reranker models combine the same bidirectional Llama + SigLIP
+backbone with a sequence-classification head for cross-encoder scoring and reranking.
+
+| Architecture | Backbone | Example HF Models |
+| - | - | - |
+| `LlamaNemotronVLForSequenceClassification` | Bidirectional Llama + SigLIP | `nvidia/llama-nemotron-rerank-vl-1b-v2` |
+
+Start the server:
+
+```shell
+vllm serve nvidia/llama-nemotron-rerank-vl-1b-v2 \
+    --runner pooling \
+    --trust-remote-code \
+    --chat-template examples/pooling/score/template/nemotron-vl-rerank.jinja
+```
+
+!!! note
+    The chat template bundled with this checkpoint's tokenizer is not suitable
+    for the Score/Rerank APIs. Use the provided override template when serving:
+    `examples/pooling/score/template/nemotron-vl-rerank.jinja`.
+
+Score a text query against an image document:
+
+```shell
+curl -s http://localhost:8000/score -H "Content-Type: application/json" -d '{
+    "model": "nvidia/llama-nemotron-rerank-vl-1b-v2",
+    "data_1": "Find diagrams about autonomous robots",
+    "data_2": [
+        {
+            "content": [
+                {"type": "image_url", "image_url": {"url": "data:image/png;base64,<BASE64>"}},
+                {"type": "text", "text": "Robotics workflow diagram."}
+            ]
+        }
+    ]
+}'
+```
+
+Rerank image documents by a text query:
+
+```shell
+curl -s http://localhost:8000/rerank -H "Content-Type: application/json" -d '{
+    "model": "nvidia/llama-nemotron-rerank-vl-1b-v2",
+    "query": "Find diagrams about autonomous robots",
+    "documents": [
+        {
+            "content": [
+                {"type": "image_url", "image_url": {"url": "data:image/png;base64,<BASE64_1>"}},
+                {"type": "text", "text": "Robotics workflow diagram."}
+            ]
+        },
+        {
+            "content": [
+                {"type": "image_url", "image_url": {"url": "data:image/png;base64,<BASE64_2>"}},
+                {"type": "text", "text": "General skyline photo."}
+            ]
+        }
+    ],
+    "top_n": 2
+}'
+```
+
+## BAAI/bge-m3
+
+The `BAAI/bge-m3` model comes with extra weights for sparse and colbert embeddings but unfortunately in its `config.json`
+the architecture is declared as `XLMRobertaModel`, which makes `vLLM` load it as a vanilla ROBERTA model without the
+extra weights. To load the full model weights, override its architecture like this:
+
+```shell
+vllm serve BAAI/bge-m3 --hf-overrides '{"architectures": ["BgeM3EmbeddingModel"]}'
+```
+
+Then you obtain the sparse embeddings like this:
+
+```shell
+curl -s http://localhost:8000/pooling -H "Content-Type: application/json" -d '{
+     "model": "BAAI/bge-m3",
+     "task": "token_classify",
+     "input": ["What is BGE M3?", "Definition of BM25"]
+}'
+```
+
+Due to limitations in the output schema, the output consists of a list of
+token scores for each token for each input. This means that you'll have to call
+`/tokenize` as well to be able to pair tokens with scores.
+Refer to the tests in  `tests/models/language/pooling/test_bge_m3.py` to see how
+to do that.
+
+You can obtain the colbert embeddings like this:
+
+```shell
+curl -s http://localhost:8000/pooling -H "Content-Type: application/json" -d '{
+     "model": "BAAI/bge-m3",
+     "task": "token_embed",
+     "input": ["What is BGE M3?", "Definition of BM25"]
+}'
+```
@@ -0,0 +1,89 @@
+# Token Classification Usages
+
+## Summary
+
+- Model Usage: token classification
+- Pooling Tasks: `token_classify`
+- Offline APIs:
+    - `LLM.encode(..., pooling_task="token_classify")`
+- Online APIs:
+    - Pooling API (`/pooling`)
+
+The key distinction between (sequence) classification and token classification lies in their output granularity: (sequence) classification produces a single result for an entire input sequence, whereas token classification yields a result for each individual token within the sequence.
+
+Many classification models support both (sequence) classification and token classification. For further details on (sequence) classification, please refer to [this page](classify.md).
+
+## Typical Use Cases
+
+### Named Entity Recognition (NER)
+
+For implementation examples, see:
+
+Offline: [examples/pooling/token_classify/ner_offline.py](../../../examples/pooling/token_classify/ner_offline.py)
+
+Online: [examples/pooling/token_classify/ner_online.py](../../../examples/pooling/token_classify/ner_online.py)
+
+### Sparse retrieval (lexical matching)
+
+The BAAI/bge-m3 model leverages token classification for sparse retrieval. For more information, see [this page](specific_models.md#baaibge-m3).
+
+## Supported Models
+
+| Architecture | Models | Example HF Models | [LoRA](../../features/lora.md) | [PP](../../serving/parallelism_scaling.md) |
+| ------------ | ------ | ----------------- | --------------------------- | --------------------------------------- |
+| `BertForTokenClassification` | bert-based | `boltuix/NeuroBERT-NER` (see note), etc. | | |
+| `ErnieForTokenClassification` | BERT-like Chinese ERNIE | `gyr66/Ernie-3.0-base-chinese-finetuned-ner` | | |
+| `ModernBertForTokenClassification` | ModernBERT-based | `disham993/electrical-ner-ModernBERT-base` | | |
+| `Qwen3ForTokenClassification`<sup>C</sup> | Qwen3-based | `bd2lcco/Qwen3-0.6B-finetuned` | | |
+| `*Model`<sup>C</sup>, `*ForCausalLM`<sup>C</sup>, etc. | Generative models | N/A | \* | \* |
+
+<sup>C</sup> Automatically converted into a classification model via `--convert classify`. ([details](./README.md#model-conversion))  
+\* Feature support is the same as that of the original model.
+
+If your model is not in the above list, we will try to automatically convert the model using
+[as_seq_cls_model][vllm.model_executor.models.adapters.as_seq_cls_model]. By default, the class probabilities are extracted from the softmaxed hidden state corresponding to the last token.
+
+### As Reward Models
+
+Using token classification models as reward models. For details on reward models, see [Reward Models](reward.md).
+
+--8<-- "docs/models/pooling_models/reward.md:supported-token-reward-models"
+
+## Offline Inference
+
+### Pooling Parameters
+
+The following [pooling parameters][vllm.PoolingParams] are supported.
+
+```python
+--8<-- "vllm/pooling_params.py:common-pooling-params"
+--8<-- "vllm/pooling_params.py:classify-pooling-params"
+```
+
+### `LLM.encode`
+
+The [encode][vllm.LLM.encode] method is available to all pooling models in vLLM.
+
+Set `pooling_task="token_classify"` when using `LLM.encode` for token classification Models:
+
+```python
+from vllm import LLM
+
+llm = LLM(model="boltuix/NeuroBERT-NER", runner="pooling")
+(output,) = llm.encode("Hello, my name is", pooling_task="token_classify")
+
+data = output.outputs.data
+print(f"Data: {data!r}")
+```
+
+## Online Serving
+
+Please refer to the [pooling API](README.md#pooling-api) and use `"task":"token_classify"`.
+
+## More examples
+
+More examples can be found here: [examples/pooling/token_classify](../../../examples/pooling/token_classify)
+
+## Supported Features
+
+Token classification features should be consistent with (sequence) classification. For more information, see [this page](classify.md#supported-features).
@@ -0,0 +1,126 @@
+# Token Embedding Usages
+
+## Summary
+
+- Model Usage: Token classification models
+- Pooling Tasks: `token_embed`
+- Offline APIs:
+    - `LLM.encode(..., pooling_task="token_embed")`
+- Online APIs:
+    - Pooling API (`/pooling`)
+
+The difference between the (sequence) embedding task and the token embedding task is that (sequence) embedding outputs one embedding for each sequence, while token embedding outputs a embedding for each token.
+
+Many embedding models support both (sequence) embedding and token embedding. For further details on (sequence) embedding, please refer to [this page](embed.md).
+
+## Typical Use Cases
+
+### Multi-Vector Retrieval
+
+For implementation examples, see:
+
+Offline: [examples/pooling/token_embed/multi_vector_retrieval_offline.py](../../../examples/pooling/token_embed/multi_vector_retrieval_offline.py)
+
+Online: [examples/pooling/token_embed/multi_vector_retrieval_online.py](../../../examples/pooling/token_embed/multi_vector_retrieval_online.py)
+
+### Late interaction
+
+Similarity scores can be computed using late interaction between two input prompts via the score API. For more information, see [Score API](scoring.md).
+
+### Extract last hidden states
+
+Models of any architecture can be converted into embedding models using `--convert embed`. Token embedding can then be used to extract the last hidden states from these models.
+
+## Supported Models
+
+--8<-- [start:supported-token-embed-models]
+
+### Text-only Models
+
+| Architecture | Models | Example HF Models | [LoRA](../../features/lora.md) | [PP](../../serving/parallelism_scaling.md) |
+| ------------ | ------ | ----------------- | -------------------- | ------------------------- |
+| `ColBERTLfm2Model` | LFM2 | `LiquidAI/LFM2-ColBERT-350M` | | |
+| `ColBERTModernBertModel` | ModernBERT | `lightonai/GTE-ModernColBERT-v1` | | |
+| `ColBERTJinaRobertaModel` | Jina XLM-RoBERTa | `jinaai/jina-colbert-v2` | | |
+| `HF_ColBERT` | BERT | `answerdotai/answerai-colbert-small-v1`, `colbert-ir/colbertv2.0` | | |
+| `*Model`<sup>C</sup>, `*ForCausalLM`<sup>C</sup>, etc. | Generative models | N/A | \* | \* |
+
+### Multimodal Models
+
+!!! note
+    For more information about multimodal models inputs, see [this page](../supported_models.md#list-of-multimodal-language-models).
+
+| Architecture | Models | Inputs | Example HF Models | [LoRA](../../features/lora.md) | [PP](../../serving/parallelism_scaling.md) |
+| ------------ | ------ | ----- | ----------------- | ------------------------------ | ------------------------------------------ |
+| `ColModernVBertForRetrieval` | ColModernVBERT | T / I | `ModernVBERT/colmodernvbert-merged` | | |
+| `ColPaliForRetrieval` | ColPali | T / I | `vidore/colpali-v1.3-hf` | | |
+| `ColQwen3` | Qwen3-VL | T / I | `TomoroAI/tomoro-colqwen3-embed-4b`, `TomoroAI/tomoro-colqwen3-embed-8b` | | |
+| `ColQwen3_5` | ColQwen3.5 | T + I + V | `athrael-soju/colqwen3.5-4.5B-v3` | | |
+| `OpsColQwen3Model` | Qwen3-VL | T / I | `OpenSearch-AI/Ops-Colqwen3-4B`, `OpenSearch-AI/Ops-Colqwen3-8B` | | |
+| `Qwen3VLNemotronEmbedModel` | Qwen3-VL | T / I | `nvidia/nemotron-colembed-vl-4b-v2`, `nvidia/nemotron-colembed-vl-8b-v2` | ✅︎ | ✅︎ |
+| `*ForConditionalGeneration`<sup>C</sup>, `*ForCausalLM`<sup>C</sup>, etc. | Generative models | \* | N/A | \* | \* |
+
+<sup>C</sup> Automatically converted into an embedding model via `--convert embed`. ([details](./README.md#model-conversion))  
+\* Feature support is the same as that of the original model.
+
+If your model is not in the above list, we will try to automatically convert the model using [as_embedding_model][vllm.model_executor.models.adapters.as_embedding_model].
+
+--8<-- [end:supported-token-embed-models]
+
+## Offline Inference
+
+### Pooling Parameters
+
+The following [pooling parameters][vllm.PoolingParams] are supported.
+
+```python
+--8<-- "vllm/pooling_params.py:common-pooling-params"
+--8<-- "vllm/pooling_params.py:embed-pooling-params"
+```
+
+### `LLM.encode`
+
+The [encode][vllm.LLM.encode] method is available to all pooling models in vLLM.
+
+Set `pooling_task="token_embed"` when using `LLM.encode` for token embedding Models:
+
+```python
+from vllm import LLM
+
+llm = LLM(model="answerdotai/answerai-colbert-small-v1", runner="pooling")
+(output,) = llm.encode("Hello, my name is", pooling_task="token_embed")
+
+data = output.outputs.data
+print(f"Data: {data!r}")
+```
+
+### `LLM.score`
+
+The [score][vllm.LLM.score] method outputs similarity scores between sentence pairs.
+
+All models that support token embedding task also support using the score API to compute similarity scores by calculating the late interaction of two input prompts.
+
+```python
+from vllm import LLM
+
+llm = LLM(model="answerdotai/answerai-colbert-small-v1", runner="pooling")
+(output,) = llm.score(
+    "What is the capital of France?",
+    "The capital of Brazil is Brasilia.",
+)
+
+score = output.outputs.score
+print(f"Score: {score}")
+```
+
+## Online Serving
+
+Please refer to the [pooling API](README.md#pooling-api) and use `"task":"token_embed"`.
+
+## More examples
+
+More examples can be found here: [examples/pooling/token_embed](../../../examples/pooling/token_embed)
+
+## Supported Features
+
+Token embedding features should be consistent with (sequence) embedding. For more information, see [this page](embed.md#supported-features).
@@ -1,6 +1,6 @@
 # Supported Models

-vLLM supports [generative](./generative_models.md) and [pooling](./pooling_models.md) models across various tasks.
+vLLM supports [generative](./generative_models.md) and [pooling](./pooling_models/README.md) models across various tasks.

 For each task, we list the model architectures that have been implemented in vLLM.
 Alongside each architecture, we include some popular models that use it.
@@ -499,156 +499,6 @@ Some models are supported only via the [Transformers modeling backend](#transfor
 !!! note
    Currently, the ROCm version of vLLM supports Mistral and Mixtral only for context lengths up to 4096.

-### Pooling Models
-
-See [this page](./pooling_models.md) for more information on how to use pooling models.
-
-!!! important
-    Since some model architectures support both generative and pooling tasks,
-    you should explicitly specify `--runner pooling` to ensure that the model is used in pooling mode instead of generative mode.
-
-#### Embedding
-
-These models primarily support the [`LLM.embed`](./pooling_models.md#llmembed) API.
-
-| Architecture | Models | Example HF Models | [LoRA](../features/lora.md) | [PP](../serving/parallelism_scaling.md) |
-| ------------ | ------ | ----------------- | -------------------- | ------------------------- |
-| `BertModel`<sup>C</sup> | BERT-based | `BAAI/bge-base-en-v1.5`, `Snowflake/snowflake-arctic-embed-xs`, etc. | | |
-| `BertSpladeSparseEmbeddingModel` | SPLADE | `naver/splade-v3` | | |
-| `ErnieModel` | BERT-like Chinese ERNIE | `shibing624/text2vec-base-chinese-sentence` | | |
-| `Gemma2Model`<sup>C</sup> | Gemma 2-based | `BAAI/bge-multilingual-gemma2`, etc. | ✅︎ | ✅︎ |
-| `Gemma3TextModel`<sup>C</sup> | Gemma 3-based | `google/embeddinggemma-300m`, etc. | ✅︎ | ✅︎ |
-| `GritLM` | GritLM | `parasail-ai/GritLM-7B-vllm`. | ✅︎ | ✅︎ |
-| `GteModel`<sup>C</sup> | Arctic-Embed-2.0-M | `Snowflake/snowflake-arctic-embed-m-v2.0`. | | |
-| `GteNewModel`<sup>C</sup> | mGTE-TRM (see note) | `Alibaba-NLP/gte-multilingual-base`, etc. | | |
-| `ModernBertModel`<sup>C</sup> | ModernBERT-based | `Alibaba-NLP/gte-modernbert-base`, etc. | | |
-| `NomicBertModel`<sup>C</sup> | Nomic BERT | `nomic-ai/nomic-embed-text-v1`, `nomic-ai/nomic-embed-text-v2-moe`, `Snowflake/snowflake-arctic-embed-m-long`, etc. | | |
-| `LlamaBidirectionalModel`<sup>C</sup> | Llama-based with bidirectional attention | `nvidia/llama-nemotron-embed-1b-v2`, etc. | ✅︎ | ✅︎ |
-| `LlamaModel`<sup>C</sup>, `LlamaForCausalLM`<sup>C</sup>, `MistralModel`<sup>C</sup>, etc. | Llama-based | `intfloat/e5-mistral-7b-instruct`, etc. | ✅︎ | ✅︎ |
-| `Qwen2Model`<sup>C</sup>, `Qwen2ForCausalLM`<sup>C</sup> | Qwen2-based | `ssmits/Qwen2-7B-Instruct-embed-base` (see note), `Alibaba-NLP/gte-Qwen2-7B-instruct` (see note), etc. | ✅︎ | ✅︎ |
-| `Qwen3Model`<sup>C</sup>, `Qwen3ForCausalLM`<sup>C</sup> | Qwen3-based | `Qwen/Qwen3-Embedding-0.6B`, etc. | ✅︎ | ✅︎ |
-| `VoyageQwen3BidirectionalEmbedModel`<sup>C</sup> | Voyage Qwen3-based with bidirectional attention | `voyageai/voyage-4-nano`, etc. | ✅︎ | ✅︎ |
-| `RobertaModel`, `RobertaForMaskedLM` | RoBERTa-based | `sentence-transformers/all-roberta-large-v1`, etc. | | |
-| `*Model`<sup>C</sup>, `*ForCausalLM`<sup>C</sup>, etc. | Generative models | N/A | \* | \* |
-
-<sup>C</sup> Automatically converted into an embedding model via `--convert embed`. ([details](./pooling_models.md#model-conversion))  
-\* Feature support is the same as that of the original model.
-
-!!! note
-    `ssmits/Qwen2-7B-Instruct-embed-base` has an improperly defined Sentence Transformers config.
-    You need to manually set mean pooling by passing `--pooler-config '{"pooling_type": "MEAN"}'`.
-
-!!! note
-    For `Alibaba-NLP/gte-Qwen2-*`, you need to enable `--trust-remote-code` for the correct tokenizer to be loaded.
-    See [relevant issue on HF Transformers](https://github.com/huggingface/transformers/issues/34882).
-
-!!! note
-    `jinaai/jina-embeddings-v3` supports multiple tasks through LoRA, while vllm temporarily only supports text-matching tasks by merging LoRA weights.
-
-!!! note
-    The second-generation GTE model (mGTE-TRM) is named `NewModel`. The name `NewModel` is too generic, you should set `--hf-overrides '{"architectures": ["GteNewModel"]}'` to specify the use of the `GteNewModel` architecture.
-
-If your model is not in the above list, we will try to automatically convert the model using
-[as_embedding_model][vllm.model_executor.models.adapters.as_embedding_model]. By default, the embeddings
-of the whole prompt are extracted from the normalized hidden state corresponding to the last token.
-
-#### Classification
-
-These models primarily support the [`LLM.classify`](./pooling_models.md#llmclassify) API.
-
-| Architecture | Models | Example HF Models | [LoRA](../features/lora.md) | [PP](../serving/parallelism_scaling.md) |
-| ------------ | ------ | ----------------- | -------------------- | ------------------------- |
-| `ErnieForSequenceClassification` | BERT-like Chinese ERNIE | `Forrest20231206/ernie-3.0-base-zh-cls` | | |
-| `GPT2ForSequenceClassification` | GPT2 | `nie3e/sentiment-polish-gpt2-small` | | |
-| `JambaForSequenceClassification` | Jamba | `ai21labs/Jamba-tiny-reward-dev`, etc. | ✅︎ | ✅︎ |
-| `*Model`<sup>C</sup>, `*ForCausalLM`<sup>C</sup>, etc. | Generative models | N/A | \* | \* |
-
-<sup>C</sup> Automatically converted into a classification model via `--convert classify`. ([details](./pooling_models.md#model-conversion))  
-\* Feature support is the same as that of the original model.
-
-If your model is not in the above list, we will try to automatically convert the model using
-[as_seq_cls_model][vllm.model_executor.models.adapters.as_seq_cls_model]. By default, the class probabilities are extracted from the softmaxed hidden state corresponding to the last token.
-
-#### Cross-encoder / Reranker
-
-Cross-encoder and reranker models are a subset of classification models that accept two prompts as input.
-These models primarily support the [`LLM.score`](./pooling_models.md#llmscore) API.
-
-| Architecture | Models | Example HF Models | Score template (see note) | [LoRA](../features/lora.md) | [PP](../serving/parallelism_scaling.md) |
-| ------------ | ------ | ----------------- | ------------------------- | --------------------------- | --------------------------------------- |
-| `BertForSequenceClassification` | BERT-based | `cross-encoder/ms-marco-MiniLM-L-6-v2`, etc. | N/A | | |
-| `ErnieForSequenceClassification` | BERT-like Chinese ERNIE | `Forrest20231206/ernie-3.0-base-zh-cls` | N/A | | |
-| `GemmaForSequenceClassification` | Gemma-based | `BAAI/bge-reranker-v2-gemma`(see note), etc. | [bge-reranker-v2-gemma.jinja](../../examples/pooling/score/template/bge-reranker-v2-gemma.jinja) | ✅︎ | ✅︎ |
-| `GteNewForSequenceClassification` | mGTE-TRM (see note) | `Alibaba-NLP/gte-multilingual-reranker-base`, etc. | N/A | | |
-| `LlamaBidirectionalForSequenceClassification`<sup>C</sup> | Llama-based with bidirectional attention | `nvidia/llama-nemotron-rerank-1b-v2`, etc. | [nemotron-rerank.jinja](../../examples/pooling/score/template/nemotron-rerank.jinja) | ✅︎ | ✅︎ |
-| `Qwen2ForSequenceClassification`<sup>C</sup> | Qwen2-based | `mixedbread-ai/mxbai-rerank-base-v2`(see note), etc. | [mxbai_rerank_v2.jinja](../../examples/pooling/score/template/mxbai_rerank_v2.jinja) | ✅︎ | ✅︎ |
-| `Qwen3ForSequenceClassification`<sup>C</sup> | Qwen3-based | `tomaarsen/Qwen3-Reranker-0.6B-seq-cls`, `Qwen/Qwen3-Reranker-0.6B`(see note), etc. | [qwen3_reranker.jinja](../../examples/pooling/score/template/qwen3_reranker.jinja) | ✅︎ | ✅︎ |
-| `RobertaForSequenceClassification` | RoBERTa-based | `cross-encoder/quora-roberta-base`, etc. | N/A | | |
-| `XLMRobertaForSequenceClassification` | XLM-RoBERTa-based | `BAAI/bge-reranker-v2-m3`, etc. | N/A | | |
-| `*Model`<sup>C</sup>, `*ForCausalLM`<sup>C</sup>, etc. | Generative models | N/A | N/A | \* | \* |
-
-<sup>C</sup> Automatically converted into a classification model via `--convert classify`. ([details](./pooling_models.md#model-conversion))  
-\* Feature support is the same as that of the original model.
-
-!!! note
-    Some models require a specific prompt format to work correctly.
-
-    You can find Example HF Models's corresponding score template in [examples/pooling/score/template/](../../examples/pooling/score/template)
-
-    Examples : [examples/pooling/score/using_template_offline.py](../../examples/pooling/score/using_template_offline.py) [examples/pooling/score/using_template_online.py](../../examples/pooling/score/using_template_online.py)
-
-!!! note
-    Load the official original `BAAI/bge-reranker-v2-gemma` by using the following command.
-
-    ```bash
-    vllm serve BAAI/bge-reranker-v2-gemma --hf_overrides '{"architectures": ["GemmaForSequenceClassification"],"classifier_from_token": ["Yes"],"method": "no_post_processing"}'
-    ```
-
-!!! note
-    The second-generation GTE model (mGTE-TRM) is named `NewForSequenceClassification`. The name `NewForSequenceClassification` is too generic, you should set `--hf-overrides '{"architectures": ["GteNewForSequenceClassification"]}'` to specify the use of the `GteNewForSequenceClassification` architecture.
-
-!!! note
-    Load the official original `mxbai-rerank-v2` by using the following command.
-
-    ```bash
-    vllm serve mixedbread-ai/mxbai-rerank-base-v2 --hf_overrides '{"architectures": ["Qwen2ForSequenceClassification"],"classifier_from_token": ["0", "1"], "method": "from_2_way_softmax"}'
-    ```
-
-!!! note
-    Load the official original `Qwen3 Reranker` by using the following command. More information can be found at: [examples/pooling/score/qwen3_reranker_offline.py](../../examples/pooling/score/qwen3_reranker_offline.py) [examples/pooling/score/qwen3_reranker_online.py](../../examples/pooling/score/qwen3_reranker_online.py).
-
-    ```bash
-    vllm serve Qwen/Qwen3-Reranker-0.6B --hf_overrides '{"architectures": ["Qwen3ForSequenceClassification"],"classifier_from_token": ["no", "yes"],"is_original_qwen3_reranker": true}'
-    ```
-
-#### Reward Modeling
-
-These models primarily support the [`LLM.reward`](./pooling_models.md#llmreward) API.
-
-| Architecture | Models | Example HF Models | [LoRA](../features/lora.md) | [PP](../serving/parallelism_scaling.md) |
-| ------------ | ------ | ----------------- | -------------------- | ------------------------- |
-| `InternLM2ForRewardModel` | InternLM2-based | `internlm/internlm2-1_8b-reward`, `internlm/internlm2-7b-reward`, etc. | ✅︎ | ✅︎ |
-| `LlamaForCausalLM` | Llama-based | `peiyi9979/math-shepherd-mistral-7b-prm`, etc. | ✅︎ | ✅︎ |
-| `Qwen2ForRewardModel` | Qwen2-based | `Qwen/Qwen2.5-Math-RM-72B`, etc. | ✅︎ | ✅︎ |
-| `Qwen2ForProcessRewardModel` | Qwen2-based | `Qwen/Qwen2.5-Math-PRM-7B`, etc. | ✅︎ | ✅︎ |
-
-!!! important
-    For process-supervised reward models such as `peiyi9979/math-shepherd-mistral-7b-prm`, the pooling config should be set explicitly,
-    e.g.: `--pooler-config '{"pooling_type": "STEP", "step_tag_id": 123, "returned_token_ids": [456, 789]}'`.
-
-#### Token Classification
-
-These models primarily support the [`LLM.encode`](./pooling_models.md#llmencode) API.
-
-| Architecture | Models | Example HF Models | [LoRA](../features/lora.md) | [PP](../serving/parallelism_scaling.md) |
-| ------------ | ------ | ----------------- | --------------------------- | --------------------------------------- |
-| `BertForTokenClassification` | bert-based | `boltuix/NeuroBERT-NER` (see note), etc. | | |
-| `ErnieForTokenClassification` | BERT-like Chinese ERNIE | `gyr66/Ernie-3.0-base-chinese-finetuned-ner` | | |
-| `ModernBertForTokenClassification` | ModernBERT-based | `disham993/electrical-ner-ModernBERT-base` | | |
-
-!!! note
-    Named Entity Recognition (NER) usage, please refer to [examples/pooling/token_classify/ner_offline.py](../../examples/pooling/token_classify/ner_offline.py), [examples/pooling/token_classify/ner_online.py](../../examples/pooling/token_classify/ner_online.py).
-
 ## List of Multimodal Language Models

 The following modalities are supported depending on the model:
@@ -685,7 +535,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
 | Architecture | Models | Inputs | Example HF Models | [LoRA](../features/lora.md) | [PP](../serving/parallelism_scaling.md) |
 | ------------ | ------ | ------ | ----------------- | -------------------- | ------------------------- |
 | `AriaForConditionalGeneration` | Aria | T + I<sup>+</sup> | `rhymes-ai/Aria` | | |
-| `AudioFlamingo3ForConditionalGeneration` | AudioFlamingo3 | T + A | `nvidia/audio-flamingo-3-hf`, `nvidia/music-flamingo-2601-hf` | ✅︎ | ✅︎ |
+| `AudioFlamingo3ForConditionalGeneration` | AudioFlamingo3 | T + A | `nvidia/audio-flamingo-3-hf`, `nvidia/music-flamingo-hf` | ✅︎ | ✅︎ |
 | `AyaVisionForConditionalGeneration` | Aya Vision | T + I<sup>+</sup> | `CohereLabs/aya-vision-8b`, `CohereLabs/aya-vision-32b`, etc. | | ✅︎ |
 | `BagelForConditionalGeneration` | BAGEL | T + I<sup>+</sup> | `ByteDance-Seed/BAGEL-7B-MoT` | ✅︎ | ✅︎ |
 | `BeeForConditionalGeneration` | Bee-8B | T + I<sup>E+</sup> | `Open-Bee/Bee-8B-RL`, `Open-Bee/Bee-8B-SFT` | | ✅︎ |
@@ -736,6 +586,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
 | `Mistral3ForConditionalGeneration` | Mistral3 (HF Transformers) | T + I<sup>+</sup> | `mistralai/Mistral-Small-3.1-24B-Instruct-2503`, etc. | ✅︎ | ✅︎ |
 | `MolmoForCausalLM` | Molmo | T + I<sup>+</sup> | `allenai/Molmo-7B-D-0924`, `allenai/Molmo-7B-O-0924`, etc. | ✅︎ | ✅︎ |
 | `Molmo2ForConditionalGeneration` | Molmo2 | T + I<sup>+</sup> / V | `allenai/Molmo2-4B`, `allenai/Molmo2-8B`, `allenai/Molmo2-O-7B` | ✅︎ | ✅︎ |
+| `MusicFlamingoForConditionalGeneration` | MusicFlamingo | T + A | `nvidia/music-flamingo-2601-hf`, `nvidia/music-flamingo-think-2601-hf` | ✅︎ | ✅︎ |
 | `NVLM_D_Model` | NVLM-D 1.0 | T + I<sup>+</sup> | `nvidia/NVLM-D-72B`, etc. | | ✅︎ |
 | `OpenCUAForConditionalGeneration` | OpenCUA-7B | T + I<sup>E+</sup> | `xlangai/OpenCUA-7B` | ✅︎ | ✅︎ |
 | `OpenPanguVLForConditionalGeneration` | openpangu-VL | T + I<sup>E+</sup> + V<sup>E+</sup> | `FreedomIntelligence/openPangu-VL-7B` | ✅︎ | ✅︎ |
@@ -816,57 +667,23 @@ Speech2Text models trained specifically for Automatic Speech Recognition.
 !!! note
    `VoxtralForConditionalGeneration` requires `mistral-common[audio]` to be installed.

-### Pooling Models
+## Pooling Models

-See [this page](./pooling_models.md) for more information on how to use pooling models.
+See [this page](pooling_models/README.md) for more information on how to use pooling models.

-#### Embedding
+!!! important
+    Since some model architectures support both generative and pooling tasks,
+    you should explicitly specify `--runner pooling` to ensure that the model is used in pooling mode instead of generative mode.

-These models primarily support the [`LLM.embed`](./pooling_models.md#llmembed) API.
+See the link below for more information on the models supported for specific pooling tasks.

-!!! note
-    To get the best results, you should use pooling models that are specifically trained as such.
-
-The following table lists those that are tested in vLLM.
-
-| Architecture | Models | Inputs | Example HF Models | [LoRA](../features/lora.md) | [PP](../serving/parallelism_scaling.md) |
-| ------------ | ------ | ------ | ----------------- | -------------------- | ------------------------- |
-| `CLIPModel` | CLIP | T / I | `openai/clip-vit-base-patch32`, `openai/clip-vit-large-patch14`, etc. | | |
-| `ColModernVBertForRetrieval` | ColModernVBERT | T / I | `ModernVBERT/colmodernvbert-merged` | | |
-| `ColPaliForRetrieval` | ColPali | T / I | `vidore/colpali-v1.3-hf` | | |
-| `ColQwen3_5` | ColQwen3.5 | T + I + V | `athrael-soju/colqwen3.5-4.5B-v3` | | |
-| `LlamaNemotronVLModel` | Llama Nemotron Embedding + SigLIP | T + I | `nvidia/llama-nemotron-embed-vl-1b-v2` | | |
-| `LlavaNextForConditionalGeneration`<sup>C</sup> | LLaVA-NeXT-based | T / I | `royokong/e5-v` | | ✅︎ |
-| `Phi3VForCausalLM`<sup>C</sup> | Phi-3-Vision-based | T + I | `TIGER-Lab/VLM2Vec-Full` | | ✅︎ |
-| `Qwen3VLForConditionalGeneration`<sup>C</sup> | Qwen3-VL | T + I + V | `Qwen/Qwen3-VL-Embedding-2B`, etc. | ✅︎ | ✅︎ |
-| `SiglipModel` | SigLIP, SigLIP2 | T / I | `google/siglip-base-patch16-224`, `google/siglip2-base-patch16-224` | | |
-| `*ForConditionalGeneration`<sup>C</sup>, `*ForCausalLM`<sup>C</sup>, etc. | Generative models | \* | N/A | \* | \* |
-
-<sup>C</sup> Automatically converted into an embedding model via `--convert embed`. ([details](./pooling_models.md#model-conversion))  
-\* Feature support is the same as that of the original model.
-
---
-
-#### Cross-encoder / Reranker
-
-Cross-encoder and reranker models are a subset of classification models that accept two prompts as input.
-These models primarily support the [`LLM.score`](./pooling_models.md#llmscore) API.
-
-| Architecture | Models | Inputs | Example HF Models | [LoRA](../features/lora.md) | [PP](../serving/parallelism_scaling.md) |
-| ------------ | ------ | ------ | ----------------- | -------------------- | ------------------------- |
-| `JinaVLForSequenceClassification` | JinaVL-based | T + I<sup>E+</sup> | `jinaai/jina-reranker-m0`, etc. | ✅︎ | ✅︎ |
-| `LlamaNemotronVLForSequenceClassification` | Llama Nemotron Reranker + SigLIP | T + I<sup>E+</sup> | `nvidia/llama-nemotron-rerank-vl-1b-v2` | | |
-| `Qwen3VLForSequenceClassification` | Qwen3-VL-Reranker | T + I<sup>E+</sup> + V<sup>E+</sup> | `Qwen/Qwen3-VL-Reranker-2B`(see note), etc. | ✅︎ | ✅︎ |
-
-<sup>C</sup> Automatically converted into a classification model via `--convert classify`. ([details](./pooling_models.md#model-conversion))  
-\* Feature support is the same as that of the original model.
-
-!!! note
-    Similar to Qwen3-Reranker, you need to use the following `--hf_overrides` to load the official original `Qwen3-VL-Reranker`.
-
-    ```bash
-    vllm serve Qwen/Qwen3-VL-Reranker-2B --hf_overrides '{"architectures": ["Qwen3VLForSequenceClassification"],"classifier_from_token": ["no", "yes"],"is_original_qwen3_reranker": true}'
-    ```
+- [Classification Usages](pooling_models/classify.md)
+- [Embedding Usages](pooling_models/embed.md)
+- [Reward Usages](pooling_models/reward.md)
+- [Token Classification Usages](pooling_models/token_classify.md)
+- [Token Embedding Usages](pooling_models/token_embed.md)
+- [Scoring Usages](pooling_models/scoring.md)
+- [Specific Model Examples](pooling_models/specific_models.md)

 ## Model Support Policy

@@ -23,7 +23,6 @@ vLLM provides multiple communication backends for EP. Use `--all2all-backend` to
 | `deepep_low_latency` | Multi-node decode | CUDA graph support, masked layout, optimized for decode | Decode-dominated workloads, low-latency scenarios |
 | `flashinfer_nvlink_one_sided` | MNNVL systems | FlashInfer's one-sided A2A strategy for multi-node NVLink | High-throughput workloads |
 | `flashinfer_nvlink_two_sided` | MNNVL systems | FlashInfer's two-sided A2A strategy for multi-node NVLink | Systems with NVLink across nodes |
-| `naive` | Testing/debugging | Simple broadcast-based implementation | Debugging, not recommended for production |

 ## Single Node Deployment

@@ -16,7 +16,7 @@ After initializing the `LLM` instance, use the available APIs to perform model i
 The available APIs depend on the model type:

 - [Generative models](../models/generative_models.md) output logprobs which are sampled from to obtain the final output text.
- [Pooling models](../models/pooling_models.md) output their hidden states directly.
+- [Pooling models](../models/pooling_models/README.md) output their hidden states directly.

 !!! info
    [API Reference](../api/README.md#offline-inference)
@@ -53,8 +53,8 @@ We currently support the following OpenAI APIs:
    - Only applicable to [text generation models](../models/generative_models.md) with a [chat template](../serving/openai_compatible_server.md#chat-template).
    - *Note: `user` parameter is ignored.*
    - *Note:* Setting the `parallel_tool_calls` parameter to `false` ensures vLLM only returns zero or one tool call per request. Setting it to `true` (the default) allows returning more than one tool call per request. There is no guarantee more than one tool call will be returned if this is set to `true`, as that behavior is model dependent and not all models are designed to support parallel tool calls.
- [Embeddings API](#embeddings-api) (`/v1/embeddings`)
-    - Only applicable to [embedding models](../models/pooling_models.md).
+- [Embeddings API](../models/pooling_models/embed.md#openai-compatible-embeddings-api) (`/v1/embeddings`)
+    - Only applicable to [embedding models](../models/pooling_models/embed.md).
 - [Transcriptions API](#transcriptions-api) (`/v1/audio/transcriptions`)
    - Only applicable to [Automatic Speech Recognition (ASR) models](../models/supported_models.md#transcription).
 - [Translation API](#translations-api) (`/v1/audio/translations`)
@@ -66,20 +66,19 @@ In addition, we have the following custom APIs:

 - [Tokenizer API](#tokenizer-api) (`/tokenize`, `/detokenize`)
    - Applicable to any model with a tokenizer.
- [Pooling API](#pooling-api) (`/pooling`)
-    - Applicable to all [pooling models](../models/pooling_models.md).
- [Classification API](#classification-api) (`/classify`)
-    - Only applicable to [classification models](../models/pooling_models.md).
- [Score API](#score-api) (`/score`)
-    - Applicable to [embedding models and cross-encoder models](../models/pooling_models.md).
- [Cohere Embed API](#cohere-embed-api) (`/v2/embed`)
+- [pooling API](../models/pooling_models/README.md#pooling-api) (`/pooling`)
+    - Applicable to all [pooling models](../models/pooling_models/README.md).
+- [Classification API](../models/pooling_models/classify.md#classification-api) (`/classify`)
+    - Only applicable to [classification models](../models/pooling_models/classify.md).
+- [Cohere Embed API](../models/pooling_models/embed.md#cohere-embed-api) (`/v2/embed`)
    - Compatible with [Cohere's Embed API](https://docs.cohere.com/reference/embed)
-    - Works with any [embedding model](../models/pooling_models.md), including multimodal models.
- [Re-rank API](#re-rank-api) (`/rerank`, `/v1/rerank`, `/v2/rerank`)
-    - Implements [Jina AI's v1 re-rank API](https://jina.ai/reranker/)
-    - Also compatible with [Cohere's v1 & v2 re-rank APIs](https://docs.cohere.com/v2/reference/rerank)
+    - Works with any [embedding model](../models/pooling_models/embed.md#supported-models), including multimodal models.
+- [Score API](../models/pooling_models/scoring.md#score-api) (`/score`)
+    - Applicable to [score models](../models/pooling_models/scoring.md).
+- [Rerank API](../models/pooling_models/scoring.md#rerank-api) (`/rerank`, `/v1/rerank`, `/v2/rerank`)
+    - Implements [Jina AI's v1 rerank API](https://jina.ai/reranker/)
+    - Also compatible with [Cohere's v1 & v2 rerank APIs](https://docs.cohere.com/v2/reference/rerank)
    - Jina and Cohere's APIs are very similar; Jina's includes extra information in the rerank endpoint's response.
-    - Only applicable to [cross-encoder models](../models/pooling_models.md).

 ## Chat Template

@@ -269,300 +268,6 @@ The following extra parameters in the response object are supported:
    --8<-- "vllm/entrypoints/openai/responses/protocol.py:responses-response-extra-params"
    ```

-### Embeddings API
-
-Our Embeddings API is compatible with [OpenAI's Embeddings API](https://platform.openai.com/docs/api-reference/embeddings);
-you can use the [official OpenAI Python client](https://github.com/openai/openai-python) to interact with it.
-
-Code example: [examples/pooling/embed/openai_embedding_client.py](../../examples/pooling/embed/openai_embedding_client.py)
-
-If the model has a [chat template](../serving/openai_compatible_server.md#chat-template), you can replace `inputs` with a list of `messages` (same schema as [Chat API](#chat-api))
-which will be treated as a single prompt to the model. Here is a convenience function for calling the API while retaining OpenAI's type annotations:
-
-??? code
-
-    ```python
-    from openai import OpenAI
-    from openai._types import NOT_GIVEN, NotGiven
-    from openai.types.chat import ChatCompletionMessageParam
-    from openai.types.create_embedding_response import CreateEmbeddingResponse
-
-    def create_chat_embeddings(
-        client: OpenAI,
-        *,
-        messages: list[ChatCompletionMessageParam],
-        model: str,
-        encoding_format: Union[Literal["base64", "float"], NotGiven] = NOT_GIVEN,
-    ) -> CreateEmbeddingResponse:
-        return client.post(
-            "/embeddings",
-            cast_to=CreateEmbeddingResponse,
-            body={"messages": messages, "model": model, "encoding_format": encoding_format},
-        )
-    ```
-
-#### Multi-modal inputs
-
-You can pass multi-modal inputs to embedding models by defining a custom chat template for the server
-and passing a list of `messages` in the request. Refer to the examples below for illustration.
-
-=== "VLM2Vec"
-
-    To serve the model:
-
-    ```bash
-    vllm serve TIGER-Lab/VLM2Vec-Full --runner pooling \
-      --trust-remote-code \
-      --max-model-len 4096 \
-      --chat-template examples/pooling/embed/template/vlm2vec_phi3v.jinja
-    ```
-
-    !!! important
-        Since VLM2Vec has the same model architecture as Phi-3.5-Vision, we have to explicitly pass `--runner pooling`
-        to run this model in embedding mode instead of text generation mode.
-
-        The custom chat template is completely different from the original one for this model,
-        and can be found here: [examples/pooling/embed/template/vlm2vec_phi3v.jinja](../../examples/pooling/embed/template/vlm2vec_phi3v.jinja)
-
-    Since the request schema is not defined by OpenAI client, we post a request to the server using the lower-level `requests` library:
-
-    ??? code
-
-        ```python
-        from openai import OpenAI
-        client = OpenAI(
-            base_url="http://localhost:8000/v1",
-            api_key="EMPTY",
-        )
-        image_url = "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
-
-        response = create_chat_embeddings(
-            client,
-            model="TIGER-Lab/VLM2Vec-Full",
-            messages=[
-                {
-                    "role": "user",
-                    "content": [
-                        {"type": "image_url", "image_url": {"url": image_url}},
-                        {"type": "text", "text": "Represent the given image."},
-                    ],
-                }
-            ],
-            encoding_format="float",
-        )
-
-        print("Image embedding output:", response.data[0].embedding)
-        ```
-
-=== "DSE-Qwen2-MRL"
-
-    To serve the model:
-
-    ```bash
-    vllm serve MrLight/dse-qwen2-2b-mrl-v1 --runner pooling \
-      --trust-remote-code \
-      --max-model-len 8192 \
-      --chat-template examples/pooling/embed/template/dse_qwen2_vl.jinja
-    ```
-
-    !!! important
-        Like with VLM2Vec, we have to explicitly pass `--runner pooling`.
-
-        Additionally, `MrLight/dse-qwen2-2b-mrl-v1` requires an EOS token for embeddings, which is handled
-        by a custom chat template: [examples/pooling/embed/template/dse_qwen2_vl.jinja](../../examples/pooling/embed/template/dse_qwen2_vl.jinja)
-
-    !!! important
-        `MrLight/dse-qwen2-2b-mrl-v1` requires a placeholder image of the minimum image size for text query embeddings. See the full code
-        example below for details.
-
-Full example: [examples/pooling/embed/vision_embedding_online.py](../../examples/pooling/embed/vision_embedding_online.py)
-
-#### Extra parameters
-
-The following [pooling parameters][vllm.PoolingParams] are supported.
-
-```python
--8<-- "vllm/pooling_params.py:common-pooling-params"
--8<-- "vllm/pooling_params.py:embed-pooling-params"
-```
-
-The following Embeddings API parameters are supported:
-
-??? code
-
-    ```python
-    --8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-params"
-    --8<-- "vllm/entrypoints/pooling/base/protocol.py:completion-params"
-    --8<-- "vllm/entrypoints/pooling/base/protocol.py:encoding-params"
-    --8<-- "vllm/entrypoints/pooling/base/protocol.py:embed-params"
-    ```
-
-The following extra parameters are supported:
-
-??? code
-
-    ```python
-    --8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-extra-params"
-    --8<-- "vllm/entrypoints/pooling/base/protocol.py:completion-extra-params"
-    --8<-- "vllm/entrypoints/pooling/base/protocol.py:encoding-extra-params"
-    --8<-- "vllm/entrypoints/pooling/base/protocol.py:embed-extra-params"
-    ```
-
-For chat-like input (i.e. if `messages` is passed), the following parameters are supported:
-
-The following parameters are supported by default:
-
-??? code
-
-    ```python
-    --8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-params"
-    --8<-- "vllm/entrypoints/pooling/base/protocol.py:chat-params"
-    --8<-- "vllm/entrypoints/pooling/base/protocol.py:encoding-params"
-    --8<-- "vllm/entrypoints/pooling/base/protocol.py:embed-params"
-    ```
-
-these extra parameters are supported instead:
-
-??? code
-
-    ```python
-    --8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-extra-params"
-    --8<-- "vllm/entrypoints/pooling/base/protocol.py:chat-extra-params"
-    --8<-- "vllm/entrypoints/pooling/base/protocol.py:encoding-extra-params"
-    --8<-- "vllm/entrypoints/pooling/base/protocol.py:embed-extra-params"
-    ```
-
-### Cohere Embed API
-
-Our API is also compatible with [Cohere's Embed v2 API](https://docs.cohere.com/reference/embed) which adds support for some modern embedding feature such as truncation, output dimensions, embedding types, and input types. This endpoint works with any embedding model (including multimodal models).
-
-#### Cohere Embed API request parameters
-
-| Parameter | Type | Required | Description |
-| --------- | ---- | -------- | ----------- |
-| `model` | string | Yes | Model name |
-| `input_type` | string | No | Prompt prefix key (model-dependent, see below) |
-| `texts` | list[string] | No | Text inputs (use one of `texts`, `images`, or `inputs`) |
-| `images` | list[string] | No | Base64 data URI images |
-| `inputs` | list[object] | No | Mixed text and image content objects |
-| `embedding_types` | list[string] | No | Output types (default: `["float"]`) |
-| `output_dimension` | int | No | Truncate embeddings to this dimension (Matryoshka) |
-| `truncate` | string | No | `END`, `START`, or `NONE` (default: `END`) |
-
-#### Text embedding
-
-```bash
-curl -X POST "http://localhost:8000/v2/embed" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "model": "Snowflake/snowflake-arctic-embed-m-v1.5",
-    "input_type": "query",
-    "texts": ["Hello world", "How are you?"],
-    "embedding_types": ["float"]
-  }'
-```
-
-??? console "Response"
-
-    ```json
-    {
-      "id": "embd-...",
-      "embeddings": {
-        "float": [
-          [0.012, -0.034, ...],
-          [0.056, 0.078, ...]
-        ]
-      },
-      "texts": ["Hello world", "How are you?"],
-      "meta": {
-        "api_version": {"version": "2"},
-        "billed_units": {"input_tokens": 12}
-      }
-    }
-    ```
-
-#### Mixed text and image inputs
-
-For multimodal models, you can embed images by passing base64 data URIs. The `inputs` field accepts a list of objects with mixed text and image content:
-
-```bash
-curl -X POST "http://localhost:8000/v2/embed" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "model": "google/siglip-so400m-patch14-384",
-    "inputs": [
-      {
-        "content": [
-          {"type": "text", "text": "A photo of a cat"},
-          {"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBOR..."}}
-        ]
-      }
-    ],
-    "embedding_types": ["float"]
-  }'
-```
-
-#### Embedding types
-
-The `embedding_types` parameter controls the output format. Multiple types can be requested in a single call:
-
-| Type | Description |
-| ---- | ----------- |
-| `float` | Raw float32 embeddings (default) |
-| `binary` | Bit-packed signed binary |
-| `ubinary` | Bit-packed unsigned binary |
-| `base64` | Little-endian float32 encoded as base64 |
-
-```bash
-curl -X POST "http://localhost:8000/v2/embed" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "model": "Snowflake/snowflake-arctic-embed-m-v1.5",
-    "input_type": "query",
-    "texts": ["What is machine learning?"],
-    "embedding_types": ["float", "binary"]
-  }'
-```
-
-??? console "Response"
-
-    ```json
-    {
-      "id": "embd-...",
-      "embeddings": {
-        "float": [[0.012, -0.034, ...]],
-        "binary": [[42, -117, ...]]
-      },
-      "texts": ["What is machine learning?"],
-      "meta": {
-        "api_version": {"version": "2"},
-        "billed_units": {"input_tokens": 8}
-      }
-    }
-    ```
-
-#### Truncation
-
-The `truncate` parameter controls how inputs exceeding the model's maximum sequence length are handled:
-
-| Value | Behavior |
-| ----- | --------- |
-| `END` (default) | Keep the first tokens, drop the end |
-| `START` | Keep the last tokens, drop the beginning |
-| `NONE` | Return an error if the input is too long |
-
-#### Input type and prompt prefixes
-
-The `input_type` field selects a prompt prefix to prepend to each text input. The available values
-depend on the model:
-
- **Models with `task_instructions` in `config.json`**: The keys from the `task_instructions` dict are
-  the valid `input_type` values and the corresponding value is prepended to each text.
- **Models with `config_sentence_transformers.json` prompts**: The keys from the `prompts` dict are
-  the valid `input_type` values. For example, `Snowflake/snowflake-arctic-embed-xs` defines `"query"`,
-  so setting `input_type: "query"` prepends `"Represent this sentence for searching relevant passages: "`.
- **Other models**: `input_type` is not accepted and will raise a validation error if passed.
-
 ### Transcriptions API

 Our Transcriptions API is compatible with [OpenAI's Transcriptions API](https://platform.openai.com/docs/api-reference/audio/createTranscription);
@@ -759,172 +464,8 @@ It consists of two endpoints:
 - `/tokenize` corresponds to calling `tokenizer.encode()`.
 - `/detokenize` corresponds to calling `tokenizer.decode()`.

-### Pooling API
-
-Our Pooling API encodes input prompts using a [pooling model](../models/pooling_models.md) and returns the corresponding hidden states.
-
-The input format is the same as [Embeddings API](#embeddings-api), but the output data can contain an arbitrary nested list, not just a 1-D list of floats.
-
-Code example: [examples/pooling/pooling/pooling_online.py](../../examples/pooling/pooling/pooling_online.py)
-
-### Classification API
-
-Our Classification API directly supports Hugging Face sequence-classification models such as [ai21labs/Jamba-tiny-reward-dev](https://huggingface.co/ai21labs/Jamba-tiny-reward-dev) and [jason9693/Qwen2.5-1.5B-apeach](https://huggingface.co/jason9693/Qwen2.5-1.5B-apeach).
-
-We automatically wrap any other transformer via `as_seq_cls_model()`, which pools on the last token, attaches a `RowParallelLinear` head, and applies a softmax to produce per-class probabilities.
-
-Code example: [examples/pooling/classify/classification_online.py](../../examples/pooling/classify/classification_online.py)
-
-#### Example Requests
-
-You can classify multiple texts by passing an array of strings:
-
-```bash
-curl -v "http://127.0.0.1:8000/classify" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "model": "jason9693/Qwen2.5-1.5B-apeach",
-    "input": [
-      "Loved the new café—coffee was great.",
-      "This update broke everything. Frustrating."
-    ]
-  }'
-```
-
-??? console "Response"
-
-    ```json
-    {
-      "id": "classify-7c87cac407b749a6935d8c7ce2a8fba2",
-      "object": "list",
-      "created": 1745383065,
-      "model": "jason9693/Qwen2.5-1.5B-apeach",
-      "data": [
-        {
-          "index": 0,
-          "label": "Default",
-          "probs": [
-            0.565970778465271,
-            0.4340292513370514
-          ],
-          "num_classes": 2
-        },
-        {
-          "index": 1,
-          "label": "Spoiled",
-          "probs": [
-            0.26448777318000793,
-            0.7355121970176697
-          ],
-          "num_classes": 2
-        }
-      ],
-      "usage": {
-        "prompt_tokens": 20,
-        "total_tokens": 20,
-        "completion_tokens": 0,
-        "prompt_tokens_details": null
-      }
-    }
-    ```
-
-You can also pass a string directly to the `input` field:
-
-```bash
-curl -v "http://127.0.0.1:8000/classify" \
-  -H "Content-Type: application/json" \
-  -d '{
-    "model": "jason9693/Qwen2.5-1.5B-apeach",
-    "input": "Loved the new café—coffee was great."
-  }'
-```
-
-??? console "Response"
-
-    ```json
-    {
-      "id": "classify-9bf17f2847b046c7b2d5495f4b4f9682",
-      "object": "list",
-      "created": 1745383213,
-      "model": "jason9693/Qwen2.5-1.5B-apeach",
-      "data": [
-        {
-          "index": 0,
-          "label": "Default",
-          "probs": [
-            0.565970778465271,
-            0.4340292513370514
-          ],
-          "num_classes": 2
-        }
-      ],
-      "usage": {
-        "prompt_tokens": 10,
-        "total_tokens": 10,
-        "completion_tokens": 0,
-        "prompt_tokens_details": null
-      }
-    }
-    ```
-
-#### Extra parameters
-
-The following [pooling parameters][vllm.PoolingParams] are supported.
-
-```python
--8<-- "vllm/pooling_params.py:common-pooling-params"
--8<-- "vllm/pooling_params.py:classify-pooling-params"
-```
-
-The following Classification API parameters are supported:
-
-??? code
-
-    ```python
-    --8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-params"
-    --8<-- "vllm/entrypoints/pooling/base/protocol.py:completion-params"
-    --8<-- "vllm/entrypoints/pooling/base/protocol.py:classify-params"
-    ```
-
-The following extra parameters are supported:
-
-??? code
-
-    ```python
-    --8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-extra-params"
-    --8<-- "vllm/entrypoints/pooling/base/protocol.py:completion-extra-params"
-    --8<-- "vllm/entrypoints/pooling/base/protocol.py:classify-extra-params"
-    ```
-
-For chat-like input (i.e. if `messages` is passed), the following parameters are supported:
-
-??? code
-
-    ```python
-    --8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-params"
-    --8<-- "vllm/entrypoints/pooling/base/protocol.py:chat-params"
-    --8<-- "vllm/entrypoints/pooling/base/protocol.py:classify-params"
-    ```
-
-these extra parameters are supported instead:
-
-??? code
-
-    ```python
-    --8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-extra-params"
-    --8<-- "vllm/entrypoints/pooling/base/protocol.py:chat-extra-params"
-    --8<-- "vllm/entrypoints/pooling/base/protocol.py:classify-extra-params"
-    ```
-
 ### Score API

-Our Score API can apply a cross-encoder model or an embedding model to predict scores for sentence or multimodal pairs. When using an embedding model the score corresponds to the cosine similarity between each embedding pair.
-Usually, the score for a sentence pair refers to the similarity between two sentences, on a scale of 0 to 1.
-
-You can find the documentation for cross encoder models at [sbert.net](https://www.sbert.net/docs/package_reference/cross_encoder/cross_encoder.html).
-
-Code example: [examples/pooling/score/score_api_online.py](../../examples/pooling/score/score_api_online.py)
-
 #### Score Template

 Some scoring models require a specific prompt format to work correctly. You can specify a custom score template using the `--chat-template` parameter (see [Chat Template](#chat-template)).
@@ -940,307 +481,6 @@ This approach is more robust than index-based access (`messages[0]`, `messages[1

 Example template file: [examples/pooling/score/template/nemotron-rerank.jinja](../../examples/pooling/score/template/nemotron-rerank.jinja)

-#### Single inference
-
-You can pass a string to both `queries` and `documents`, forming a single sentence pair.
-
-```bash
-curl -X 'POST' \
-  'http://127.0.0.1:8000/score' \
-  -H 'accept: application/json' \
-  -H 'Content-Type: application/json' \
-  -d '{
-  "model": "BAAI/bge-reranker-v2-m3",
-  "encoding_format": "float",
-  "queries": "What is the capital of France?",
-  "documents": "The capital of France is Paris."
-}'
-```
-
-??? console "Response"
-
-    ```json
-    {
-      "id": "score-request-id",
-      "object": "list",
-      "created": 693447,
-      "model": "BAAI/bge-reranker-v2-m3",
-      "data": [
-        {
-          "index": 0,
-          "object": "score",
-          "score": 1
-        }
-      ],
-      "usage": {}
-    }
-    ```
-
-#### Batch inference
-
-You can pass a string to `queries` and a list to `documents`, forming multiple sentence pairs
-where each pair is built from `queries` and a string in `documents`.
-The total number of pairs is `len(documents)`.
-
-??? console "Request"
-
-    ```bash
-    curl -X 'POST' \
-      'http://127.0.0.1:8000/score' \
-      -H 'accept: application/json' \
-      -H 'Content-Type: application/json' \
-      -d '{
-      "model": "BAAI/bge-reranker-v2-m3",
-      "queries": "What is the capital of France?",
-      "documents": [
-        "The capital of Brazil is Brasilia.",
-        "The capital of France is Paris."
-      ]
-    }'
-    ```
-
-??? console "Response"
-
-    ```json
-    {
-      "id": "score-request-id",
-      "object": "list",
-      "created": 693570,
-      "model": "BAAI/bge-reranker-v2-m3",
-      "data": [
-        {
-          "index": 0,
-          "object": "score",
-          "score": 0.001094818115234375
-        },
-        {
-          "index": 1,
-          "object": "score",
-          "score": 1
-        }
-      ],
-      "usage": {}
-    }
-    ```
-
-You can pass a list to both `queries` and `documents`, forming multiple sentence pairs
-where each pair is built from a string in `queries` and the corresponding string in `documents` (similar to `zip()`).
-The total number of pairs is `len(documents)`.
-
-??? console "Request"
-
-    ```bash
-    curl -X 'POST' \
-      'http://127.0.0.1:8000/score' \
-      -H 'accept: application/json' \
-      -H 'Content-Type: application/json' \
-      -d '{
-      "model": "BAAI/bge-reranker-v2-m3",
-      "encoding_format": "float",
-      "queries": [
-        "What is the capital of Brazil?",
-        "What is the capital of France?"
-      ],
-      "documents": [
-        "The capital of Brazil is Brasilia.",
-        "The capital of France is Paris."
-      ]
-    }'
-    ```
-
-??? console "Response"
-
-    ```json
-    {
-      "id": "score-request-id",
-      "object": "list",
-      "created": 693447,
-      "model": "BAAI/bge-reranker-v2-m3",
-      "data": [
-        {
-          "index": 0,
-          "object": "score",
-          "score": 1
-        },
-        {
-          "index": 1,
-          "object": "score",
-          "score": 1
-        }
-      ],
-      "usage": {}
-    }
-    ```
-
-#### Multi-modal inputs
-
-You can pass multi-modal inputs to scoring models by passing `content` including a list of multi-modal input (image, etc.) in the request. Refer to the examples below for illustration.
-
-=== "JinaVL-Reranker"
-
-    To serve the model:
-
-    ```bash
-    vllm serve jinaai/jina-reranker-m0
-    ```
-
-    Since the request schema is not defined by OpenAI client, we post a request to the server using the lower-level `requests` library:
-
-    ??? Code
-
-        ```python
-        import requests
-        
-        response = requests.post(
-            "http://localhost:8000/v1/score",
-            json={
-                "model": "jinaai/jina-reranker-m0",
-                "queries": "slm markdown",
-                "documents": [
-                    {
-                        "content": [
-                            {
-                                "type": "image_url",
-                                "image_url": {
-                                    "url": "https://raw.githubusercontent.com/jina-ai/multimodal-reranker-test/main/handelsblatt-preview.png"
-                                },
-                            }
-                        ],
-                    },
-                    {
-                        "content": [
-                            {
-                                "type": "image_url",
-                                "image_url": {
-                                    "url": "https://raw.githubusercontent.com/jina-ai/multimodal-reranker-test/main/handelsblatt-preview.png"
-                                },
-                            }
-                        ]
-                    },
-                ],
-            },
-        )
-        response.raise_for_status()
-        response_json = response.json()
-        print("Scoring output:", response_json["data"][0]["score"])
-        print("Scoring output:", response_json["data"][1]["score"])
-        ```
-Full example:
-
- [examples/pooling/score/vision_score_api_online.py](../../examples/pooling/score/vision_score_api_online.py)
- [examples/pooling/score/vision_rerank_api_online.py](../../examples/pooling/score/vision_rerank_api_online.py)
-
-#### Extra parameters
-
-The following [pooling parameters][vllm.PoolingParams] are supported.
-
-```python
--8<-- "vllm/pooling_params.py:common-pooling-params"
--8<-- "vllm/pooling_params.py:classify-pooling-params"
-```
-
-The following Score API parameters are supported:
-
-```python
--8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-params"
-```
-
-The following extra parameters are supported:
-
-```python
--8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-extra-params"
--8<-- "vllm/entrypoints/pooling/base/protocol.py:classify-extra-params"
-```
-
-### Re-rank API
-
-Our Re-rank API can apply an embedding model or a cross-encoder model to predict relevant scores between a single query, and
-each of a list of documents. Usually, the score for a sentence pair refers to the similarity between two sentences or multi-modal inputs (image, etc.), on a scale of 0 to 1.
-
-You can find the documentation for cross encoder models at [sbert.net](https://www.sbert.net/docs/package_reference/cross_encoder/cross_encoder.html).
-
-The rerank endpoints support popular re-rank models such as `BAAI/bge-reranker-base` and other models supporting the
-`score` task. Additionally, `/rerank`, `/v1/rerank`, and `/v2/rerank`
-endpoints are compatible with both [Jina AI's re-rank API interface](https://jina.ai/reranker/) and
-[Cohere's re-rank API interface](https://docs.cohere.com/v2/reference/rerank) to ensure compatibility with
-popular open-source tools.
-
-Code example: [examples/pooling/score/rerank_api_online.py](../../examples/pooling/score/rerank_api_online.py)
-
-#### Example Request
-
-Note that the `top_n` request parameter is optional and will default to the length of the `documents` field.
-Result documents will be sorted by relevance, and the `index` property can be used to determine original order.
-
-??? console "Request"
-
-    ```bash
-    curl -X 'POST' \
-      'http://127.0.0.1:8000/v1/rerank' \
-      -H 'accept: application/json' \
-      -H 'Content-Type: application/json' \
-      -d '{
-      "model": "BAAI/bge-reranker-base",
-      "query": "What is the capital of France?",
-      "documents": [
-        "The capital of Brazil is Brasilia.",
-        "The capital of France is Paris.",
-        "Horses and cows are both animals"
-      ]
-    }'
-    ```
-
-??? console "Response"
-
-    ```json
-    {
-      "id": "rerank-fae51b2b664d4ed38f5969b612edff77",
-      "model": "BAAI/bge-reranker-base",
-      "usage": {
-        "total_tokens": 56
-      },
-      "results": [
-        {
-          "index": 1,
-          "document": {
-            "text": "The capital of France is Paris."
-          },
-          "relevance_score": 0.99853515625
-        },
-        {
-          "index": 0,
-          "document": {
-            "text": "The capital of Brazil is Brasilia."
-          },
-          "relevance_score": 0.0005860328674316406
-        }
-      ]
-    }
-    ```
-
-#### Extra parameters
-
-The following [pooling parameters][vllm.PoolingParams] are supported.
-
-```python
--8<-- "vllm/pooling_params.py:common-pooling-params"
--8<-- "vllm/pooling_params.py:classify-pooling-params"
-```
-
-The following Re-rank API parameters are supported:
-
-```python
--8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-params"
--8<-- "vllm/entrypoints/pooling/base/protocol.py:classify-extra-params"
-```
-
-The following extra parameters are supported:
-
-```python
--8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-extra-params"
--8<-- "vllm/entrypoints/pooling/base/protocol.py:classify-extra-params"
-```
-
 ## Ray Serve LLM

 Ray Serve LLM enables scalable, production-grade serving of the vLLM engine. It integrates tightly with vLLM and extends it with features such as auto-scaling, load balancing, and back-pressure.
@@ -104,12 +104,22 @@ def run_musicflamingo(question: str, audio_count: int) -> ModelRequestData:
        enforce_eager=True,
    )

-    # MusicFlamingo uses <sound> token for audio
+    # MusicFlamingo prompt placeholders use <sound>; vLLM's MusicFlamingo
+    # multimodal processor expands each one into <|sound_bos|> + audio tokens +
+    # <|sound_eos|> based on extracted audio feature lengths.
    audio_placeholder = "<sound>" * audio_count
+    system_prompt = (
+        "You are Music Flamingo, a multimodal assistant for language and music. "
+        "On each turn you receive an audio clip which contains music and optional "
+        "text, you will receive at least one or both; use your world knowledge and "
+        "reasoning to help the user with any task. Interpret the entirety of the "
+        "content any input music--regardlenss of whether the user calls it audio, "
+        "music, or sound."
+    )

    prompt = (
        "<|im_start|>system\n"
-        "You are a helpful assistant.<|im_end|>\n"
+        f"{system_prompt}<|im_end|>\n"
        "<|im_start|>user\n"
        f"{audio_placeholder}{question}<|im_end|>\n"
        "<|im_start|>assistant\n"
@@ -2,25 +2,38 @@
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 """
 Demonstrates async reinforcement learning using vLLM and Ray,
-with native weight syncing APIs at engine instance.
+with native weight syncing APIs and batch-invariant generation.

 The script separates training and inference workloads onto distinct GPUs
 so that Ray can manage process placement and inter-process communication.
-A Hugging Face Transformer model occupies one GPU for training, whereas a
-2x tensor-parallel vLLM inference engine occupies two GPUs.
+A Hugging Face Transformer model occupies one GPU for training, and a
+vLLM AsyncLLMEngine occupies another GPU for inference.
+
+Batch invariance is enabled so that generation output is deterministic
+regardless of how many requests are batched together. This is required
+for the validation phase to succeed. Batch invariance currently requires
+NVIDIA GPUs with compute capability 9.0 or higher:
+  - H-series: H100, H200
+  - B-series: B100, B200

 The example performs the following steps:
-* Load the training model on one gpu (scheduled via ray)
-* Initialize the inference model with dummy weights across
-  two gpus using vLLM's tensor parallelism and Ray placement groups.
-* Generate gibberish from a list of prompts using the randomly initialized
-  inference engine.
-* Pause generation once generation completes for one sequence
-* Update the weights of the training model and broadcast the updated weights
-  to the inference engine by using a Ray collective RPC group.
-* Resume generation and print out the results
+* Load the training model (Qwen3-1.7B) on one GPU via a Ray actor.
+* Initialize the inference engine with a base model (Qwen3-1.7B-Base)
+  on a separate GPU using vLLM's AsyncLLMEngine with Ray as the
+  distributed executor backend.
+* Set up an NCCL-based weight transfer channel between the trainer
+  and the inference engine.
+* Submit generation requests for a batch of prompts.
+* Pause generation once any request reaches a token threshold.
+* Broadcast the training model's weights to the inference engine
+  via the NCCL weight transfer engine, replacing the base weights.
+* Resume generation and collect results, noting which tokens were
+  generated before vs. after the weight swap.
+* Validate correctness by launching a fresh vLLM instance loaded
+  directly with the training model and comparing its output to the
+  post-swap tokens from the weight-synced engine.

-This example assumes a single-node cluster with three GPUs, but Ray
+This example assumes a single-node cluster with two GPUs, but Ray
 supports multi-node clusters. vLLM expects the GPUs are only used for vLLM
 workloads. Residual GPU activity interferes with vLLM memory profiling and
 causes unexpected behavior.
@@ -121,7 +121,7 @@ python = "./.venv"
 # these files may be written in non english words
 extend-exclude = ["tests/models/fixtures/*", "tests/prompts/*", "tests/tokenizers_/*",
    "benchmarks/sonnet.txt", "tests/lora/data/*", "examples/pooling/token_embed/*", "build/*",
-    "vllm/third_party/*", "vllm/entrypoints/serve/instrumentator/static/*", "tests/entrypoints/openai/test_transcription_validation.py",
+    "vllm/third_party/*", "vllm/entrypoints/serve/instrumentator/static/*", "tests/entrypoints/openai/speech_to_text/test_transcription_validation.py",
    "docs/governance/process.md", "tests/v1/engine/test_fast_incdec_prefix_err.py", ".git/*"]
 ignore-hidden = false

@@ -47,6 +47,8 @@ pystemmer==3.0.0
 # Multi-modal processing
 av==16.1.0
    # required for audio_in_video tests
+resampy==0.4.3
+    # audio processing, required for audio_in_video tests
 blobfile==3.0.0
    # Multi-Modal Models Test
 decord==0.6.0
@@ -21,6 +21,7 @@ vocos # required for minicpmo_26 test
 peft>=0.15.0 # required for phi-4-mm test
 pqdm
 ray[cgraph,default]>=2.48.0 # Ray Compiled Graph, required by pipeline parallelism tests
+resampy # required for audio tests
 sentence-transformers>=5.2.0 # required for embedding tests
 soundfile # required for audio tests
 jiwer # required for audio tests
@@ -544,6 +544,7 @@ numba==0.61.2
    # via
    #   -r requirements/test.in
    #   librosa
+    #   resampy
 numpy==2.2.6
    # via
    #   -r requirements/test.in
@@ -584,6 +585,7 @@ numpy==2.2.6
    #   pyogrio
    #   pywavelets
    #   rasterio
+    #   resampy
    #   rioxarray
    #   rouge-score
    #   runai-model-streamer
@@ -995,6 +997,8 @@ requests==2.32.3
    #   tiktoken
    #   transformers
    #   wandb
+resampy==0.4.3
+    # via -r requirements/test.in
 responses==0.25.3
    # via genai-perf
 rfc3339-validator==0.1.4
@@ -15,4 +15,4 @@ torch==2.10.0+xpu
 torchaudio
 torchvision

-vllm_xpu_kernels @ https://github.com/vllm-project/vllm-xpu-kernels/releases/download/v0.1.3/vllm_xpu_kernels-0.1.3-cp38-abi3-linux_x86_64.whl
+vllm_xpu_kernels @ https://github.com/vllm-project/vllm-xpu-kernels/releases/download/v0.1.4/vllm_xpu_kernels-0.1.4-cp38-abi3-manylinux_2_28_x86_64.whl
@@ -54,6 +54,9 @@ elif sys.platform.startswith("linux") and os.getenv("VLLM_TARGET_DEVICE") is Non
    if torch.version.hip is not None:
        VLLM_TARGET_DEVICE = "rocm"
        logger.info("Auto-detected ROCm")
+    elif torch.version.xpu is not None:
+        VLLM_TARGET_DEVICE = "xpu"
+        logger.info("Auto-detected XPU")
    elif torch.version.cuda is not None:
        VLLM_TARGET_DEVICE = "cuda"
        logger.info("Auto-detected CUDA")
@@ -597,6 +600,7 @@ class precompiled_wheel_utils:
            with zipfile.ZipFile(wheel_path) as wheel:
                files_to_copy = [
                    "vllm/_C.abi3.so",
+                    "vllm/_C_stable_libtorch.abi3.so",
                    "vllm/_moe_C.abi3.so",
                    "vllm/_flashmla_C.abi3.so",
                    "vllm/_flashmla_extension_C.abi3.so",
@@ -932,6 +936,10 @@ if _is_cpu():

 if _build_custom_ops():
    ext_modules.append(CMakeExtension(name="vllm._C"))
+    # also _is_hip() once https://github.com/vllm-project/vllm/issues/35163 is
+    # fixed
+    if _is_cuda():
+        ext_modules.append(CMakeExtension(name="vllm._C_stable_libtorch"))

 package_data = {
    "vllm": [
@@ -979,11 +987,11 @@ setup(
        "instanttensor": ["instanttensor >= 0.1.5"],
        "runai": ["runai-model-streamer[s3,gcs,azure] >= 0.15.7"],
        "audio": [
-            "librosa",
+            "av",
+            "resampy",
            "scipy",
            "soundfile",
            "mistral_common[audio]",
-            "av",
        ],  # Required for audio processing
        "video": [],  # Kept for backwards compatibility
        "flashinfer": [],  # Kept for backwards compatibility
@@ -0,0 +1,208 @@
+# SM103 NVFP4 Programmatic Dependent Launch (PDL) Summary
+
+## Overview
+
+This document summarizes the addition of **Programmatic Dependent Launch (PDL)** to the SM103 (B300 Blackwell Ultra) NVFP4 quantization and CUTLASS GEMM kernels in vLLM. PDL is a CUDA 12+ feature that allows consecutive kernels on the same stream to overlap execution, reducing the gap between a producer kernel's tail and a consumer kernel's head.
+
+## What is PDL?
+
+In standard CUDA stream semantics, Kernel B cannot begin until Kernel A fully completes. PDL relaxes this constraint:
+
+```
+Without PDL:
+  [==== quant kernel ====]  [==== GEMM kernel ====]
+                          ^ idle gap
+
+With PDL:
+  [==== quant kernel ====]
+              [==== GEMM kernel ====]
+              ^ overlap region
+```
+
+The CUDA attribute `cudaLaunchAttributeProgrammaticStreamSerialization` is set on the **producer** kernel, telling the driver that the next kernel on the stream may begin before the producer fully completes. This is safe when the consumer's early thread blocks operate on data that the producer has already finished writing (which is the typical case for tile-based execution).
+
+## Changes Made
+
+### 1. Quant kernel PDL (`csrc/quantization/fp4/nvfp4_quant_kernels.cu`)
+
+- Refactored `scaled_fp4_quant_sm103a` into `scaled_fp4_quant_sm103a_impl` with a `use_pdl` parameter
+- When `use_pdl=true`, the kernel is launched via `cudaLaunchKernelEx` with `ProgrammaticStreamSerialization = 1`
+- Added `scaled_fp4_quant_sm103a_pdl()` entry point
+
+### 2. CUTLASS GEMM PDL (`csrc/quantization/fp4/nvfp4_scaled_mm_kernels.cu`)
+
+- Added `launch_with_pdl` parameter to `runGemm()` template, forwarded to CUTLASS's `GemmUniversalAdapter::run(..., launch_with_pdl)`
+- CUTLASS internally sets `ProgrammaticStreamSerialization` on the GEMM launch via `ClusterLauncher`
+- Added `cutlass_scaled_fp4_mm_sm103a_pdl()` entry point
+
+### 3. Op registration (`csrc/torch_bindings.cpp`, `csrc/quantization/fp4/nvfp4_*_entry.cu`)
+
+New torch ops registered:
+- `torch.ops._C.scaled_fp4_quant_sm103_pdl` -- PDL-enabled SM103 quant
+- `torch.ops._C.scaled_fp4_quant_sm103_pdl.out` -- out-variant
+- `torch.ops._C.cutlass_scaled_fp4_mm_sm103a_pdl` -- PDL-enabled SM103 GEMM
+
+### 4. Benchmark (`benchmarks/kernels/benchmark_nvfp4_sm103.py`)
+
+Updated benchmark with new modes:
+- `--mode gemm`: SM100 vs SM103 vs SM103+PDL GEMM-only
+- `--mode e2e`: End-to-end quant+GEMM with PDL comparison
+- `--mode pdl`: Multi-layer pipeline benchmark (back-to-back quant+GEMM pairs)
+
+## PDL Pipeline Analysis
+
+### Single kernel pair (quant + GEMM)
+
+For a single quant->GEMM pair, PDL enables:
+1. **Quant kernel** with `ProgrammaticStreamSerialization`: GEMM can start before quant finishes
+2. **GEMM kernel** with `ProgrammaticStreamSerialization` (via CUTLASS): the next layer's kernel can start before GEMM finishes
+
+### Multi-layer pipeline
+
+In a real transformer, the pattern repeats:
+```
+Layer 1: quant_1 -> GEMM_1
+Layer 2: quant_2 -> GEMM_2
+...
+```
+
+With PDL on both kernels, each transition overlaps:
+```
+[quant_1]--->[GEMM_1]--->[quant_2]--->[GEMM_2]--->  (without PDL)
+
+[quant_1]--[GEMM_1]--[quant_2]--[GEMM_2]--          (with PDL)
+          ^^        ^^         ^^
+          overlap at each transition
+```
+
+### Measured performance impact (B300 SXM6, CUDA 12.9)
+
+| Scenario | PDL benefit | Explanation |
+|----------|-------------|-------------|
+| GEMM-only (isolated) | ~0-3% | Marginal; no meaningful consumer overlap for a single kernel |
+| Single quant+GEMM (decode, M=1-16) | 0-3% | Quant is tiny, GEMM dominates wall time |
+| Single quant+GEMM (prefill, M=64-512) | 2-3% | Moderate overlap window |
+| Single quant+GEMM (prefill, M=1024+) | 2-5% | Larger quant = more overlap-able tail |
+| 4-layer pipeline (decode, M=1-16) | 4-6% | Cumulative overlap across 8 kernel transitions |
+| 4-layer pipeline (M=1024) | **12%** | Best case: sustained overlap on compute-heavy layers |
+| 4-layer pipeline (prefill, M=4096) | 4% | GEMM dominates; quant tail is proportionally smaller |
+
+The PDL benefit is proportional to the **ratio of overlap-able tail time to total kernel time**. The sweet spot is M=256-1024 where the quant kernel is large enough to provide meaningful overlap but doesn't yet dominate the pipeline.
+
+## SM100 vs SM103 vs SM103+PDL Comparison
+
+### Kernel architecture differences
+
+| Aspect | SM100 (B200) | SM103 (B300) | SM103+PDL |
+|--------|-------------|-------------|-----------|
+| MMA instructions | FP4 BlockScaled | FP4 Ultra (UltraVs16) | Same as SM103 |
+| Tile K size | 256 | 768 (3x larger) | Same as SM103 |
+| Cooperative SMs | 1-2 per tile | 2 per tile (default) | Same as SM103 |
+| SF layout | Sm1xxBlockScaledConfig | Sm103BlockScaledConfig | Same as SM103 |
+| Kernel overlap | None (stream-serialized) | None | Quant tail overlaps GEMM head |
+| Epilogue | TmaWarpSpecialized | NoSmemWarpSpecialized | Same as SM103 |
+
+### Measured performance on B300 SXM6, N=K=7168
+
+**SM103 vs SM100 (GEMM-only):** SM103 Ultra MMA provides 3-7% higher throughput for small-to-medium M (decode/small batch). At large M (2048+), both achieve similar throughput as the problem becomes compute-bound on both paths. The SM103 advantage comes from:
+- 3x larger K-tile (768 vs 256): fewer mainloop iterations
+- UltraVs16 instructions: higher throughput per clock
+- NoSmem epilogue: more shared memory for mainloop double-buffering
+
+**PDL benefit (E2E):** PDL provides a consistent 2-5% speedup on the end-to-end quant+GEMM path for most M sizes, with a peak of 5% at M=1024 where the quant and GEMM are well-balanced.
+
+**PDL pipeline benefit (4 layers):** The multi-layer pipeline shows 4-12% speedup, with the best result at M=1024 (12% speedup) where cumulative overlap across 8 kernel transitions provides maximum benefit.
+
+## Files Modified
+
+| File | Change |
+|------|--------|
+| `csrc/quantization/fp4/nvfp4_quant_kernels.cu` | PDL launch via `cudaLaunchKernelEx` for SM103 quant |
+| `csrc/quantization/fp4/nvfp4_scaled_mm_kernels.cu` | `launch_with_pdl` parameter forwarded to CUTLASS |
+| `csrc/quantization/fp4/nvfp4_quant_entry.cu` | PDL entry points and forward declarations |
+| `csrc/quantization/fp4/nvfp4_scaled_mm_entry.cu` | PDL GEMM forward declaration |
+| `csrc/torch_bindings.cpp` | Op registration for `_pdl` variants |
+| `benchmarks/kernels/benchmark_nvfp4_sm103.py` | PDL benchmark modes (gemm, e2e, pdl pipeline) |
+
+## How to Run
+
+```bash
+# Build vLLM with SM103 support
+python setup.py build_ext --inplace
+
+# Run all benchmarks
+python benchmarks/kernels/benchmark_nvfp4_sm103.py --mode all
+
+# Run only the PDL pipeline benchmark with 8 layers
+python benchmarks/kernels/benchmark_nvfp4_sm103.py --mode pdl --layers 8
+
+# Run end-to-end comparison
+python benchmarks/kernels/benchmark_nvfp4_sm103.py --mode e2e
+```
+
+## Benchmark Results
+
+**Hardware:** NVIDIA B300 SXM6 AC (SM103), CUDA 12.9
+**Problem:** N=7168, K=7168 (DeepSeek-style dimensions), BF16 output
+
+### GEMM-Only: SM100 vs SM103 vs SM103+PDL
+
+| M | SM100 (us) | SM100 TFLOPS | SM103 (us) | SM103 TFLOPS | SM103+PDL (us) | SM103+PDL TFLOPS | SM103 vs SM100 |
+|---|-----------|-------------|-----------|-------------|---------------|-----------------|----------------|
+| 1 | 26.43 | 3.89 | 25.09 | 4.10 | 24.35 | 4.22 | 1.05x |
+| 16 | 27.30 | 60.23 | 24.83 | 66.21 | 25.31 | 64.96 | 1.10x |
+| 128 | 28.26 | 465.51 | 26.43 | 497.63 | 26.34 | 499.44 | 1.07x |
+| 512 | 28.19 | 1866.25 | 27.36 | 1923.00 | 26.53 | 1983.31 | 1.03x |
+| 1024 | 30.72 | 3425.35 | 34.72 | 3030.72 | 34.78 | 3025.15 | 0.88x |
+| 4096 | 93.12 | 4520.05 | 90.02 | 4675.91 | 89.89 | 4682.57 | 1.03x |
+
+**Observations:**
+- SM103 is faster than SM100 for M <= 512 (up to 10% at M=16)
+- At M=1024, SM100 is faster (tile configuration tradeoff)
+- PDL on GEMM-only has marginal effect (expected: no consumer kernel to overlap)
+
+### End-to-End: Quant + GEMM
+
+| M | SM100 (us) | SM103 (us) | SM103+PDL (us) | SM103 vs SM100 | PDL vs no-PDL | PDL vs SM100 |
+|---|-----------|-----------|---------------|----------------|---------------|--------------|
+| 1 | 36.54 | 35.74 | 35.90 | 1.02x | 1.00x | 1.02x |
+| 8 | 36.58 | 34.53 | 33.73 | 1.06x | **1.02x** | **1.08x** |
+| 64 | 38.30 | 36.64 | 35.84 | 1.05x | **1.02x** | **1.07x** |
+| 256 | 38.24 | 36.74 | 36.03 | 1.04x | **1.02x** | **1.06x** |
+| 1024 | 38.66 | 40.90 | 38.78 | 0.95x | **1.05x** | 1.00x |
+| 2048 | 65.63 | 65.41 | 63.04 | 1.00x | **1.04x** | **1.04x** |
+| 4096 | 112.38 | 110.78 | 108.38 | 1.01x | **1.02x** | **1.04x** |
+
+**Observations:**
+- PDL consistently improves E2E by 2-5% over non-PDL SM103
+- Best PDL improvement at M=1024: 5% (40.90 us -> 38.78 us)
+- Total SM103+PDL vs SM100 speedup: up to 8% at M=8
+
+### PDL Pipeline: 4 Back-to-Back Layers
+
+| M | No PDL (us) | No PDL TFLOPS | PDL (us) | PDL TFLOPS | PDL Speedup |
+|---|------------|--------------|---------|-----------|-------------|
+| 1 | 111.65 | 3.68 | 107.10 | 3.84 | 1.04x |
+| 16 | 112.42 | 58.50 | 107.07 | 61.42 | **1.05x** |
+| 64 | 120.51 | 218.29 | 115.65 | 227.47 | 1.04x |
+| 256 | 120.83 | 870.85 | 115.33 | 912.41 | **1.05x** |
+| 1024 | 151.52 | 2777.90 | 134.94 | 3119.12 | **1.12x** |
+| 2048 | 250.27 | 3363.59 | 236.35 | 3561.69 | **1.06x** |
+| 4096 | 432.38 | 3893.82 | 416.32 | 4044.07 | 1.04x |
+
+**Key finding:** At M=1024, PDL provides **12% speedup** over non-PDL SM103 in the 4-layer pipeline benchmark. This is where the quant and GEMM kernels are well-balanced in execution time, maximizing the overlap benefit across 8 kernel transitions (4 quant + 4 GEMM).
+
+### Activation Quantization: SM100 vs SM103
+
+| M | SM100 (us) | SM100 GB/s | SM103 (us) | SM103 GB/s |
+|---|-----------|-----------|-----------|-----------|
+| 1 | 17.06 | 0.84 | 17.09 | 0.84 |
+| 256 | 17.25 | 212.78 | 17.18 | 213.57 |
+| 1024 | 17.25 | 851.12 | 16.90 | 868.85 |
+| 4096 | 19.17 | 3063.45 | 18.24 | 3219.31 |
+
+**Observations:** SM103 quant is 2-5% faster than SM100 quant, primarily from the different SF swizzle pattern being more cache-friendly on SM103.
+
+## Conclusion
+
+PDL is a low-cost optimization that provides consistent 2-5% E2E improvement for single quant+GEMM pairs, scaling to **12% in multi-layer pipelines** at the M=1024 sweet spot. The implementation adds no correctness risk (PDL is a scheduling hint) and no overhead when the GPU decides not to overlap. It should be enabled by default for SM103 production workloads.
@@ -84,7 +84,10 @@ def run_e2e_fusion_test(monkeypatch, caplog_mp_spawn):

        # TODO: remove this after finishing migration from envs to model kwargs
        if model_name == "openai/gpt-oss-20b":
-            monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8", "1")
+            from .common import is_blackwell
+
+            if is_blackwell():
+                monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8", "1")

        # Disable, compile cache to make sure custom passes run.
        # Otherwise, we can't verify fusion happened through the logs.
@@ -127,7 +127,7 @@ class AttentionQuantPatternModel(torch.nn.Module):
        raw_tensor = raw_tensor.view(kv_cache_shape)
        kv_cache = raw_tensor.permute(*inv_order)

-        self.attn.kv_cache = [kv_cache]
+        self.attn.kv_cache = kv_cache

        # Build attn metadata
        self.attn_metadata = self.builder.build(
@@ -148,7 +148,7 @@ class QKRoPEKVCacheTestModel(torch.nn.Module):
        raw_tensor = raw_tensor.view(kv_cache_shape)
        kv_cache = raw_tensor.permute(*inv_order)

-        self.attn.kv_cache = [kv_cache]
+        self.attn.kv_cache = kv_cache

        # Build attn metadata
        attn_metadata = self.builder.build(
@@ -295,7 +295,7 @@ def test_rope_kvcache_fusion(
            }
            q_unfused, k_unfused, v_unfused, dummy = model(qkv_unfused, pos_unfused)
            attn_layer = forward_context.no_compile_layers[model.layer_name]
-            kv_cache_unfused = attn_layer.kv_cache[0]
+            kv_cache_unfused = attn_layer.kv_cache
        del dummy

        torch._dynamo.mark_dynamic(qkv, 0)
@@ -309,7 +309,7 @@ def test_rope_kvcache_fusion(
            }
            q_fused, k_fused, v_fused, dummy = model_fused(qkv, pos)
            attn_layer = forward_context.no_compile_layers[model.layer_name]
-            kv_cache_fused = attn_layer.kv_cache[0]
+            kv_cache_fused = attn_layer.kv_cache
        del dummy

        assert fusion_pass.matched_count == 1
@@ -14,6 +14,7 @@ from unittest.mock import Mock, patch
 import pytest
 import torch

+import vllm.envs as envs
 import vllm.model_executor.layers.activation
 from vllm.compilation.backends import VllmBackend
 from vllm.compilation.caching import (
@@ -162,6 +163,9 @@ def test_save_and_load(monkeypatch: pytest.MonkeyPatch):

@pytest.mark.skipif(not is_torch_equal_or_newer("2.10.0"), reason="requires torch 2.10")
 def test_save_and_load_slice(monkeypatch: pytest.MonkeyPatch):
+    from torch._subclasses import FakeTensorMode
+    from torch.fx.experimental.symbolic_shapes import ShapeEnv
+
    def foo(x: torch.Tensor):
        return x[slice(0, x.shape[0])]

@@ -172,12 +176,13 @@ def test_save_and_load_slice(monkeypatch: pytest.MonkeyPatch):
    gm = torch.fx.symbolic_trace(foo)
    assert "getitem_1 = x[slice(0, getitem, None)]" in gm.code
    with use_vllm_config(vllm_config):
-        payload = VllmSerializableFunction.serialize_compile_artifacts(
-            VllmSerializableFunction(gm, (example_input,), "", foo)
+        payload = VllmSerializableFunction.serialize_graph_module(gm)
+        fake_mode = FakeTensorMode(shape_env=ShapeEnv())
+        loaded_gm = VllmSerializableFunction.deserialize_graph_module(
+            payload, fake_mode
        )
-        fn = VllmSerializableFunction.deserialize_compile_artifacts(payload)

-    assert gm.code == fn.graph_module.code
+    assert gm.code == loaded_gm.code


@pytest.mark.skipif(not is_torch_equal_or_newer("2.10.0"), reason="requires torch 2.10")
@@ -725,6 +730,10 @@ class TestStandaloneCompiledArtifactsIntegration:
        ]:
            assert cache.get(submod, shape) == shared_data

+    @pytest.mark.skipif(
+        envs.VLLM_USE_MEGA_AOT_ARTIFACT,
+        reason="There's no AOT Autograd run with mega artifact",
+    )
    def test_functorch_config(self):
        vllm_config = make_vllm_config()
        example_inputs = (torch.randn(10, 10),)
@@ -23,8 +23,14 @@ from vllm.utils.torch_utils import is_torch_equal_or_newer

 def get_test_models():
    """Get list of models to test based on PyTorch version"""
-    # TODO "Qwen/Qwen3-4B-Instruct-2507" fails Fix issue and support it.
-    return ["gpt2", "Qwen/Qwen2-7B-Instruct", "meta-llama/Llama-3.1-8B"]
+    models = [
+        "gpt2",
+        "Qwen/Qwen2-7B-Instruct",
+        "meta-llama/Llama-3.1-8B",
+    ]
+    if is_torch_equal_or_newer("2.12.0"):
+        models.append("Qwen/Qwen3-4B-Instruct-2507")
+    return models


@pytest.mark.parametrize("model_name", get_test_models())
@@ -5,6 +5,8 @@ import operator

 import pytest
 import torch
+import torch._dynamo
+import torch.fx as fx
 from torch.fx.experimental.proxy_tensor import make_fx

 from vllm.compilation.backends import _is_empty_allocation_node, split_graph
@@ -327,3 +329,296 @@ def test_builtin_empty_only_partition_is_merged():
    output_original = gm(x)
    output_split = split_gm(x)
    assert torch.allclose(output_original, output_split), "Output mismatch after split"
+
+
+@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
+def test_sym_size_whole_shape_boundary():
+    """
+    Test that using x.size() (whole shape) across a split boundary can be
+    compiled by standalone_compile.
+
+    The dynamo graph looks like:
+        shape = x.size()
+        y = sigmoid(x)          # split point
+        z = y.clone().view(shape)
+
+    Which splits into:
+        subgraph0(x) -> shape          # returns torch.Size — problematic
+        subgraph1(x) -> y              # sigmoid
+        subgraph2(y, shape) -> z       # view
+
+    Two approaches to fix the torch.Size crossing:
+
+    Approach 1 — move sym_size to consumer (memory implication: x passed to
+    subgraph2 just for .size()):
+        subgraph0(x) ->                # empty
+        subgraph1(x) -> y
+        subgraph2(y, x) -> z           # computes shape locally from x
+
+    Approach 2 — decompose shape into individual int/SymInt values:
+        subgraph0(x) -> s0, val        # returns individual scalars, not Size
+        subgraph1(x) -> y
+        subgraph2(y, s0, val) -> z     # reconstructs view args from scalars
+    """
+    from torch._inductor import standalone_compile
+
+    captured_graph = None
+
+    def capturing_backend(gm: fx.GraphModule, example_inputs: list) -> fx.GraphModule:
+        nonlocal captured_graph
+        captured_graph = gm
+        return gm
+
+    def model_fn(x: torch.Tensor) -> torch.Tensor:
+        shape = x.size()
+        x = torch.ops.aten.sigmoid.default(x)
+        x = x.clone().view(shape)
+        return x
+
+    x = torch.randn(4, 8)
+    torch._dynamo.mark_dynamic(x, 0)
+    compiled_fn = torch.compile(model_fn, backend=capturing_backend)
+    compiled_fn(x)
+
+    split_gm, split_items = split_graph(captured_graph, ["aten::sigmoid"])
+    assert len(split_items) == 3
+
+    submod_0 = split_gm.submod_0
+    example_input = torch.randn(4, 8)
+    compiled = standalone_compile(
+        submod_0, [example_input, 4], dynamic_shapes="from_example_inputs"
+    )
+    assert compiled is not None
+
+
+@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
+def test_symint_crosses_split_boundary():
+    """
+    Test that SymInt placeholders from torch.compile + mark_dynamic
+    cross split boundaries safely via split_module's natural threading.
+
+    SymInt values are threaded through subgraphs by split_module and
+    handled correctly by inductor — no special replacement is needed.
+    """
+    captured_graph = None
+
+    def capturing_backend(gm: fx.GraphModule, example_inputs: list) -> fx.GraphModule:
+        nonlocal captured_graph
+        captured_graph = gm
+        return gm
+
+    def model_fn(x: torch.Tensor) -> torch.Tensor:
+        batch_size = x.shape[0]
+        hidden_size = x.shape[1]
+        x = torch.ops.aten.sigmoid.default(x)
+        x = x.clone().view(batch_size, hidden_size)
+        x = torch.ops.aten.sigmoid.default(x)
+        x = x.clone().view(batch_size, hidden_size)
+        x = torch.ops.aten.sigmoid.default(x)
+        x = x.clone().view(batch_size, hidden_size)
+        return x
+
+    x = torch.randn(4, 8)
+    torch._dynamo.mark_dynamic(x, 0)
+
+    compiled_fn = torch.compile(model_fn, backend=capturing_backend)
+    compiled_fn(x)
+
+    assert captured_graph is not None, "Graph should be captured by backend"
+
+    # SymInt placeholders should exist in the captured graph
+    symint_placeholders = [
+        node
+        for node in captured_graph.graph.nodes
+        if node.op == "placeholder"
+        and isinstance(node.meta.get("example_value"), torch.SymInt)
+    ]
+    assert len(symint_placeholders) > 0, (
+        "Captured graph should have SymInt placeholders from mark_dynamic."
+    )
+
+    # split_graph should handle SymInt placeholders without error
+    split_gm, split_items = split_graph(captured_graph, ["aten::sigmoid"])
+
+    # Should have 3 splitting subgraphs (3 sigmoids)
+    splitting_subgraphs = [item for item in split_items if item.is_splitting_graph]
+    assert len(splitting_subgraphs) == 3, (
+        f"Expected 3 splitting subgraphs (3 sigmoids), got {len(splitting_subgraphs)}"
+    )
+    assert len(split_items) >= 6, (
+        f"Expected at least 6 total subgraphs, got {len(split_items)}"
+    )
+
+
+@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
+def test_shape_boundary_standalone_compile():
+    """
+    Repro for the original production bug:
+
+        AssertionError: out_spec mismatch
+        TreeSpec(tuple, None, [*, *, TreeSpec(Size, None, [*, *]), *])
+        vs
+        TreeSpec(tuple, None, [*, *, *, *])
+
+    A subgraph outputs torch.Size (e.g. torch.Size([s72, 2048])) as one of
+    its values when shape info crosses a split boundary. aot_autograd / inductor
+    expect all submodule outputs to be flat tensors or scalars, not torch.Size.
+
+    With the fix, x.size() is decomposed into individual sym_size.int calls
+    so only scalar SymInts cross the boundary — not the torch.Size.
+    """
+    from torch._inductor import standalone_compile
+
+    captured_graph = None
+
+    def capturing_backend(gm: fx.GraphModule, example_inputs: list) -> fx.GraphModule:
+        nonlocal captured_graph
+        captured_graph = gm
+        return gm
+
+    def model_fn(x: torch.Tensor) -> torch.Tensor:
+        shape = x.size()
+        x = torch.ops.aten.sigmoid.default(x)
+        x = x.clone().view(shape)
+        return x
+
+    x = torch.randn(4, 8)
+    torch._dynamo.mark_dynamic(x, 0)
+    torch.compile(model_fn, backend=capturing_backend)(x)
+
+    split_gm, split_items = split_graph(captured_graph, ["aten::sigmoid"])
+    assert len(split_items) == 3
+
+    # Verify that the consumer subgraph only has a placeholder for the dynamic
+    # dim (SymInt) — the static dim (8) should be inlined as a literal, not
+    # threaded as a placeholder.
+    consumer = split_items[-1]  # valid since len == 3: [producer, sigmoid, consumer]
+    symint_placeholders = [
+        n
+        for n in consumer.graph.graph.nodes
+        if n.op == "placeholder"
+        and isinstance(n.meta.get("example_value"), torch.SymInt)
+    ]
+    static_int_placeholders = [
+        n
+        for n in consumer.graph.graph.nodes
+        if n.op == "placeholder"
+        and isinstance(n.meta.get("example_value"), int)
+        and not isinstance(n.meta.get("example_value"), torch.SymInt)
+    ]
+    assert len(symint_placeholders) >= 1, (
+        "Consumer should have a SymInt placeholder for the dynamic dim."
+    )
+    assert len(static_int_placeholders) == 0, (
+        "Static dims should be inlined as literals, not threaded as placeholders."
+    )
+
+    submod_0 = split_gm.submod_0
+
+    standalone_compile(
+        submod_0, [torch.randn(4, 8), 4], dynamic_shapes="from_example_inputs"
+    )
+
+
+@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
+def test_size_used_in_multiple_consumer_subgraphs():
+    """
+    Validates that x.size() (whole shape) used by multiple downstream subgraphs
+    does not cause torch.Size to cross split boundaries.
+
+    Model:
+        shape = x.size()          # whole shape — must not cross as torch.Size
+        z1 = sigmoid(x)           # split point 1
+        y1 = y.view(shape)        # consumer 1 uses shape
+        z2 = sigmoid(z1)          # split point 2
+        y2 = y.view(shape)        # consumer 2 uses shape again
+
+    Without the fix, torch.Size crosses the boundary as a submodule output,
+    which aot_autograd / standalone_compile rejects.
+    """
+    captured_graph = None
+    captured_inputs = None
+
+    def capturing_backend(gm: fx.GraphModule, example_inputs: list) -> fx.GraphModule:
+        nonlocal captured_graph, captured_inputs
+        captured_graph = gm
+        captured_inputs = example_inputs
+        return gm
+
+    def model_fn(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
+        shape = x.size()
+        z1 = torch.ops.aten.sigmoid.default(x)
+        y1 = y.view(shape)
+        z2 = torch.ops.aten.sigmoid.default(z1)
+        y2 = y.view(shape)
+        return z2 + y1 + y2
+
+    x = torch.randn(4, 8)
+    y = torch.randn(4, 8)  # same shape as x so view(shape) doesn't specialize dim 0
+    torch._dynamo.mark_dynamic(x, 0)
+    torch._dynamo.mark_dynamic(y, 0)
+    torch.compile(model_fn, backend=capturing_backend)(x, y)
+
+    split_gm, split_items = split_graph(captured_graph, ["aten::sigmoid"])
+
+    splitting_items = [item for item in split_items if item.is_splitting_graph]
+    assert len(splitting_items) == 2
+
+    # Verify functional correctness — fails without the fix because torch.Size
+    # would cross a split boundary as a submodule output
+    output_original = model_fn(x, y)
+    output_split = split_gm(*captured_inputs)
+    if isinstance(output_split, tuple):
+        output_split = next(o for o in output_split if isinstance(o, torch.Tensor))
+    assert torch.allclose(output_original, output_split), "Output mismatch after split"
+
+
+@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
+def test_sym_size_metadata_propagated():
+    """
+    Validates that new sym_size.int nodes created by the pre-pass have
+    example_value metadata set. Without it, placeholder metadata in consumer
+    subgraphs would be None, breaking any code that dynamically builds
+    example inputs from metadata (e.g. standalone_compile per-submodule).
+    """
+    from torch._inductor import standalone_compile
+
+    captured_graph = None
+
+    def capturing_backend(gm: fx.GraphModule, example_inputs: list) -> fx.GraphModule:
+        nonlocal captured_graph
+        captured_graph = gm
+        return gm
+
+    def model_fn(x: torch.Tensor) -> torch.Tensor:
+        shape = x.size()
+        x = torch.ops.aten.sigmoid.default(x)
+        x = x.clone().view(shape)
+        return x
+
+    x = torch.randn(4, 8)
+    torch._dynamo.mark_dynamic(x, 0)
+    torch.compile(model_fn, backend=capturing_backend)(x)
+
+    split_gm, split_items = split_graph(captured_graph, ["aten::sigmoid"])
+
+    # For each submodule, build example inputs purely from placeholder metadata.
+    # This fails if example_value is None on any placeholder (i.e. metadata
+    # was not propagated to the sym_size.int nodes we created).
+    for item in split_items:
+        submod = item.graph
+        example_inputs = []
+        for n in submod.graph.nodes:
+            if n.op != "placeholder":
+                continue
+            ev = n.meta.get("example_value")
+            assert ev is not None, (
+                f"Placeholder '{n.name}' in {item.submod_name} has no "
+                "example_value metadata. sym_size.int nodes must propagate "
+                "metadata so consumer subgraphs can be introspected."
+            )
+            if isinstance(ev, torch.Tensor):
+                example_inputs.append(torch.randn(*(int(d) for d in ev.shape)))
+            else:
+                example_inputs.append(int(ev))
+        standalone_compile(submod, example_inputs, dynamic_shapes="from_example_inputs")
@@ -9,10 +9,15 @@ then runs in the parent with clean in-memory state but populated caches.

 import multiprocessing as mp

+import pytest
 from torch._dynamo.utils import counters

+import vllm.envs as envs
 from vllm.compilation.counter import compilation_counter
 from vllm.config import CompilationConfig, CompilationMode, CUDAGraphMode
+from vllm.utils.torch_utils import is_torch_equal_or_newer
+
+from ..utils import fork_new_process_for_each_test

 MODEL = "microsoft/Phi-tiny-MoE-instruct"

@@ -45,8 +50,11 @@ def _cold_start(vllm_runner):
    assert counters["aot_autograd"]["autograd_cache_hit"] == 0


-def test_moe_startup(monkeypatch, vllm_runner, fresh_vllm_cache):
+@fork_new_process_for_each_test
+@pytest.mark.parametrize("mega_aot_artifact", ["0", "1"])
+def test_moe_startup(monkeypatch, vllm_runner, fresh_vllm_cache, mega_aot_artifact):
    monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
+    monkeypatch.setenv("VLLM_USE_MEGA_AOT_ARTIFACT", mega_aot_artifact)

    # Cold start in a forked child (must fork before CUDA init).
    # This model has 32 identical transformer layers which produce
@@ -64,7 +72,15 @@ def test_moe_startup(monkeypatch, vllm_runner, fresh_vllm_cache):
        num_compiled_artifacts_saved=0,
    ):
        _run_vllm(vllm_runner)
-    assert counters["aot_autograd"]["total"] == 30
+    mega_aot_active = envs.VLLM_USE_MEGA_AOT_ARTIFACT and is_torch_equal_or_newer(
+        "2.10.0"
+    )
+    if mega_aot_active:
+        # MEGA_AOT_ARTIFACT is enabled, so we expect no aot_autograd running on
+        # subgraphs.
+        assert counters["aot_autograd"]["total"] == 0
+    else:
+        assert counters["aot_autograd"]["total"] == 30
    assert counters["aot_autograd"]["autograd_cache_miss"] == 0
    assert (
        counters["aot_autograd"]["autograd_cache_hit"] == 0
@@ -6,9 +6,6 @@ from copy import deepcopy

 from tblib import pickling_support

-# Import fixture
-from tests.v1.entrypoints.conftest import sample_json_schema  # noqa
-
 # ruff: noqa

 # Install support for pickling exceptions so that we can nicely propagate
@@ -81,6 +78,55 @@ if TYPE_CHECKING:

 logger = init_logger(__name__)

+
+@pytest.fixture
+def sample_json_schema():
+    return {
+        "type": "object",
+        "properties": {
+            "name": {"type": "string"},
+            "age": {"type": "integer"},
+            "skills": {
+                "type": "array",
+                "items": {
+                    "type": "string",
+                },
+            },
+            "grade": {
+                "type": "string",
+                "pattern": "^[A-D]$",
+            },
+            "email": {
+                "type": "string",
+                "pattern": "^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$",
+            },
+            "work_history": {
+                "type": "array",
+                "items": {
+                    "type": "object",
+                    "properties": {
+                        "company": {"type": "string"},
+                        "duration": {
+                            "type": "number",
+                            "minimum": 0.0,
+                            "maximum": 100.0,
+                        },
+                        "position": {"type": "string"},
+                    },
+                    "required": ["company", "duration", "position"],
+                    "additionalProperties": False,
+                },
+                "minItems": 0,
+                "maxItems": 3,
+            },
+        },
+        "required": ["name", "age", "skills", "grade", "email", "work_history"],
+        "additionalProperties": False,
+        "minProperties": 1,
+        "maxProperties": 10,
+    }
+
+
 _TEST_DIR = os.path.dirname(__file__)
 _TEST_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "example.txt")]
 _LONG_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "summary.txt")]
@@ -319,9 +319,6 @@ def _compare_tp(
        pp_env = {
            "VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL": "1",
        }
-        # Temporary. Currently when zeromq + SPMD is used, it does not properly
-        # terminate because of a Ray Compiled Graph issue.
-        common_args.append("--disable-frontend-multiprocessing")
    elif distributed_backend == "mp":
        pp_env = None
    else:
@@ -5,7 +5,7 @@ import anthropic
 import pytest
 import pytest_asyncio

-from ...utils import RemoteOpenAIServer
+from tests.utils import RemoteOpenAIServer

 MODEL_NAME = "Qwen/Qwen3-0.6B"

@@ -24,6 +24,108 @@ from vllm.sampling_params import (
    StructuredOutputsParams,
 )

+SAMPLE_REGEX = (
+    r"((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.){3}"
+    r"(25[0-5]|(2[0-4]|1\d|[1-9]|)\d)"
+)
+
+# Note: Ensure this only uses attributes compatible with xgrammar
+SAMPLE_JSON_SCHEMA = {
+    "type": "object",
+    "properties": {
+        "name": {"type": "string"},
+        "age": {"type": "integer"},
+        "skills": {
+            "type": "array",
+            "items": {
+                "type": "string",
+            },
+        },
+        "grade": {
+            "type": "string",
+            "pattern": "^[A-D]$",  # Regex pattern
+        },
+        "email": {
+            "type": "string",
+            "pattern": "^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$",
+        },
+        "work_history": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "company": {"type": "string"},
+                    "duration": {
+                        "type": "number",
+                        "minimum": 0.0,
+                        "maximum": 100.0,  # Numeric range
+                    },
+                    "position": {"type": "string"},
+                },
+                "required": ["company", "duration", "position"],
+                "additionalProperties": False,
+            },
+            "minItems": 0,
+            "maxItems": 3,
+        },
+    },
+    "required": ["name", "age", "skills", "grade", "email", "work_history"],
+    "additionalProperties": False,
+    "minProperties": 1,
+    "maxProperties": 10,
+}
+
+# A schema unsupported by xgrammar
+UNSUPPORTED_JSON_SCHEMA = {
+    "type": "object",
+    "properties": {
+        "score": {
+            "type": "integer",
+            "multipleOf": 5,  # Numeric multiple
+        },
+        "tags": {
+            "type": "array",
+            "items": {"type": "string", "minLength": 10, "maxLength": 20},
+        },
+    },
+    "required": ["score", "tags"],
+    "additionalProperties": False,
+    "patternProperties": {
+        "^score$": {"type": "integer"},
+    },
+}
+
+SAMPLE_STRUCTURED_OUTPUTS_CHOICES = [
+    "Python",
+    "Java",
+    "JavaScript",
+    "C++",
+    "C#",
+    "PHP",
+    "TypeScript",
+    "Ruby",
+    "Swift",
+    "Kotlin",
+]
+
+SAMPLE_SQL_EBNF = """
+root ::= select_statement
+select_statement ::= "SELECT" column "from" table "where" condition
+column ::= "col_1" | "col_2"
+table ::= "table_1" | "table_2"
+condition ::= column "=" number
+number ::= "1" | "2"
+"""
+
+SAMPLE_SQL_LARK = """
+start: select_statement
+select_statement: "SELECT" column "from" table "where" condition
+column: "col_1" | "col_2"
+table: "table_1" | "table_2"
+condition: column "=" number
+number: "1" | "2"
+"""
+
 NGRAM_SPEC_CONFIG = {
    "model": "[ngram]",
    "num_speculative_tokens": 5,
@@ -110,17 +212,17 @@ class CarDescription(BaseModel):
    PARAMS_MODELS_BACKENDS_TOKENIZER_MODE,
 )
 def test_structured_output(
-    sample_json_schema: dict[str, Any],
-    unsupported_json_schema: dict[str, Any],
-    sample_sql_ebnf: str,
-    sample_sql_lark: str,
-    sample_regex: str,
-    sample_structured_outputs_choices: str,
    backend: str,
    tokenizer_mode: str,
    model_name: str,
    speculative_config: dict[str, Any],
 ):
+    sample_json_schema = SAMPLE_JSON_SCHEMA
+    unsupported_json_schema = UNSUPPORTED_JSON_SCHEMA
+    sample_sql_ebnf = SAMPLE_SQL_EBNF
+    sample_sql_lark = SAMPLE_SQL_LARK
+    sample_regex = SAMPLE_REGEX
+    sample_structured_outputs_choices = SAMPLE_STRUCTURED_OUTPUTS_CHOICES
    if current_platform.is_tpu() and speculative_config:
        pytest.skip("TPU does not support speculative decoding")

@@ -702,10 +804,10 @@ def test_structured_output_with_reasoning_matrices(

@pytest.mark.parametrize("model_name, tokenizer_mode", PARAMS_MODELS_TOKENIZER_MODE)
 def test_structured_output_auto_mode(
-    unsupported_json_schema: dict[str, Any],
    model_name: str,
    tokenizer_mode: str,
 ):
+    unsupported_json_schema = UNSUPPORTED_JSON_SCHEMA
    llm = LLM(
        model=model_name,
        max_model_len=1024,
@@ -808,9 +910,9 @@ def test_guidance_no_additional_properties():

@pytest.mark.parametrize("backend", ["guidance", "xgrammar", "outlines"])
 def test_structured_output_batched_with_non_structured_outputs_requests(
-    sample_json_schema: dict[str, Any],
    backend: str,
 ):
+    sample_json_schema = SAMPLE_JSON_SCHEMA
    # Don't use eager execution on TPUs because we want to test for no
    # recompilation at runtime
    enforce_eager = bool(not current_platform.is_tpu())
@@ -231,13 +231,14 @@ def k2_server():
        "--gpu-memory-utilization",
        "0.4",
    ] + ROCM_EXTRA_ARGS
-    # hack to test kimi_k2 tool use tool_id format.
-    # avoid error in is_deepseek_mla check by setting kv_lora_rank=null
+    # Test kimi_k2 tool use tool_id format by overriding model_type.
+    # is_deepseek_mla safely returns False via getattr when kv_lora_rank
+    # is absent from the underlying config.
    with RemoteOpenAIServer(
        MODEL_NAME,
        args,
        env_dict=ROCM_ENV_OVERRIDES,
-        override_hf_configs={"model_type": "kimi_k2", "kv_lora_rank": None},
+        override_hf_configs={"model_type": "kimi_k2"},
    ) as remote_server:
        yield remote_server

--- a/Show More
+++ b/Show More