[CI] Reorganize release pipeline: separate nightly vs release sections

Reorder the pipeline into clear sections for readability:

1. Build Python Wheels (always runs)
2. ROCm Wheel Pipeline (always runs)
3. Nightly Docker Images (NIGHTLY=1 only) - CUDA/Ubuntu builds,
   multi-arch manifests, DockerHub publish, ROCm image + publish
4. Release (manual) - version input, PyPI upload, CPU image builds,
   ROCm root index

Key changes:
- Extract CPU image builds (manual/blocked) from nightly-gated group
  into their own "Build release CPU Docker images" group so they remain
  available for actual releases without NIGHTLY=1
- Move ROCm wheel jobs (1-4) up next to CUDA wheel builds
- Remove redundant per-step NIGHTLY gates inside already-gated groups
- Rename groups: "Build release Docker images" -> "Build nightly Docker
  images", "Publish release images" -> "Publish nightly images"

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

.buildkite/hardware_tests/cpu.yaml

@@ -46,7 +46,7 @@ steps:
   - tests/models/language/pooling/
   commands:
     - |
-      bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 40m "
+      bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 30m "
       pytest -x -v -s tests/models/language/generation -m cpu_model
       pytest -x -v -s tests/models/language/pooling -m cpu_model"
 
@@ -99,7 +99,7 @@ steps:
     - |
       bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 45m "
       pytest -x -v -s tests/models/multimodal/generation --ignore=tests/models/multimodal/generation/test_pixtral.py -m cpu_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
-  parallelism: 3
+  parallelism: 2
 
 - label: "Arm CPU Test"
   depends_on: []

.buildkite/release-pipeline.yaml

@@ -1,9 +1,7 @@
 steps:
-  - input: "Provide Release version here"
-    id: input-release-version
-    fields:
-      - text: "What is the release version?"
-        key: release-version
+  # =============================================================================
+  # Build Python Wheels (runs on every pipeline trigger)
+  # =============================================================================
 
   - group: "Build Python wheels"
     key: "build-wheels"
@@ -98,15 +96,257 @@ steps:
     commands:
       - "bash .buildkite/scripts/generate-and-upload-nightly-index.sh"
 
-  - block: "Unblock to build release Docker images"
-    depends_on: ~
-    key: block-build-release-images
-    if: build.env("NIGHTLY") != "1"
+  # =============================================================================
+  # ROCm Wheel Pipeline (runs on every pipeline trigger)
+  # =============================================================================
 
-  - group: "Build release Docker images"
+  # ROCm Job 1: Build ROCm Base Wheels (with S3 caching)
+  - label: ":rocm: Build ROCm Base Image & Wheels"
+    id: build-rocm-base-wheels
+    depends_on: ~
+    agents:
+      queue: cpu_queue_release
+    commands:
+      - |
+        set -euo pipefail
+
+        # Generate cache key
+        CACHE_KEY=$$(.buildkite/scripts/cache-rocm-base-wheels.sh key)
+        ECR_CACHE_TAG="public.ecr.aws/q9t5s3a7/vllm-release-repo:$${CACHE_KEY}-rocm-base"
+
+        echo "========================================"
+        echo "ROCm Base Build Configuration"
+        echo "========================================"
+        echo "  CACHE_KEY: $${CACHE_KEY}"
+        echo "  ECR_CACHE_TAG: $${ECR_CACHE_TAG}"
+        echo "========================================"
+
+        # Login to ECR
+        aws ecr-public get-login-password --region us-east-1 | \
+          docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
+
+        IMAGE_EXISTS=false
+        WHEELS_EXIST=false
+
+        # Check ECR for Docker image
+
+        if docker manifest inspect "$${ECR_CACHE_TAG}" > /dev/null 2>&1; then
+          IMAGE_EXISTS=true
+          echo "ECR image cache HIT"
+        fi
+
+        # Check S3 for wheels
+        WHEEL_CACHE_STATUS=$(.buildkite/scripts/cache-rocm-base-wheels.sh check)
+        if [ "$${WHEEL_CACHE_STATUS}" = "hit" ]; then
+          WHEELS_EXIST=true
+          echo "S3 wheels cache HIT"
+        fi
+
+
+        # Scenario 1: Both cached (best case)
+        if [ "$${IMAGE_EXISTS}" = "true" ] && [ "$${WHEELS_EXIST}" = "true" ]; then
+          echo ""
+          echo "FULL CACHE HIT - Reusing both image and wheels"
+          echo ""
+
+          # Download wheels
+          .buildkite/scripts/cache-rocm-base-wheels.sh download
+
+          # Save ECR tag for downstream jobs
+          buildkite-agent meta-data set "rocm-base-image-tag" "$${ECR_CACHE_TAG}"
+
+        # Scenario 2: Full rebuild needed
+        else
+          echo ""
+          echo " CACHE MISS - Building from scratch..."
+          echo ""
+
+          # Build full base image and push to ECR
+          DOCKER_BUILDKIT=1 docker buildx build \
+            --file docker/Dockerfile.rocm_base \
+            --tag "$${ECR_CACHE_TAG}" \
+            --build-arg USE_SCCACHE=1 \
+            --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
+            --build-arg SCCACHE_REGION_NAME=us-west-2 \
+            --build-arg SCCACHE_S3_NO_CREDENTIALS=0 \
+            --push \
+            .
+
+          # Build wheel extraction stage
+          DOCKER_BUILDKIT=1 docker buildx build \
+            --file docker/Dockerfile.rocm_base \
+            --tag rocm-base-debs:$${BUILDKITE_BUILD_NUMBER} \
+            --target debs_wheel_release \
+            --build-arg USE_SCCACHE=1 \
+            --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
+            --build-arg SCCACHE_REGION_NAME=us-west-2 \
+            --build-arg SCCACHE_S3_NO_CREDENTIALS=0 \
+            --load \
+            .
+
+          # Extract and upload wheels
+          mkdir -p artifacts/rocm-base-wheels
+          cid=$(docker create rocm-base-debs:$${BUILDKITE_BUILD_NUMBER})
+          docker cp $${cid}:/app/debs/. artifacts/rocm-base-wheels/
+          docker rm $${cid}
+
+          .buildkite/scripts/cache-rocm-base-wheels.sh upload
+
+          # Cache base docker image to ECR
+          docker push "$${ECR_CACHE_TAG}"
+
+          buildkite-agent meta-data set "rocm-base-image-tag" "$${ECR_CACHE_TAG}"
+
+          echo ""
+          echo " Build complete - Image and wheels cached"
+        fi
+
+    artifact_paths:
+      - "artifacts/rocm-base-wheels/*.whl"
+    env:
+      DOCKER_BUILDKIT: "1"
+      S3_BUCKET: "vllm-wheels"
+
+  # ROCm Job 2: Build vLLM ROCm Wheel
+  - label: ":python: Build vLLM ROCm Wheel - x86_64"
+    id: build-rocm-vllm-wheel
+    depends_on:
+      - step: build-rocm-base-wheels
+        allow_failure: false
+    agents:
+      queue: cpu_queue_release
+    timeout_in_minutes: 180
+    commands:
+      # Download artifacts and prepare Docker image
+      - |
+        set -euo pipefail
+
+        # Ensure git tags are up-to-date (Buildkite's default fetch doesn't update tags)
+        # This fixes version detection when tags are moved/force-pushed
+        echo "Fetching latest tags from origin..."
+        git fetch --tags --force origin
+
+        # Log tag information for debugging version detection
+        echo "========================================"
+        echo "Git Tag Verification"
+        echo "========================================"
+        echo "Current HEAD: $(git rev-parse HEAD)"
+        echo "git describe --tags: $(git describe --tags 2>/dev/null || echo 'No tags found')"
+        echo ""
+        echo "Recent tags (pointing to commits near HEAD):"
+        git tag -l --sort=-creatordate | head -5
+        echo "setuptools_scm version detection:"
+        pip install -q setuptools_scm 2>/dev/null || true
+        python3 -c "import setuptools_scm; print('  Detected version:', setuptools_scm.get_version())" 2>/dev/null || echo "  (setuptools_scm not available in this environment)"
+        echo "========================================"
+
+        # Download wheel artifacts from current build
+        echo "Downloading wheel artifacts from current build"
+        buildkite-agent artifact download "artifacts/rocm-base-wheels/*.whl" .
+
+        # Get ECR image tag from metadata (set by build-rocm-base-wheels)
+        ECR_IMAGE_TAG="$$(buildkite-agent meta-data get rocm-base-image-tag 2>/dev/null || echo '')"
+        if [ -z "$${ECR_IMAGE_TAG}" ]; then
+          echo "ERROR: rocm-base-image-tag metadata not found"
+          echo "This should have been set by the build-rocm-base-wheels job"
+          exit 1
+        fi
+
+        echo "Pulling base Docker image from ECR: $${ECR_IMAGE_TAG}"
+
+        # Login to ECR
+        aws ecr-public get-login-password --region us-east-1 | \
+          docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
+
+        # Pull base Docker image from ECR
+        docker pull "$${ECR_IMAGE_TAG}"
+
+        echo "Loaded base image: $${ECR_IMAGE_TAG}"
+
+        # Prepare base wheels for Docker build context
+        mkdir -p docker/context/base-wheels
+        touch docker/context/base-wheels/.keep
+        cp artifacts/rocm-base-wheels/*.whl docker/context/base-wheels/
+        echo "Base wheels for vLLM build:"
+        ls -lh docker/context/base-wheels/
+
+        echo "========================================"
+        echo "Building vLLM wheel with:"
+        echo "  BUILDKITE_COMMIT: $${BUILDKITE_COMMIT}"
+        echo "  BUILDKITE_BRANCH: $${BUILDKITE_BRANCH}"
+        echo "  BASE_IMAGE: $${ECR_IMAGE_TAG}"
+        echo "========================================"
+
+        # Build vLLM wheel using local checkout (REMOTE_VLLM=0)
+        DOCKER_BUILDKIT=1 docker build \
+          --file docker/Dockerfile.rocm \
+          --target export_vllm_wheel_release \
+          --output type=local,dest=rocm-dist \
+          --build-arg BASE_IMAGE="$${ECR_IMAGE_TAG}" \
+          --build-arg REMOTE_VLLM=0 \
+          --build-arg GIT_REPO_CHECK=1 \
+          --build-arg USE_SCCACHE=1 \
+          --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
+          --build-arg SCCACHE_REGION_NAME=us-west-2 \
+          --build-arg SCCACHE_S3_NO_CREDENTIALS=0 \
+          .
+        echo "Built vLLM wheel:"
+        ls -lh rocm-dist/*.whl
+        # Copy wheel to artifacts directory
+        mkdir -p artifacts/rocm-vllm-wheel
+        cp rocm-dist/*.whl artifacts/rocm-vllm-wheel/
+        echo "Final vLLM wheel:"
+        ls -lh artifacts/rocm-vllm-wheel/
+    artifact_paths:
+      - "artifacts/rocm-vllm-wheel/*.whl"
+    env:
+      DOCKER_BUILDKIT: "1"
+      S3_BUCKET: "vllm-wheels"
+
+  # ROCm Job 3: Upload Wheels to S3
+  - label: ":s3: Upload ROCm Wheels to S3"
+    id: upload-rocm-wheels
+    depends_on:
+      - step: build-rocm-vllm-wheel
+        allow_failure: false
+    agents:
+      queue: cpu_queue_release
+    timeout_in_minutes: 60
+    commands:
+      # Download all wheel artifacts and run upload
+      - |
+        set -euo pipefail
+
+        # Download artifacts from current build
+        echo "Downloading artifacts from current build"
+        buildkite-agent artifact download "artifacts/rocm-base-wheels/*.whl" .
+        buildkite-agent artifact download "artifacts/rocm-vllm-wheel/*.whl" .
+
+        # Run upload script
+        bash .buildkite/scripts/upload-rocm-wheels.sh
+    env:
+      DOCKER_BUILDKIT: "1"
+      S3_BUCKET: "vllm-wheels"
+
+  # ROCm Job 4: Annotate ROCm Wheel Release
+  - label: ":memo: Annotate ROCm wheel release"
+    id: annotate-rocm-release
+    depends_on:
+      - upload-rocm-wheels
+    agents:
+      queue: cpu_queue_release
+    commands:
+      - "bash .buildkite/scripts/annotate-rocm-release.sh"
+    env:
+      S3_BUCKET: "vllm-wheels"
+
+  # =============================================================================
+  # Nightly: Build & Publish Docker Images (NIGHTLY=1 only)
+  # =============================================================================
+
+  - group: "Build nightly Docker images"
     key: "build-release-images"
-    depends_on: block-build-release-images
-    allow_dependency_failure: true
+    if: build.env("NIGHTLY") == "1"
     steps:
       - label: "Build release image - x86_64 - CUDA 12.9"
         depends_on: ~
@@ -199,44 +439,9 @@ steps:
           - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg UBUNTU_VERSION=24.04 --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 --build-arg FLASHINFER_AOT_COMPILE=true --build-arg torch_cuda_arch_list='8.7 8.9 9.0 10.0+PTX 12.0 12.1' --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu24.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130-ubuntu2404 --target vllm-openai --progress plain -f docker/Dockerfile ."
           - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130-ubuntu2404"
 
-      - block: "Build release image for x86_64 CPU"
-        key: block-cpu-release-image-build
-        depends_on: ~
-
-      - label: "Build release image - x86_64 - CPU"
-        depends_on:
-          - block-cpu-release-image-build
-          - input-release-version
-        agents:
-          queue: cpu_queue_release
-        commands:
-          - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
-          - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg GIT_REPO_CHECK=1 --build-arg VLLM_CPU_X86=true --tag public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:$(buildkite-agent meta-data get release-version) --tag public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:latest --progress plain --target vllm-openai -f docker/Dockerfile.cpu ."
-          - "docker push public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:latest"
-          - "docker push public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:$(buildkite-agent meta-data get release-version)"
-        env:
-          DOCKER_BUILDKIT: "1"
-
-      - block: "Build release image for arm64 CPU"
-        key: block-arm64-cpu-release-image-build
-        depends_on: ~
-
-      - label: "Build release image - arm64 - CPU"
-        depends_on: 
-          - block-arm64-cpu-release-image-build
-          - input-release-version
-        agents:
-          queue: arm64_cpu_queue_release
-        commands:
-          - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
-          - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg GIT_REPO_CHECK=1 --tag public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:$(buildkite-agent meta-data get release-version) --tag public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:latest --progress plain --target vllm-openai -f docker/Dockerfile.cpu ."
-          - "docker push public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:latest"
-          - "docker push public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:$(buildkite-agent meta-data get release-version)"
-        env:
-          DOCKER_BUILDKIT: "1"
-
-  - group: "Publish release images"
+  - group: "Publish nightly images"
     key: "publish-release-images"
+    if: build.env("NIGHTLY") == "1"
     steps:
       - label: "Create multi-arch manifest - CUDA 12.9"
         depends_on:
@@ -298,7 +503,6 @@ steps:
       - label: "Publish nightly multi-arch image to DockerHub"
         depends_on:
           - create-multi-arch-manifest
-        if: build.env("NIGHTLY") == "1"
         agents:
           queue: small_cpu_queue_release
         commands:
@@ -316,7 +520,6 @@ steps:
       - label: "Publish nightly multi-arch image to DockerHub - CUDA 13.0"
         depends_on:
           - create-multi-arch-manifest-cuda-13-0
-        if: build.env("NIGHTLY") == "1"
         agents:
           queue: small_cpu_queue_release
         commands:
@@ -331,301 +534,11 @@ steps:
           DOCKER_BUILDKIT: "1"
           DOCKERHUB_USERNAME: "vllmbot"
 
-  - group: "Publish wheels"
-    key: "publish-wheels"
-    steps:
-      - block: "Confirm update release wheels to PyPI (experimental, use with caution)?"
-        key: block-upload-release-wheels
-        depends_on:
-          - input-release-version
-          - build-wheels
-
-      - label: "Upload release wheels to PyPI"
-        depends_on:
-          - block-upload-release-wheels
-        id: upload-release-wheels
-        agents:
-          queue: small_cpu_queue_release
-        commands:
-          - "bash .buildkite/scripts/upload-release-wheels-pypi.sh"
-
-  # =============================================================================
-  # ROCm Release Pipeline (x86_64 only)
-  # =============================================================================
-  #
-  # vLLM version is determined by the Buildkite checkout (like CUDA pipeline).
-  # To build a specific version, trigger the build from that branch/tag.
-  #
-  # Environment variables for ROCm builds (set via Buildkite UI or schedule):
-  #
-  # Note: ROCm version is determined by BASE_IMAGE in docker/Dockerfile.rocm_base
-  #
-  # =============================================================================
-
-  # ROCm Job 1: Build ROCm Base Wheels (with S3 caching)
-  - label: ":rocm: Build ROCm Base Image & Wheels"
-    id: build-rocm-base-wheels
-    depends_on: ~
-    agents:
-      queue: cpu_queue_release
-    commands:
-      - |
-        set -euo pipefail
-
-        # Generate cache key
-        CACHE_KEY=$$(.buildkite/scripts/cache-rocm-base-wheels.sh key)
-        ECR_CACHE_TAG="public.ecr.aws/q9t5s3a7/vllm-release-repo:$${CACHE_KEY}-rocm-base"
-
-        echo "========================================"
-        echo "ROCm Base Build Configuration"
-        echo "========================================"
-        echo "  CACHE_KEY: $${CACHE_KEY}"
-        echo "  ECR_CACHE_TAG: $${ECR_CACHE_TAG}"
-        echo "========================================"
-        
-        # Login to ECR
-        aws ecr-public get-login-password --region us-east-1 | \
-          docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
-        
-        IMAGE_EXISTS=false
-        WHEELS_EXIST=false
-        
-        # Check ECR for Docker image
-
-        if docker manifest inspect "$${ECR_CACHE_TAG}" > /dev/null 2>&1; then
-          IMAGE_EXISTS=true
-          echo "ECR image cache HIT"
-        fi
-        
-        # Check S3 for wheels
-        WHEEL_CACHE_STATUS=$(.buildkite/scripts/cache-rocm-base-wheels.sh check)
-        if [ "$${WHEEL_CACHE_STATUS}" = "hit" ]; then
-          WHEELS_EXIST=true
-          echo "S3 wheels cache HIT"
-        fi
-
-        
-        # Scenario 1: Both cached (best case)
-        if [ "$${IMAGE_EXISTS}" = "true" ] && [ "$${WHEELS_EXIST}" = "true" ]; then
-          echo ""
-          echo "FULL CACHE HIT - Reusing both image and wheels"
-          echo ""
-
-          # Download wheels
-          .buildkite/scripts/cache-rocm-base-wheels.sh download
-          
-          # Save ECR tag for downstream jobs
-          buildkite-agent meta-data set "rocm-base-image-tag" "$${ECR_CACHE_TAG}"
-          
-        # Scenario 2: Full rebuild needed
-        else
-          echo ""
-          echo " CACHE MISS - Building from scratch..."
-          echo ""
-          
-          # Build full base image and push to ECR
-          DOCKER_BUILDKIT=1 docker buildx build \
-            --file docker/Dockerfile.rocm_base \
-            --tag "$${ECR_CACHE_TAG}" \
-            --build-arg USE_SCCACHE=1 \
-            --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
-            --build-arg SCCACHE_REGION_NAME=us-west-2 \
-            --build-arg SCCACHE_S3_NO_CREDENTIALS=0 \
-            --push \
-            .
-          
-          # Build wheel extraction stage
-          DOCKER_BUILDKIT=1 docker buildx build \
-            --file docker/Dockerfile.rocm_base \
-            --tag rocm-base-debs:$${BUILDKITE_BUILD_NUMBER} \
-            --target debs_wheel_release \
-            --build-arg USE_SCCACHE=1 \
-            --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
-            --build-arg SCCACHE_REGION_NAME=us-west-2 \
-            --build-arg SCCACHE_S3_NO_CREDENTIALS=0 \
-            --load \
-            .
-          
-          # Extract and upload wheels
-          mkdir -p artifacts/rocm-base-wheels
-          cid=$(docker create rocm-base-debs:$${BUILDKITE_BUILD_NUMBER})
-          docker cp $${cid}:/app/debs/. artifacts/rocm-base-wheels/
-          docker rm $${cid}
-          
-          .buildkite/scripts/cache-rocm-base-wheels.sh upload
-
-          # Cache base docker image to ECR
-          docker push "$${ECR_CACHE_TAG}"
-          
-          buildkite-agent meta-data set "rocm-base-image-tag" "$${ECR_CACHE_TAG}"
-          
-          echo ""
-          echo " Build complete - Image and wheels cached"
-        fi
-        
-    artifact_paths:
-      - "artifacts/rocm-base-wheels/*.whl"
-    env:
-      DOCKER_BUILDKIT: "1"
-      S3_BUCKET: "vllm-wheels"
-
-  # ROCm Job 2: Build vLLM ROCm Wheel
-  - label: ":python: Build vLLM ROCm Wheel - x86_64"
-    id: build-rocm-vllm-wheel
-    depends_on:
-      - step: build-rocm-base-wheels
-        allow_failure: false
-    agents:
-      queue: cpu_queue_release
-    timeout_in_minutes: 180
-    commands:
-      # Download artifacts and prepare Docker image
-      - |
-        set -euo pipefail
-
-        # Ensure git tags are up-to-date (Buildkite's default fetch doesn't update tags)
-        # This fixes version detection when tags are moved/force-pushed
-        echo "Fetching latest tags from origin..."
-        git fetch --tags --force origin
-        
-        # Log tag information for debugging version detection
-        echo "========================================"
-        echo "Git Tag Verification"
-        echo "========================================"
-        echo "Current HEAD: $(git rev-parse HEAD)"
-        echo "git describe --tags: $(git describe --tags 2>/dev/null || echo 'No tags found')"
-        echo ""
-        echo "Recent tags (pointing to commits near HEAD):"
-        git tag -l --sort=-creatordate | head -5
-        echo "setuptools_scm version detection:"
-        pip install -q setuptools_scm 2>/dev/null || true
-        python3 -c "import setuptools_scm; print('  Detected version:', setuptools_scm.get_version())" 2>/dev/null || echo "  (setuptools_scm not available in this environment)"
-        echo "========================================"
-
-        # Download wheel artifacts from current build
-        echo "Downloading wheel artifacts from current build"
-        buildkite-agent artifact download "artifacts/rocm-base-wheels/*.whl" .
-
-        # Get ECR image tag from metadata (set by build-rocm-base-wheels)
-        ECR_IMAGE_TAG="$$(buildkite-agent meta-data get rocm-base-image-tag 2>/dev/null || echo '')"
-        if [ -z "$${ECR_IMAGE_TAG}" ]; then
-          echo "ERROR: rocm-base-image-tag metadata not found"
-          echo "This should have been set by the build-rocm-base-wheels job"
-          exit 1
-        fi
-        
-        echo "Pulling base Docker image from ECR: $${ECR_IMAGE_TAG}"
-        
-        # Login to ECR
-        aws ecr-public get-login-password --region us-east-1 | \
-          docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
-        
-        # Pull base Docker image from ECR
-        docker pull "$${ECR_IMAGE_TAG}"
-        
-        echo "Loaded base image: $${ECR_IMAGE_TAG}"
-        
-        # Prepare base wheels for Docker build context
-        mkdir -p docker/context/base-wheels
-        touch docker/context/base-wheels/.keep
-        cp artifacts/rocm-base-wheels/*.whl docker/context/base-wheels/
-        echo "Base wheels for vLLM build:"
-        ls -lh docker/context/base-wheels/
-
-        echo "========================================"
-        echo "Building vLLM wheel with:"
-        echo "  BUILDKITE_COMMIT: $${BUILDKITE_COMMIT}"
-        echo "  BUILDKITE_BRANCH: $${BUILDKITE_BRANCH}"
-        echo "  BASE_IMAGE: $${ECR_IMAGE_TAG}"
-        echo "========================================"
-
-        # Build vLLM wheel using local checkout (REMOTE_VLLM=0)
-        DOCKER_BUILDKIT=1 docker build \
-          --file docker/Dockerfile.rocm \
-          --target export_vllm_wheel_release \
-          --output type=local,dest=rocm-dist \
-          --build-arg BASE_IMAGE="$${ECR_IMAGE_TAG}" \
-          --build-arg REMOTE_VLLM=0 \
-          --build-arg GIT_REPO_CHECK=1 \
-          --build-arg USE_SCCACHE=1 \
-          --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
-          --build-arg SCCACHE_REGION_NAME=us-west-2 \
-          --build-arg SCCACHE_S3_NO_CREDENTIALS=0 \
-          .
-        echo "Built vLLM wheel:"
-        ls -lh rocm-dist/*.whl
-        # Copy wheel to artifacts directory
-        mkdir -p artifacts/rocm-vllm-wheel
-        cp rocm-dist/*.whl artifacts/rocm-vllm-wheel/
-        echo "Final vLLM wheel:"
-        ls -lh artifacts/rocm-vllm-wheel/
-    artifact_paths:
-      - "artifacts/rocm-vllm-wheel/*.whl"
-    env:
-      DOCKER_BUILDKIT: "1"
-      S3_BUCKET: "vllm-wheels"
-
-  # ROCm Job 3: Upload Wheels to S3
-  - label: ":s3: Upload ROCm Wheels to S3"
-    id: upload-rocm-wheels
-    depends_on:
-      - step: build-rocm-vllm-wheel
-        allow_failure: false
-    agents:
-      queue: cpu_queue_release
-    timeout_in_minutes: 60
-    commands:
-      # Download all wheel artifacts and run upload
-      - |
-        set -euo pipefail
-
-        # Download artifacts from current build
-        echo "Downloading artifacts from current build"
-        buildkite-agent artifact download "artifacts/rocm-base-wheels/*.whl" .
-        buildkite-agent artifact download "artifacts/rocm-vllm-wheel/*.whl" .
-
-        # Run upload script
-        bash .buildkite/scripts/upload-rocm-wheels.sh
-    env:
-      DOCKER_BUILDKIT: "1"
-      S3_BUCKET: "vllm-wheels"
-
-  # ROCm Job 4: Annotate ROCm Wheel Release
-  - label: ":memo: Annotate ROCm wheel release"
-    id: annotate-rocm-release
-    depends_on:
-      - upload-rocm-wheels
-    agents:
-      queue: cpu_queue_release
-    commands:
-      - "bash .buildkite/scripts/annotate-rocm-release.sh"
-    env:
-      S3_BUCKET: "vllm-wheels"
-
-  # ROCm Job 5: Generate Root Index for ROCm Wheels (for release only)
-  # This is the job to create https://wheels.vllm.ai/rocm/ index allowing
-  # users to install with `uv pip install vllm --extra-index-url https://wheels.vllm.ai/rocm/`
-  - block: "Generate Root Index for ROCm Wheels for Release"
-    key: block-generate-root-index-rocm-wheels
-    depends_on: upload-rocm-wheels
-
-  - label: ":package: Generate Root Index for ROCm Wheels for Release"
-    depends_on: block-generate-root-index-rocm-wheels
-    id: generate-root-index-rocm-wheels
-    agents:
-      queue: cpu_queue_release
-    commands:
-      - "bash tools/vllm-rocm/generate-rocm-wheels-root-index.sh"
-    env:
-      S3_BUCKET: "vllm-wheels"
-      VARIANT: "rocm721"
-
-  # ROCm Job 6: Build ROCm Release Docker Image
+  # ROCm nightly Docker image
   - label: ":docker: Build release image - x86_64 - ROCm"
     id: build-rocm-release-image
+    if: build.env("NIGHTLY") == "1"
     depends_on:
-      - step: block-build-release-images
-        allow_failure: true
       - step: build-rocm-base-wheels
         allow_failure: false
     agents:
@@ -634,11 +547,11 @@ steps:
     commands:
       - |
         set -euo pipefail
-        
+
         # Login to ECR
         aws ecr-public get-login-password --region us-east-1 | \
           docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
-        
+
         # Get ECR image tag from metadata (set by build-rocm-base-wheels)
         ECR_IMAGE_TAG="$$(buildkite-agent meta-data get rocm-base-image-tag 2>/dev/null || echo '')"
         if [ -z "$${ECR_IMAGE_TAG}" ]; then
@@ -646,23 +559,23 @@ steps:
           echo "This should have been set by the build-rocm-base-wheels job"
           exit 1
         fi
-        
+
         echo "Pulling base Docker image from ECR: $${ECR_IMAGE_TAG}"
-        
+
         # Pull base Docker image from ECR
         docker pull "$${ECR_IMAGE_TAG}"
-        
+
         echo "Loaded base image: $${ECR_IMAGE_TAG}"
-        
+
         # Pass the base image ECR tag to downstream steps (nightly publish)
         buildkite-agent meta-data set "rocm-base-ecr-tag" "$${ECR_IMAGE_TAG}"
-        
+
         echo "========================================"
         echo "Building vLLM ROCm release image with:"
         echo "  BASE_IMAGE: $${ECR_IMAGE_TAG}"
         echo "  BUILDKITE_COMMIT: $${BUILDKITE_COMMIT}"
         echo "========================================"
-        
+
         # Build vLLM ROCm release image using cached base
         DOCKER_BUILDKIT=1 docker build \
           --build-arg max_jobs=16 \
@@ -675,10 +588,10 @@ steps:
           --target vllm-openai \
           --progress plain \
           -f docker/Dockerfile.rocm .
-        
+
         # Push to ECR
         docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rocm
-        
+
         echo ""
         echo " Successfully built and pushed ROCm release image"
         echo "   Image: public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rocm"
@@ -705,3 +618,84 @@ steps:
     env:
       DOCKER_BUILDKIT: "1"
       DOCKERHUB_USERNAME: "vllmbot"
+
+  # =============================================================================
+  # Release: Publish Wheels & Build CPU Images (manual, requires release version)
+  # =============================================================================
+
+  - input: "Provide Release version here"
+    id: input-release-version
+    fields:
+      - text: "What is the release version?"
+        key: release-version
+
+  - group: "Publish release wheels"
+    key: "publish-wheels"
+    steps:
+      - block: "Confirm update release wheels to PyPI (experimental, use with caution)?"
+        key: block-upload-release-wheels
+        depends_on:
+          - input-release-version
+          - build-wheels
+
+      - label: "Upload release wheels to PyPI"
+        depends_on:
+          - block-upload-release-wheels
+        id: upload-release-wheels
+        agents:
+          queue: small_cpu_queue_release
+        commands:
+          - "bash .buildkite/scripts/upload-release-wheels-pypi.sh"
+
+  - group: "Build release CPU Docker images"
+    steps:
+      - block: "Build release image for x86_64 CPU"
+        key: block-cpu-release-image-build
+        depends_on: ~
+
+      - label: "Build release image - x86_64 - CPU"
+        depends_on:
+          - block-cpu-release-image-build
+          - input-release-version
+        agents:
+          queue: cpu_queue_release
+        commands:
+          - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
+          - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg GIT_REPO_CHECK=1 --build-arg VLLM_CPU_X86=true --tag public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:$(buildkite-agent meta-data get release-version) --tag public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:latest --progress plain --target vllm-openai -f docker/Dockerfile.cpu ."
+          - "docker push public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:latest"
+          - "docker push public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:$(buildkite-agent meta-data get release-version)"
+        env:
+          DOCKER_BUILDKIT: "1"
+
+      - block: "Build release image for arm64 CPU"
+        key: block-arm64-cpu-release-image-build
+        depends_on: ~
+
+      - label: "Build release image - arm64 - CPU"
+        depends_on:
+          - block-arm64-cpu-release-image-build
+          - input-release-version
+        agents:
+          queue: arm64_cpu_queue_release
+        commands:
+          - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
+          - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg GIT_REPO_CHECK=1 --tag public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:$(buildkite-agent meta-data get release-version) --tag public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:latest --progress plain --target vllm-openai -f docker/Dockerfile.cpu ."
+          - "docker push public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:latest"
+          - "docker push public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:$(buildkite-agent meta-data get release-version)"
+        env:
+          DOCKER_BUILDKIT: "1"
+
+  - block: "Generate Root Index for ROCm Wheels for Release"
+    key: block-generate-root-index-rocm-wheels
+    depends_on: upload-rocm-wheels
+
+  - label: ":package: Generate Root Index for ROCm Wheels for Release"
+    depends_on: block-generate-root-index-rocm-wheels
+    id: generate-root-index-rocm-wheels
+    agents:
+      queue: cpu_queue_release
+    commands:
+      - "bash tools/vllm-rocm/generate-rocm-wheels-root-index.sh"
+    env:
+      S3_BUCKET: "vllm-wheels"
+      VARIANT: "rocm721"

.buildkite/scripts/hardware_ci/run-cpu-test-arm.sh

@@ -51,7 +51,6 @@ function cpu_tests() {
     set -e
     pytest -x -v -s tests/kernels/test_onednn.py
     pytest -x -v -s tests/kernels/attention/test_cpu_attn.py
-    pytest -x -v -s tests/kernels/core/test_cpu_activation.py
     pytest -x -v -s tests/kernels/moe/test_moe.py -k test_cpu_fused_moe_basic"
 
   # basic online serving

.buildkite/scripts/hardware_ci/run-cpu-test.sh

@@ -16,5 +16,5 @@ echo "--- :docker: Building Docker image"
 docker build --progress plain --tag "$IMAGE_NAME" --target vllm-test -f docker/Dockerfile.cpu .
 
 # Run the image, setting --shm-size=4g for tensor parallel.
-docker run --rm --cpuset-cpus="$CORE_RANGE" --cpuset-mems="$NUMA_NODE" -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true -e HF_TOKEN -e VLLM_CPU_KVCACHE_SPACE=16 -e VLLM_CPU_CI_ENV=1 -e VLLM_CPU_SIM_MULTI_NUMA=1 -e VLLM_CPU_ATTN_SPLIT_KV=0 --shm-size=4g "$IMAGE_NAME" \
+docker run --rm --cpuset-cpus="$CORE_RANGE" --cpuset-mems="$NUMA_NODE" -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true -e HF_TOKEN -e VLLM_CPU_KVCACHE_SPACE=16 -e VLLM_CPU_CI_ENV=1 -e VLLM_CPU_SIM_MULTI_NUMA=1 --shm-size=4g "$IMAGE_NAME" \
         timeout "$TIMEOUT_VAL" bash -c "set -euox pipefail; echo \"--- Print packages\"; pip list; echo \"--- Running tests\"; ${TEST_COMMAND}"

.buildkite/test-amd.yaml

@@ -769,7 +769,7 @@ steps:
   - tests/kernels/helion/
   - vllm/platforms/rocm.py
   commands:
-  - pip install helion==1.0.0
+  - pip install helion==0.3.3
   - pytest -v -s kernels/helion/
 
 

.buildkite/test_areas/distributed.yaml

@@ -196,6 +196,7 @@ steps:
     - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_async_new_apis.py
     - VLLM_USE_DEEP_GEMM=1 VLLM_LOGGING_LEVEL=DEBUG python3 examples/offline_inference/data_parallel.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=deepep_high_throughput
     - pytest -v -s tests/v1/distributed/test_dbo.py
+    - TP_SIZE=1 DP_SIZE=2 pytest -v -s tests/v1/distributed/test_eagle_dp.py
 
 - label: Distributed Tests (2 GPUs)(B200)
   device: b200

.buildkite/test_areas/kernels.yaml

@@ -155,7 +155,7 @@ steps:
   - vllm/utils/import_utils.py
   - tests/kernels/helion/
   commands:
-    - pip install helion==1.0.0
+    - pip install helion==0.3.3
     - pytest -v -s kernels/helion/
 
  

.buildkite/test_areas/models_basic.yaml

@@ -4,6 +4,7 @@ depends_on:
 steps:
 - label: Basic Models Tests (Initialization)
   timeout_in_minutes: 45
+  device: h200_18gb
   torch_nightly: true
   source_file_dependencies:
   - vllm/
@@ -72,18 +73,3 @@ steps:
     - python3 examples/offline_inference/vision_language.py --model-type qwen2_5_vl
     # Whisper needs spawn method to avoid deadlock
     - VLLM_WORKER_MULTIPROC_METHOD=spawn python3 examples/offline_inference/audio_language.py --model-type whisper
-
-- label: Transformers Backward Compatibility Models Test
-  working_dir: "/vllm-workspace/"
-  optional: true
-  soft_fail: true
-  commands:
-    - pip install transformers==4.57.5
-    - pytest -v -s tests/models/test_initialization.py
-    - pytest -v -s tests/models/test_transformers.py
-    - pytest -v -s tests/models/multimodal/processing/
-    - pytest -v -s tests/models/multimodal/test_mapping.py
-    - python3 examples/offline_inference/basic/chat.py
-    - python3 examples/offline_inference/vision_language.py --model-type qwen2_5_vl
-    # Whisper needs spawn method to avoid deadlock
-    - VLLM_WORKER_MULTIPROC_METHOD=spawn python3 examples/offline_inference/audio_language.py --model-type whisper

.buildkite/test_areas/spec_decode.yaml

@@ -42,16 +42,3 @@ steps:
     - tests/v1/e2e/spec_decode/
   commands:
     - pytest -v -s v1/e2e/spec_decode -k "draft_model or no_sync or batch_inference"
-
-- label: DFlash Speculators Correctness
-  timeout_in_minutes: 30
-  device: h100
-  optional: true
-  num_devices: 1
-  source_file_dependencies:
-    - vllm/v1/spec_decode/
-    - vllm/model_executor/models/qwen3_dflash.py
-    - tests/v1/spec_decode/test_speculators_dflash.py
-  commands:
-    - export VLLM_ALLOW_INSECURE_SERIALIZATION=1
-    - pytest -v -s v1/spec_decode/test_speculators_dflash.py -m slow_test

.github/PULL_REQUEST_TEMPLATE.md

@@ -15,6 +15,7 @@ PLEASE FILL IN THE PR DESCRIPTION HERE ENSURING ALL CHECKLIST ITEMS (AT THE BOTT
 - [ ] The test plan, such as providing test command.
 - [ ] The test results, such as pasting the results comparison before and after, or e2e results
 - [ ] (Optional) The necessary documentation update, such as updating `supported_models.md` and `examples` for a new model.
+- [ ] (Optional) Release notes update. If your change is user facing, please update the release notes draft in the [Google Doc](https://docs.google.com/document/d/1YyVqrgX4gHTtrstbq8oWUImOyPCKSGnJ7xtTpmXzlRs/edit?tab=t.0).
 </details>
 
 **BEFORE SUBMITTING, PLEASE READ <https://docs.vllm.ai/en/latest/contributing>** (anything written below this line will be removed by GitHub Actions)

cmake/cpu_extension.cmake

@@ -360,7 +360,6 @@ set(VLLM_EXT_SRC
 if (ASIMD_FOUND AND NOT APPLE_SILICON_FOUND)
     set(VLLM_EXT_SRC
         "csrc/cpu/shm.cpp"
-        "csrc/cpu/activation_lut_bf16.cpp"
         ${VLLM_EXT_SRC})
 endif()
 

csrc/cpu/activation_lut_bf16.cpp

@@ -1,71 +0,0 @@
-#include "cpu_types.hpp"
-
-#include <array>
-#include <cstdint>
-#include <mutex>
-#include <string>
-
-#include <ATen/ops/empty.h>
-#include <ATen/ops/gelu.h>
-#include <c10/util/BFloat16.h>
-
-constexpr uint32_t ActivationLutSize = 1u << 16;
-
-at::Tensor gelu_reference(const at::Tensor& x) { return at::gelu(x, "none"); }
-
-void maybe_init_activation_lut_bf16(
-    uint16_t* lut, std::once_flag& once,
-    at::Tensor (*activation)(const at::Tensor&)) {
-  std::call_once(once, [&]() {
-    auto lut_input =
-        at::empty({static_cast<int64_t>(ActivationLutSize)},
-                  at::TensorOptions().device(at::kCPU).dtype(at::kFloat));
-    auto* lut_input_ptr = lut_input.data_ptr<float>();
-#pragma omp parallel for
-    for (uint32_t i = 0; i < ActivationLutSize; ++i) {
-      lut_input_ptr[i] = c10::detail::f32_from_bits(static_cast<uint16_t>(i));
-    }
-
-    auto lut_output = activation(lut_input);
-    const auto* lut_output_ptr = lut_output.data_ptr<float>();
-#pragma omp parallel for
-    for (uint32_t i = 0; i < ActivationLutSize; ++i) {
-      lut[i] = c10::detail::round_to_nearest_even(lut_output_ptr[i]);
-    }
-  });
-}
-
-void activation_lut_bf16(torch::Tensor& out, torch::Tensor& input,
-                         const uint16_t* lut, const char* op_name) {
-  TORCH_CHECK(input.scalar_type() == at::kBFloat16, op_name,
-              ": input must be bfloat16");
-  TORCH_CHECK(out.scalar_type() == at::kBFloat16, op_name,
-              ": out must be bfloat16");
-  TORCH_CHECK(input.is_contiguous(), op_name, ": input must be contiguous");
-  TORCH_CHECK(out.is_contiguous(), op_name, ": out must be contiguous");
-
-  const auto* src =
-      reinterpret_cast<const uint16_t*>(input.data_ptr<at::BFloat16>());
-  auto* dst = reinterpret_cast<uint16_t*>(out.data_ptr<at::BFloat16>());
-  const int64_t n = input.numel();
-
-  CPU_KERNEL_GUARD_IN(activation_lut_bf16_impl)
-#pragma omp parallel for
-  for (int64_t i = 0; i < n; ++i) {
-    dst[i] = lut[src[i]];
-  }
-  CPU_KERNEL_GUARD_OUT(activation_lut_bf16_impl)
-}
-
-void activation_lut_bf16(torch::Tensor& out, torch::Tensor& input,
-                         const std::string& activation) {
-  if (activation == "gelu") {
-    static std::array<uint16_t, ActivationLutSize> lut{};
-    static std::once_flag once;
-    maybe_init_activation_lut_bf16(lut.data(), once, gelu_reference);
-    activation_lut_bf16(out, input, lut.data(), "gelu_lut");
-    return;
-  }
-
-  TORCH_CHECK(false, "Unsupported activation: ", activation);
-}

csrc/cpu/cpu_attn_impl.hpp

@@ -147,9 +147,6 @@ struct AttentionMetadata {
       case ISA::NEON:
         ss << "NEON, ";
         break;
-      case ISA::VXE:
-        ss << "VXE, ";
-        break;
     }
     ss << "workitem_group_num: " << workitem_group_num
        << ", reduction_item_num: " << reduction_item_num

csrc/cpu/torch_bindings.cpp

@@ -85,9 +85,6 @@ at::Tensor int4_scaled_mm_cpu(at::Tensor& x, at::Tensor& w, at::Tensor& w_zeros,
                               at::Tensor& w_scales,
                               std::optional<at::Tensor> bias);
 
-void activation_lut_bf16(torch::Tensor& out, torch::Tensor& input,
-                         const std::string& activation);
-
 torch::Tensor get_scheduler_metadata(
     const int64_t num_req, const int64_t num_heads_q,
     const int64_t num_heads_kv, const int64_t head_dim,
@@ -234,15 +231,6 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
   ops.def("gelu_quick(Tensor! out, Tensor input) -> ()");
   ops.impl("gelu_quick", torch::kCPU, &gelu_quick);
 
-#if (defined(__aarch64__) && !defined(__APPLE__))
-
-  ops.def(
-      "activation_lut_bf16(Tensor! out, Tensor input, str activation)"
-      " -> ()");
-  ops.impl("activation_lut_bf16", torch::kCPU, &activation_lut_bf16);
-
-#endif  // (defined(__aarch64__) && !defined(__APPLE__))
-
   // Layernorm
   // Apply Root Mean Square (RMS) Normalization to the input tensor.
   ops.def(

csrc/cpu/utils.hpp

@@ -54,34 +54,12 @@ struct Counter {
 };
 
 inline int64_t get_available_l2_size() {
-#if defined(__s390x__)
-  static int64_t size = []() {
-    uint32_t l2_cache_size = 0;
-    auto caps = at::cpu::get_cpu_capabilities();
-    auto it = caps.find("l2_cache_size");
-    if (it != caps.end()) {
-      l2_cache_size = static_cast<uint32_t>(it->second.toInt());
-    }
-    if (l2_cache_size == 0) {
-      long sys_l2 = sysconf(_SC_LEVEL2_CACHE_SIZE);
-      if (sys_l2 > 0) {
-        l2_cache_size = static_cast<uint32_t>(sys_l2);
-      }
-    }
-    if (l2_cache_size == 0) {
-      l2_cache_size = 256 * 1024;
-    }
-    return static_cast<int64_t>(l2_cache_size) >> 1;  // use 50% of L2 cache
-  }();
-  return size;
-#else
   static int64_t size = []() {
     auto caps = at::cpu::get_cpu_capabilities();
     const uint32_t l2_cache_size = caps.at("l2_cache_size").toInt();
     return l2_cache_size >> 1;  // use 50% of L2 cache
   }();
   return size;
-#endif
 }
 
 template <int32_t alignment_v, typename T>

csrc/ops.h

@@ -1,7 +1,6 @@
 #pragma once
 
 #include <optional>
-#include <string>
 #include <torch/library.h>
 #include <tuple>
 

docker/Dockerfile

@@ -642,7 +642,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
     else \
         BITSANDBYTES_VERSION="${BITSANDBYTES_VERSION_X86}"; \
     fi; \
-    uv pip install --system accelerate modelscope \
+    uv pip install --system accelerate hf_transfer modelscope \
         "bitsandbytes>=${BITSANDBYTES_VERSION}" "timm${TIMM_VERSION}" "runai-model-streamer[s3,gcs,azure]${RUNAI_MODEL_STREAMER_VERSION}"
 
 # ============================================================
@@ -756,10 +756,9 @@ RUN --mount=type=cache,target=/root/.cache/uv \
     uv pip install --system -e tests/vllm_test_utils
 
 # enable fast downloads from hf (for testing)
-ENV HF_XET_HIGH_PERFORMANCE 1
-
-# increase timeout for hf downloads (for testing)
-ENV HF_HUB_DOWNLOAD_TIMEOUT 60
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv pip install --system hf_transfer
+ENV HF_HUB_ENABLE_HF_TRANSFER 1
 
 # Copy in the v1 package for testing (it isn't distributed yet)
 COPY vllm/v1 /usr/local/lib/python${PYTHON_VERSION}/dist-packages/vllm/v1

docker/Dockerfile.cpu

@@ -197,12 +197,6 @@ ADD ./.buildkite/ ./.buildkite/
 RUN --mount=type=cache,target=/root/.cache/uv \
     uv pip install -e tests/vllm_test_utils
 
-# enable fast downloads from hf (for testing)
-ENV HF_XET_HIGH_PERFORMANCE 1
-
-# increase timeout for hf downloads (for testing)
-ENV HF_HUB_DOWNLOAD_TIMEOUT 60
-
 ######################### RELEASE IMAGE #########################
 FROM base AS vllm-openai
 

docker/Dockerfile.nightly_torch

@@ -272,10 +272,9 @@ RUN --mount=type=cache,target=/root/.cache/uv \
     uv pip install --system -e tests/vllm_test_utils
 
 # enable fast downloads from hf (for testing)
-ENV HF_XET_HIGH_PERFORMANCE 1
-
-# increase timeout for hf downloads (for testing)
-ENV HF_HUB_DOWNLOAD_TIMEOUT 60
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv pip install --system hf_transfer
+ENV HF_HUB_ENABLE_HF_TRANSFER 1
 
 RUN --mount=type=cache,target=/root/.cache/uv \
     uv pip install --system -r requirements/test/nightly-torch.txt

docker/Dockerfile.rocm

@@ -365,10 +365,9 @@ RUN cd /vllm-workspace \
     && python3 -m pip install pytest-shard
 
 # enable fast downloads from hf (for testing)
-ENV HF_XET_HIGH_PERFORMANCE=1
-
-# increase timeout for hf downloads (for testing)
-ENV HF_HUB_DOWNLOAD_TIMEOUT 60
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv pip install --system hf_transfer
+ENV HF_HUB_ENABLE_HF_TRANSFER=1
 
 # install audio decode package `torchcodec` from source (required due to 
 # ROCm and torch version mismatch) for tests with datasets package

docker/Dockerfile.s390x

@@ -42,7 +42,7 @@ FROM python-install AS pyarrow
 # Build Apache Arrow
 WORKDIR /tmp
 RUN --mount=type=cache,target=/root/.cache/uv \
-    git clone https://github.com/apache/arrow.git  -b maint-19.0.1 && \
+    git clone https://github.com/apache/arrow.git && \
     cd arrow/cpp && \
     mkdir release && cd release && \
     cmake -DCMAKE_BUILD_TYPE=Release \
@@ -68,6 +68,19 @@ RUN --mount=type=cache,target=/root/.cache/uv \
     uv pip install -r requirements-build.txt && \
     python setup.py build_ext --build-type=$ARROW_BUILD_TYPE --bundle-arrow-cpp bdist_wheel
 
+FROM python-install AS numa-build
+# Install numactl (needed for numa.h dependency)
+WORKDIR /tmp
+RUN curl -LO https://github.com/numactl/numactl/archive/refs/tags/v2.0.16.tar.gz && \
+    tar -xvzf v2.0.16.tar.gz && \
+    cd numactl-2.0.16 && \
+    ./autogen.sh && \
+    ./configure && \
+    make
+
+# Set include path
+ENV C_INCLUDE_PATH="/usr/local/include:$C_INCLUDE_PATH"
+
 FROM python-install AS rust
 ENV CARGO_HOME=/root/.cargo
 ENV RUSTUP_HOME=/root/.rustup
@@ -78,18 +91,6 @@ RUN curl https://sh.rustup.rs -sSf | sh -s -- -y && \
     rustup default stable && \
     rustup show
 
-FROM python-install AS numa-build
-WORKDIR /tmp
-RUN curl -LO https://github.com/numactl/numactl/archive/refs/tags/v2.0.19.tar.gz && \
-    tar -xvzf v2.0.19.tar.gz && \
-    cd numactl-2.0.19 && \
-    ./autogen.sh && \
-    ./configure && \
-    make
-
-# Set include path
-ENV C_INCLUDE_PATH="/usr/local/include:$C_INCLUDE_PATH"
-
 FROM python-install AS torch-vision
 # Install torchvision
 ARG TORCH_VISION_VERSION=v0.26.0
@@ -132,7 +133,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
     git clone --recursive https://github.com/numba/llvmlite.git -b v0.44.0 && \
     git clone --recursive https://github.com/numba/numba.git -b ${NUMBA_VERSION} && \
     cd llvm-project && mkdir build && cd  build && \
-    uv pip install 'cmake<4' 'setuptools<70' numpy && \
+    uv pip install 'cmake<4' setuptools numpy && \
     export PREFIX=/usr/local && CMAKE_ARGS="${CMAKE_ARGS} -DLLVM_ENABLE_PROJECTS=lld;libunwind;compiler-rt" \
     CFLAGS="$(echo $CFLAGS | sed 's/-fno-plt //g')" \
     CXXFLAGS="$(echo $CXXFLAGS | sed 's/-fno-plt //g')" \
@@ -192,22 +193,27 @@ RUN --mount=type=cache,target=/root/.cache/uv \
     cd opencv-python && \
     python -m build --wheel --installer=uv --outdir /tmp/opencv-python/dist
 
-## Todo(r3hankhan123): Remove guidance-builder stage once vLLM upgrades to new version of llguidance that fixes s390x issues. See https://github.com/guidance-ai/llguidance/issues/330
-FROM python-install AS guidance-builder
+# Build Outlines Core
+FROM python-install AS outlines-core-builder
 WORKDIR /tmp
 ENV CARGO_HOME=/root/.cargo
 ENV RUSTUP_HOME=/root/.rustup
 ENV PATH="$CARGO_HOME/bin:$RUSTUP_HOME/bin:$PATH"
+COPY requirements/common.txt /tmp/requirements/common.txt
+ARG OUTLINES_CORE_VERSION
 RUN --mount=type=cache,target=/root/.cache/uv \
     --mount=type=bind,from=rust,source=/root/.cargo,target=/root/.cargo,rw \
     --mount=type=bind,from=rust,source=/root/.rustup,target=/root/.rustup,rw \
-    git clone https://github.com/guidance-ai/llguidance.git && \
-    cd llguidance && \
-    git checkout s390x-fix-v2  && \
+    OUTLINES_CORE_VERSION=${OUTLINES_CORE_VERSION:-$(grep -E '^outlines_core\s*==\s*[0-9.]+' /tmp/requirements/common.txt | grep -Eo '[0-9.]+')} && \
+    if [ -z "${OUTLINES_CORE_VERSION}" ]; then echo "ERROR: Could not determine outlines_core version"; exit 1; fi && \
+    git clone https://github.com/dottxt-ai/outlines-core.git && \
+    cd outlines-core && \
+    git checkout tags/${OUTLINES_CORE_VERSION} && \
+    sed -i "s/version = \"0.0.0\"/version = \"${OUTLINES_CORE_VERSION}\"/" Cargo.toml && \
     uv pip install maturin && \
-    python -m maturin build --release --out dist --compatibility linux
+    python -m maturin build --release --out dist
 
-# # Final build stage
+# Final build stage
 FROM python-install AS vllm-cpu
 ARG PYTHON_VERSION
 ARG PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl/cpu"
@@ -223,12 +229,10 @@ ENV PKG_CONFIG_PATH="/opt/rh/gcc-toolset-14/root/usr/lib64/pkgconfig:/usr/local/
 ENV PATH="${VIRTUAL_ENV:+${VIRTUAL_ENV}/bin}:/opt/rh/gcc-toolset-14/root/usr/bin:/usr/local/bin:$CARGO_HOME/bin:$RUSTUP_HOME/bin:$PATH"
 ENV PIP_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
 ENV UV_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
-# Force pure Python protobuf to avoid s390x C++ extension crashes
-ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python
 COPY . /workspace/vllm
 WORKDIR /workspace/vllm
 
-RUN --mount=type=bind,from=numa-build,src=/tmp/numactl-2.0.19,target=/numactl \
+RUN --mount=type=bind,from=numa-build,src=/tmp/numactl-2.0.16,target=/numactl \
     make -C /numactl install
 
 # Install dependencies, including PyTorch and Apache Arrow
@@ -241,22 +245,22 @@ RUN --mount=type=cache,target=/root/.cache/uv \
     --mount=type=bind,from=numba-builder,source=/tmp/llvmlite/dist,target=/tmp/llvmlite-wheels/ \
     --mount=type=bind,from=numba-builder,source=/tmp/numba/dist,target=/tmp/numba-wheels/ \
     --mount=type=bind,from=opencv-builder,source=/tmp/opencv-python/dist,target=/tmp/opencv-wheels/ \
-    --mount=type=bind,from=guidance-builder,source=/tmp/llguidance/dist,target=/tmp/guidance-wheels/ \
-     ARROW_WHL_FILE=$(ls /tmp/arrow-wheels/*.whl) && \
+    --mount=type=bind,from=outlines-core-builder,source=/tmp/outlines-core/dist,target=/tmp/outlines-core/dist/ \
+     ARROW_WHL_FILE=$(ls /tmp/arrow-wheels/pyarrow-*.whl) && \
      VISION_WHL_FILE=$(ls /tmp/vision-wheels/*.whl) && \
      HF_XET_WHL_FILE=$(ls /tmp/hf-xet-wheels/*.whl) && \
      LLVM_WHL_FILE=$(ls /tmp/llvmlite-wheels/*.whl) && \
      NUMBA_WHL_FILE=$(ls /tmp/numba-wheels/*.whl) && \
      OPENCV_WHL_FILE=$(ls /tmp/opencv-wheels/*.whl) && \
-     GUIDANCE_WHL_FILE=$(ls /tmp/guidance-wheels/*.whl) && \
-     uv pip install -v \    
-        $ARROW_WHL_FILE \
+     OUTLINES_CORE_WHL_FILE=$(ls /tmp/outlines-core/dist/*.whl) && \
+     uv pip install -v \
+        $ARROW_WHL_FILE  \
         $VISION_WHL_FILE \
         $HF_XET_WHL_FILE \
         $LLVM_WHL_FILE \
         $NUMBA_WHL_FILE \
         $OPENCV_WHL_FILE \
-        $GUIDANCE_WHL_FILE \
+        $OUTLINES_CORE_WHL_FILE \
         --index-strategy unsafe-best-match \
         -r requirements/build/cpu.txt \
         -r requirements/cpu.txt
@@ -267,10 +271,6 @@ RUN --mount=type=cache,target=/root/.cache/uv \
     VLLM_TARGET_DEVICE=cpu VLLM_CPU_MOE_PREPACK=0 python setup.py bdist_wheel && \
     uv pip install "$(echo dist/*.whl)[tensorizer]"
 
-# Remove protobuf C++ extension that crashes on s390x
-RUN rm -rf /opt/vllm/lib64/python${PYTHON_VERSION}/site-packages/google/_upb/*.so \
-           /opt/vllm/lib64/python${PYTHON_VERSION}/site-packages/google/protobuf/pyext/*.so 2>/dev/null || true
-
 # setup non-root user for vllm
 RUN umask 002 && \
     /usr/sbin/useradd --uid 2000 --gid 0 vllm && \

docs/benchmarking/cli.md

@@ -37,7 +37,6 @@ th {
 | HuggingFace-Blazedit | ✅ | ✅ | `vdaita/edit_5k_char`, `vdaita/edit_10k_char` |
 | HuggingFace-ASR | ✅ | ✅ | `openslr/librispeech_asr`, `facebook/voxpopuli`,  `LIUM/tedlium`, `edinburghcstr/ami`,        `speechcolab/gigaspeech`,        `kensho/spgispeech` |
 | Spec Bench | ✅ | ✅ | `wget https://raw.githubusercontent.com/hemingkx/Spec-Bench/refs/heads/main/data/spec_bench/question.jsonl` |
-| SPEED-Bench | ✅ | ✅ | `curl -LsSf https://raw.githubusercontent.com/NVIDIA-NeMo/Skills/refs/heads/main/nemo_skills/dataset/speed-bench/prepare.py \| python3 -` |
 | Custom | ✅ | ✅ | Local file: `data.jsonl` |
 | Custom MM | ✅ | ✅ | Local file: `mm_data.jsonl` |
 
@@ -240,69 +239,6 @@ vllm bench serve \
     --spec-bench-category "summarization"
 ```
 
-#### SPEED-Bench Benchmark with Speculative Decoding
-
-[SPEED-Bench](https://huggingface.co/datasets/nvidia/SPEED-Bench) is a unified and diverse dataset for speculative decoding, supporting acceptance rate and length measurements using the Qualitative split and throughput measurements using the Throughput splits in 5 configuration of input sequence length (1k, 2k, 8k, 16k, 32k).
-
-!!! note
-    This dataset is governed by the [NVIDIA Evaluation Dataset License Agreement](https://huggingface.co/datasets/nvidia/SPEED-Bench/blob/main/License.pdf). For each dataset a user elects to use, the user is responsible for checking if the dataset license is fit for the intended purpose. The `prepare.py` script automatically fetches data from all the source datasets.
-
-First, download the dataset to a folder, using this one liner:
-
-```bash
-curl -LsSf https://raw.githubusercontent.com/NVIDIA-NeMo/Skills/refs/heads/main/nemo_skills/dataset/speed-bench/prepare.py \| python3 -
-```
-
-The command supports also the following arguments:
-
-- `--config`: download only a subset of the dataset: `qualitative`, `throughput_1k`, `throughput_2k`, `throughput_8k`, `throughput_16k` and `throughput_32k`. By default, it will download all subsets.
-- `--output_dir`: download to a specified folder. By default, it will download to the current directory.
-
-Start a server with speculative decoding:
-
-```bash
-vllm serve meta-llama/Llama-3.3-70B-Instruct \
-    --speculative-config $'{"method": "eagle3",
-    "num_speculative_tokens": 3,
-    "model": "nvidia/Llama-3.3-70B-Instruct-Eagle3"}'
-```
-
-Run all categories in the Qualitative split:
-
-```bash
-vllm bench serve \
-    --model meta-llama/Llama-3.3-70B-Instruct \
-    --dataset-name speed_bench \
-    --dataset-path "<YOUR_DOWNLOADED_PATH>/data/speed_bench" \
-    --num-prompts -1
-```
-
-Available categories include `[writing, roleplay, reasoning, math, coding, stem, humanities, multilingual, summarization, qa, rag]`.
-
-Run only a specific category like "multilingual":
-
-```bash
-vllm bench serve \
-    --model meta-llama/Llama-3.3-70B-Instruct \
-    --dataset-name speed_bench \
-    --dataset-path "<YOUR_DOWNLOADED_PATH>/data/speed_bench" \
-    --num-prompts -1
-    --speed-bench-category "multilingual"
-```
-
-Run all categories in the Throughput split (2k ISL):
-
-```bash
-vllm bench serve \
-    --model meta-llama/Llama-3.3-70B-Instruct \
-    --dataset-name speed_bench \
-    --speed-bench-dataset-subset throughput_2k
-    --dataset-path "<YOUR_DOWNLOADED_PATH>/data/speed_bench/" \
-    --num-prompts -1
-```
-
-Available categories include `[high_entropy, mixed, low_entropy]`, where high entropy data contains unstructued data such as creative writing while low entropy data contains more structured data such as coding, more details are in the dataset card.
-
 #### Other HuggingFaceDataset Examples
 
 ```bash

docs/features/quantization/README.md

@@ -16,7 +16,6 @@ The following are the supported quantization formats for vLLM:
 - [INT8 W8A8](int8.md)
 - [FP8 W8A8](fp8.md)
 - [NVIDIA Model Optimizer](modelopt.md)
-- [Online Quantization](online.md)
 - [AMD Quark](quark.md)
 - [Quantized KV Cache](quantized_kvcache.md)
 - [TorchAO](torchao.md)

docs/features/quantization/online.md

@@ -1,94 +0,0 @@
-# Online Quantization
-
-Online quantization lets you take a BF16/FP16 model and quantize its Linear
-and MoE weights to lower precision (such as FP8) at load time, without needing
-a pre-quantized checkpoint or calibration data. Weights are converted during
-model loading and activations are dynamically scaled during each forward pass.
-
-## Quick Start
-
-Pass a scheme name to the `quantization` parameter:
-
-```python
-from vllm import LLM
-
-# Per-tensor FP8 quantization (one scale per weight tensor)
-llm = LLM("meta-llama/Llama-3.1-8B", quantization="fp8_per_tensor")
-
-# Per-block FP8 quantization (128x128 block scaling for weights and 1x128 block scaling for activations)
-llm = LLM("meta-llama/Llama-3.1-8B", quantization="fp8_per_block")
-```
-
-Or with the CLI:
-
-```bash
-vllm serve meta-llama/Llama-3.1-8B --quantization fp8_per_tensor
-vllm serve meta-llama/Llama-3.1-8B --quantization fp8_per_block
-```
-
-## Supported Schemes
-
-| Scheme | Weight recipe | Activation recipe | Notes |
-| ------ | ------------- | ------------------ | ----- |
-| `fp8_per_tensor` | fp8_e4m3 data, fp32 per-tensor scale | fp8_e4m3 data, fp32 per-tensor scale | On some GPUs (Ada, Hopper) linear activations use per-token scaling for better performance |
-| `fp8_per_block` | fp8_e4m3 data, fp32 per-128x128-block scale | fp8_e4m3 data, fp32 per-1x128-block scale | |
-
-Support for additional schemes will be added in future versions of vllm.
-
-## Advanced Configuration
-
-For fine-grained control, use a `quantization_config` dictionary.
-
-### Separate Schemes for Dense and MoE Layers
-
-You can apply different quantization schemes to dense linear layers and MoE expert layers:
-
-```python
-from vllm import LLM
-
-llm = LLM(
-    "ibm-granite/granite-3.0-1b-a400m-base",
-    quantization="fp8_per_tensor",
-    quantization_config={
-        "linear_scheme_override": "fp8_per_block",
-    },
-)
-```
-
-Or,
-
-```python
-from vllm import LLM
-
-llm = LLM(
-    "ibm-granite/granite-3.0-1b-a400m-base",
-    quantization="fp8_per_tensor",
-    quantization_config={
-        "moe_scheme_override": "fp8_per_block",
-    },
-)
-```
-
-### Excluding Layers from Quantization
-
-Use the `ignore` parameter to skip specific layers. It accepts exact layer names and regex patterns (prefixed with `re:`):
-
-```python
-from vllm import LLM
-
-llm = LLM(
-    "ibm-granite/granite-3.0-1b-a400m-base",
-    quantization="fp8_per_tensor",
-    quantization_config={
-        "ignore": [
-            # exact layer name
-            "model.layers.1.self_attn.o_proj",
-            # regex: skip all QKV projections
-            "re:.*[qkv]_proj",
-        ],
-    },
-)
-```
-
-!!! note
-    For fused layers (e.g., `qkv_proj` which fuses `q_proj`, `k_proj`, `v_proj`), the ignore pattern must match the **unfused** shard names (`q_proj`, `k_proj`, `v_proj`), not the fused name.

docs/getting_started/installation/cpu.s390x.inc.md

@@ -3,15 +3,15 @@
 
 vLLM has experimental support for s390x architecture on IBM Z platform. For now, users must build from source to natively run on IBM Z platform.
 
-Currently, the CPU implementation for s390x architecture supports FP32, BF16 and FP16.
+Currently, the CPU implementation for s390x architecture supports FP32 datatype only.
 
 --8<-- [end:installation]
 --8<-- [start:requirements]
 
 - OS: `Linux`
-- SDK: `gcc/g++ >= 14.0.0` or later with Command Line Tools
+- SDK: `gcc/g++ >= 12.3.0` or later with Command Line Tools
 - Instruction Set Architecture (ISA): VXE support is required. Works with Z14 and above.
-- Build install python packages: `torchvision`, `llvmlite`, `numba`, `pyarrow (for testing)`, `opencv-headless`
+- Build install python packages: `pyarrow`, `torch` and `torchvision`
 
 --8<-- [end:requirements]
 --8<-- [start:set-up-using-python]
@@ -24,14 +24,13 @@ Currently, there are no pre-built IBM Z CPU wheels.
 --8<-- [end:pre-built-wheels]
 --8<-- [start:build-wheel-from-source]
 
-Install the following packages from the package manager before building the vLLM. For example on RHEL 9.6:
+Install the following packages from the package manager before building the vLLM. For example on RHEL 9.4:
 
 ```bash
 dnf install -y \
-    which procps findutils tar vim git gcc-toolset-14 gcc-toolset-14-binutils gcc-toolset-14-libatomic-devel zlib-devel \
+    which procps findutils tar vim git gcc g++ make patch make cython zlib-devel \
     libjpeg-turbo-devel libtiff-devel libpng-devel libwebp-devel freetype-devel harfbuzz-devel \
-    openssl-devel openblas openblas-devel autoconf automake libtool cmake numpy libsndfile \
-    clang llvm-devel llvm-static clang-devel
+    openssl-devel openblas openblas-devel wget autoconf automake libtool cmake numactl-devel
 ```
 
 Install rust>=1.80 which is needed for `outlines-core` and `uvloop` python packages installation.
@@ -44,13 +43,13 @@ curl https://sh.rustup.rs -sSf | sh -s -- -y && \
 Execute the following commands to build and install vLLM from source.
 
 !!! tip
-    Please build the following dependencies, `torchvision`, `llvmlite`, `numba`, `llguidance`, `pyarrow`, `opencv-headless` from source before building vLLM.
+    Please build the following dependencies, `torchvision`, `pyarrow` from source before building vLLM.
 
 ```bash
+    sed -i '/^torch/d' requirements/build/cuda.txt    # remove torch from requirements/build/cuda.txt since we use nightly builds
     uv pip install -v \
-        --extra-index-url https://download.pytorch.org/whl/cpu \
         --torch-backend auto \
-        -r requirements/build/cpu.txt \
+        -r requirements/build/cuda.txt \
         -r requirements/cpu.txt \
     VLLM_TARGET_DEVICE=cpu python setup.py bdist_wheel && \
         uv pip install dist/*.whl
@@ -58,9 +57,10 @@ Execute the following commands to build and install vLLM from source.
 
 ??? console "pip"
     ```bash
+        sed -i '/^torch/d' requirements/build/cuda.txt    # remove torch from requirements/build/cuda.txt since we use nightly builds
         pip install -v \
-            --extra-index-url https://download.pytorch.org/whl/cpu \
-            -r requirements/build/cpu.txt \
+            --extra-index-url https://download.pytorch.org/whl/nightly/cpu \
+            -r requirements/build/cuda.txt \
             -r requirements/cpu.txt \
         VLLM_TARGET_DEVICE=cpu python setup.py bdist_wheel && \
             pip install dist/*.whl

docs/getting_started/installation/gpu.rocm.inc.md

@@ -240,7 +240,7 @@ uv pip install vllm==${VLLM_VERSION} \
         # Install dependencies
         pip install --upgrade numba \
             scipy \
-            huggingface-hub[cli] \
+            huggingface-hub[cli,hf_transfer] \
             setuptools_scm
         pip install -r requirements/rocm.txt
 

docs/models/pooling_models/README.md

@@ -59,16 +59,6 @@ please refer to [IO Processor Plugins](../../design/io_processor_plugins.md).
     Within classification tasks, there is a specialized subcategory: Cross-encoder (aka reranker) models. These models
 are a subset of classification models that accept two prompts as input and output num_labels equal to 1.
 
-### Pooling Types
-
-| Pooling Tasks  | Granularity   | Description                                                                                                                                                                                       |
-|----------------|---------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `CLS` pooling  | Sequence-wise | For BERT‑like (bidirectional self‑attention) models, CLS pooling is used by default. This means the last_hidden_states corresponding to the first token (the [CLS] token) is taken as the output. |
-| `LAST` pooling | Sequence-wise | For GPT‑like (causal self‑attention) models, LAST pooling is used by default. This means the last_hidden_states corresponding to the last token is taken as the output.                           |
-| `MEAN` pooling | Sequence-wise | Many studies have shown that averaging the last_hidden_states over all input tokens performs better on certain downstream tasks. Therefore, more and more models are using MEAN pooling.          |
-| `ALL` pooling  | Token-wise    | Outputs the last_hidden_states for all input tokens.                                                                                                                                              |
-| `STEP` pooling | Token-wise    | Filters and outputs the last_hidden_states corresponding to the token IDs returned by returned_token_ids.                                                                                         |
-
 ### Score Types
 
 The scoring models is designed to compute similarity scores between two input prompts. It supports three model types

docs/models/pooling_models/embed.md

@@ -45,7 +45,6 @@ You can compute pairwise similarity scores to build a similarity matrix using th
 | `GritLM` | GritLM | `parasail-ai/GritLM-7B-vllm`. | ✅︎ | ✅︎ |
 | `GteModel` | Arctic-Embed-2.0-M | `Snowflake/snowflake-arctic-embed-m-v2.0`. | | |
 | `GteNewModel` | mGTE-TRM (see note) | `Alibaba-NLP/gte-multilingual-base`, etc. | | |
-| `JinaEmbeddingsV5Model`<sup>C</sup> | Qwen3-based with task-specific LoRA adapters | `jinaai/jina-embeddings-v5-text-small` (see note) | ✅︎ | ✅︎ |
 | `LlamaBidirectionalModel`<sup>C</sup> | Llama-based with bidirectional attention | `nvidia/llama-nemotron-embed-1b-v2`, etc. | ✅︎ | ✅︎ |
 | `LlamaModel`<sup>C</sup>, `LlamaForCausalLM`<sup>C</sup>, `MistralModel`<sup>C</sup>, etc. | Llama-based | `intfloat/e5-mistral-7b-instruct`, etc. | ✅︎ | ✅︎ |
 | `ModernBertModel` | ModernBERT-based | `Alibaba-NLP/gte-modernbert-base`, etc. | | |
@@ -74,12 +73,6 @@ You can compute pairwise similarity scores to build a similarity matrix using th
 !!! note
     `jinaai/jina-embeddings-v3` supports multiple tasks through LoRA, while vllm temporarily only supports text-matching tasks by merging LoRA weights.
 
-!!! note
-    `jinaai/jina-embeddings-v5-text-small` ships with four task-specific LoRA adapters
-    (`retrieval`, `text-matching`, `classification`, `clustering`). vLLM merges the
-    selected adapter into the base weights at load time. Choose the task with
-    `--hf-overrides '{"jina_task": "<task>"}'`; the default is `retrieval`.
-
 ### Multimodal Models
 
 !!! note

docs/models/pooling_models/scoring.md

@@ -160,8 +160,6 @@ The following Score API parameters are supported:
 --8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-params"
 --8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-extra-params"
 --8<-- "vllm/entrypoints/pooling/base/protocol.py:classify-extra-params"
---8<-- "vllm/entrypoints/pooling/scoring/protocol.py:scoring-common-params"
---8<-- "vllm/entrypoints/pooling/scoring/protocol.py:score-request-params"
 ```
 
 #### Examples
@@ -372,8 +370,6 @@ The following rerank api parameters are supported:
 --8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-params"
 --8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-extra-params"
 --8<-- "vllm/entrypoints/pooling/base/protocol.py:classify-extra-params"
---8<-- "vllm/entrypoints/pooling/scoring/protocol.py:scoring-common-params"
---8<-- "vllm/entrypoints/pooling/scoring/protocol.py:rerank-request-params"
 ```
 
 #### Examples

docs/models/pooling_models/token_classify.md

@@ -68,7 +68,7 @@ If your model is not in the above list, we will try to automatically convert the
     Forced alignment usage requires `--hf-overrides '{"architectures": ["Qwen3ASRForcedAlignerForTokenClassification"]}'`.
     Please refer to [examples/pooling/token_classify/forced_alignment_offline.py](../../../examples/pooling/token_classify/forced_alignment_offline.py).
 
-### Reward Models
+### As Reward Models
 
 Using token classification models as reward models. For details on reward models, see [Reward Models](reward.md).
 

docs/serving/openai_compatible_server.md

@@ -467,11 +467,28 @@ It consists of two endpoints:
 - `/tokenize` corresponds to calling `tokenizer.encode()`.
 - `/detokenize` corresponds to calling `tokenizer.decode()`.
 
+### Score API
+
+#### Score Template
+
+Some scoring models require a specific prompt format to work correctly. You can specify a custom score template using the `--chat-template` parameter (see [Chat Template](#chat-template)).
+
+Score templates are supported for **cross-encoder** models only. If you are using an **embedding** model for scoring, vLLM does not apply a score template.
+
+Like chat templates, the score template receives a `messages` list. For scoring, each message has a `role` attribute—either `"query"` or `"document"`. For the usual kind of point-wise cross-encoder, you can expect exactly two messages: one query and one document. To access the query and document content, use Jinja's `selectattr` filter:
+
+- **Query**: `{{ (messages | selectattr("role", "eq", "query") | first).content }}`
+- **Document**: `{{ (messages | selectattr("role", "eq", "document") | first).content }}`
+
+This approach is more robust than index-based access (`messages[0]`, `messages[1]`) because it selects messages by their semantic role. It also avoids assumptions about message ordering if additional message types are added to `messages` in the future.
+
+Example template file: [examples/pooling/score/template/nemotron-rerank.jinja](../../examples/pooling/score/template/nemotron-rerank.jinja)
+
 ### Generative Scoring API
 
 The `/generative_scoring` endpoint uses a CausalLM model (e.g., Llama, Qwen, Mistral) to compute the probability of specified token IDs appearing as the next token. Each item (document) is concatenated with the query to form a prompt, and the model predicts how likely each label token is as the next token after that prompt. This lets you score items against a query — for example, asking "Is this the capital of France?" and scoring each city by how likely the model is to answer "Yes".
 
-This endpoint is automatically available when the server is started with a generative model (task `"generate"`). It is separate from the pooling-based [Score API](../models/pooling_models/scoring.md#score-api), which uses cross-encoder, bi-encoder, or late-interaction models.
+This endpoint is automatically available when the server is started with a generative model (task `"generate"`). It is separate from the pooling-based [Score API](#score-api), which uses cross-encoder, bi-encoder, or late-interaction models.
 
 **Requirements:**
 

requirements/common.txt

@@ -7,7 +7,7 @@ requests >= 2.26.0
 tqdm
 blake3
 py-cpuinfo
-transformers >= 4.56.0, != 5.0.*, != 5.1.*, != 5.2.*, != 5.3.*, != 5.4.*, != 5.5.0
+transformers >= 4.56.0, < 5
 tokenizers >= 0.21.1  # Required for fast incremental detokenization.
 protobuf >= 5.29.6, !=6.30.*, !=6.31.*, !=6.32.*, !=6.33.0.*, !=6.33.1.*, !=6.33.2.*, !=6.33.3.*, !=6.33.4.* # Required by LlamaTokenizer, gRPC. CVE-2026-0994
 fastapi[standard] >= 0.115.0 # Required by FastAPI's form models in the OpenAI API server's audio transcriptions endpoint.
@@ -19,7 +19,7 @@ pillow  # Required for image processing
 prometheus-fastapi-instrumentator >= 7.0.0
 tiktoken >= 0.6.0  # Required for DBRX tokenizer
 lm-format-enforcer == 0.11.3
-llguidance >= 1.3.0, < 1.4.0; platform_machine == "x86_64" or platform_machine == "arm64" or platform_machine == "aarch64" or platform_machine == "ppc64le"
+llguidance >= 1.3.0, < 1.4.0; platform_machine == "x86_64" or platform_machine == "arm64" or platform_machine == "aarch64" or platform_machine == "s390x" or platform_machine == "ppc64le"
 outlines_core == 0.2.11
 # required for outlines backend disk cache
 diskcache == 5.6.3
@@ -32,14 +32,12 @@ pyzmq >= 25.0.0
 msgspec
 gguf >= 0.17.0
 mistral_common[image] >= 1.11.0
-av  # required for audio in video IO
 opencv-python-headless >= 4.13.0    # required for video IO
-soundfile  # required for audio IO
 pyyaml
 six>=1.16.0; python_version > '3.11' # transitive dependency of pandas that needs to be the latest version for python 3.12
 setuptools>=77.0.3,<81.0.0; python_version > '3.11' # Setuptools is used by triton, we need to ensure a modern version is installed for 3.12+ so that it does not try to import distutils, which was removed in 3.12
 einops # Required for Qwen2-VL.
-compressed-tensors == 0.15.0.1 # required for compressed-tensors
+compressed-tensors == 0.14.0.1 # required for compressed-tensors
 depyf==0.20.0 # required for profiling and debugging with compilation config
 cloudpickle # allows pickling lambda functions in model_executor/models/registry.py
 watchfiles # required for http server to monitor the updates of TLS files

requirements/kv_connectors.txt

@@ -1,5 +1,5 @@
 lmcache >= 0.3.9
-nixl[cu13] >= 0.7.1, <= 0.10.1 # Required for disaggregated prefill
-nixl-cu12 >= 0.7.1, <= 0.10.1
-nixl-cu13 >= 0.7.1, <= 0.10.1
+nixl[cu13] >= 0.7.1, < 0.10.0 # Required for disaggregated prefill
+nixl-cu12 >= 0.7.1, < 0.10.0
+nixl-cu13 >= 0.7.1, < 0.10.0
 mooncake-transfer-engine >= 0.3.8

requirements/test/cuda.in

@@ -18,9 +18,10 @@ httpx
 librosa # required for audio tests
 vector_quantize_pytorch # required for minicpmo_26 test
 vocos # required for minicpmo_26 test
-peft>=0.18.1 # required for phi-4-mm test
+peft>=0.15.0 # required for phi-4-mm test
 pqdm
 ray[cgraph,default]>=2.48.0 # Ray Compiled Graph, required by pipeline parallelism tests
+resampy # required for audio tests
 sentence-transformers>=5.2.0 # required for embedding tests
 soundfile # required for audio tests
 jiwer # required for audio tests
@@ -38,8 +39,8 @@ opencv-python-headless >= 4.13.0 # required for video test
 datamodel_code_generator # required for minicpm3 test
 lm-eval[api]>=0.4.11 # required for model evaluation test
 mteb[bm25s]>=2, <3 # required for mteb test
-transformers==5.5.3
-tokenizers==0.22.2
+transformers==4.57.5
+tokenizers==0.22.0
 schemathesis>=3.39.15 # Required for openai schema test.
 # quantization
 bitsandbytes==0.49.2

requirements/test/cuda.txt

@@ -4,7 +4,7 @@ absl-py==2.1.0
     # via
     #   rouge-score
     #   tensorboard
-accelerate==1.13.0
+accelerate==1.0.1
     # via peft
 aenum==3.1.16
     # via lightly
@@ -248,6 +248,7 @@ filelock==3.16.1
     #   huggingface-hub
     #   ray
     #   torch
+    #   transformers
     #   virtualenv
 fiona==1.10.1
     # via torchgeo
@@ -330,7 +331,7 @@ h5py==3.13.0
     # via terratorch
 harfile==0.3.0
     # via schemathesis
-hf-xet==1.4.3
+hf-xet==1.1.7
     # via huggingface-hub
 hiredis==3.0.0
     # via tensorizer
@@ -344,10 +345,9 @@ httpx==0.27.2
     # via
     #   -r requirements/test/cuda.in
     #   diffusers
-    #   huggingface-hub
     #   perceptron
     #   schemathesis
-huggingface-hub==1.10.2
+huggingface-hub==0.36.2
     # via
     #   accelerate
     #   datasets
@@ -555,6 +555,7 @@ numba==0.61.2
     #   -c requirements/cuda.txt
     #   -r requirements/test/cuda.in
     #   librosa
+    #   resampy
 numpy==2.2.6
     # via
     #   -r requirements/test/cuda.in
@@ -595,6 +596,7 @@ numpy==2.2.6
     #   pyogrio
     #   pywavelets
     #   rasterio
+    #   resampy
     #   rioxarray
     #   rouge-score
     #   runai-model-streamer
@@ -754,7 +756,7 @@ pathvalidate==3.2.1
     # via pytablewriter
 patsy==1.0.1
     # via statsmodels
-peft==0.18.1
+peft==0.16.0
     # via -r requirements/test/cuda.in
 perceptron==0.1.4
     # via -r requirements/test/cuda.in
@@ -980,7 +982,7 @@ referencing==0.35.1
     # via
     #   jsonschema
     #   jsonschema-specifications
-regex==2026.2.28
+regex==2024.9.11
     # via
     #   diffusers
     #   nltk
@@ -1000,6 +1002,7 @@ requests==2.32.3
     #   google-api-core
     #   google-cloud-storage
     #   gpt-oss
+    #   huggingface-hub
     #   lightly
     #   lm-eval
     #   mistral-common
@@ -1012,7 +1015,10 @@ requests==2.32.3
     #   starlette-testclient
     #   tacoreader
     #   tiktoken
+    #   transformers
     #   wandb
+resampy==0.4.3
+    # via -r requirements/test/cuda.in
 responses==0.25.3
     # via genai-perf
 rfc3339-validator==0.1.4
@@ -1210,7 +1216,7 @@ timm==1.0.17
     #   segmentation-models-pytorch
     #   terratorch
     #   torchgeo
-tokenizers==0.22.2
+tokenizers==0.22.0
     # via
     #   -c requirements/common.txt
     #   -r requirements/test/cuda.in
@@ -1289,7 +1295,7 @@ tqdm==4.67.3
     #   tacoreader
     #   terratorch
     #   transformers
-transformers==5.5.3
+transformers==4.57.5
     # via
     #   -c requirements/common.txt
     #   -r requirements/test/cuda.in
@@ -1311,9 +1317,7 @@ typepy==1.3.2
 typer==0.15.2
     # via
     #   fastsafetensors
-    #   huggingface-hub
     #   perceptron
-    #   transformers
 types-python-dateutil==2.9.0.20241206
     # via arrow
 typeshed-client==2.8.2

requirements/test/nightly-torch.txt

@@ -29,8 +29,8 @@ opencv-python-headless >= 4.13.0 # required for video test
 datamodel_code_generator # required for minicpm3 test
 lm-eval[api]>=0.4.11 # required for model evaluation test
 mteb[bm25s]>=2, <3 # required for mteb test
-transformers==5.5.3
-tokenizers==0.22.2
+transformers==4.57.5
+tokenizers==0.22.0
 schemathesis>=3.39.15 # Required for openai schema test.
 # quantization
 bitsandbytes>=0.49.2

requirements/test/rocm.in

@@ -23,6 +23,7 @@ vocos # required for minicpmo_26 test
 peft>=0.15.0 # required for phi-4-mm test
 pqdm
 ray[cgraph,default]>=2.48.0 # Ray Compiled Graph, required by pipeline parallelism tests
+resampy # required for audio tests
 sentence-transformers>=5.2.0 # required for embedding tests
 soundfile # required for audio tests
 jiwer # required for audio tests
@@ -37,8 +38,8 @@ opencv-python-headless>=4.13.0 # required for video test
 datamodel_code_generator # required for minicpm3 test
 lm-eval[api]>=0.4.11 # required for model evaluation test
 mteb[bm25s]>=2, <3 # required for mteb test
-transformers==5.5.3
-tokenizers==0.22.2
+transformers==4.57.5
+tokenizers==0.22.0
 schemathesis>=3.39.15 # Required for openai schema test
 # quantization
 bitsandbytes==0.49.2
@@ -81,3 +82,4 @@ plotly # required for perf comparison html report
 rapidfuzz
 torchgeo==0.7.0
 multiprocess==0.70.16
+huggingface-hub==0.36.2

requirements/test/rocm.txt

@@ -39,7 +39,7 @@ annotated-doc==0.0.4
     #   typer
 annotated-types==0.7.0
     # via pydantic
-anthropic==0.93.0
+anthropic==0.89.0
     # via
     #   -c requirements/common.txt
     #   -r requirements/test/../common.txt
@@ -76,9 +76,7 @@ attrs==26.1.0
 audioread==3.0.1
     # via librosa
 av==16.1.0
-    # via
-    #   -r requirements/test/../common.txt
-    #   -r requirements/test/rocm.in
+    # via -r requirements/test/rocm.in
 azure-core==1.39.0
     # via
     #   azure-identity
@@ -174,7 +172,7 @@ colorful==0.5.8
     # via ray
 colorlog==6.10.1
     # via optuna
-compressed-tensors==0.15.0.1
+compressed-tensors==0.14.0.1
     # via
     #   -c requirements/common.txt
     #   -r requirements/test/../common.txt
@@ -271,9 +269,9 @@ fastapi==0.135.2
     #   model-hosting-container-standards
 fastapi-cli==0.0.24
     # via fastapi
-fastapi-cloud-cli==0.16.1
+fastapi-cloud-cli==0.15.1
     # via fastapi-cli
-fastar==0.10.0
+fastar==0.9.0
     # via fastapi-cloud-cli
 fastparquet==2026.3.0
     # via genai-perf
@@ -292,6 +290,7 @@ filelock==3.25.2
     #   python-discovery
     #   ray
     #   torch
+    #   transformers
     #   virtualenv
 fiona==1.10.1
     # via torchgeo
@@ -385,7 +384,7 @@ h5py==3.16.0
     # via terratorch
 harfile==0.4.0
     # via schemathesis
-hf-xet==1.4.3
+hf-xet==1.4.2
     # via huggingface-hub
 hiredis==3.3.1
     # via tensorizer
@@ -404,7 +403,6 @@ httpx==0.27.2
     #   diffusers
     #   fastapi
     #   fastapi-cloud-cli
-    #   huggingface-hub
     #   mcp
     #   model-hosting-container-standards
     #   openai
@@ -412,8 +410,9 @@ httpx==0.27.2
     #   schemathesis
 httpx-sse==0.4.3
     # via mcp
-huggingface-hub==1.10.2
+huggingface-hub==0.36.2
     # via
+    #   -r requirements/test/rocm.in
     #   accelerate
     #   datasets
     #   diffusers
@@ -485,7 +484,7 @@ jinja2==3.1.6
     #   genai-perf
     #   lm-eval
     #   torch
-jiter==0.14.0
+jiter==0.13.0
     # via
     #   anthropic
     #   openai
@@ -632,7 +631,7 @@ msgpack==1.1.2
     # via
     #   librosa
     #   ray
-msgspec==0.21.0
+msgspec==0.20.0
     # via -r requirements/test/../common.txt
 mteb==2.11.5
     # via -r requirements/test/rocm.in
@@ -664,6 +663,7 @@ numba==0.61.2
     #   -c requirements/rocm.txt
     #   -r requirements/test/rocm.in
     #   librosa
+    #   resampy
 numkong==7.1.1
     # via albucore
 numpy==2.2.6
@@ -709,6 +709,7 @@ numpy==2.2.6
     #   pytrec-eval-terrier
     #   pywavelets
     #   rasterio
+    #   resampy
     #   rioxarray
     #   rouge-score
     #   runai-model-streamer
@@ -741,7 +742,7 @@ omegaconf==2.3.0
     #   lightning
 open-clip-torch==2.32.0
     # via -r requirements/test/rocm.in
-openai==2.31.0
+openai==2.30.0
     # via
     #   -c requirements/common.txt
     #   -r requirements/test/../common.txt
@@ -1092,7 +1093,7 @@ python-dotenv==1.2.2
     #   uvicorn
 python-json-logger==4.1.0
     # via -r requirements/test/../common.txt
-python-multipart==0.0.26
+python-multipart==0.0.22
     # via
     #   fastapi
     #   mcp
@@ -1179,6 +1180,7 @@ requests==2.32.5
     #   google-api-core
     #   google-cloud-storage
     #   gpt-oss
+    #   huggingface-hub
     #   lightly
     #   lm-eval
     #   mistral-common
@@ -1192,7 +1194,10 @@ requests==2.32.5
     #   starlette-testclient
     #   tacoreader
     #   tiktoken
+    #   transformers
     #   wandb
+resampy==0.4.3
+    # via -r requirements/test/rocm.in
 responses==0.26.0
     # via genai-perf
 rfc3339-validator==0.1.4
@@ -1333,7 +1338,6 @@ sortedcontainers==2.4.0
     # via hypothesis
 soundfile==0.13.1
     # via
-    #   -r requirements/test/../common.txt
     #   -r requirements/test/rocm.in
     #   genai-perf
     #   librosa
@@ -1424,7 +1428,7 @@ timm==1.0.17
     #   segmentation-models-pytorch
     #   terratorch
     #   torchgeo
-tokenizers==0.22.2
+tokenizers==0.22.0
     # via
     #   -c requirements/common.txt
     #   -r requirements/test/../common.txt
@@ -1467,7 +1471,7 @@ tqdm==4.67.3
     #   tacoreader
     #   terratorch
     #   transformers
-transformers==5.5.3
+transformers==4.57.5
     # via
     #   -c requirements/common.txt
     #   -r requirements/test/../common.txt
@@ -1494,9 +1498,7 @@ typer==0.24.1
     #   fastapi-cli
     #   fastapi-cloud-cli
     #   fastsafetensors
-    #   huggingface-hub
     #   perceptron
-    #   transformers
 typeshed-client==2.9.0
     # via jsonargparse
 typing-extensions==4.15.0

requirements/test/xpu.in

@@ -13,6 +13,7 @@ pytest-shard
 absl-py
 accelerate
 arctic-inference
+hf_transfer
 lm_eval[api]
 modelscope
 

requirements/test/xpu.txt

@@ -19,9 +19,7 @@ aiosignal==1.4.0
 albumentations==1.4.6
     # via -r requirements/test/xpu.in
 annotated-doc==0.0.4
-    # via
-    #   fastapi
-    #   typer
+    # via fastapi
 annotated-types==0.7.0
     # via pydantic
 anyio==4.13.0
@@ -66,7 +64,6 @@ click==8.3.1
     #   jiwer
     #   nltk
     #   schemathesis
-    #   typer
     #   uvicorn
 colorama==0.4.6
     # via sacrebleu
@@ -115,6 +112,7 @@ filelock==3.25.2
     #   huggingface-hub
     #   modelscope
     #   torch
+    #   transformers
 frozenlist==1.8.0
     # via
     #   aiohttp
@@ -135,7 +133,9 @@ h11==0.16.0
     #   uvicorn
 harfile==0.4.0
     # via schemathesis
-hf-xet==1.4.3
+hf-transfer==0.1.9
+    # via -r requirements/test/xpu.in
+hf-xet==1.4.2
     # via huggingface-hub
 html2text==2025.4.15
     # via gpt-oss
@@ -144,9 +144,8 @@ httpcore==1.0.9
 httpx==0.28.1
     # via
     #   datasets
-    #   huggingface-hub
     #   schemathesis
-huggingface-hub==1.10.2
+huggingface-hub==0.36.2
     # via
     #   accelerate
     #   datasets
@@ -516,6 +515,7 @@ requests==2.33.1
     #   docker
     #   evaluate
     #   gpt-oss
+    #   huggingface-hub
     #   lm-eval
     #   mistral-common
     #   modelscope
@@ -524,11 +524,11 @@ requests==2.33.1
     #   schemathesis
     #   starlette-testclient
     #   tiktoken
+    #   transformers
 rich==14.3.3
     # via
     #   mteb
     #   schemathesis
-    #   typer
 rouge-score==0.1.2
     # via lm-eval
 rpds-py==0.30.0
@@ -572,8 +572,6 @@ setuptools==80.10.2
     #   modelscope
     #   pytablewriter
     #   torch
-shellingham==1.5.4
-    # via typer
 six==1.17.0
     # via
     #   -c requirements/common.txt
@@ -667,7 +665,7 @@ tqdm==4.67.3
     #   pqdm
     #   sentence-transformers
     #   transformers
-transformers==5.5.3
+transformers==4.57.6
     # via
     #   -c requirements/common.txt
     #   sentence-transformers
@@ -678,10 +676,6 @@ typepy==1.3.4
     #   dataproperty
     #   pytablewriter
     #   tabledata
-typer==0.24.1
-    # via
-    #   huggingface-hub
-    #   transformers
 typing-extensions==4.15.0
     # via
     #   -c requirements/common.txt

setup.py

@@ -693,12 +693,6 @@ class precompiled_wheel_utils:
                 flash_attn_regex = re.compile(
                     r"vllm/vllm_flash_attn/(?:[^/.][^/]*/)*(?!\.)[^/]*\.py"
                 )
-                # __init__.py and flash_attn_interface.py are source-controlled
-                # in vllm and should not be overwritten (matches cmake exclusions)
-                flash_attn_files_to_skip = {
-                    "vllm/vllm_flash_attn/__init__.py",
-                    "vllm/vllm_flash_attn/flash_attn_interface.py",
-                }
                 triton_kernels_regex = re.compile(
                     r"vllm/third_party/triton_kernels/(?:[^/.][^/]*/)*(?!\.)[^/]*\.py"
                 )
@@ -711,11 +705,7 @@ class precompiled_wheel_utils:
                     filter(lambda x: x.filename in files_to_copy, wheel.filelist)
                 )
                 file_members += list(
-                    filter(
-                        lambda x: flash_attn_regex.match(x.filename)
-                        and x.filename not in flash_attn_files_to_skip,
-                        wheel.filelist,
-                    )
+                    filter(lambda x: flash_attn_regex.match(x.filename), wheel.filelist)
                 )
                 file_members += list(
                     filter(
@@ -1092,7 +1082,10 @@ setup(
         "instanttensor": ["instanttensor >= 0.1.5"],
         "runai": ["runai-model-streamer[s3,gcs,azure] >= 0.15.7"],
         "audio": [
+            "av",
+            "resampy",
             "scipy",
+            "soundfile",
             "mistral_common[audio]",
         ],  # Required for audio processing
         "video": [],  # Kept for backwards compatibility
@@ -1101,7 +1094,7 @@ setup(
         # NOTE: When updating helion version, also update CI files:
         #   - .buildkite/test_areas/kernels.yaml
         #   - .buildkite/test-amd.yaml
-        "helion": ["helion==1.0.0"],
+        "helion": ["helion==0.3.3"],
         # Optional deps for gRPC server (vllm serve --grpc)
         "grpc": ["smg-grpc-servicer[vllm] >= 0.5.0"],
         # Optional deps for OpenTelemetry tracing

tests/compile/test_dynamic_shapes_compilation.py

@@ -222,47 +222,3 @@ def test_model_specialization_with_evaluate_guards(
         torch.randn(1, 10).cuda(),
         is_01_specialization=True,
     )
-
-
-@pytest.mark.skipif(not is_torch_equal_or_newer("2.10.0"), reason="requires torch 2.10")
-def test_piecewise_backend_empty_sym_shape_indices():
-    """Test that PiecewiseBackend handles empty sym_shape_indices correctly.
-
-    When all inputs have static shapes (no torch.SymInt), sym_shape_indices
-    will be empty. The fix in PiecewiseBackend.__call__ handles this case
-    by using the first compiled range_entry.
-    """
-    gc.collect()
-    torch.accelerator.empty_cache()
-    torch.accelerator.synchronize()
-
-    # Use small max_model_len and max_num_batched_tokens to encourage
-    # static shape compilation with empty sym_shape_indices
-    llm = LLM(
-        model="Qwen/Qwen3-0.6B",
-        max_model_len=512,
-        max_num_batched_tokens=1,
-        compilation_config={
-            "mode": CompilationMode.VLLM_COMPILE,
-            "dynamic_shapes_config": {
-                "type": DynamicShapesType.BACKED.value,
-            },
-        },
-    )
-
-    sampling_params = SamplingParams(temperature=0, top_p=0.95, max_tokens=10)
-
-    # Generate with static shape inputs
-    output = llm.generate("Hello, my name is", sampling_params=sampling_params)
-    result = output[0].outputs[0].text
-    assert len(result) > 0, "Should generate non-empty output"
-
-    # Generate again to verify compilation works with empty sym_shape_indices
-    output = llm.generate("The capital of France is", sampling_params=sampling_params)
-    result = output[0].outputs[0].text
-    assert len(result) > 0, "Should generate non-empty output on second run"
-
-    del llm
-    gc.collect()
-    torch.accelerator.empty_cache()
-    torch.accelerator.synchronize()

tests/config/base_model_arch_groundtruth.json

@@ -356,23 +356,6 @@
         "is_multimodal_model": false,
         "dtype": "torch.float32"
     },
-    "stepfun-ai/Step-3.5-Flash": {
-        "architectures": [
-            "Step3p5ForCausalLM"
-        ],
-        "model_type": "step3p5",
-        "text_model_type": "step3p5",
-        "hidden_size": 4096,
-        "total_num_hidden_layers": 45,
-        "total_num_attention_heads": 64,
-        "head_size": 128,
-        "vocab_size": 128896,
-        "total_num_kv_heads": 8,
-        "num_experts": 288,
-        "is_deepseek_mla": false,
-        "is_multimodal_model": false,
-        "dtype": "torch.bfloat16"
-    },
     "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16": {
         "architectures": [
             "NemotronHForCausalLM"

tests/config/test_model_arch_config.py

@@ -16,7 +16,6 @@ BASE_TRUST_REMOTE_CODE_MODELS = {
     "nvidia/Llama-3_3-Nemotron-Super-49B-v1",
     "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16",
     "XiaomiMiMo/MiMo-7B-RL",
-    "stepfun-ai/Step-3.5-Flash",
     # Excluded: Not available online right now
     # "FreedomIntelligence/openPangu-Ultra-MoE-718B-V1.1",
     "meituan-longcat/LongCat-Flash-Chat",

tests/conftest.py

@@ -364,7 +364,6 @@ class HfRunner:
         model_name: str,
         dtype: str = "auto",
         *,
-        revision: str | None = None,
         model_kwargs: dict[str, Any] | None = None,
         trust_remote_code: bool = True,
         is_sentence_transformer: bool = False,
@@ -384,7 +383,6 @@ class HfRunner:
             self._init(
                 model_name=model_name,
                 dtype=dtype,
-                revision=revision,
                 model_kwargs=model_kwargs,
                 trust_remote_code=trust_remote_code,
                 is_sentence_transformer=is_sentence_transformer,
@@ -398,7 +396,6 @@ class HfRunner:
         model_name: str,
         dtype: str = "auto",
         *,
-        revision: str | None = None,
         model_kwargs: dict[str, Any] | None = None,
         trust_remote_code: bool = True,
         is_sentence_transformer: bool = False,
@@ -413,15 +410,6 @@ class HfRunner:
             model_name,
             trust_remote_code=trust_remote_code,
         )
-        # HF runner should use the HF config so that it's consistent with the HF model
-        if self.config.__module__.startswith("vllm.transformers_utils.configs"):
-            from transformers.models.auto.configuration_auto import CONFIG_MAPPING
-
-            del CONFIG_MAPPING._extra_content[self.config.model_type]
-            self.config = AutoConfig.from_pretrained(
-                model_name,
-                trust_remote_code=trust_remote_code,
-            )
         self.device = self.get_default_device()
         self.dtype = dtype = _get_and_verify_dtype(
             self.model_name,
@@ -440,7 +428,6 @@ class HfRunner:
 
             self.model = SentenceTransformer(
                 model_name,
-                revision=revision,
                 device=self.device,
                 model_kwargs=model_kwargs,
                 trust_remote_code=trust_remote_code,
@@ -451,7 +438,6 @@ class HfRunner:
 
             self.model = CrossEncoder(
                 model_name,
-                revision=revision,
                 device=self.device,
                 automodel_args=model_kwargs,
                 trust_remote_code=trust_remote_code,
@@ -461,7 +447,6 @@ class HfRunner:
                 nn.Module,
                 auto_cls.from_pretrained(
                     model_name,
-                    revision=revision,
                     trust_remote_code=trust_remote_code,
                     **model_kwargs,
                 ),

tests/entrypoints/llm/test_generate.py

@@ -91,12 +91,6 @@ def test_multiple_priority(llm: LLM):
         outputs = llm.generate(PROMPTS, sampling_params=None, priority=[])
 
 
-def test_single_prompt_priority(llm: LLM):
-    # Single string prompts should be normalized to one request.
-    outputs = llm.generate(PROMPTS[0], sampling_params=None, priority=[0])
-    assert len(outputs) == 1
-
-
 def test_max_model_len():
     max_model_len = 20
     llm = LLM(

tests/entrypoints/openai/test_tool_calls_serialization.py

@@ -1,150 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""Unit tests for tool_calls Iterable → list materialisation.
-
-Regression tests for https://github.com/vllm-project/vllm/issues/34792.
-
-Setting VLLM_LOGGING_LEVEL=debug caused tool calling to break for Mistral
-models because:
-  1. The OpenAI Python SDK types tool_calls as Iterable[...] in
-     ChatCompletionAssistantMessageParam.
-  2. Pydantic v2, when validating from Python objects (not from raw JSON),
-     wraps Iterable fields in a one-shot lazy iterator.
-  3. Debug logging called model_dump_json() which consumed that iterator.
-  4. The Mistral tokenizer then saw empty tool_calls and raised
-     "ValueError: Unexpected tool call id ...".
-"""
-
-import pytest
-
-from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
-
-
-def _make_tool_call(tc_id: str, name: str, args: str) -> dict:
-    return {
-        "id": tc_id,
-        "type": "function",
-        "function": {"name": name, "arguments": args},
-    }
-
-
-def _make_request(messages: list) -> ChatCompletionRequest:
-    return ChatCompletionRequest(
-        model="test-model",
-        messages=messages,
-    )
-
-
-def test_tool_calls_list_preserved_after_model_dump():
-    """tool_calls in assistant messages must be readable after model_dump_json.
-
-    When the request is built from Python dicts (as in the Anthropic → OpenAI
-    conversion path), Pydantic v2 previously wrapped the Iterable tool_calls
-    in a one-shot iterator.  model_dump_json() consumed it, leaving subsequent
-    readers (e.g. the Mistral tokenizer) with an empty sequence.
-    """
-    tool_call = _make_tool_call("call_abc123", "get_weather", '{"city": "Paris"}')
-    messages = [
-        {"role": "user", "content": "What is the weather in Paris?"},
-        {"role": "assistant", "content": None, "tool_calls": [tool_call]},
-        {
-            "role": "tool",
-            "tool_call_id": "call_abc123",
-            "content": '{"temperature": 20}',
-        },
-    ]
-
-    req = _make_request(messages)
-
-    # Simulate debug logging: serialize the model (this was the trigger)
-    _ = req.model_dump_json()
-
-    # The assistant message must still have accessible tool_calls afterwards
-    assistant_msg = req.messages[1]
-    assert isinstance(assistant_msg, dict)
-    tool_calls = assistant_msg.get("tool_calls")
-    assert tool_calls is not None, "tool_calls must not be None after model_dump_json"
-    assert isinstance(tool_calls, list), "tool_calls must be a list"
-    assert len(tool_calls) > 0, "tool_calls must not be empty after model_dump_json"
-
-
-def test_tool_calls_from_generator_are_materialised():
-    """tool_calls passed as a generator must be converted to list on validation."""
-    tool_call = _make_tool_call("call_gen1", "search", '{"query": "vllm"}')
-
-    def tool_calls_gen():
-        yield tool_call
-
-    messages = [
-        {"role": "user", "content": "Search for vllm"},
-        {
-            "role": "assistant",
-            "content": None,
-            "tool_calls": tool_calls_gen(),  # one-shot generator
-        },
-    ]
-
-    req = _make_request(messages)
-    assistant_msg = req.messages[1]
-    assert isinstance(assistant_msg, dict)
-
-    # Iterate twice — must not raise or return empty on second pass
-    tool_calls_first = list(assistant_msg.get("tool_calls", []))
-    tool_calls_second = list(assistant_msg.get("tool_calls", []))
-
-    assert len(tool_calls_first) == 1, "First read must return the tool call"
-    assert len(tool_calls_second) == 1, "Second read must also return the tool call"
-
-
-def test_tool_calls_list_passthrough():
-    """tool_calls already provided as a list must remain a list."""
-    tool_call = _make_tool_call("call_list1", "calculate", '{"expr": "2+2"}')
-    messages = [
-        {"role": "user", "content": "Calculate 2+2"},
-        {"role": "assistant", "content": None, "tool_calls": [tool_call]},
-    ]
-
-    req = _make_request(messages)
-    assistant_msg = req.messages[1]
-    assert isinstance(assistant_msg, dict)
-    assert isinstance(assistant_msg.get("tool_calls"), list)
-
-
-def test_messages_without_tool_calls_unaffected():
-    """Messages without tool_calls must be handled correctly."""
-    messages = [
-        {"role": "system", "content": "You are a helpful assistant."},
-        {"role": "user", "content": "Hello!"},
-        {"role": "assistant", "content": "Hi there!"},
-    ]
-
-    req = _make_request(messages)
-    # None of the messages should have tool_calls injected
-    for msg in req.messages:
-        assert isinstance(msg, dict)
-        assert msg.get("tool_calls") is None or msg.get("tool_calls") == []
-
-
-@pytest.mark.parametrize("num_tool_calls", [1, 3])
-def test_multiple_tool_calls_materialised(num_tool_calls: int):
-    """Multiple tool calls in a single message are all preserved."""
-    tool_calls = [
-        _make_tool_call(f"call_{i}", f"func_{i}", f'{{"arg": {i}}}')
-        for i in range(num_tool_calls)
-    ]
-    messages = [
-        {"role": "user", "content": "Do things"},
-        {"role": "assistant", "content": None, "tool_calls": iter(tool_calls)},
-    ]
-
-    req = _make_request(messages)
-    assistant_msg = req.messages[1]
-    assert isinstance(assistant_msg, dict)
-
-    result_tool_calls = assistant_msg.get("tool_calls")
-    assert isinstance(result_tool_calls, list)
-    assert len(result_tool_calls) == num_tool_calls
-
-    # Verify after model_dump_json too
-    _ = req.model_dump_json()
-    assert len(assistant_msg.get("tool_calls", [])) == num_tool_calls

tests/kernels/core/test_cpu_activation.py

@@ -1,111 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-import pytest
-import torch
-
-from tests.kernels.allclose_default import get_default_atol, get_default_rtol
-from tests.kernels.utils import opcheck
-from vllm.platforms import CpuArchEnum, current_platform
-from vllm.utils.torch_utils import set_random_seed
-
-if not current_platform.is_cpu():
-    pytest.skip("skipping CPU-only tests", allow_module_level=True)
-
-from vllm.model_executor.layers.activation import (
-    GELU,
-    FastGELU,
-    GeluAndMul,
-    NewGELU,
-    QuickGELU,
-    SiluAndMul,
-)
-
-DTYPES = [torch.bfloat16, torch.float32]
-NUM_TOKENS = [7, 83]
-D = [512, 2048]
-SEEDS = [0]
-
-
-@pytest.mark.parametrize(
-    ("activation_cls", "fn"),
-    [
-        (SiluAndMul, torch.ops._C.silu_and_mul),
-        (GeluAndMul, torch.ops._C.gelu_and_mul),
-        (GeluAndMul, torch.ops._C.gelu_tanh_and_mul),
-    ],
-)
-@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
-@pytest.mark.parametrize("d", D)
-@pytest.mark.parametrize("dtype", DTYPES)
-@pytest.mark.parametrize("seed", SEEDS)
-@torch.inference_mode()
-def test_cpu_act_and_mul(
-    default_vllm_config,
-    activation_cls: type[torch.nn.Module],
-    fn: object,
-    num_tokens: int,
-    d: int,
-    dtype: torch.dtype,
-    seed: int,
-) -> None:
-    set_random_seed(seed)
-    x = torch.randn(num_tokens, 2 * d, dtype=dtype)
-
-    layer = activation_cls()
-    out = layer(x)
-    ref_out = layer.forward_native(x)
-
-    torch.testing.assert_close(
-        out, ref_out, atol=get_default_atol(out), rtol=get_default_rtol(out)
-    )
-
-    output_shape = x.shape[:-1] + (x.shape[-1] // 2,)
-    raw_out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
-    opcheck(fn, (raw_out, x))
-
-
-@pytest.mark.parametrize(
-    ("activation_cls", "fn", "op_args"),
-    [
-        (NewGELU, torch.ops._C.gelu_new, ()),
-        (FastGELU, torch.ops._C.gelu_fast, ()),
-        (QuickGELU, torch.ops._C.gelu_quick, ()),
-        pytest.param(
-            GELU,
-            getattr(torch.ops._C, "activation_lut_bf16", None),
-            ("gelu",),
-            marks=pytest.mark.skipif(
-                current_platform.get_cpu_architecture() != CpuArchEnum.ARM,
-                reason="activation_lut_bf16 is only built on Arm CPU",
-            ),
-        ),
-    ],
-)
-@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
-@pytest.mark.parametrize("d", D)
-@pytest.mark.parametrize("dtype", DTYPES)
-@pytest.mark.parametrize("seed", SEEDS)
-@torch.inference_mode()
-def test_cpu_unary_activation(
-    default_vllm_config,
-    activation_cls: type[torch.nn.Module],
-    fn: object,
-    op_args: tuple[str, ...],
-    num_tokens: int,
-    d: int,
-    dtype: torch.dtype,
-    seed: int,
-) -> None:
-    set_random_seed(seed)
-    x = torch.randn(num_tokens, d, dtype=dtype)
-    layer = activation_cls()
-    out = layer(x)
-    ref_out = layer.forward_native(x)
-    torch.testing.assert_close(
-        out, ref_out, atol=get_default_atol(out), rtol=get_default_rtol(out)
-    )
-    # gelu with activation_lut_bf16 only makes sense for BF16
-    if not (activation_cls is GELU and dtype != torch.bfloat16):
-        raw_out = torch.empty_like(x)
-        opcheck(fn, (raw_out, x, *op_args))

tests/kernels/helion/test_register.py

@@ -36,11 +36,9 @@ from vllm.kernels.helion.register import (
 )
 
 if _HOP_AVAILABLE:
-    from helion._compat import supports_torch_compile_fusion
     from helion._compiler._dynamo.higher_order_ops import (
         helion_kernel_wrapper_mutation,
     )
-    from torch._inductor.utils import run_and_get_code
 
 
 def _add_kernel(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
@@ -1005,49 +1003,3 @@ class TestTorchCompileHOP:
             "Compiled execution result doesn't match eager execution. "
             f"Max difference: {torch.max(torch.abs(compiled_result - eager_result))}"
         )
-
-    @pytest.mark.skipif(
-        not (_HOP_AVAILABLE and supports_torch_compile_fusion()),
-        reason="Requires PyTorch with Helion inductor fusion support",
-    )
-    def test_inductor_backend_compiles_helion_hop(self):
-        """Test torch.compile with inductor backend and Helion fusion enabled."""
-
-        configs = {"default": helion.Config(block_sizes=[4, 4])}
-
-        with dummy_kernel_registry(configs=configs) as register:
-            add_helion_kernel = register(
-                op_name="test_inductor_add_kernel",
-                config_picker=lambda args, keys: "default",
-                helion_settings=helion.Settings(
-                    torch_compile_fusion=True, static_shapes=False
-                ),
-            )(_add_kernel)
-
-        def f(x, y):
-            x = x * 2.0
-            y = y + 1.0
-            out = add_helion_kernel(x, y)
-            return out.relu()
-
-        torch._dynamo.reset()
-        compiled_f = torch.compile(f, backend="inductor", fullgraph=True)
-
-        x = torch.randn(4, 4, device="cuda")
-        y = torch.randn(4, 4, device="cuda")
-
-        compiled_result, source_codes = run_and_get_code(compiled_f, x, y)
-        eager_result = f(x, y)
-
-        assert torch.allclose(compiled_result, eager_result, atol=1e-5, rtol=1e-5), (
-            "Inductor-compiled result doesn't match eager execution. "
-            f"Max difference: {torch.max(torch.abs(compiled_result - eager_result))}"
-        )
-
-        # With fusion enabled, prologue/epilogue ops should be fused into
-        # a single triton kernel rather than generating separate kernels.
-        kernel_count = sum(code.count("@triton.jit") for code in source_codes)
-        assert kernel_count == 1, (
-            f"Expected 1 fused triton kernel, got {kernel_count}. "
-            "Prologue/epilogue ops were not fused into the Helion kernel."
-        )

tests/lora/conftest.py

@@ -3,7 +3,6 @@
 
 import tempfile
 from collections import OrderedDict
-from importlib import reload
 from unittest.mock import MagicMock
 
 import pytest
@@ -48,11 +47,6 @@ def cleanup_fixture(should_do_global_cleanup_after_test: bool):
 def maybe_enable_lora_dual_stream(monkeypatch: pytest.MonkeyPatch):
     if current_platform.is_cuda():
         monkeypatch.setenv("VLLM_LORA_ENABLE_DUAL_STREAM", "1")
-        import vllm.lora.layers.base_linear
-
-        if not hasattr(vllm.lora.layers.base_linear, "lora_linear_async"):
-            # Reload the module to ensure the environment variable takes effect.
-            reload(vllm.lora.layers.base_linear)
     yield
 
 

tests/lora/test_minicpmv_tp.py

@@ -1,10 +1,7 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 
-from importlib.metadata import version
-
 import pytest
-from packaging.version import Version
 
 import vllm
 from vllm.assets.image import ImageAsset
@@ -13,14 +10,6 @@ from vllm.platforms import current_platform
 
 from ..utils import multi_gpu_test
 
-pytestmark = pytest.mark.skipif(
-    Version("5.0") <= Version(version("transformers")),
-    reason=(
-        "MiniCPMV custom processor uses tokenizer.im_start_id which is not "
-        "available on TokenizersBackend in transformers v5.0+"
-    ),
-)
-
 MODEL_PATH = "openbmb/MiniCPM-Llama3-V-2_5"
 
 PROMPT_TEMPLATE = (

tests/model_executor/model_loader/test_reload.py

@@ -164,34 +164,6 @@ def test_reload_weights(base_model, mul_model, add_model, tp_size, vllm_runner):
         assert add_perp < mul_perp
 
 
-def test_kv_scale_reload(vllm_runner):
-    """Test reloading a checkpoint that contains k_scale/v_scale weights."""
-    if not current_platform.supports_fp8():
-        pytest.skip(reason="Requires FP8 support")
-
-    model = "nm-testing/Llama-3.2-1B-Instruct-FP8-KV"
-
-    # Load dummy weights, then reload real checkpoint
-    with vllm_runner(
-        model_name=model,
-        load_format="dummy",
-        enable_prefix_caching=False,
-        max_model_len=16,
-        max_num_seqs=1,
-    ) as llm:
-        llm.collective_rpc(
-            "update_config",
-            kwargs={"overrides": {"load_config": {"load_format": "auto"}}},
-        )
-        llm.collective_rpc("reload_weights", kwargs={"weights_path": model})
-        reloaded_perp = llm.generate_prompt_perplexity(
-            ["The capital of France is the city of Paris"],
-            mask=["The capital of France is"],
-        )[0]
-
-    assert reloaded_perp < 10
-
-
 @pytest.mark.parametrize(
     "tp_size", [pytest.param(1), pytest.param(2, marks=[pytest.mark.slow_test])]
 )

tests/model_executor/test_keye_mrope.py

@@ -1,145 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-from dataclasses import dataclass, field
-
-import pytest
-import torch
-
-from vllm.model_executor.models.keye import KeyeForConditionalGeneration
-from vllm.multimodal.inputs import (
-    MultiModalFeatureSpec,
-    MultiModalFieldElem,
-    MultiModalKwargsItem,
-    PlaceholderRange,
-)
-
-pytestmark = pytest.mark.skip_global_cleanup
-
-
-@pytest.fixture(autouse=True, scope="module")
-def _force_cpu_default_device():
-    original = torch.get_default_device()
-    torch.set_default_device("cpu")
-    yield
-    torch.set_default_device(original)
-
-
-@dataclass
-class DummyVisionConfig:
-    spatial_merge_size: int = 2
-
-
-@dataclass
-class DummyConfig:
-    vision_config: DummyVisionConfig = field(default_factory=DummyVisionConfig)
-
-
-def make_model(config: DummyConfig) -> KeyeForConditionalGeneration:
-    model = object.__new__(KeyeForConditionalGeneration)
-    model.config = config
-    return model
-
-
-def make_mm_feature(
-    *,
-    modality: str,
-    offset: int,
-    length: int,
-    grid_thw: tuple[int, int, int] | list[tuple[int, int, int]],
-) -> MultiModalFeatureSpec:
-    field_name = "image_grid_thw" if modality == "image" else "video_grid_thw"
-    return MultiModalFeatureSpec(
-        data=MultiModalKwargsItem(
-            {
-                field_name: MultiModalFieldElem(
-                    data=torch.tensor(grid_thw),
-                    field=None,  # HACK.
-                ),
-            }
-        ),
-        modality=modality,
-        identifier="DUMMY",
-        mm_position=PlaceholderRange(offset=offset, length=length),
-    )
-
-
-def test_get_mrope_input_positions_text_only():
-    model = make_model(DummyConfig())
-
-    positions, delta = model.get_mrope_input_positions(
-        input_tokens=[11, 12, 13, 14, 15],
-        mm_features=[],
-    )
-
-    expected = torch.tensor(
-        [
-            [0, 1, 2, 3, 4],
-            [0, 1, 2, 3, 4],
-            [0, 1, 2, 3, 4],
-        ]
-    )
-
-    assert torch.equal(positions, expected)
-    assert delta == 0
-
-
-def test_get_mrope_input_positions_single_image():
-    model = make_model(DummyConfig())
-    mm_features = [
-        make_mm_feature(
-            modality="image",
-            offset=1,
-            length=4,
-            grid_thw=(1, 4, 4),
-        )
-    ]
-
-    positions, delta = model.get_mrope_input_positions(
-        input_tokens=[10, 20, 21, 22, 23, 30, 31],
-        mm_features=mm_features,
-    )
-
-    expected = torch.tensor(
-        [
-            [0, 1, 1, 1, 1, 3, 4],
-            [0, 1, 1, 2, 2, 3, 4],
-            [0, 1, 2, 1, 2, 3, 4],
-        ]
-    )
-
-    assert torch.equal(positions, expected)
-    assert delta == -2
-
-
-def test_get_mrope_input_positions_interleaved_image_and_video():
-    model = make_model(DummyConfig())
-    mm_features = [
-        make_mm_feature(
-            modality="image",
-            offset=1,
-            length=4,
-            grid_thw=(1, 4, 4),
-        ),
-        make_mm_feature(
-            modality="video",
-            offset=7,
-            length=4,
-            grid_thw=[(2, 4, 2)],
-        ),
-    ]
-
-    positions, delta = model.get_mrope_input_positions(
-        input_tokens=[10, 20, 21, 22, 23, 30, 31, 40, 41, 42, 43, 50, 51],
-        mm_features=mm_features,
-    )
-
-    expected = torch.tensor(
-        [
-            [0, 1, 1, 1, 1, 3, 4, 5, 5, 7, 7, 9, 10],
-            [0, 1, 1, 2, 2, 3, 4, 5, 6, 7, 8, 9, 10],
-            [0, 1, 2, 1, 2, 3, 4, 5, 5, 7, 7, 9, 10],
-        ]
-    )
-
-    assert torch.equal(positions, expected)
-    assert delta == -2

tests/model_executor/test_keye_vl1_5_mrope.py

@@ -1,145 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-from dataclasses import dataclass, field
-
-import pytest
-import torch
-
-from vllm.model_executor.models.keye_vl1_5 import KeyeVL1_5ForConditionalGeneration
-from vllm.multimodal.inputs import (
-    MultiModalFeatureSpec,
-    MultiModalFieldElem,
-    MultiModalKwargsItem,
-    PlaceholderRange,
-)
-
-pytestmark = pytest.mark.skip_global_cleanup
-
-
-@pytest.fixture(autouse=True, scope="module")
-def _force_cpu_default_device():
-    original = torch.get_default_device()
-    torch.set_default_device("cpu")
-    yield
-    torch.set_default_device(original)
-
-
-@dataclass
-class DummyVisionConfig:
-    spatial_merge_size: int = 2
-
-
-@dataclass
-class DummyConfig:
-    vision_config: DummyVisionConfig = field(default_factory=DummyVisionConfig)
-
-
-def make_model(config: DummyConfig) -> KeyeVL1_5ForConditionalGeneration:
-    model = object.__new__(KeyeVL1_5ForConditionalGeneration)
-    model.config = config
-    return model
-
-
-def make_mm_feature(
-    *,
-    modality: str,
-    offset: int,
-    length: int,
-    grid_thw: tuple[int, int, int] | list[tuple[int, int, int]],
-    is_embed: list[bool] | None = None,
-) -> MultiModalFeatureSpec:
-    field_name = "image_grid_thw" if modality == "image" else "video_grid_thw"
-    return MultiModalFeatureSpec(
-        data=MultiModalKwargsItem(
-            {
-                field_name: MultiModalFieldElem(
-                    data=torch.tensor(grid_thw),
-                    field=None,  # HACK.
-                ),
-            }
-        ),
-        modality=modality,
-        identifier="DUMMY",
-        mm_position=PlaceholderRange(
-            offset=offset,
-            length=length,
-            is_embed=None if is_embed is None else torch.tensor(is_embed),
-        ),
-    )
-
-
-def test_get_mrope_input_positions_text_only():
-    model = make_model(DummyConfig())
-
-    positions, delta = model.get_mrope_input_positions(
-        input_tokens=[11, 12, 13, 14, 15],
-        mm_features=[],
-    )
-
-    expected = torch.tensor(
-        [
-            [0, 1, 2, 3, 4],
-            [0, 1, 2, 3, 4],
-            [0, 1, 2, 3, 4],
-        ]
-    )
-
-    assert torch.equal(positions, expected)
-    assert delta == 0
-
-
-def test_get_mrope_input_positions_single_image():
-    model = make_model(DummyConfig())
-    mm_features = [
-        make_mm_feature(
-            modality="image",
-            offset=1,
-            length=4,
-            grid_thw=(1, 4, 4),
-        )
-    ]
-
-    positions, delta = model.get_mrope_input_positions(
-        input_tokens=[10, 20, 21, 22, 23, 30, 31],
-        mm_features=mm_features,
-    )
-
-    expected = torch.tensor(
-        [
-            [0, 1, 1, 1, 1, 3, 4],
-            [0, 1, 1, 2, 2, 3, 4],
-            [0, 1, 2, 1, 2, 3, 4],
-        ]
-    )
-
-    assert torch.equal(positions, expected)
-    assert delta == -2
-
-
-def test_get_mrope_input_positions_video_uses_embed_ranges():
-    model = make_model(DummyConfig())
-    mm_features = [
-        make_mm_feature(
-            modality="video",
-            offset=1,
-            length=8,
-            grid_thw=[(2, 4, 2)],
-            is_embed=[False, False, True, True, False, False, True, True],
-        )
-    ]
-
-    positions, delta = model.get_mrope_input_positions(
-        input_tokens=[10, 101, 102, 20, 21, 103, 104, 30, 31, 40, 41],
-        mm_features=mm_features,
-    )
-
-    expected = torch.tensor(
-        [
-            [0, 1, 2, 3, 3, 5, 6, 7, 7, 9, 10],
-            [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
-            [0, 1, 2, 3, 3, 5, 6, 7, 7, 9, 10],
-        ]
-    )
-
-    assert torch.equal(positions, expected)
-    assert delta == 0

tests/model_executor/test_paddleocr_vl_mrope.py

@@ -1,210 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-from dataclasses import dataclass, field
-
-import pytest
-import torch
-
-from vllm.model_executor.models.paddleocr_vl import (
-    PaddleOCRVLForConditionalGeneration,
-)
-from vllm.multimodal.inputs import (
-    MultiModalFeatureSpec,
-    MultiModalFieldElem,
-    MultiModalKwargsItem,
-    PlaceholderRange,
-)
-
-pytestmark = pytest.mark.skip_global_cleanup
-
-
-@pytest.fixture(autouse=True, scope="module")
-def _force_cpu_default_device():
-    original = torch.get_default_device()
-    torch.set_default_device("cpu")
-    yield
-    torch.set_default_device(original)
-
-
-@dataclass
-class DummyVisionConfig:
-    spatial_merge_size: int = 2
-    patch_size: int = 14
-
-
-@dataclass
-class DummyConfig:
-    image_token_id: int = 151655
-    video_token_id: int = 151654
-    vision_start_token_id: int = 151652
-    vision_end_token_id: int = 151653
-    vision_config: DummyVisionConfig = field(default_factory=DummyVisionConfig)
-
-
-def make_model(config: DummyConfig) -> PaddleOCRVLForConditionalGeneration:
-    model = object.__new__(PaddleOCRVLForConditionalGeneration)
-    model.config = config
-    return model
-
-
-def make_mm_feature(
-    *,
-    offset: int,
-    length: int,
-    image_grid_thw: tuple[int, int, int],
-) -> MultiModalFeatureSpec:
-    return MultiModalFeatureSpec(
-        data=MultiModalKwargsItem(
-            {
-                "image_grid_thw": MultiModalFieldElem(
-                    data=torch.tensor(image_grid_thw),
-                    field=None,
-                ),
-            }
-        ),
-        modality="image",
-        identifier="DUMMY",
-        mm_position=PlaceholderRange(offset=offset, length=length),
-    )
-
-
-def test_get_mrope_input_positions_text_only():
-    model = make_model(DummyConfig())
-    input_tokens = [11, 12, 13, 14, 15]
-    positions, delta = model.get_mrope_input_positions(
-        input_tokens=input_tokens,
-        mm_features=[],
-    )
-    expected = torch.tensor(
-        [
-            [0, 1, 2, 3, 4],
-            [0, 1, 2, 3, 4],
-            [0, 1, 2, 3, 4],
-        ]
-    )
-    assert torch.equal(positions, expected)
-    assert delta == 0
-
-
-def test_get_mrope_input_positions_single_image():
-    model = make_model(DummyConfig())
-    spatial_merge_size = model.config.vision_config.spatial_merge_size
-
-    t, h, w = 1, 2, 2
-    num_image_tokens = t * h * w
-
-    input_tokens = (
-        [10]
-        + [model.config.vision_start_token_id]
-        + [model.config.image_token_id] * num_image_tokens
-        + [model.config.vision_end_token_id]
-        + [30, 31]
-    )
-
-    mm_features = [
-        make_mm_feature(
-            offset=2,  # 1 (text) + 1 (vision_start)
-            length=num_image_tokens,
-            image_grid_thw=(t, h * spatial_merge_size, w * spatial_merge_size),
-        )
-    ]
-
-    positions, delta = model.get_mrope_input_positions(
-        input_tokens=input_tokens,
-        mm_features=mm_features,
-    )
-
-    expected = torch.tensor(
-        [
-            [0, 1, 2, 2, 2, 2, 4, 5, 6],
-            [0, 1, 2, 2, 3, 3, 4, 5, 6],
-            [0, 1, 2, 3, 2, 3, 4, 5, 6],
-        ]
-    )
-
-    assert torch.equal(positions, expected)
-    expected_delta = (positions.max().item() + 1) - len(input_tokens)
-    assert delta == expected_delta
-
-
-def test_get_mrope_input_positions_multiple_images():
-    model = make_model(DummyConfig())
-    spatial_merge_size = model.config.vision_config.spatial_merge_size
-
-    t1, h1, w1 = 1, 2, 2
-    num1 = t1 * h1 * w1
-
-    t2, h2, w2 = 1, 1, 3
-    num2 = t2 * h2 * w2
-
-    input_tokens = (
-        [10]
-        + [model.config.vision_start_token_id]
-        + [model.config.image_token_id] * num1
-        + [model.config.vision_end_token_id]
-        + [20, 21]
-        + [model.config.vision_start_token_id]
-        + [model.config.image_token_id] * num2
-        + [model.config.vision_end_token_id]
-        + [30]
-    )
-
-    mm_features = [
-        make_mm_feature(
-            offset=2,
-            length=num1,
-            image_grid_thw=(t1, h1 * spatial_merge_size, w1 * spatial_merge_size),
-        ),
-        make_mm_feature(
-            offset=2 + num1 + 1 + 2 + 1,
-            length=num2,
-            image_grid_thw=(t2, h2 * spatial_merge_size, w2 * spatial_merge_size),
-        ),
-    ]
-
-    positions, delta = model.get_mrope_input_positions(
-        input_tokens=input_tokens,
-        mm_features=mm_features,
-    )
-
-    assert positions.shape == (3, 15)
-    assert not torch.equal(positions[:, 2:6], torch.arange(4).expand(3, 4) + 2)
-    assert not torch.equal(positions[:, 10:13], torch.arange(3).expand(3, 3) + 10)
-
-
-def test_get_mrope_input_positions_image_at_start():
-    model = make_model(DummyConfig())
-    spatial_merge_size = model.config.vision_config.spatial_merge_size
-
-    t, h, w = 1, 2, 2
-    num_tokens = t * h * w
-
-    input_tokens = (
-        [model.config.vision_start_token_id]
-        + [model.config.image_token_id] * num_tokens
-        + [model.config.vision_end_token_id]
-        + [10, 11]
-    )
-
-    mm_features = [
-        make_mm_feature(
-            offset=1,  # start token at index 0
-            length=num_tokens,
-            image_grid_thw=(t, h * spatial_merge_size, w * spatial_merge_size),
-        )
-    ]
-
-    positions, delta = model.get_mrope_input_positions(
-        input_tokens=input_tokens,
-        mm_features=mm_features,
-    )
-
-    expected = torch.tensor(
-        [
-            [0, 1, 1, 1, 1, 3, 4, 5],
-            [0, 1, 1, 2, 2, 3, 4, 5],
-            [0, 1, 2, 1, 2, 3, 4, 5],
-        ]
-    )
-
-    assert torch.equal(positions, expected)

tests/model_executor/test_weight_utils.py

@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 
+import os
 import tempfile
 
 import huggingface_hub.constants
@@ -9,10 +10,26 @@ from huggingface_hub.utils import LocalEntryNotFoundError
 
 from vllm.model_executor.model_loader.weight_utils import (
     download_weights_from_hf,
+    enable_hf_transfer,
     maybe_remap_kv_scale_name,
 )
 
 
+def test_hf_transfer_auto_activation():
+    if "HF_HUB_ENABLE_HF_TRANSFER" in os.environ:
+        # in case it is already set, we can't test the auto activation
+        pytest.skip("HF_HUB_ENABLE_HF_TRANSFER is set, can't test auto activation")
+    enable_hf_transfer()
+    try:
+        # enable hf hub transfer if available
+        import hf_transfer  # type: ignore # noqa
+
+        HF_TRANSFER_ACTIVE = True
+    except ImportError:
+        HF_TRANSFER_ACTIVE = False
+    assert huggingface_hub.constants.HF_HUB_ENABLE_HF_TRANSFER == HF_TRANSFER_ACTIVE
+
+
 def test_download_weights_from_hf():
     with tempfile.TemporaryDirectory() as tmpdir:
         # assert LocalEntryNotFoundError error is thrown
@@ -161,4 +178,5 @@ class TestMaybeRemapKvScaleName:
 
 
 if __name__ == "__main__":
+    test_hf_transfer_auto_activation()
     test_download_weights_from_hf()

tests/models/language/generation/test_common.py

@@ -100,7 +100,7 @@ AITER_MODEL_LIST = [
         pytest.param("bigcode/starcoder2-3b"),  # starcoder2
         pytest.param(
             "TitanML/tiny-mixtral",  # mixtral
-            marks=[pytest.mark.core_model],
+            marks=[pytest.mark.core_model, pytest.mark.cpu_model],
         ),
         pytest.param("swiss-ai/Apertus-8B-Instruct-2509"),  # apertus
         pytest.param(
@@ -143,11 +143,6 @@ def test_models(
         # in parts of the operators
         pytest.skip(f"Skipping '{model}' model test with AITER kernel.")
 
-    if current_platform.is_cpu() and model in ("openai-community/gpt2",):
-        # These models are sensitive to the rounding error
-        # Fuse ops to reduce rounding
-        monkeypatch.setenv("VLLM_CPU_CI_ENV", "0")
-
     with hf_runner(model) as hf_model:
         hf_outputs = hf_model.generate_greedy_logprobs_limit(
             example_prompts, max_tokens, num_logprobs

tests/models/language/generation/test_granite.py

@@ -15,7 +15,6 @@ MODELS = [
 @pytest.mark.parametrize("dtype", ["bfloat16"])
 @pytest.mark.parametrize("max_tokens", [64])
 @pytest.mark.parametrize("num_logprobs", [5])
-@pytest.mark.cpu_model
 def test_models(
     hf_runner,
     vllm_runner,

tests/models/language/pooling_mteb_test/mteb_embed_utils.py

@@ -74,25 +74,10 @@ class MtebEmbedMixin(mteb.EncoderProtocol):
         return sim
 
 
-class HfMtebEncoder(MtebEmbedMixin):
-    def __init__(self, model):
-        self.model = model
-
-    def encode(
-        self,
-        inputs: DataLoader[mteb.types.BatchedInput],
-        *args,
-        **kwargs,
-    ) -> np.ndarray:
-        sentences = [text for batch in inputs for text in batch["text"]]
-        return self.model.encode(sentences)
-
-
 class VllmMtebEncoder(MtebEmbedMixin):
-    def __init__(self, vllm_model, prompt_prefix: str | None = None):
+    def __init__(self, vllm_model):
         self.llm = vllm_model
         self.rng = np.random.default_rng(seed=42)
-        self.prompt_prefix = prompt_prefix
 
     def encode(
         self,
@@ -102,11 +87,7 @@ class VllmMtebEncoder(MtebEmbedMixin):
     ) -> np.ndarray:
         # Hoping to discover potential scheduling
         # issues by randomizing the order.
-        sentences = [
-            self.prompt_prefix + text if self.prompt_prefix else text
-            for batch in inputs
-            for text in batch["text"]
-        ]
+        sentences = [text for batch in inputs for text in batch["text"]]
         r = self.rng.permutation(len(sentences))
         sentences = [sentences[i] for i in r]
         outputs = self.llm.embed(sentences, use_tqdm=False)
@@ -162,7 +143,6 @@ def mteb_test_embed_models(
     vllm_extra_kwargs=None,
     hf_model_callback=None,
     atol=MTEB_EMBED_TOL,
-    prompt_prefix: str | None = None,
 ):
     vllm_extra_kwargs = get_vllm_extra_kwargs(model_info, vllm_extra_kwargs)
 
@@ -202,7 +182,7 @@ def mteb_test_embed_models(
             )
 
         vllm_main_score = run_mteb_embed_task(
-            VllmMtebEncoder(vllm_model, prompt_prefix=prompt_prefix), MTEB_EMBED_TASKS
+            VllmMtebEncoder(vllm_model), MTEB_EMBED_TASKS
         )
         vllm_dtype = vllm_model.llm.llm_engine.model_config.dtype
         head_dtype = model_config.head_dtype
@@ -230,9 +210,7 @@ def mteb_test_embed_models(
             if hf_model_callback is not None:
                 hf_model_callback(hf_model)
 
-            st_main_score = run_mteb_embed_task(
-                HfMtebEncoder(hf_model), MTEB_EMBED_TASKS
-            )
+            st_main_score = run_mteb_embed_task(hf_model, MTEB_EMBED_TASKS)
             st_dtype = next(hf_model.model.parameters()).dtype
 
             # Check embeddings close to hf outputs

tests/models/language/pooling_mteb_test/test_baai.py

@@ -69,10 +69,7 @@ MODELS = [
         attn_type="decoder",
         is_prefix_caching_supported=True,
         is_chunked_prefill_supported=True,
-        # Skip: model's custom tokenizer on HF hub is incompatible with
-        # transformers v5 (sets attrs before super().__init__, triggering
-        # AttributeError on 'verbose' in __getattr__).
-        enable_test=False,
+        enable_test=True,
     ),
 ]
 

tests/models/language/pooling_mteb_test/test_gte.py

@@ -72,8 +72,7 @@ MODELS = [
         attn_type="encoder_only",
         is_prefix_caching_supported=False,
         is_chunked_prefill_supported=False,
-        # Skip: numerical regression with transformers v5.
-        enable_test=False,
+        enable_test=True,
     ),
     ########## ModernBertModel
     EmbedModelInfo(

tests/models/language/pooling_mteb_test/test_jina.py

@@ -28,16 +28,7 @@ EMBEDDING_MODELS = [
         attn_type="encoder_only",
         is_prefix_caching_supported=False,
         is_chunked_prefill_supported=False,
-    ),
-    EmbedModelInfo(
-        "jinaai/jina-embeddings-v5-text-small",
-        mteb_score=0.794535707854956,
-        architecture="JinaEmbeddingsV5Model",
-        seq_pooling_type="LAST",
-        attn_type="decoder",
-        is_prefix_caching_supported=True,
-        is_chunked_prefill_supported=True,
-    ),
+    )
 ]
 
 RERANK_MODELS = [
@@ -55,18 +46,11 @@ RERANK_MODELS = [
 
 @pytest.mark.parametrize("model_info", EMBEDDING_MODELS)
 def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None:
-    task = "retrieval" if "v5" in model_info.name else "text-matching"
-    prompt_prefix: str | None = "Document: " if "v5" in model_info.name else None
-
     def hf_model_callback(model):
-        model.encode = partial(model.encode, task=task)
+        model.encode = partial(model.encode, task="text-matching")
 
     mteb_test_embed_models(
-        hf_runner,
-        vllm_runner,
-        model_info,
-        hf_model_callback=hf_model_callback,
-        prompt_prefix=prompt_prefix,
+        hf_runner, vllm_runner, model_info, hf_model_callback=hf_model_callback
     )
 
 
@@ -74,10 +58,8 @@ def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -
 def test_embed_models_correctness(
     hf_runner, vllm_runner, model_info: EmbedModelInfo, example_prompts
 ) -> None:
-    task = "retrieval" if "v5" in model_info.name else "text-matching"
-
     def hf_model_callback(model):
-        model.encode = partial(model.encode, task=task)
+        model.encode = partial(model.encode, task="text-matching")
 
     correctness_test_embed_models(
         hf_runner,
@@ -93,10 +75,6 @@ def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None:
     mteb_test_rerank_models(vllm_runner, model_info)
 
 
-@pytest.mark.skip(
-    reason="jinaai/jina-embeddings-v3 custom XLMRobertaLoRA model on HF hub "
-    "is incompatible with transformers v5 (missing all_tied_weights_keys)"
-)
 @pytest.mark.parametrize("model_info", EMBEDDING_MODELS)
 @pytest.mark.parametrize("dtype", ["half"])
 @pytest.mark.parametrize("dimensions", [16, 32])
@@ -115,14 +93,12 @@ def test_matryoshka(
     # ST will strip the input texts, see test_embedding.py
     example_prompts = [str(s).strip() for s in example_prompts]
 
-    task = "retrieval" if "v5" in model_info.name else "text-matching"
-
     with hf_runner(
         model_info.name,
         dtype=dtype,
         is_sentence_transformer=True,
     ) as hf_model:
-        hf_outputs = hf_model.encode(example_prompts, task=task)
+        hf_outputs = hf_model.encode(example_prompts, task="text-matching")
         hf_outputs = matryoshka_fy(hf_outputs, dimensions)
 
     with vllm_runner(

tests/models/multimodal/generation/test_common.py

@@ -186,14 +186,7 @@ VLM_TEST_SETTINGS = {
         max_num_seqs=2,
         auto_cls=AutoModel,
         hf_output_post_proc=model_utils.ultravox_trunc_hf_output,
-        marks=[
-            pytest.mark.core_model,
-            pytest.mark.cpu_model,
-            # TODO: Remove skip once model has been upstreamed to Transformers
-            pytest.mark.skip(
-                reason="Custom model code is not compatible with Transformers v5"
-            ),
-        ],
+        marks=[pytest.mark.core_model, pytest.mark.cpu_model],
     ),
     #### Transformers fallback to test
     ## To reduce test burden, we only test batching arbitrary image size
@@ -404,14 +397,14 @@ VLM_TEST_SETTINGS = {
     "gemma4": VLMTestInfo(
         models=["google/gemma-4-E2B-it"],
         test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
-        prompt_formatter=lambda img_prompt: f"<bos><|turn>user\n{img_prompt}<turn|>\n<|turn>model\n",  # noqa: E501
+        prompt_formatter=lambda img_prompt: f"<bos><start_of_turn>user\n{img_prompt}<end_of_turn>\n<start_of_turn>model\n",  # noqa: E501
         single_image_prompts=IMAGE_ASSETS.prompts(
             {
-                "stop_sign": "<|image|>What's the content in the center of the image?",  # noqa: E501
-                "cherry_blossom": "<|image|>What is the season?",
+                "stop_sign": "What's the content in the center of the image?",
+                "cherry_blossom": "What is the season?",
             }
         ),
-        multi_image_prompt="<|image|><|image|>Describe the two images in detail.",  # noqa: E501
+        multi_image_prompt="Describe the two images in detail.",
         max_model_len=4096,
         max_num_seqs=2,
         auto_cls=AutoModelForImageTextToText,
@@ -540,12 +533,6 @@ VLM_TEST_SETTINGS = {
         max_model_len=4096,
         use_tokenizer_eos=True,
         patch_hf_runner=model_utils.internvl_patch_hf_runner,
-        # TODO: Remove skip once model has been upstreamed to Transformers
-        marks=[
-            pytest.mark.skip(
-                reason="Custom model code tries to access data from meta-tensor"
-            )
-        ],
     ),
     "intern_vl-video": VLMTestInfo(
         models=[
@@ -558,12 +545,6 @@ VLM_TEST_SETTINGS = {
         use_tokenizer_eos=True,
         patch_hf_runner=model_utils.internvl_patch_hf_runner,
         num_logprobs=10 if current_platform.is_rocm() else 5,
-        # TODO: Remove skip once model has been upstreamed to Transformers
-        marks=[
-            pytest.mark.skip(
-                reason="Custom model code tries to access data from meta-tensor"
-            )
-        ],
     ),
     "intern_vl-hf": VLMTestInfo(
         models=["OpenGVLab/InternVL3-1B-hf"],
@@ -610,8 +591,6 @@ VLM_TEST_SETTINGS = {
         hf_model_kwargs={"device_map": "auto"},
         patch_hf_runner=model_utils.isaac_patch_hf_runner,
         image_size_factors=[(0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
-        # TODO: Remove skip once model has been upstreamed to Transformers
-        marks=[pytest.mark.skip(reason="Custom model imports deleted object")],  # noqa: E501
     ),
     "kimi_vl": VLMTestInfo(
         models=["moonshotai/Kimi-VL-A3B-Instruct"],
@@ -827,12 +806,7 @@ VLM_TEST_SETTINGS = {
             pytest.mark.skipif(
                 Version(TRANSFORMERS_VERSION) == Version("4.57.3"),
                 reason="This model is broken in Transformers v4.57.3",
-            ),
-            pytest.mark.skipif(
-                Version(TRANSFORMERS_VERSION) >= Version("5.0.0"),
-                reason="Model's custom code uses ROPE_INIT_FUNCTIONS"
-                "['default'] which was removed in transformers v5",
-            ),
+            )
         ],
     ),
     "phi3v": VLMTestInfo(
@@ -986,12 +960,6 @@ VLM_TEST_SETTINGS = {
             )
             for inp in custom_inputs.different_patch_input_cases_internvl()
         ],
-        # TODO: Remove skip once model has been upstreamed to Transformers
-        marks=[
-            pytest.mark.skip(
-                reason="Custom model code tries to access data from meta-tensor"
-            )
-        ],
     ),
     "llava_onevision-multiple-images": VLMTestInfo(
         models=["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"],

tests/models/multimodal/generation/test_nemotron_parse.py

@@ -103,10 +103,6 @@ def run_test(
         )
 
 
-@pytest.mark.skip(
-    reason="Model's custom MBart decoder has head count mismatch with "
-    "transformers v5's GQA-aware cross-attention (8 vs 16 heads)"
-)
 @pytest.mark.parametrize("model", ["nvidia/NVIDIA-Nemotron-Parse-v1.1"])
 @pytest.mark.parametrize("dtype", ["bfloat16"])
 @pytest.mark.parametrize("num_logprobs", [5])

tests/models/multimodal/generation/test_phi4siglip.py

@@ -2,11 +2,9 @@
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 
 from collections.abc import Sequence
-from importlib.metadata import version
 
 import pytest
 import regex as re
-from packaging.version import Version
 from transformers import AutoModelForCausalLM, AutoTokenizer
 
 from vllm.logprobs import SampleLogprobs
@@ -21,15 +19,6 @@ from ....conftest import (
 from ....utils import multi_gpu_test
 from ...utils import check_logprobs_close
 
-pytestmark = pytest.mark.skipif(
-    Version("5.0") <= Version(version("transformers")),
-    reason=(
-        "vllm upgraded transformers above v5.4 where HF model custom code uses siglip2 "
-        "internals (filter_out_non_signature_kwargs) removed by "
-        "huggingface/transformers#43514"
-    ),
-)
-
 MODEL_ID = "microsoft/Phi-4-reasoning-vision-15B"
 
 HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(

tests/models/multimodal/generation/test_voxtral.py

@@ -149,10 +149,6 @@ def test_online_serving(vllm_runner, audio_assets: AudioTestAssets):
     )
 
 
-@pytest.mark.skip(
-    reason="VoxtralProcessor.apply_chat_template() in transformers v5 "
-    "doesn't resolve chat_template=None to the default template"
-)
 def test_hf_reference(hf_runner, vllm_runner, audio_assets: AudioTestAssets):
     """Compare vLLM Mistral-format output against HF Transformers reference.
 

tests/models/multimodal/generation/vlm_utils/core.py

@@ -80,11 +80,6 @@ def run_test(
     if vllm_runner_kwargs:
         vllm_runner_kwargs_.update(vllm_runner_kwargs)
 
-    # Avoid passing limit_mm_per_prompt twice when vllm_runner_kwargs
-    # already contains it (e.g. gemma4 sets it via vllm_runner_kwargs).
-    if "limit_mm_per_prompt" in vllm_runner_kwargs_:
-        limit_mm_per_prompt = vllm_runner_kwargs_.pop("limit_mm_per_prompt")
-
     with vllm_runner(
         model,
         max_model_len=max_model_len,

tests/models/multimodal/pooling/test_colqwen3.py

@@ -22,11 +22,6 @@ from vllm.entrypoints.pooling.scoring.typing import ScoreMultiModalParam
 
 from ....conftest import VllmRunner
 
-pytestmark = pytest.mark.skip(
-    reason="ColQwen3 model's weight tying is incompatible with "
-    "transformers v5 (missing all_tied_weights_keys)"
-)
-
 MODELS = [
     "TomoroAI/tomoro-colqwen3-embed-4b",
     "OpenSearch-AI/Ops-Colqwen3-4B",

tests/models/multimodal/pooling/test_intern_vit.py

@@ -12,11 +12,6 @@ from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
 
 from ....conftest import ImageTestAssets
 
-pytestmark = pytest.mark.skip(
-    reason="InternVisionModel's custom code is incompatible with "
-    "transformers v5 (missing all_tied_weights_keys)"
-)
-
 # we use snapshot_download to prevent conflicts between
 # dynamic_module and trust_remote_code for hf_runner
 DOWNLOAD_PATTERN = ["*.json", "*.py", "*.safetensors", "*.txt", "*.model"]

tests/models/multimodal/pooling/test_jinavl_reranker.py

@@ -15,11 +15,6 @@ from vllm.entrypoints.pooling.scoring.typing import ScoreMultiModalParam
 
 from ....conftest import HfRunner, VllmRunner
 
-pytestmark = pytest.mark.skip(
-    reason="jinaai/jina-reranker-m0 custom code is incompatible with "
-    "transformers v5 (missing all_tied_weights_keys)"
-)
-
 MODELS = ["jinaai/jina-reranker-m0"]
 
 MM_PROCESSOR_KWARGS = {

tests/models/multimodal/processing/test_musicflamingo.py

@@ -17,13 +17,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from importlib.metadata import version
 from unittest.mock import MagicMock
 
 import numpy as np
 import pytest
 import torch
-from packaging.version import Version
 from transformers import PretrainedConfig
 
 from tests.models.registry import HF_EXAMPLE_MODELS
@@ -124,11 +122,6 @@ def test_musicflamingo_dummy_text_uses_plain_audio_tokens(mock_ctx):
     assert builder.get_dummy_text({"audio": 2}) == "<sound><sound>"
 
 
-@pytest.mark.skipif(
-    Version(version("transformers")) >= Version("5.5"),
-    reason="transformers v5.5 added native MusicFlamingoForConditionalGeneration "
-    "with a different get_audio_features signature (requires input_ids)",
-)
 def test_musicflamingo_audio_feature_pipeline_matches_hf_small_config():
     from transformers.models.musicflamingo import (
         modeling_musicflamingo as hf_musicflamingo_modeling,

tests/models/registry.py

@@ -335,15 +335,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
         "internlm/internlm2-chat-7b", trust_remote_code=True
     ),
     "InternLM2VEForCausalLM": _HfExamplesInfo(
-        "OpenGVLab/Mono-InternVL-2B",
-        trust_remote_code=True,
-        max_transformers_version="4.57",
-        transformers_version_reason={
-            "vllm": (
-                "Custom config cannot be loaded with Transformers "
-                "v5 because `vision_config` is not always set"
-            )
-        },
+        "OpenGVLab/Mono-InternVL-2B", trust_remote_code=True
     ),
     "InternLM3ForCausalLM": _HfExamplesInfo(
         "internlm/internlm3-8b-instruct", trust_remote_code=True
@@ -483,13 +475,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
     "Plamo2ForCausalLM": _HfExamplesInfo(
         "pfnet/plamo-2-1b",
         trust_remote_code=True,
-        max_transformers_version="4.57",
-        transformers_version_reason={
-            "hf": (
-                "Custom model code uses `_tied_weight_keys: list[str]` but "
-                "Transformers v5 now expects `_tied_weight_keys: dict[str, str]`"
-            )
-        },
     ),
     "Plamo3ForCausalLM": _HfExamplesInfo(
         "pfnet/plamo-3-nict-2b-base",
@@ -530,13 +515,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
         trust_remote_code=True,
         max_model_len=4096,
         is_available_online=True,
-        max_transformers_version="5.3",
-        transformers_version_reason={
-            "vllm": (
-                "vllm upgraded transformers above v5.4 where "
-                "validate_rope() no longer accepts ignore_keys param"
-            )
-        },
     ),
     "SeedOssForCausalLM": _HfExamplesInfo(
         "ByteDance-Seed/Seed-OSS-36B-Instruct",
@@ -575,11 +553,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
         "xverse/XVERSE-7B-Chat",
         tokenizer="meta-llama/Llama-2-7b",
         trust_remote_code=True,
-        max_transformers_version="4.57",
-        transformers_version_reason={
-            "vllm": "XVERSE tokenizer is incompatible with transformers v5 "
-            "(add_prefix_space / prepend_scheme mismatch).",
-        },
     ),
     "Zamba2ForCausalLM": _HfExamplesInfo("Zyphra/Zamba2-7B-instruct"),
     "MiMoForCausalLM": _HfExamplesInfo("XiaomiMiMo/MiMo-7B-RL", trust_remote_code=True),
@@ -609,10 +582,6 @@ _EMBEDDING_EXAMPLE_MODELS = {
         trust_remote_code=True,
         hf_overrides={"architectures": ["GteNewModel"]},
     ),
-    "JinaEmbeddingsV5Model": _HfExamplesInfo(
-        "jinaai/jina-embeddings-v5-text-small",
-        trust_remote_code=True,
-    ),
     "LlamaModel": _HfExamplesInfo("llama", is_available_online=False),
     "LlamaBidirectionalModel": _HfExamplesInfo(
         "nvidia/llama-nemotron-embed-1b-v2", trust_remote_code=True
@@ -794,18 +763,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
     # [Decoder-only]
     "AriaForConditionalGeneration": _HfExamplesInfo("rhymes-ai/Aria"),
     "AudioFlamingo3ForConditionalGeneration": _HfExamplesInfo(
-        "nvidia/audio-flamingo-3-hf",
-        min_transformers_version="5.3.0",
-        transformers_version_reason={
-            "vllm": "Needs https://github.com/huggingface/transformers/pull/43538"
-        },
+        "nvidia/audio-flamingo-3-hf", min_transformers_version="5.0.0"
     ),
     "MusicFlamingoForConditionalGeneration": _HfExamplesInfo(
-        "nvidia/music-flamingo-2601-hf",
-        min_transformers_version="5.3.0",
-        transformers_version_reason={
-            "vllm": "Needs https://github.com/huggingface/transformers/pull/43538"
-        },
+        "nvidia/music-flamingo-2601-hf", min_transformers_version="5.3.0"
     ),
     "AyaVisionForConditionalGeneration": _HfExamplesInfo("CohereLabs/aya-vision-8b"),
     "BagelForConditionalGeneration": _HfExamplesInfo("ByteDance-Seed/BAGEL-7B-MoT"),
@@ -860,30 +821,12 @@ _MULTIMODAL_EXAMPLE_MODELS = {
     ),
     "FireRedASR2ForConditionalGeneration": _HfExamplesInfo(
         "allendou/FireRedASR2-LLM-vllm",
-        trust_remote_code=True,
-        max_transformers_version="5.1",
-        transformers_version_reason={
-            "vllm": "Incompatible with transformers v5.2+ "
-            "(dict object has no attribute '__name__').",
-        },
     ),
     "FireRedLIDForConditionalGeneration": _HfExamplesInfo(
         "PatchyTisa/FireRedLID-vllm",
-        trust_remote_code=True,
-        max_transformers_version="5.1",
-        transformers_version_reason={
-            "vllm": "Incompatible with transformers v5.2+ "
-            "(dict object has no attribute '__name__').",
-        },
     ),
     "FunASRForConditionalGeneration": _HfExamplesInfo(
         "allendou/Fun-ASR-Nano-2512-vllm",
-        trust_remote_code=True,
-        max_transformers_version="5.1",
-        transformers_version_reason={
-            "vllm": "Incompatible with transformers v5.2+ "
-            "(dict object has no attribute '__name__').",
-        },
     ),
     "FunAudioChatForConditionalGeneration": _HfExamplesInfo(
         "funaudiochat", is_available_online=False
@@ -925,13 +868,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
     "HCXVisionForCausalLM": _HfExamplesInfo(
         "naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B",
         trust_remote_code=True,
-        max_transformers_version="4.57",
-        transformers_version_reason={
-            "vllm": (
-                "Custom config cannot be loaded with Transformers "
-                "v5 because `text_config` is not always set"
-            )
-        },
     ),
     "HCXVisionV2ForCausalLM": _HfExamplesInfo(
         "naver-hyperclovax/HyperCLOVAX-SEED-Think-32B",
@@ -951,12 +887,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {
         extras={"0.2-2B-Preview": "PerceptronAI/Isaac-0.2-2B-Preview"},
     ),
     "InternS1ForConditionalGeneration": _HfExamplesInfo(
-        "internlm/Intern-S1",
-        trust_remote_code=True,
-        max_transformers_version="4.57",
-        transformers_version_reason={
-            "vllm": "Custom tokenizer code is not compatible with Transformers v5."
-        },
+        "internlm/Intern-S1", trust_remote_code=True
     ),
     "InternS1ProForConditionalGeneration": _HfExamplesInfo(
         "internlm/Intern-S1-Pro",
@@ -1045,14 +976,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {
     "MiDashengLMModel": _HfExamplesInfo(
         "mispeech/midashenglm-7b", trust_remote_code=True
     ),
-    "MiniCPMO": _HfExamplesInfo(
-        "openbmb/MiniCPM-o-2_6",
-        trust_remote_code=True,
-        max_transformers_version="4.57",
-        transformers_version_reason={
-            "hf": "Custom processor code is not compatible with Transformers v5."
-        },
-    ),
+    "MiniCPMO": _HfExamplesInfo("openbmb/MiniCPM-o-2_6", trust_remote_code=True),
     "MiniCPMV": _HfExamplesInfo(
         "openbmb/MiniCPM-Llama3-V-2_5",
         extras={
@@ -1060,13 +984,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
             "4.0": "openbmb/MiniCPM-V-4",
             "4.5": "openbmb/MiniCPM-V-4_5",
         },
-        max_transformers_version="4.57",
-        transformers_version_reason={
-            "vllm": (
-                "MiniCPMVBatchFeature is incompatible with its base class in "
-                "Transformers v5. See https://huggingface.co/openbmb/MiniCPM-Llama3-V-2_5/discussions/78"
-            )
-        },
         trust_remote_code=True,
     ),
     "MiniMaxVL01ForConditionalGeneration": _HfExamplesInfo(
@@ -1121,70 +1038,14 @@ _MULTIMODAL_EXAMPLE_MODELS = {
         },
         trust_remote_code=True,
     ),
-    # NemotronH_Nano_Omni_Reasoning_V3 is an alias for NemotronH_Nano_VL_V2
-    # Use the same registry test as NemotronH_Nano_VL_V2 above
-    "NemotronH_Nano_Omni_Reasoning_V3": _HfExamplesInfo(
-        "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16",
-        max_model_len=4096,
-        use_original_num_layers=True,
-        hf_overrides={
-            "vision_config": PretrainedConfig(
-                args={
-                    "min_num_patches": 1,
-                    "max_num_patches": 12,
-                    "model": "vit_huge_patch16_224",
-                },
-                video_temporal_patch_size=2,
-            ),
-            "text_config": {
-                "num_hidden_layers": 2,
-                "hybrid_override_pattern": "M*",
-            },
-        },
-        trust_remote_code=True,
-    ),
-    # NemotronH_Super_Omni_Reasoning_V3 is an alias for NemotronH_Nano_VL_V2 as well
-    # Use the same registry test as NemotronH_Nano_VL_V2 above
-    "NemotronH_Super_Omni_Reasoning_V3": _HfExamplesInfo(
-        "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16",
-        max_model_len=4096,
-        use_original_num_layers=True,
-        hf_overrides={
-            "vision_config": PretrainedConfig(
-                args={
-                    "min_num_patches": 1,
-                    "max_num_patches": 12,
-                    "model": "vit_huge_patch16_224",
-                },
-                video_temporal_patch_size=2,
-            ),
-            "text_config": {
-                "num_hidden_layers": 2,
-                "hybrid_override_pattern": "M*",
-            },
-        },
-        trust_remote_code=True,
-    ),
     "OpenCUAForConditionalGeneration": _HfExamplesInfo(
-        "xlangai/OpenCUA-7B",
-        trust_remote_code=True,
-        max_transformers_version="4.57",
-        transformers_version_reason={
-            "vllm": "Tokenizer cannot be initialised in Transformers v5."
-        },
+        "xlangai/OpenCUA-7B", trust_remote_code=True
     ),
     "OpenPanguVLForConditionalGeneration": _HfExamplesInfo(
         "FreedomIntelligence/openPangu-VL-7B",
         trust_remote_code=True,
         max_model_len=4096,
         enforce_eager=True,
-        max_transformers_version="4.57",
-        transformers_version_reason={
-            "vllm": (
-                "OpenPanguVLVideoProcessorInitKwargs does not specify total=False, "
-                "making all kwargs required. See https://huggingface.co/FreedomIntelligence/openPangu-VL-7B/discussions/2"
-            )
-        },
     ),
     "Ovis": _HfExamplesInfo(
         "AIDC-AI/Ovis2-1B",
@@ -1196,24 +1057,12 @@ _MULTIMODAL_EXAMPLE_MODELS = {
             "1.6-gemma": "AIDC-AI/Ovis1.6-Gemma2-9B",
         },
     ),
-    "Ovis2_5": _HfExamplesInfo(
-        "AIDC-AI/Ovis2.5-2B",
-        trust_remote_code=True,
-        max_transformers_version="4.57",
-        transformers_version_reason={
-            "vllm": "Custom processor code is not compatible with Transformers v5."
-        },
-    ),
+    "Ovis2_5": _HfExamplesInfo("AIDC-AI/Ovis2.5-2B", trust_remote_code=True),
     "Ovis2_6ForCausalLM": _HfExamplesInfo(
         "AIDC-AI/Ovis2.6-2B", is_available_online=False, trust_remote_code=True
     ),
     "Ovis2_6_MoeForCausalLM": _HfExamplesInfo(
-        "AIDC-AI/Ovis2.6-30B-A3B",
-        trust_remote_code=True,
-        max_transformers_version="4.57",
-        transformers_version_reason={
-            "vllm": "Custom processor code is not compatible with Transformers v5."
-        },
+        "AIDC-AI/Ovis2.6-30B-A3B", trust_remote_code=True
     ),
     "PaddleOCRVLForConditionalGeneration": _HfExamplesInfo(
         "PaddlePaddle/PaddleOCR-VL",
@@ -1233,17 +1082,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {
         extras={"phi3.5": "microsoft/Phi-3.5-vision-instruct"},
     ),
     "Phi4ForCausalLMV": _HfExamplesInfo(
-        "microsoft/Phi-4-reasoning-vision-15B",
-        trust_remote_code=True,
-        max_transformers_version="5.3",
-        transformers_version_reason={
-            "vllm": (
-                "vllm upgraded transformers above v5.4 where HF model "
-                "custom code uses siglip2 internals "
-                "(filter_out_non_signature_kwargs) removed "
-                "by huggingface/transformers#43514"
-            )
-        },
+        "microsoft/Phi-4-reasoning-vision-15B", trust_remote_code=True
     ),
     "Phi4MMForCausalLM": _HfExamplesInfo(
         "microsoft/Phi-4-multimodal-instruct", trust_remote_code=True
@@ -1340,14 +1179,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
             "architectures": ["Tarsier2ForConditionalGeneration"],
             "model_type": "tarsier2",
         },
-        max_transformers_version="5.3",
-        transformers_version_reason={
-            "vllm": (
-                "Qwen2VLConfig was split into Qwen2VLConfig + "
-                "Qwen2VLTextConfig in transformers v5, breaking "
-                "attribute access (num_attention_heads, hidden_size, etc.)"
-            )
-        },
     ),
     "VoxtralForConditionalGeneration": _HfExamplesInfo(
         "mistralai/Voxtral-Mini-3B-2507",

tests/models/utils.py

@@ -476,16 +476,7 @@ def dummy_hf_overrides(
     else:
         # Use minimal layers for testing
         num_layers = 1
-        num_hidden_layers = (
-            3
-            if model_arch
-            in (
-                "Gemma3nForConditionalGeneration",
-                "Gemma4ForCausalLM",
-                "Gemma4ForConditionalGeneration",
-            )
-            else 1
-        )
+        num_hidden_layers = 3 if model_arch == "Gemma3nForConditionalGeneration" else 1
 
     update_dict = {
         "num_layers": num_layers,

tests/quantization/test_cpu_wna16.py

@@ -12,7 +12,6 @@ MODELS = [
     "TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ",  # with g_idx
     "Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int4",  # without g_idx
     "RedHatAI/Qwen3-1.7B-quantized.w4a16",  # with zp
-    "OPEA/Qwen2.5-0.5B-Instruct-int4-sym-inc",
 ]
 DTYPE = ["bfloat16"]
 

tests/reasoning/test_step3p5_reasoning_parser.py

@@ -2,10 +2,10 @@
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 
 import pytest
+from transformers import AutoTokenizer
 
 from tests.reasoning.utils import run_reasoning_extraction
 from vllm.reasoning import ReasoningParser, ReasoningParserManager
-from vllm.tokenizers import get_tokenizer
 
 parser_name = "step3p5"
 start_token = "<think>"
@@ -16,7 +16,7 @@ REASONING_MODEL_NAME = "stepfun-ai/Step-3.5-Flash"
 
 @pytest.fixture(scope="module")
 def step3p5_tokenizer():
-    return get_tokenizer(tokenizer_name=REASONING_MODEL_NAME)
+    return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME)
 
 
 SIMPLE_REASONING = {

tests/tool_parsers/test_ernie45_moe_tool_parser.py

@@ -328,7 +328,7 @@ def test_extract_tool_calls_streaming_incremental(
     expected_content,
 ):
     """Verify the Ernie45 Parser streaming behavior by verifying each chunk is as expected."""  # noqa: E501
-    request = ChatCompletionRequest(model=MODEL, messages=[])
+    request = ChatCompletionRequest(model=MODEL, messages=[], tools=[])
 
     tool_calls_dict = {}
     for delta_message in stream_delta_message_generator(

tests/tool_parsers/test_xlam_tool_parser.py

@@ -484,7 +484,7 @@ def test_extract_tool_calls_streaming_incremental(
     expected_content,
 ):
     """Verify the XLAM Parser streaming behavior by verifying each chunk is as expected."""  # noqa: E501
-    request = ChatCompletionRequest(model=MODEL, messages=[])
+    request = ChatCompletionRequest(model=MODEL, messages=[], tools=[])
 
     chunks = []
     for delta_message in stream_delta_message_generator(

tests/tool_use/mistral/test_mistral_tool_calls.py

@@ -1,198 +1,25 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 
-import json
-from dataclasses import dataclass, field
-
 import openai
 import pytest
 
-from tests.tool_use.utils import (
-    MESSAGES_ASKING_FOR_PARALLEL_TOOLS,
-    MESSAGES_ASKING_FOR_TOOLS,
-    MESSAGES_WITH_TOOL_RESPONSE,
-    MESSAGES_WITHOUT_TOOLS,
-    SEARCH_TOOL,
-    SEED,
-    WEATHER_TOOL,
-    ensure_system_prompt,
-)
-
-from .utils import ServerConfig
-
-
-def _requires_tool_parser(server_config: ServerConfig) -> None:
-    r"""Skip test if server was not started with --tool-call-parser."""
-    if "--tool-call-parser" not in server_config.get("arguments", []):
-        pytest.skip(
-            f"Skipping: {server_config['model']} not configured with --tool-call-parser"
-        )
-
-
-def _is_pre_v11(server_config: ServerConfig) -> bool:
-    r"""Pre-v11 Mistral models lack grammar-based tool call enforcement."""
-    return "7B" in server_config.get("model", "")
-
-
-@dataclass
-class StreamedToolCallResult:
-    r"""Accumulated result from streaming a single tool call."""
-
-    function_name: str | None = None
-    function_args_str: str = ""
-    tool_call_id: str | None = None
-    role_name: str | None = None
-    finish_reason_count: int = 0
-    finish_reason: str | None = None
-
-
-async def _collect_streamed_tool_call(
-    stream: openai.AsyncStream,
-    *,
-    expected_finish_reason: str = "tool_calls",
-) -> StreamedToolCallResult:
-    result = StreamedToolCallResult()
-
-    async for chunk in stream:
-        if chunk.choices[0].finish_reason:
-            result.finish_reason_count += 1
-            result.finish_reason = chunk.choices[0].finish_reason
-            assert chunk.choices[0].finish_reason == expected_finish_reason
-
-        if chunk.choices[0].delta.role:
-            assert not result.role_name or result.role_name == "assistant"
-            result.role_name = "assistant"
-
-        streamed_tool_calls = chunk.choices[0].delta.tool_calls
-        if streamed_tool_calls and len(streamed_tool_calls) > 0:
-            assert len(streamed_tool_calls) == 1
-            tool_call = streamed_tool_calls[0]
-
-            if tool_call.id:
-                assert not result.tool_call_id
-                result.tool_call_id = tool_call.id
-
-            if tool_call.function:
-                if tool_call.function.name:
-                    assert result.function_name is None
-                    result.function_name = tool_call.function.name
-                if tool_call.function.arguments:
-                    result.function_args_str += tool_call.function.arguments
-
-    return result
-
-
-@dataclass
-class StreamedContentResult:
-    r"""Accumulated result from streaming a content-only response."""
-
-    chunks: list[str] = field(default_factory=list)
-    finish_reason_count: int = 0
-    finish_reason: str | None = None
-    role_sent: bool = False
-
-
-async def _collect_streamed_content(
-    stream: openai.AsyncStream,
-    *,
-    expected_finish_reason: str | None = None,
-    no_tool_calls: bool = True,
-) -> StreamedContentResult:
-    r"""Consume a streaming response and collect text content."""
-    result = StreamedContentResult()
-
-    async for chunk in stream:
-        delta = chunk.choices[0].delta
-
-        if delta.role:
-            assert not result.role_sent
-            assert delta.role == "assistant"
-            result.role_sent = True
-
-        if delta.content:
-            result.chunks.append(delta.content)
-
-        if chunk.choices[0].finish_reason is not None:
-            result.finish_reason_count += 1
-            result.finish_reason = chunk.choices[0].finish_reason
-            if expected_finish_reason is not None:
-                assert result.finish_reason == expected_finish_reason
-
-        if no_tool_calls:
-            assert not delta.tool_calls or len(delta.tool_calls) == 0
-
-    return result
-
-
-@dataclass
-class StreamedParallelToolCallResult:
-    r"""Accumulated result from streaming parallel tool calls."""
-
-    function_names: list[str] = field(default_factory=list)
-    function_args_strs: list[str] = field(default_factory=list)
-    tool_call_ids: list[str] = field(default_factory=list)
-    role_name: str | None = None
-    finish_reason_count: int = 0
-
-
-async def _collect_streamed_parallel_tool_calls(
-    stream: openai.AsyncStream,
-) -> StreamedParallelToolCallResult:
-    r"""Consume a streaming response and collect parallel tool calls."""
-    result = StreamedParallelToolCallResult()
-    tool_call_idx: int = -1
-
-    async for chunk in stream:
-        if chunk.choices[0].finish_reason:
-            result.finish_reason_count += 1
-            assert chunk.choices[0].finish_reason == "tool_calls"
-
-        if chunk.choices[0].delta.role:
-            assert not result.role_name or result.role_name == "assistant"
-            result.role_name = "assistant"
-
-        streamed_tool_calls = chunk.choices[0].delta.tool_calls
-        if streamed_tool_calls and len(streamed_tool_calls) > 0:
-            assert len(streamed_tool_calls) == 1
-            tool_call = streamed_tool_calls[0]
-
-            if tool_call.index != tool_call_idx:
-                tool_call_idx = tool_call.index
-                result.function_args_strs.append("")
-                result.tool_call_ids.append("")
-
-            if tool_call.id:
-                result.tool_call_ids[tool_call.index] = tool_call.id
-
-            if tool_call.function:
-                if tool_call.function.name:
-                    result.function_names.append(tool_call.function.name)
-                if tool_call.function.arguments:
-                    result.function_args_strs[tool_call.index] += (
-                        tool_call.function.arguments
-                    )
-
-    return result
+from tests.tool_use.utils import MESSAGES_ASKING_FOR_TOOLS, WEATHER_TOOL
 
 
 # test: a tool_choice with mistral-tokenizer results in an ID of length 9
 @pytest.mark.asyncio
-async def test_tool_call_with_tool_choice(
-    client: openai.AsyncOpenAI, server_config: ServerConfig
-) -> None:
-    _requires_tool_parser(server_config)
-
+async def test_tool_call_with_tool_choice(client: openai.AsyncOpenAI):
     models = await client.models.list()
     model_name: str = models.data[0].id
     chat_completion = await client.chat.completions.create(
-        messages=ensure_system_prompt(MESSAGES_ASKING_FOR_TOOLS, server_config),
+        messages=MESSAGES_ASKING_FOR_TOOLS,
         temperature=0,
         max_completion_tokens=100,
         model=model_name,
         tools=[WEATHER_TOOL],
         tool_choice=WEATHER_TOOL,
         logprobs=False,
-        seed=SEED,
     )
 
     choice = chat_completion.choices[0]
@@ -201,307 +28,3 @@ async def test_tool_call_with_tool_choice(
     assert choice.message.role == "assistant"
     assert choice.message.tool_calls is None or len(choice.message.tool_calls) == 1
     assert len(choice.message.tool_calls[0].id) == 9  # length of 9 for mistral
-
-
-_NOT_SET = object()
-
-
-@pytest.mark.asyncio
-@pytest.mark.parametrize(
-    "tools, tool_choice, streaming_id_len_pre_v11",
-    [
-        pytest.param(
-            [WEATHER_TOOL, SEARCH_TOOL],
-            _NOT_SET,
-            9,
-            id="auto",
-        ),
-        pytest.param(
-            [WEATHER_TOOL],
-            "required",
-            30,
-            id="required",
-        ),
-    ],
-)
-async def test_tool_call_auto_or_required(
-    client: openai.AsyncOpenAI,
-    server_config: ServerConfig,
-    tools: list,
-    tool_choice: object,
-    streaming_id_len_pre_v11: int,
-) -> None:
-    _requires_tool_parser(server_config)
-
-    models = await client.models.list()
-    model_name: str = models.data[0].id
-
-    create_kwargs: dict = {
-        "messages": ensure_system_prompt(MESSAGES_ASKING_FOR_TOOLS, server_config),
-        "temperature": 0,
-        "max_completion_tokens": 100,
-        "model": model_name,
-        "tools": tools,
-        "logprobs": False,
-        "seed": SEED,
-    }
-    if tool_choice is not _NOT_SET:
-        create_kwargs["tool_choice"] = tool_choice
-
-    # --- non-streaming ---
-    chat_completion = await client.chat.completions.create(**create_kwargs)
-
-    choice = chat_completion.choices[0]
-    tool_calls = choice.message.tool_calls
-
-    assert choice.finish_reason == "tool_calls"
-    assert tool_calls is not None and len(tool_calls) >= 1
-    assert tool_calls[0].function.name == "get_current_weather"
-    parsed_arguments = json.loads(tool_calls[0].function.arguments)
-    assert "city" in parsed_arguments
-    assert len(tool_calls[0].id) == 9
-
-    # --- streaming ---
-    stream = await client.chat.completions.create(**create_kwargs, stream=True)
-
-    result = await _collect_streamed_tool_call(stream)
-
-    assert result.finish_reason_count == 1
-    assert result.role_name == "assistant"
-    assert result.function_name == "get_current_weather"
-    streamed_args = json.loads(result.function_args_str)
-    assert isinstance(result.tool_call_id, str)
-    if _is_pre_v11(server_config):
-        assert len(result.tool_call_id) == streaming_id_len_pre_v11
-    else:
-        assert len(result.tool_call_id) == 9
-    assert parsed_arguments == streamed_args
-
-
-@pytest.mark.asyncio
-async def test_tool_call_none_with_tools(
-    client: openai.AsyncOpenAI, server_config: ServerConfig
-) -> None:
-    _requires_tool_parser(server_config)
-
-    models = await client.models.list()
-    model_name: str = models.data[0].id
-
-    # --- non-streaming ---
-    chat_completion = await client.chat.completions.create(
-        messages=ensure_system_prompt(MESSAGES_ASKING_FOR_TOOLS, server_config),
-        temperature=0,
-        max_completion_tokens=100,
-        model=model_name,
-        tools=[WEATHER_TOOL],
-        tool_choice="none",
-        logprobs=False,
-        seed=SEED,
-    )
-
-    choice = chat_completion.choices[0]
-
-    assert choice.finish_reason != "tool_calls"
-    assert choice.message.tool_calls is None or len(choice.message.tool_calls) == 0
-    assert choice.message.content is not None
-    # Without grammar enforcement, pre-v11 models may still emit [TOOL_CALLS]
-    if not _is_pre_v11(server_config):
-        assert "[TOOL_CALLS]" not in choice.message.content
-
-    non_streaming_content = choice.message.content
-
-    # --- streaming ---
-    stream = await client.chat.completions.create(
-        messages=ensure_system_prompt(MESSAGES_ASKING_FOR_TOOLS, server_config),
-        temperature=0,
-        max_completion_tokens=100,
-        model=model_name,
-        tools=[WEATHER_TOOL],
-        tool_choice="none",
-        logprobs=False,
-        seed=SEED,
-        stream=True,
-    )
-
-    # Pre-v11 models lack grammar enforcement, so the model may still
-    # emit tool calls even with tool_choice="none".
-    pre_v11 = _is_pre_v11(server_config)
-    result = await _collect_streamed_content(stream, no_tool_calls=not pre_v11)
-
-    assert result.finish_reason_count == 1
-    if not pre_v11:
-        assert result.finish_reason != "tool_calls"
-    streamed_content = "".join(result.chunks)
-    if not pre_v11:
-        assert "[TOOL_CALLS]" not in streamed_content
-        assert streamed_content == non_streaming_content
-
-
-@pytest.mark.asyncio
-async def test_chat_without_tools(
-    client: openai.AsyncOpenAI, server_config: ServerConfig
-) -> None:
-    models = await client.models.list()
-    model_name: str = models.data[0].id
-
-    # --- non-streaming ---
-    chat_completion = await client.chat.completions.create(
-        messages=ensure_system_prompt(MESSAGES_WITHOUT_TOOLS, server_config),
-        temperature=0,
-        max_completion_tokens=150,
-        model=model_name,
-        logprobs=False,
-        seed=SEED,
-    )
-
-    choice = chat_completion.choices[0]
-    output_text = choice.message.content
-
-    assert output_text is not None and len(output_text) > 0
-    assert choice.finish_reason != "tool_calls"
-    assert choice.message.tool_calls is None or len(choice.message.tool_calls) == 0
-
-    # --- streaming ---
-    stream = await client.chat.completions.create(
-        messages=ensure_system_prompt(MESSAGES_WITHOUT_TOOLS, server_config),
-        temperature=0,
-        max_completion_tokens=150,
-        model=model_name,
-        logprobs=False,
-        seed=SEED,
-        stream=True,
-    )
-
-    result = await _collect_streamed_content(
-        stream, expected_finish_reason=choice.finish_reason
-    )
-
-    assert result.role_sent
-    assert result.finish_reason_count == 1
-    assert len(result.chunks)
-    assert "".join(result.chunks) == output_text
-
-
-@pytest.mark.asyncio
-async def test_tool_call_with_results(
-    client: openai.AsyncOpenAI, server_config: ServerConfig
-) -> None:
-    _requires_tool_parser(server_config)
-
-    models = await client.models.list()
-    model_name: str = models.data[0].id
-
-    # --- non-streaming ---
-    chat_completion = await client.chat.completions.create(
-        messages=ensure_system_prompt(MESSAGES_WITH_TOOL_RESPONSE, server_config),
-        temperature=0,
-        max_completion_tokens=100,
-        model=model_name,
-        tools=[WEATHER_TOOL, SEARCH_TOOL],
-        logprobs=False,
-        seed=SEED,
-    )
-
-    choice = chat_completion.choices[0]
-
-    assert choice.finish_reason != "tool_calls"
-    assert choice.message.tool_calls is None or len(choice.message.tool_calls) == 0
-    assert choice.message.content is not None
-    assert "98" in choice.message.content
-
-    # --- streaming ---
-    stream = await client.chat.completions.create(
-        messages=ensure_system_prompt(MESSAGES_WITH_TOOL_RESPONSE, server_config),
-        temperature=0,
-        max_completion_tokens=100,
-        model=model_name,
-        tools=[WEATHER_TOOL, SEARCH_TOOL],
-        logprobs=False,
-        seed=SEED,
-        stream=True,
-    )
-
-    result = await _collect_streamed_content(
-        stream, expected_finish_reason=choice.finish_reason
-    )
-
-    assert result.role_sent
-    assert result.finish_reason_count == 1
-    assert len(result.chunks)
-    assert "".join(result.chunks) == choice.message.content
-
-
-def _requires_parallel(server_config: ServerConfig) -> None:
-    r"""Skip test if the model does not support parallel tool calls."""
-    if not server_config.get("supports_parallel"):
-        pytest.skip(
-            f"Skipping: {server_config['model']} does not support parallel tool calls"
-        )
-
-
-@pytest.mark.asyncio
-async def test_tool_call_parallel(
-    client: openai.AsyncOpenAI, server_config: ServerConfig
-) -> None:
-    _requires_tool_parser(server_config)
-    _requires_parallel(server_config)
-
-    models = await client.models.list()
-    model_name: str = models.data[0].id
-
-    # --- non-streaming ---
-    chat_completion = await client.chat.completions.create(
-        messages=ensure_system_prompt(
-            MESSAGES_ASKING_FOR_PARALLEL_TOOLS, server_config
-        ),
-        temperature=0,
-        max_completion_tokens=200,
-        model=model_name,
-        tools=[WEATHER_TOOL],
-        logprobs=False,
-        seed=SEED,
-    )
-
-    choice = chat_completion.choices[0]
-    tool_calls = choice.message.tool_calls
-
-    assert choice.finish_reason == "tool_calls"
-    assert tool_calls is not None and len(tool_calls) >= 2
-    for tc in tool_calls:
-        assert tc.type == "function"
-        assert tc.function.name == "get_current_weather"
-        assert isinstance(tc.function.arguments, str)
-        parsed = json.loads(tc.function.arguments)
-        assert "city" in parsed
-        assert len(tc.id) == 9
-
-    non_streaming_tool_calls = tool_calls
-
-    # --- streaming ---
-    stream = await client.chat.completions.create(
-        messages=ensure_system_prompt(
-            MESSAGES_ASKING_FOR_PARALLEL_TOOLS, server_config
-        ),
-        temperature=0,
-        max_completion_tokens=200,
-        model=model_name,
-        tools=[WEATHER_TOOL],
-        logprobs=False,
-        seed=SEED,
-        stream=True,
-    )
-
-    result = await _collect_streamed_parallel_tool_calls(stream)
-
-    assert result.finish_reason_count == 1
-    assert result.role_name == "assistant"
-    assert len(result.function_names) >= 2
-    assert all(name == "get_current_weather" for name in result.function_names)
-    assert len(result.tool_call_ids) >= 2
-    assert all(isinstance(tid, str) and len(tid) == 9 for tid in result.tool_call_ids)
-
-    for args_str in result.function_args_strs:
-        streamed_args = json.loads(args_str)
-        assert "city" in streamed_args
-
-    assert len(result.function_names) == len(non_streaming_tool_calls)

tests/tool_use/mistral/utils.py

@@ -2,7 +2,16 @@
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 
 
-from tests.tool_use.utils import ServerConfig
+from typing_extensions import TypedDict
+
+
+class ServerConfig(TypedDict, total=False):
+    model: str
+    arguments: list[str]
+    system_prompt: str | None
+    supports_parallel: bool | None
+    supports_rocm: bool | None
+
 
 ARGS: list[str] = ["--max-model-len", "1024"]
 
@@ -12,11 +21,6 @@ CONFIGS: dict[str, ServerConfig] = {
         "arguments": [
             "--tokenizer-mode",
             "mistral",
-            "--tool-call-parser",
-            "mistral",
-            "--enable-auto-tool-choice",
-            "--enforce-eager",
-            "--no-enable-prefix-caching",
             '--ignore-patterns="consolidated.safetensors"',
         ],
         "system_prompt": "You are a helpful assistant with access to tools. If a tool"
@@ -25,22 +29,4 @@ CONFIGS: dict[str, ServerConfig] = {
         "without calling a tool. DO NOT CALL A TOOL THAT IS IRRELEVANT "
         "to the user's question - just respond to it normally.",
     },
-    "ministral-3b": {
-        "model": "mistralai/Ministral-3-3B-Instruct-2512",
-        "arguments": [
-            "--tokenizer-mode",
-            "mistral",
-            "--tool-call-parser",
-            "mistral",
-            "--enable-auto-tool-choice",
-            "--enforce-eager",
-            "--no-enable-prefix-caching",
-        ],
-        "system_prompt": "You are a helpful assistant with access to tools. If a tool"
-        " that you have would be helpful to answer a user query, "
-        "call the tool. Otherwise, answer the user's query directly "
-        "without calling a tool. DO NOT CALL A TOOL THAT IS IRRELEVANT "
-        "to the user's question - just respond to it normally.",
-        "supports_parallel": True,
-    },
 }

tests/tool_use/test_chat_completion_request_validations.py

@@ -26,15 +26,15 @@ def test_chat_completion_request_with_no_tools():
     )
     assert request.tool_choice == "none"
 
-    # tools key present but empty -- should be rejected
-    with pytest.raises(ValueError, match="must not be an empty array"):
-        ChatCompletionRequest.model_validate(
-            {
-                "messages": [{"role": "user", "content": "Hello"}],
-                "model": "facebook/opt-125m",
-                "tools": [],
-            }
-        )
+    # tools key present but empty
+    request = ChatCompletionRequest.model_validate(
+        {
+            "messages": [{"role": "user", "content": "Hello"}],
+            "model": "facebook/opt-125m",
+            "tools": [],
+        }
+    )
+    assert request.tool_choice == "none"
 
 
 @pytest.mark.parametrize("tool_choice", ["auto", "required"])

tests/utils.py

@@ -1261,6 +1261,9 @@ def fork_new_process_for_each_test(func: Callable[_P, None]) -> Callable[_P, Non
 
     @functools.wraps(func)
     def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> None:
+        # Make the process the leader of its own process group
+        # to avoid sending SIGTERM to the parent process
+        os.setpgrp()
         from _pytest.outcomes import Skipped
 
         # Create a unique temporary file to store exception info from child
@@ -1280,9 +1283,6 @@ def fork_new_process_for_each_test(func: Callable[_P, None]) -> Callable[_P, Non
             pid = os.fork()
             print(f"Fork a new process to run a test {pid}")
             if pid == 0:
-                # Make the child process the leader of its own process group
-                # to avoid sending SIGTERM to the parent process
-                os.setpgrp()
                 # Parent process responsible for deleting, don't delete
                 # in child.
                 delete_after.pop_all()
@@ -1322,12 +1322,14 @@ def fork_new_process_for_each_test(func: Callable[_P, None]) -> Callable[_P, Non
                 else:
                     os._exit(0)
             else:
-                # After setpgrp(), the child's pgid equals its pid
-                pgid = pid
+                pgid = os.getpgid(pid)
                 _pid, _exitcode = os.waitpid(pid, 0)
-                # kill all child processes - but they may already have exited cleanly
-                with contextlib.suppress(ProcessLookupError):
-                    os.killpg(pgid, signal.SIGTERM)
+                # ignore SIGTERM signal itself
+                old_signal_handler = signal.signal(signal.SIGTERM, signal.SIG_IGN)
+                # kill all child processes
+                os.killpg(pgid, signal.SIGTERM)
+                # restore the signal handler
+                signal.signal(signal.SIGTERM, old_signal_handler)
                 if _exitcode != 0:
                     # Try to read the exception from the child process
                     exc_info = {}
@@ -1856,7 +1858,6 @@ class TestFP8Layer(torch.nn.Module):
             out_dtype=out_dtype,
             force_kernel=force_kernel,
         )
-        self.kernel.process_weights_after_loading(self)
 
     def is_quant_fp8_enabled(self) -> bool:
         return self.kernel.quant_fp8.enabled()

tests/v1/determinism/test_batch_invariance.py

@@ -36,10 +36,10 @@ def test_v1_generation_is_deterministic_across_batch_sizes_with_needle(
     using the high-level v1 LLM() API only (no manual batching).
 
     Strategy:
-    - Create a single LLM engine configured for the larger batch limit (N).
-    - Compute a baseline output for the needle prompt when it is run alone.
-    - For many trials, generate a mixed batch (size N) where the needle appears
-      at a random position among random filler prompts using the same engine.
+    - Create two LLM engines with identical config except max_num_seqs: 1 vs N.
+    - Compute a baseline output for the needle prompt with the bs=1 engine.
+    - For many trials, generate a batch (size N) where the needle appears at a
+      random position among random filler prompts using the bs=N engine.
     - Track how many trials match vs mismatch, and report totals at the end.
       The test fails if any mismatches occur, but we still dump pass/fail
       counts.
@@ -83,9 +83,11 @@ def test_v1_generation_is_deterministic_across_batch_sizes_with_needle(
 
     needle_prompt = "There once was a "
 
-    llm = None
+    llm_bs1 = None
+    llm_bsN = None
     try:
-        llm = LLM_with_max_seqs(
+        # Engine with bs=1 behavior
+        llm_bs1 = LLM_with_max_seqs(
             model=model,
             max_num_seqs=max_batch_size,
             gpu_memory_utilization=gpu_mem_util,
@@ -94,11 +96,20 @@ def test_v1_generation_is_deterministic_across_batch_sizes_with_needle(
         )
 
         # Baseline generation for the needle prompt alone.
-        baseline_out = llm.generate([needle_prompt], sampling)
+        baseline_out = llm_bs1.generate([needle_prompt], sampling)
         assert len(baseline_out) == 1
         assert len(baseline_out[0].outputs) >= 1
         baseline_text = baseline_out[0].outputs[0].text
 
+        # Engine with larger batch limit (e.g., 64)
+        llm_bsN = LLM_with_max_seqs(
+            model=model,
+            max_num_seqs=max_batch_size,
+            gpu_memory_utilization=gpu_mem_util,
+            max_model_len=max_model_len,
+            attention_config=attention_config,
+        )
+
         mismatches = 0
 
         for trial in range(num_trials):
@@ -113,8 +124,8 @@ def test_v1_generation_is_deterministic_across_batch_sizes_with_needle(
                 else:
                     prompts.append(_random_prompt(min_random_prompt, max_random_prompt))
 
-            # Generate with the same engine but in a larger batch.
-            outputs = llm.generate(prompts, sampling)
+            # Generate with the larger-batch engine
+            outputs = llm_bsN.generate(prompts, sampling)
             # Find the needle output by position
             needle_output = outputs[needle_pos]
             assert needle_output.prompt == needle_prompt
@@ -140,9 +151,12 @@ def test_v1_generation_is_deterministic_across_batch_sizes_with_needle(
 
     finally:
         # Ensure engines are shutdown to free GPU/VRAM across test sessions
-        if llm is not None:
+        if llm_bs1 is not None:
             with contextlib.suppress(Exception):
-                llm.shutdown()
+                llm_bs1.shutdown()
+        if llm_bsN is not None:
+            with contextlib.suppress(Exception):
+                llm_bsN.shutdown()
 
 
 @skip_unsupported

tests/v1/e2e/spec_decode/test_spec_decode.py

@@ -557,16 +557,12 @@ def test_eagle_correctness_light(
             "auto",
             0.8,
         ),
-        pytest.param(
+        (
             ("eagle3", "Qwen/Qwen3-8B", "AngelSlim/Qwen3-8B_eagle3", 1),
             False,
             False,
             "transformers",
             0.8,
-            # TODO(hmellor): figure out why memory usage is so high
-            marks=pytest.mark.skip(
-                reason="Feature is experimental and uses too much memory in CI",
-            ),
         ),
         pytest.param(
             (

tests/v1/executor/test_ray_utils.py

@@ -1,54 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-import numpy as np
-
-from vllm.v1.executor.ray_utils import detach_zero_copy_from_model_runner_output
-from vllm.v1.outputs import LogprobsLists, LogprobsTensors, ModelRunnerOutput
-
-
-def _make_readonly(arr: np.ndarray) -> np.ndarray:
-    arr.setflags(write=False)
-    return arr
-
-
-def test_detach_zero_copy_from_model_runner_output_copies_only_numpy_views():
-    cu_num_generated_tokens = [0, 2]
-    prompt_logprobs = LogprobsTensors.empty_cpu(1, 2)
-    output = ModelRunnerOutput(
-        req_ids=["req-0"],
-        req_id_to_index={"req-0": 0},
-        logprobs=LogprobsLists(
-            logprob_token_ids=_make_readonly(
-                np.array([[1, 2], [3, 4]], dtype=np.int32)
-            ),
-            logprobs=_make_readonly(
-                np.array([[0.1, 0.2], [0.3, 0.4]], dtype=np.float32)
-            ),
-            sampled_token_ranks=_make_readonly(np.array([1, 2], dtype=np.int32)),
-            cu_num_generated_tokens=cu_num_generated_tokens,
-        ),
-        prompt_logprobs_dict={"req-0": prompt_logprobs},
-    )
-
-    original_logprobs = output.logprobs
-    assert original_logprobs is not None
-
-    detach_zero_copy_from_model_runner_output(output)
-
-    detached_logprobs = output.logprobs
-    assert detached_logprobs is not None
-    assert detached_logprobs is not original_logprobs
-    assert (
-        detached_logprobs.logprob_token_ids is not original_logprobs.logprob_token_ids
-    )
-    assert detached_logprobs.logprobs is not original_logprobs.logprobs
-    assert (
-        detached_logprobs.sampled_token_ranks
-        is not original_logprobs.sampled_token_ranks
-    )
-    assert detached_logprobs.logprob_token_ids.flags.writeable
-    assert detached_logprobs.logprobs.flags.writeable
-    assert detached_logprobs.sampled_token_ranks.flags.writeable
-    assert detached_logprobs.cu_num_generated_tokens is cu_num_generated_tokens
-    assert output.prompt_logprobs_dict["req-0"] is prompt_logprobs

tests/v1/kv_connector/nixl_integration/toy_proxy_server.py

@@ -173,9 +173,6 @@ async def send_request_to_service(
         req_data["max_completion_tokens"] = 1
     if "stream_options" in req_data:
         del req_data["stream_options"]
-    # These args are not supported for P
-    min_tokens = req_data.pop("min_tokens", None)
-    min_completion_tokens = req_data.pop("min_completion_tokens", None)
     headers = {
         "Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}",
         "X-Request-Id": request_id,
@@ -190,10 +187,6 @@ async def send_request_to_service(
     # otherwise, it would http.ReadError
     await response.aread()
 
-    # Add back the min_tokens and min_completion_tokens so D can use them
-    req_data["min_tokens"] = min_tokens
-    req_data["min_completion_tokens"] = min_completion_tokens
-
     return response
 
 

tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py

@@ -124,8 +124,12 @@ def test_offloading_connector(request_runner, async_scheduling: bool):
         return [BlockHash(str(i).encode()) for i in int_hashes]
 
     def take_events() -> Iterable[OffloadingEvent]:
-        yield OffloadingEvent(keys=to_keys([1, 2, 3]), medium="A", removed=False)
-        yield OffloadingEvent(keys=to_keys([4, 5, 6]), medium="B", removed=True)
+        yield OffloadingEvent(
+            keys=to_keys([1, 2, 3]), block_size=16, medium="A", removed=False
+        )
+        yield OffloadingEvent(
+            keys=to_keys([4, 5, 6]), block_size=32, medium="B", removed=True
+        )
 
     runner.manager.take_events.side_effect = take_events
     events = list(runner.scheduler_connector.take_events())
@@ -133,7 +137,7 @@ def test_offloading_connector(request_runner, async_scheduling: bool):
     event = events[0]
     assert isinstance(event, BlockStored)
     assert event.block_hashes == to_hashes([1, 2, 3])
-    assert event.block_size == 0
+    assert event.block_size == 16
     assert event.medium == "A"
     assert event.token_ids == []
     assert event.parent_block_hash is None

tests/v1/kv_connector/unit/test_mooncake_connector.py

@@ -609,51 +609,6 @@ def test_register_kv_caches():
                 assert bl == tensor1[0].nbytes // tensor1.shape[1]
 
 
-def test_register_kv_caches_supports_mixed_mla_and_eagle_shapes():
-    """Mixed MLA+Eagle caches should register by byte length, not shape."""
-
-    vllm_config = create_vllm_config(
-        kv_connector="MooncakeConnector", kv_role="kv_consumer"
-    )
-
-    with (
-        set_current_vllm_config(vllm_config),
-        patch_worker_dependencies(),
-        patch(
-            "vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector.threading.Event"
-        ),
-        patch(
-            "vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector.threading.Thread"
-        ) as mock_thread,
-    ):
-        connector = MooncakeConnector(vllm_config, KVConnectorRole.WORKER)
-        worker = connector.connector_worker
-        mock_thread.return_value.is_alive.return_value = False
-
-        worker.use_mla = True
-        worker.kv_topo.is_mla = True
-
-        # MLA cache tensor: shape[-2] is the block size.
-        mla_cache = torch.zeros((2, 16, 96), dtype=torch.float16)
-        # Eagle3/GQA-like cache tensor: shape[-2] is num_kv_heads, not block size.
-        eagle_cache = torch.zeros((2, 16, 8, 64), dtype=torch.float16)
-        kv_caches = {"mla_layer": mla_cache, "eagle_layer": eagle_cache}
-
-        with patch.object(
-            worker.engine, "batch_register_memory", return_value=0
-        ) as mock_batch_register:
-            connector.register_kv_caches(kv_caches)
-
-        mock_batch_register.assert_called_once()
-        registered_ptrs, registered_lens = mock_batch_register.call_args[0]
-        assert registered_ptrs == [mla_cache.data_ptr(), eagle_cache.data_ptr()]
-        assert registered_lens == [mla_cache.nbytes, eagle_cache.nbytes]
-        assert worker.block_len_per_layer == [
-            mla_cache.nbytes // mla_cache.shape[0],
-            eagle_cache.nbytes // eagle_cache.shape[0],
-        ]
-
-
 @pytest.mark.asyncio
 @patch(
     "vllm.distributed.kv_transfer.kv_connector.v1.mooncake."

tests/v1/kv_connector/unit/test_nixl_connector_hma.py

@@ -89,99 +89,6 @@ def test_logical_to_kernel_block_ids_with_hma():
     )
 
 
-@pytest.mark.cpu_test
-@pytest.mark.parametrize(
-    "has_mamba,swa_enabled,mamba_enabled,remote_ratio,"
-    "remote_block_ids,expected_remote_block_ids",
-    [
-        # Non-mamba (FA+SWA): both groups expanded via _logical_to_kernel_block_ids.
-        # Regression for https://github.com/vllm-project/vllm/pull/39724
-        (
-            False,
-            True,
-            False,
-            1,
-            ([0, 1, 2], [3, 4]),
-            [[0, 1, 2, 3, 4, 5], [6, 7, 8, 9]],
-        ),
-        # Mamba (FA+Mamba): FA expanded via _logical_to_remote_kernel_block_ids,
-        # Mamba passed through unchanged.
-        # remote_ratio=261 (Nemotron 30B TP=1) != local_ratio=2 so that using
-        # the wrong conversion method produces different FA results.
-        (
-            True,
-            False,
-            True,
-            261,
-            ([0, 1, 2], [10, 11]),
-            [[0, 1, 261, 262, 522, 523], [10, 11]],
-        ),
-    ],
-    ids=["non_mamba_fa_swa", "mamba_fa_ssm"],
-)
-def test_read_blocks_for_req_expands_remote_ids(
-    has_mamba,
-    swa_enabled,
-    mamba_enabled,
-    remote_ratio,
-    remote_block_ids,
-    expected_remote_block_ids,
-):
-    """_read_blocks_for_req must expand remote logical block IDs to kernel
-    block IDs when kernel block size != logical block size.
-
-    Non-mamba path uses _logical_to_kernel_block_ids (all groups expanded).
-    Mamba path uses _logical_to_remote_kernel_block_ids (FA expanded, Mamba
-    passed through).
-    """
-    from unittest.mock import MagicMock
-
-    from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
-        NixlConnectorMetadata,
-    )
-    from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
-        NixlConnectorWorker,
-    )
-
-    worker = object.__new__(NixlConnectorWorker)
-    worker._has_mamba = has_mamba
-    worker._physical_blocks_per_logical_kv_block = 2
-    worker.kv_cache_config = make_kv_cache_config(
-        block_size=16, swa_enabled=swa_enabled, mamba_enabled=mamba_enabled
-    )
-
-    remote_engine_id = "remote-engine"
-    if has_mamba:
-        worker._mamba_phys_ratio = {remote_engine_id: remote_ratio}
-
-    # Mock kv_topo: empty remote ranks skips the transfer machinery entirely,
-    # isolating the block-ID expansion logic.
-    worker.kv_topo = MagicMock()
-    worker.kv_topo.get_target_remote_ranks_from_engine_id.return_value = []
-    worker.kv_topo.tp_ratio_from_engine_id.return_value = 1
-
-    metadata = NixlConnectorMetadata()
-    metadata.add_new_req_to_recv(
-        request_id="test-req",
-        local_block_ids=([0, 1], [2, 3]),
-        kv_transfer_params={
-            "remote_block_ids": remote_block_ids,
-            "remote_engine_id": remote_engine_id,
-            "remote_request_id": "prefill-test-req",
-            "remote_host": "localhost",
-            "remote_port": 1234,
-            "tp_size": 1,
-        },
-    )
-
-    meta = metadata.reqs_to_recv["test-req"]
-    worker._read_blocks_for_req("test-req", meta)
-
-    assert meta.remote.block_ids == expected_remote_block_ids, (
-        f"Expected {expected_remote_block_ids}, got {meta.remote.block_ids}"
-    )
-
-
 @pytest.mark.parametrize("model_name, sw_size", [("google/gemma-3-1b-it", 512)])
 def test_fewer_blocks_with_hma(monkeypatch, model_name, sw_size):
     """Test that a prefill instance returns fewer "remote blocks" for the SWA groups
@@ -195,7 +102,7 @@ def test_fewer_blocks_with_hma(monkeypatch, model_name, sw_size):
     llm_kwargs = {
         "model": model_name,
         "enforce_eager": True,
-        "gpu_memory_utilization": 0.47,
+        "gpu_memory_utilization": 0.5,
         "kv_transfer_config": kv_transfer_config,
         "max_model_len": 2048,
         # NOTE: Make sure HMA is enabled

tests/v1/kv_offload/test_cpu_manager.py

@@ -59,6 +59,7 @@ def verify_load_output(
 
 def verify_events(
     events: Iterable[OffloadingEvent],
+    block_size: int,
     expected_stores: tuple[set[int], ...] = (),
     expected_evictions: tuple[set[int], ...] = (),
 ):
@@ -66,6 +67,7 @@ def verify_events(
     evictions: list[set[OffloadKey]] = []
     for event in events:
         assert event.medium == CPULoadStoreSpec.medium()
+        assert event.block_size == block_size
         if event.removed:
             evictions.append(set(event.keys))
         else:
@@ -96,7 +98,9 @@ def test_already_stored_block_not_evicted_during_prepare_store(eviction_policy):
               candidate to make room for [3, 4, 5]
         - After complete_store([2, 3, 4, 5]), block 2 must still be present.
     """
+    block_size = 256
     manager = CPUOffloadingManager(
+        block_size=block_size,
         num_blocks=4,
         cache_policy=eviction_policy,
         enable_events=True,
@@ -134,9 +138,10 @@ def test_cpu_manager():
     """
     Tests CPUOffloadingManager with lru policy.
     """
-    # initialize a CPU manager with a capacity of 4 blocks
+    # initialize a CPU backend with a capacity of 4 blocks
+    block_size = 256
     cpu_manager = CPUOffloadingManager(
-        num_blocks=4, cache_policy="lru", enable_events=True
+        block_size=block_size, num_blocks=4, cache_policy="lru", enable_events=True
     )
 
     # prepare store [1, 2]
@@ -158,7 +163,9 @@ def test_cpu_manager():
 
     # complete store [1, 2]
     cpu_manager.complete_store(to_keys([1, 2]))
-    verify_events(cpu_manager.take_events(), expected_stores=({1, 2},))
+    verify_events(
+        cpu_manager.take_events(), block_size=block_size, expected_stores=({1, 2},)
+    )
 
     # lookup [1, 2]
     assert cpu_manager.lookup(to_keys([1])) == 1
@@ -177,7 +184,9 @@ def test_cpu_manager():
     )
 
     # verify eviction event
-    verify_events(cpu_manager.take_events(), expected_evictions=({1},))
+    verify_events(
+        cpu_manager.take_events(), block_size=block_size, expected_evictions=({1},)
+    )
 
     # prepare store with no space
     assert cpu_manager.prepare_store(to_keys([1, 6])) is None
@@ -232,6 +241,7 @@ def test_cpu_manager():
 
     verify_events(
         cpu_manager.take_events(),
+        block_size=block_size,
         expected_stores=({3, 4, 5}, {6, 7, 8}),
         expected_evictions=({2, 3, 4}, {8}),
     )
@@ -244,6 +254,7 @@ class TestARCPolicy:
         self, num_blocks: int = 4, enable_events: bool = True
     ) -> tuple[CPUOffloadingManager, ARCCachePolicy]:
         manager = CPUOffloadingManager(
+            block_size=256,
             num_blocks=num_blocks,
             cache_policy="arc",
             enable_events=enable_events,
@@ -278,7 +289,9 @@ class TestARCPolicy:
 
         # complete store [1, 2]
         cpu_manager.complete_store(to_keys([1, 2]))
-        verify_events(cpu_manager.take_events(), expected_stores=({1, 2},))
+        verify_events(
+            cpu_manager.take_events(), block_size=256, expected_stores=({1, 2},)
+        )
 
         # lookup [1, 2]
         assert cpu_manager.lookup(to_keys([1])) == 1
@@ -534,8 +547,9 @@ def test_filter_reused_manager():
     """
     Tests FilterReusedOffloadingManager with a CPUOffloadingManager.
     """
+    block_size = 256
     lru_manager = CPUOffloadingManager(
-        num_blocks=4, cache_policy="lru", enable_events=True
+        block_size=block_size, num_blocks=4, cache_policy="lru", enable_events=True
     )
 
     manager = FilterReusedOffloadingManager(

tests/v1/metrics/test_stats.py

@@ -26,7 +26,6 @@ def test_prefill_kv_computed_with_cache():
     # Case 1: With prefix cache (1200 tokens cached)
     iteration_stats.update_from_finished_request(
         finish_reason=FinishReason.STOP,
-        request_id="test-req-001",
         num_prompt_tokens=10000,
         max_tokens_param=100,
         req_stats=req_stats,
@@ -36,7 +35,6 @@ def test_prefill_kv_computed_with_cache():
     finished_req = iteration_stats.finished_requests[0]
     assert finished_req.num_prompt_tokens == 10000
     assert finished_req.num_cached_tokens == 1200
-    assert finished_req.request_id == "test-req-001"
 
     # Verify calculation: prefill KV = prompt tokens - cached tokens
     prefill_kv_computed = finished_req.num_prompt_tokens - max(
@@ -57,7 +55,6 @@ def test_prefill_kv_computed_no_cache():
     # Case 2: No prefix cache
     iteration_stats.update_from_finished_request(
         finish_reason=FinishReason.STOP,
-        request_id="test-req-002",
         num_prompt_tokens=2000,
         max_tokens_param=100,
         req_stats=req_stats,
@@ -67,7 +64,6 @@ def test_prefill_kv_computed_no_cache():
     finished_req = iteration_stats.finished_requests[0]
     assert finished_req.num_prompt_tokens == 2000
     assert finished_req.num_cached_tokens == 0
-    assert finished_req.request_id == "test-req-002"
 
     # Verify calculation: prefill KV = full prompt when no cache
     prefill_kv_computed = finished_req.num_prompt_tokens - max(
@@ -88,7 +84,6 @@ def test_prefill_kv_computed_edge_cases():
     # Case 3: Negative num_cached_tokens (shouldn't happen, but handle gracefully)
     iteration_stats.update_from_finished_request(
         finish_reason=FinishReason.STOP,
-        request_id="test-req-003",
         num_prompt_tokens=100,
         max_tokens_param=10,
         req_stats=req_stats,
@@ -101,13 +96,11 @@ def test_prefill_kv_computed_edge_cases():
         finished_req.num_cached_tokens, 0
     )
     assert prefill_kv_computed == 100  # Should treat negative as 0
-    assert finished_req.request_id == "test-req-003"
 
     # Case 4: All tokens cached (shouldn't happen in practice)
     iteration_stats2 = IterationStats()
     iteration_stats2.update_from_finished_request(
         finish_reason=FinishReason.STOP,
-        request_id="test-req-004",
         num_prompt_tokens=100,
         max_tokens_param=10,
         req_stats=req_stats,
@@ -119,7 +112,6 @@ def test_prefill_kv_computed_edge_cases():
         finished_req2.num_cached_tokens, 0
     )
     assert prefill_kv_computed2 == 0  # All cached, nothing computed
-    assert finished_req2.request_id == "test-req-004"
 
 
 def test_prompt_token_stats_all_computed():

tests/v1/spec_decode/test_speculators_dflash.py

@@ -1,171 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-import pytest
-import torch
-
-from tests.evals.gsm8k.gsm8k_eval import evaluate_gsm8k_offline
-from tests.utils import large_gpu_mark
-from vllm import LLM
-from vllm.config import SpeculativeConfig
-from vllm.distributed import cleanup_dist_env_and_memory
-
-MODEL_PATH = "nm-testing/dflash-qwen3-8b-speculators"
-
-EXPECTED_GSM8K_ACCURACY = 0.885
-ACCURACY_RTOL = 0.03
-EXPECTED_ACCEPTANCE_LEN = 3.45
-ACCEPTANCE_LEN_RTOL = 0.15
-
-# Expected per-position acceptance rates (accepted_at_pos / num_drafts)
-# Based on GSM8K evaluation with Qwen3-8B dflash speculators.
-EXPECTED_PER_POS_ACCEPTANCE_RATES = [0.795, 0.611, 0.429, 0.282]
-PER_POS_RTOL = 0.15
-
-
-def compute_spec_decode_stats(
-    metrics,
-) -> dict:
-    """Extract all spec-decode metrics and compute derived stats."""
-    name2metric = {m.name: m for m in metrics}
-
-    n_drafts = name2metric["vllm:spec_decode_num_drafts"].value
-    n_draft_tokens = name2metric["vllm:spec_decode_num_draft_tokens"].value
-    n_accepted = name2metric["vllm:spec_decode_num_accepted_tokens"].value
-
-    per_pos_vec = name2metric["vllm:spec_decode_num_accepted_tokens_per_pos"].values
-
-    acceptance_len = 1 + (n_accepted / n_drafts) if n_drafts > 0 else 1.0
-    draft_tokens_per_step = (n_draft_tokens / n_drafts) if n_drafts > 0 else 0
-    overall_acceptance_rate = (n_accepted / n_draft_tokens) if n_draft_tokens > 0 else 0
-    per_pos_rates = [v / n_drafts for v in per_pos_vec] if n_drafts > 0 else []
-
-    return {
-        "num_drafts": n_drafts,
-        "num_draft_tokens": n_draft_tokens,
-        "num_accepted_tokens": n_accepted,
-        "acceptance_len": acceptance_len,
-        "draft_tokens_per_step": draft_tokens_per_step,
-        "overall_acceptance_rate": overall_acceptance_rate,
-        "per_pos_accepted": list(per_pos_vec),
-        "per_pos_acceptance_rates": per_pos_rates,
-    }
-
-
-def print_spec_decode_stats(stats: dict) -> None:
-    """Print all spec-decode metrics and derived values."""
-    print("\n===== Spec Decode Metrics =====")
-    print(f"  num_drafts:              {stats['num_drafts']}")
-    print(f"  num_draft_tokens:        {stats['num_draft_tokens']}")
-    print(f"  num_accepted_tokens:     {stats['num_accepted_tokens']}")
-    print(f"  draft_tokens_per_step:   {stats['draft_tokens_per_step']:.2f}")
-    print(f"  overall_acceptance_rate: {stats['overall_acceptance_rate']:.4f}")
-    print(f"  acceptance_len (1+acc/drafts): {stats['acceptance_len']:.4f}")
-    print("  per-position accepted tokens:", stats["per_pos_accepted"])
-    print("  per-position acceptance rates:")
-    for i, rate in enumerate(stats["per_pos_acceptance_rates"]):
-        print(f"    pos {i}: {rate:.4f}")
-    print("===============================\n")
-
-
-def test_dflash_speculators_model(vllm_runner, example_prompts, monkeypatch):
-    """
-    Test DFlash speculators model properly initializes speculative decoding.
-
-    Verifies:
-    1. Speculative config is automatically initialized from speculators config
-    2. Method is detected as 'dflash'
-    3. The draft model path is correctly set
-    4. Speculative tokens count is valid (num_speculative_tokens=8)
-    5. Text generation works with speculative decoding enabled
-    """
-    monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
-
-    with vllm_runner(
-        MODEL_PATH,
-        dtype=torch.bfloat16,
-        enforce_eager=True,
-        quantization="fp8",
-    ) as vllm_model:
-        vllm_config = vllm_model.llm.llm_engine.vllm_config
-
-        assert isinstance(vllm_config.speculative_config, SpeculativeConfig), (
-            "Speculative config should be initialized for speculators model"
-        )
-
-        spec_config = vllm_config.speculative_config
-        assert spec_config.method == "dflash", (
-            f"Expected method='dflash', got '{spec_config.method}'"
-        )
-        assert spec_config.num_speculative_tokens > 0, (
-            f"Expected positive speculative tokens, "
-            f"got {spec_config.num_speculative_tokens}"
-        )
-        assert spec_config.model == MODEL_PATH, (
-            f"Draft model should be {MODEL_PATH}, got {spec_config.model}"
-        )
-
-        vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens=20)
-        assert vllm_outputs, f"No outputs generated for speculators model {MODEL_PATH}"
-
-
-@pytest.mark.slow_test
-@large_gpu_mark(min_gb=40)
-def test_dflash_speculators_correctness(monkeypatch):
-    """
-    E2E correctness test for DFlash via the speculators auto-detect path.
-
-    Evaluates GSM8k accuracy to ensure the speculators-format model produces
-    correct outputs, and checks that acceptance length does not collapse under
-    batched inference (lm-eval style).
-
-    Observed per-position acceptance rates on GSM8K (1319 prompts):
-        pos 0: 0.795, pos 1: 0.611, pos 2: 0.429, pos 3: 0.282,
-        pos 4: 0.169, pos 5: 0.093, pos 6: 0.048, pos 7: 0.023
-    Observed mean AL: 3.45 (GSM8K dataset, max_num_seqs=128)
-    """
-    monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
-
-    spec_llm = LLM(
-        model=MODEL_PATH,
-        trust_remote_code=True,
-        max_model_len=4096,
-        max_num_seqs=128,
-        gpu_memory_utilization=0.85,
-        enforce_eager=False,
-        disable_log_stats=False,
-    )
-
-    results = evaluate_gsm8k_offline(spec_llm)
-    accuracy = results["accuracy"]
-    accuracy_threshold = EXPECTED_GSM8K_ACCURACY * (1 - ACCURACY_RTOL)
-    assert accuracy >= accuracy_threshold, (
-        f"Expected GSM8K accuracy >= {accuracy_threshold:.3f}, got {accuracy:.3f}"
-    )
-
-    current_metrics = spec_llm.get_metrics()
-    stats = compute_spec_decode_stats(current_metrics)
-    print_spec_decode_stats(stats)
-
-    acceptance_len = stats["acceptance_len"]
-    al_threshold = EXPECTED_ACCEPTANCE_LEN * (1 - ACCEPTANCE_LEN_RTOL)
-    assert acceptance_len >= al_threshold, (
-        f"DFlash speculators acceptance length too low: "
-        f"{acceptance_len:.2f} < {al_threshold:.2f}"
-    )
-
-    # Check per-position acceptance rates for the first few positions.
-    per_pos_rates = stats["per_pos_acceptance_rates"]
-    for i, expected_rate in enumerate(EXPECTED_PER_POS_ACCEPTANCE_RATES):
-        assert i < len(per_pos_rates), (
-            f"Missing per-position acceptance rate for position {i}"
-        )
-        threshold = expected_rate * (1 - PER_POS_RTOL)
-        assert per_pos_rates[i] >= threshold, (
-            f"Per-position acceptance rate at pos {i} too low: "
-            f"{per_pos_rates[i]:.4f} < {threshold:.4f} "
-            f"(expected ~{expected_rate:.4f})"
-        )
-
-    del spec_llm
-    torch.accelerator.empty_cache()
-    cleanup_dist_env_and_memory()

vllm/_aiter_ops.py

@@ -8,18 +8,12 @@ from torch._ops import OpOverload
 
 import vllm.envs as envs
 from vllm.platforms import current_platform
-from vllm.utils.import_utils import PlaceholderModule
 from vllm.utils.torch_utils import direct_register_custom_op
 from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
     rocm_aiter_sparse_attn_indexer,
     rocm_aiter_sparse_attn_indexer_fake,
 )
 
-try:
-    import pandas as pd
-except ImportError:
-    pd = PlaceholderModule("pandas")
-
 # fp8_dtype is not cached.
 # on ROCm the fp8_dtype always calls is_fp8_fnuz
 # which is a host op, so we cache it once here.
@@ -62,29 +56,6 @@ def is_aiter_found_and_supported() -> bool:
     return False
 
 
-@functools.cache
-def _load_gemm_tuned_configs(
-    q_dtype_w: torch.dtype, csv_path: str
-) -> set[tuple[int, int, int]]:
-    try:
-        df = pd.read_csv(csv_path).drop_duplicates()
-        df = df[df["q_dtype_w"] == str(q_dtype_w)]
-        return set(zip(df["N"].astype(int), df["K"].astype(int), df["M"].astype(int)))
-    except Exception:
-        return set()
-
-
-def _check_kernel_tuned(N: int, K: int, q_dtype_w: torch.dtype, csv_path: str) -> bool:
-    configs = _load_gemm_tuned_configs(q_dtype_w, csv_path)
-    l_m = (
-        [1, 2, 4]
-        + list(range(8, 513, 8))
-        + [1024, 1536]
-        + [2**i for i in range(11, 19)]
-    )
-    return any((N, K, M) in configs for M in l_m)
-
-
 def if_aiter_supported(func: Callable) -> Callable:
     """Decorator that only executes the function if
     ROCm AITER package is supported and enabled on gfx9 archs.
@@ -497,7 +468,7 @@ def _rocm_aiter_mla_decode_fwd_fake(
     pass
 
 
-def _rocm_aiter_w8a8_gemm_impl(
+def _rocm_aiter_gemm_a8w8_impl(
     A: torch.Tensor,
     B: torch.Tensor,
     As: torch.Tensor,
@@ -514,7 +485,7 @@ def _rocm_aiter_w8a8_gemm_impl(
     return gemm_a8w8_CK(A, B, As, Bs, bias, output_dtype)
 
 
-def _rocm_aiter_w8a8_gemm_fake(
+def _rocm_aiter_gemm_a8w8_fake(
     A: torch.Tensor,
     B: torch.Tensor,
     As: torch.Tensor,
@@ -528,35 +499,6 @@ def _rocm_aiter_w8a8_gemm_fake(
     return Y
 
 
-def _rocm_aiter_preshuffled_per_token_w8a8_gemm_impl(
-    A: torch.Tensor,
-    B: torch.Tensor,
-    As: torch.Tensor,
-    Bs: torch.Tensor,
-    bias: torch.Tensor | None = None,
-    output_dtype: torch.dtype = torch.float16,
-) -> torch.Tensor:
-    from aiter import gemm_a8w8_bpreshuffle
-
-    output = gemm_a8w8_bpreshuffle(A, B, As, Bs, None, output_dtype)
-    if bias is not None:
-        output.add_(bias)
-    return output
-
-
-def _rocm_aiter_preshuffled_per_token_w8a8_gemm_fake(
-    A: torch.Tensor,
-    B: torch.Tensor,
-    As: torch.Tensor,
-    Bs: torch.Tensor,
-    bias: torch.Tensor | None = None,
-    output_dtype: torch.dtype = torch.float16,
-) -> torch.Tensor:
-    m = A.shape[0]
-    n = B.shape[0]
-    return torch.empty(m, n, dtype=output_dtype, device=A.device)
-
-
 def _rocm_aiter_triton_gemm_a8w8_blockscale_impl(
     A: torch.Tensor,
     B: torch.Tensor,
@@ -1371,15 +1313,11 @@ class rocm_aiter_ops:
             )
 
             direct_register_custom_op(
-                op_name="rocm_aiter_w8a8_gemm",
-                op_func=_rocm_aiter_w8a8_gemm_impl,
-                fake_impl=_rocm_aiter_w8a8_gemm_fake,
-            )
-
-            direct_register_custom_op(
-                op_name="_rocm_aiter_preshuffled_per_token_w8a8_gemm",
-                op_func=_rocm_aiter_preshuffled_per_token_w8a8_gemm_impl,
-                fake_impl=_rocm_aiter_preshuffled_per_token_w8a8_gemm_fake,
+                op_name="rocm_aiter_gemm_a8w8",
+                op_func=_rocm_aiter_gemm_a8w8_impl,
+                mutates_args=[],
+                fake_impl=_rocm_aiter_gemm_a8w8_fake,
+                dispatch_key=current_platform.dispatch_key,
             )
 
             direct_register_custom_op(
@@ -1555,7 +1493,7 @@ class rocm_aiter_ops:
         )
 
     @staticmethod
-    def w8a8_gemm(
+    def gemm_a8w8(
         A: torch.Tensor,
         B: torch.Tensor,
         As: torch.Tensor,
@@ -1563,20 +1501,7 @@ class rocm_aiter_ops:
         bias: torch.Tensor | None = None,
         output_dtype: torch.dtype = torch.float16,
     ) -> torch.Tensor:
-        return torch.ops.vllm.rocm_aiter_w8a8_gemm(A, B, As, Bs, bias, output_dtype)
-
-    @staticmethod
-    def preshuffled_per_token_w8a8_gemm(
-        A: torch.Tensor,
-        B: torch.Tensor,
-        As: torch.Tensor,
-        Bs: torch.Tensor,
-        bias: torch.Tensor | None = None,
-        output_dtype: torch.dtype = torch.float16,
-    ) -> torch.Tensor:
-        return torch.ops.vllm._rocm_aiter_preshuffled_per_token_w8a8_gemm(
-            A, B, As, Bs, bias, output_dtype
-        )
+        return torch.ops.vllm.rocm_aiter_gemm_a8w8(A, B, As, Bs, bias, output_dtype)
 
     @staticmethod
     def triton_gemm_a8w8_blockscale(
@@ -1995,24 +1920,6 @@ class rocm_aiter_ops:
             (8192, 3584),
         ]
 
-    @staticmethod
-    def is_shuffled_per_token_w8a8_gemm_tuned(
-        N: int, K: int, q_dtype_w: torch.dtype
-    ) -> bool:
-        import aiter.ops.gemm_op_a8w8 as aiter_gemm_a8w8_ops
-
-        csv_path = (
-            aiter_gemm_a8w8_ops.AITER_CONFIGS.AITER_CONFIG_GEMM_A8W8_BPRESHUFFLE_FILE
-        )
-        return _check_kernel_tuned(N, K, q_dtype_w, csv_path)
-
-    @staticmethod
-    def is_per_token_w8a8_gemm_tuned(N: int, K: int, q_dtype_w: torch.dtype) -> bool:
-        import aiter.ops.gemm_op_a8w8 as aiter_gemm_a8w8_ops
-
-        csv_path = aiter_gemm_a8w8_ops.AITER_CONFIGS.AITER_CONFIG_GEMM_A8W8_FILE
-        return _check_kernel_tuned(N, K, q_dtype_w, csv_path)
-
     @staticmethod
     def shuffle_weight(
         tensor: torch.Tensor, layout: tuple[int, int] = (16, 16)

vllm/_custom_ops.py

@@ -3300,12 +3300,6 @@ def cpu_gemm_wna16(
     return output
 
 
-def cpu_activation_lut_bf16(input: torch.Tensor, activation: str) -> torch.Tensor:
-    out = torch.empty_like(input)
-    torch.ops._C.activation_lut_bf16(out, input, activation)
-    return out
-
-
 def cpu_prepack_moe_weight(
     weight: torch.Tensor,
     isa: str,

vllm/_xpu_ops.py

@@ -144,46 +144,6 @@ def _xpu_mxfp8_quantize_fake(
     return x.to(dtype), x_s.to(torch.float8_e8m0fnu)
 
 
-def _xpu_mxfp4_quantize_impl(
-    x: torch.Tensor,
-) -> tuple[torch.Tensor, torch.Tensor]:
-    MXFP4_BLOCK_SIZE = 32
-    eps = 1e-10
-    assert x.ndim == 2, "input must be 2-D"
-    assert x.shape[-1] % MXFP4_BLOCK_SIZE == 0, (
-        f"last dimension {x.shape[-1]} must be divisible by group_size "
-        f"{MXFP4_BLOCK_SIZE}"
-    )
-    assert x.is_contiguous(), "input groups must be contiguous"
-
-    M, N = x.shape
-
-    # Packed FP4 output: two nibbles per byte
-    x_q = torch.empty(M, N // 2, device=x.device, dtype=torch.uint8)
-    x_s = torch.empty(M, N // MXFP4_BLOCK_SIZE, device=x.device, dtype=torch.float32)
-
-    torch.ops._C.per_token_group_quant_mxfp4(x, x_q, x_s, MXFP4_BLOCK_SIZE, eps)
-
-    x_q = x_q.view(torch.float4_e2m1fn_x2)
-    x_s = x_s.to(dtype=torch.float8_e8m0fnu, memory_format=torch.preserve_format)
-    return x_q, x_s
-
-
-def _xpu_mxfp4_quantize_fake(
-    x: torch.Tensor,
-) -> tuple[torch.Tensor, torch.Tensor]:
-    MXFP4_BLOCK_SIZE = 32
-    M, N = x.shape
-
-    # Packed FP4 output: two nibbles per byte
-    x_q = torch.empty(M, N // 2, device=x.device, dtype=torch.uint8)
-    x_s = torch.empty(M, N // MXFP4_BLOCK_SIZE, device=x.device, dtype=torch.float32)
-
-    x_q = x_q.view(torch.float4_e2m1fn_x2)
-    x_s = x_s.to(dtype=torch.float8_e8m0fnu, memory_format=torch.preserve_format)
-    return x_q, x_s
-
-
 # Global flag to ensure ops are registered only once
 _OPS_REGISTERED = False
 
@@ -595,12 +555,6 @@ class xpu_ops:
                 fake_impl=_xpu_mxfp8_quantize_fake,
             )
 
-            direct_register_custom_op(
-                op_name="xpu_mxfp4_quantize",
-                op_func=_xpu_mxfp4_quantize_impl,
-                fake_impl=_xpu_mxfp4_quantize_fake,
-            )
-
             _OPS_REGISTERED = True
 
 

vllm/benchmarks/datasets/datasets.py

@@ -25,7 +25,6 @@ from contextlib import suppress
 from dataclasses import dataclass, replace
 from functools import cache
 from io import BytesIO
-from pathlib import Path
 from tempfile import NamedTemporaryFile
 from typing import Any, cast
 
@@ -1423,7 +1422,6 @@ def add_dataset_parser(parser: FlexibleArgumentParser):
             "custom_mm",
             "prefix_repetition",
             "spec_bench",
-            "speed_bench",
         ],
         help="Name of the dataset to benchmark on.",
     )
@@ -1608,34 +1606,6 @@ def add_dataset_parser(parser: FlexibleArgumentParser):
         "repetition dataset.",
     )
 
-    speed_bench_group = parser.add_argument_group("speed bench dataset options")
-    speed_bench_group.add_argument(
-        "--speed-bench-dataset-subset",
-        type=str,
-        default="qualitative",
-        choices={
-            "qualitative",
-            "throughput_1k",
-            "throughput_2k",
-            "throughput_8k",
-            "throughput_16k",
-            "throughput_32k",
-        },
-        help="Subset of the SPEED-Bench dataset.",
-    )
-    speed_bench_group.add_argument(
-        "--speed-bench-output-len",
-        type=int,
-        default=4096,
-        help="Num of output tokens per request, used only for speed bench dataset.",
-    )
-    speed_bench_group.add_argument(
-        "--speed-bench-category",
-        type=str,
-        default=None,
-        help="Category for speed bench dataset. If None, use all categories.",
-    )
-
 
 def add_random_dataset_base_args(
     parser_or_group: FlexibleArgumentParser | argparse._ArgumentGroup,
@@ -2104,19 +2074,6 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
                 request_id_prefix=args.request_id_prefix,
                 no_oversample=args.no_oversample,
             ),
-            "speed_bench": lambda: SpeedBench(
-                dataset_path=args.dataset_path,
-                dataset_subset=args.speed_bench_dataset_subset,
-                category=args.speed_bench_category,
-                disable_shuffle=args.disable_shuffle,
-            ).sample(
-                num_requests=args.num_prompts,
-                tokenizer=tokenizer,
-                output_len=args.speed_bench_output_len,
-                enable_multimodal_chat=args.enable_multimodal_chat,
-                request_id_prefix=args.request_id_prefix,
-                no_oversample=args.no_oversample,
-            ),
         }
 
         try:
@@ -3594,48 +3551,3 @@ class MMStarDataset(HuggingFaceDataset):
             sampled_requests, num_requests, request_id_prefix, no_oversample
         )
         return sampled_requests
-
-
-# -----------------------------------------------------------------------------
-# Speed Bench Dataset Implementation
-# -----------------------------------------------------------------------------
-
-
-class SpeedBench(CustomDataset):
-    """
-    Implements the SPEED-Bench dataset: https://huggingface.co/datasets/nvidia/SPEED-Bench
-    Download the dataset using:
-    curl -LsSf https://raw.githubusercontent.com/NVIDIA-NeMo/Skills/refs/heads/main/nemo_skills/dataset/speed-bench/prepare.py | python3 -
-    """  # noqa: E501
-
-    def __init__(self, **kwargs) -> None:
-        self.dataset_subset = kwargs.pop("dataset_subset", "qualitative")
-        self.category = kwargs.pop("category", None)
-        super().__init__(**kwargs)
-        self.load_data()
-
-    def load_data(self) -> None:
-        if self.dataset_path is None:
-            raise ValueError("dataset_path must be provided for loading data.")
-
-        self.data = []
-
-        # Load the JSONL file
-        jsonl_data = pd.read_json(
-            path_or_buf=Path(self.dataset_path) / f"{self.dataset_subset}.jsonl",
-            lines=True,
-        )
-
-        # check if the JSONL file has a 'turns' column
-        if "messages" not in jsonl_data.columns:
-            raise ValueError("JSONL file must contain a 'messages' column.")
-
-        for _, row in jsonl_data.iterrows():
-            # sample only from a specific category if specified
-            if (not self.category) or (self.category == row["category"]):
-                prompt = row["messages"][0]["content"]
-                self.data.append({"prompt": prompt})
-
-        random.seed(self.random_seed)
-        if not getattr(self, "disable_shuffle", False):
-            random.shuffle(self.data)

vllm/compilation/backends.py

@@ -1263,23 +1263,6 @@ class VllmBackend:
             original_split_gm if envs.VLLM_USE_MEGA_AOT_ARTIFACT else self.graph
         )
 
-        from vllm.compilation.codegen import (
-            compile_execution_fn,
-            generate_execution_code,
-        )
-
-        execution_code, submod_names = generate_execution_code(self.split_gm)
-        # Use getattr to get correct callables: __dict__ has PiecewiseBackend
-        # instances (from PiecewiseCompileInterpreter), _modules has originals.
-        # getattr checks __dict__ first, then falls back to _modules.
-        submod_callables = {
-            name: getattr(self.split_gm, name)
-            for name, _ in self.split_gm.named_children()
-        }
-        runtime_callable = compile_execution_fn(
-            execution_code, submod_callables, submod_names
-        )
-
         if (
             self.compilation_config.cudagraph_mode == CUDAGraphMode.NONE
             or not self.compilation_config.cudagraph_copy_inputs
@@ -1288,11 +1271,9 @@ class VllmBackend:
                 graph_to_serialize,
                 example_inputs,
                 self.prefix,
-                runtime_callable,
+                self.split_gm,
                 is_encoder=self.is_encoder,
                 vllm_backend=self,
-                execution_code=execution_code,
-                submod_names=submod_names,
             )
 
         # index of tensors that have symbolic shapes (batch size)
@@ -1313,7 +1294,7 @@ class VllmBackend:
         copy_and_call = make_copy_and_call(
             sym_tensor_indices,
             [example_inputs[x].clone() for x in sym_tensor_indices],
-            runtime_callable,
+            self.split_gm,
         )
 
         return VllmSerializableFunction(
@@ -1324,6 +1305,4 @@ class VllmBackend:
             is_encoder=self.is_encoder,
             vllm_backend=self,
             sym_tensor_indices=sym_tensor_indices,
-            execution_code=execution_code,
-            submod_names=submod_names,
         )

vllm/compilation/caching.py

@@ -184,8 +184,6 @@ class VllmSerializableFunction(SerializableCallable):  # type: ignore[misc]
         vllm_backend: Any | None = None,
         sym_tensor_indices: list[int] | None = None,
         aot_autograd_config: dict[str, Any] | None = None,
-        execution_code: str | None = None,
-        submod_names: list[str] | None = None,
     ) -> None:
         assert isinstance(graph_module, torch.fx.GraphModule)
         self.graph_module = graph_module
@@ -196,8 +194,6 @@ class VllmSerializableFunction(SerializableCallable):  # type: ignore[misc]
         self.shape_env = None
         self.vllm_backend = vllm_backend
         self.sym_tensor_indices = sym_tensor_indices
-        self.execution_code = execution_code
-        self.submod_names = submod_names
         self._fake_mode: Any | None = None
 
         import torch._functorch.config as functorch_config
@@ -457,7 +453,7 @@ def reconstruct_serializable_fn_from_mega_artifact(
 
     standalone_compile_artifacts.load_all()
 
-    piecewise_submod_names = standalone_compile_artifacts.submodule_names()
+    submod_names = standalone_compile_artifacts.submodule_names()
     compiled_callables: dict[str, dict[str, Callable[..., Any]]] = {}
 
     for cache_key in standalone_compile_artifacts.submodule_bytes:
@@ -477,13 +473,13 @@ def reconstruct_serializable_fn_from_mega_artifact(
 
     # spot check that cached submodules exist in the graph structure
     graph_children = {name for name, _ in split_gm.named_children()}
-    missing = set(piecewise_submod_names) - graph_children
+    missing = set(submod_names) - graph_children
     assert not missing, (
         f"artifacts reference submodules not in graph: {missing}. "
         f"graph has: {sorted(graph_children)}"
     )
 
-    for i, submod_name in enumerate(piecewise_submod_names):
+    for i, submod_name in enumerate(submod_names):
         assert submod_name in sym_shape_indices_map and submod_name in returns_tuple_map
 
         sym_shape_indices = sym_shape_indices_map[submod_name]
@@ -494,7 +490,7 @@ def reconstruct_serializable_fn_from_mega_artifact(
             graph=None,  # not needed for cached artifacts
             vllm_config=vllm_config,
             piecewise_compile_index=i,
-            total_piecewise_compiles=len(piecewise_submod_names),
+            total_piecewise_compiles=len(submod_names),
             sym_shape_indices=sym_shape_indices,
             vllm_backend=vllm_backend,
             returns_tuple=returns_tuple,
@@ -502,7 +498,7 @@ def reconstruct_serializable_fn_from_mega_artifact(
         )
 
         is_first = i == 0
-        is_last = i == len(piecewise_submod_names) - 1
+        is_last = i == len(submod_names) - 1
         wrapped_backend = wrap_with_cudagraph_if_needed(
             piecewise_backend,
             vllm_config,
@@ -517,21 +513,6 @@ def reconstruct_serializable_fn_from_mega_artifact(
             submod_name,
         )
 
-    # Use codegen'd execution code if available, fall back to split_gm
-    execution_code = state.get("execution_code")
-    submod_names = state.get("submod_names")
-    if execution_code is not None and submod_names is not None:
-        from vllm.compilation.codegen import compile_execution_fn
-
-        submod_callables = {
-            name: getattr(split_gm, name) for name, _ in split_gm.named_children()
-        }
-        runtime_callable = compile_execution_fn(
-            execution_code, submod_callables, submod_names
-        )
-    else:
-        runtime_callable = split_gm
-
     if compilation_config.cudagraph_copy_inputs:
         sym_tensor_indices = state["sym_tensor_indices"]
         input_buffers = [
@@ -540,11 +521,9 @@ def reconstruct_serializable_fn_from_mega_artifact(
             )
             for idx in sym_tensor_indices
         ]
-        optimized_call = make_copy_and_call(
-            sym_tensor_indices, input_buffers, runtime_callable
-        )
+        optimized_call = make_copy_and_call(sym_tensor_indices, input_buffers, split_gm)
     else:
-        optimized_call = runtime_callable
+        optimized_call = split_gm
 
     fn = VllmSerializableFunction(
         **state,

vllm/compilation/codegen.py

@@ -1,155 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""Code generation for split_gm stitching graph execution.
-
-Generates a plain Python function that replaces the FX GraphModule's
-interpreter-based execution of the stitching graph, eliminating
-nn.Module.__call__ overhead and __getattr__ dispatch.
-"""
-
-import operator
-from collections.abc import Callable
-from functools import partial
-from typing import Any
-
-import torch.fx
-from torch._dynamo.utils import dynamo_timed
-from torch._logging import trace_structured
-
-
-@dynamo_timed("vllm.generate_execution_code")
-def generate_execution_code(
-    split_gm: torch.fx.GraphModule,
-) -> tuple[str, list[str]]:
-    """Generate Python source code from a split_gm's stitching graph.
-
-    Walks split_gm.graph.nodes and produces a function that calls
-    submodules via a __vllm_submods__ list, avoiding FX GraphModule overhead
-    and dict lookup cost.
-
-    Args:
-        split_gm: The split graph module produced by split_graph().
-
-    Returns:
-        A tuple of (code, submod_names) where code is the Python source
-        and submod_names is the ordered list of submodule target names
-        corresponding to list indices used in the generated code.
-    """
-    lines: list[str] = []
-    param_names: list[str] = []
-    submod_names: list[str] = []
-    submod_index: dict[str, int] = {}
-
-    # Build node ordering for liveness analysis.
-    nodes = list(split_gm.graph.nodes)
-    node_order = {node: i for i, node in enumerate(nodes)}
-
-    # For each value-producing node, find the position of its last consumer.
-    # If the last consumer is the output node, skip (return handles cleanup).
-    # Otherwise, schedule a del after that consumer to free memory early.
-    del_after: dict[int, list[str]] = {}  # position -> names to delete
-    for node in nodes:
-        if node.op == "output":
-            continue
-        users = list(node.users.keys())
-        if not users:
-            continue
-        last_user = max(users, key=lambda u: node_order[u])
-        if last_user.op == "output":
-            continue
-        del_after.setdefault(node_order[last_user], []).append(node.name)
-
-    for i, node in enumerate(nodes):
-        if node.op == "placeholder":
-            param_names.append(node.name)
-
-        elif node.op == "call_module":
-            target = node.target
-            if target not in submod_index:
-                submod_index[target] = len(submod_names)
-                submod_names.append(target)
-            idx = submod_index[target]
-            args_str = ", ".join(_node_ref(a) for a in node.args)
-            kwargs_str = ", ".join(
-                f"{k}={_node_ref(v)}" for k, v in node.kwargs.items()
-            )
-            all_args = ", ".join(filter(None, [args_str, kwargs_str]))
-            lines.append(f"    {node.name} = __vllm_submods__[{idx}]({all_args})")
-
-        elif node.op == "call_function" and node.target is operator.getitem:
-            source = _node_ref(node.args[0])
-            index = node.args[1]
-            assert isinstance(index, int)
-            lines.append(f"    {node.name} = {source}[{index}]")
-
-        elif node.op == "output":
-            assert len(node.args) == 1
-            ret = _node_ref(node.args[0])
-            lines.append(f"    return {ret}")
-
-        else:
-            raise RuntimeError(f"Unsupported node from codegen: {node.format_node()}")
-
-        # Emit del for variables whose last use was this node.
-        if i in del_after:
-            names = sorted(del_after[i])
-            lines.append(f"    del {', '.join(names)}")
-
-    assert len(param_names) > 0
-    params = ", ".join(param_names)
-    header = f"def execution_fn({params}, *, __vllm_submods__):"
-    return "import torch\n" + "\n".join([header] + lines) + "\n", submod_names
-
-
-@dynamo_timed("vllm.compile_execution_fn")
-def compile_execution_fn(
-    code: str,
-    submod_callables: dict[str, Callable[..., Any]],
-    submod_names: list[str],
-) -> Callable[..., Any]:
-    """Compile execution code and bind submodule callables.
-
-    Args:
-        code: Python source from generate_execution_code().
-        submod_callables: Mapping of submodule names to their callables.
-        submod_names: Ordered list of submodule names matching the indices
-            used in the generated code.
-
-    Returns:
-        A callable that executes the stitching logic.
-    """
-    trace_structured(
-        "artifact",
-        metadata_fn=lambda: {
-            "name": "vllm_execution_code",
-            "encoding": "string",
-        },
-        payload_fn=lambda: code,
-    )
-    namespace: dict[str, Any] = {}
-    exec(code, namespace)  # noqa: S102
-    fn = namespace["execution_fn"]
-    # Use .forward() directly to avoid nn.Module.__call__ overhead.
-    submods_list = [
-        c.forward if isinstance(c, torch.fx.GraphModule) else c
-        for c in (submod_callables[name] for name in submod_names)
-    ]
-    return partial(fn, __vllm_submods__=submods_list)
-
-
-def _node_ref(arg: Any) -> str:
-    """Convert an FX node argument to a source code reference recursively."""
-    if isinstance(arg, torch.fx.Node):
-        return arg.name
-    if isinstance(arg, list):
-        return f"[{', '.join(_node_ref(x) for x in arg)}]"
-    if isinstance(arg, tuple):
-        items = ", ".join(_node_ref(x) for x in arg)
-        return f"({items},)" if len(arg) == 1 else f"({items})"
-    if isinstance(arg, dict):
-        return (
-            "{"
-            + ", ".join(f"{_node_ref(k)}: {_node_ref(v)}" for k, v in arg.items())
-            + "}"
-        )
-    return repr(arg)

vllm/compilation/cuda_graph.py

@@ -290,14 +290,9 @@ class CUDAGraphWrapper:
                     # across layers will make the cudagraph capture very slow.
                     # therefore, we only run gc for the first graph,
                     # and disable gc for the rest of the graphs.
+                    stack.enter_context(patch("gc.collect", lambda: None))
                     stack.enter_context(
-                        patch("gc.collect", lambda *args, **kwargs: None)
-                    )
-                    stack.enter_context(
-                        patch(
-                            "torch.accelerator.empty_cache",
-                            lambda *args, **kwargs: None,
-                        )
+                        patch("torch.accelerator.empty_cache", lambda: None)
                     )
 
                 if self.graph_pool is not None: