[CI] Inline env vars lost by CONTINUE_ON_FAILURE subshell wrapping

Postmerge/nightly/daily builds set CONTINUE_ON_FAILURE=1, which wraps
each YAML command in `(cmd) || CI_OVERALL_STATUS=1`. A standalone
`export VAR=val` executes inside the subshell and the variable is
immediately lost — subsequent commands never see it.

This caused real failures (model_executor fastsafetensors crash from
missing VLLM_WORKER_MULTIPROC_METHOD=spawn) and silent coverage gaps
(rust_frontend tests silently running the Python frontend because
VLLM_USE_RUST_FRONTEND=1 was never set).

Fix: inline env vars as command prefixes (`VAR=val command`) instead
of standalone `export` lines across all 11 affected CI YAML files.

Co-authored-by: Claude <noreply@anthropic.com>
Signed-off-by: Tyler Michael Smith <tyler@tylermsmith.com>
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>

.buildkite/ci_config_intel.yaml

@@ -2,17 +2,17 @@ name: vllm_intel_ci
 job_dirs:
   - ".buildkite/intel_jobs"
 run_all_patterns:
-  - ".buildkite/ci_config_intel.yaml"
-  - ".buildkite/scripts/hardware_ci/run-intel-test.sh"
   - "docker/Dockerfile"
-  - "docker/Dockerfile.xpu"
   - "CMakeLists.txt"
   - "requirements/common.txt"
   - "requirements/xpu.txt"
+  - "requirements/build/cuda.txt"
+  - "requirements/test/cuda.txt"
   - "setup.py"
   - "csrc/"
   - "cmake/"
 run_all_exclude_patterns:
+  - "docker/Dockerfile."
   - "csrc/cpu/"
   - "csrc/rocm/"
   - "cmake/hipify.py"

.buildkite/hardware_tests/amd.yaml

@@ -6,7 +6,6 @@ steps:
   # differ ci_base is rebuilt and pushed automatically.
   - label: "AMD: :docker: ensure ci_base"
     key: ensure-ci-base-amd
-    soft_fail: false
     depends_on: []
     device: amd_cpu
     no_plugin: true
@@ -27,7 +26,6 @@ steps:
 
   - label: "AMD: :docker: build test image and artifacts"
     key: image-build-amd
-    soft_fail: false
     depends_on:
       - ensure-ci-base-amd
     device: amd_cpu

.buildkite/hardware_tests/cpu.yaml

@@ -17,14 +17,12 @@ steps:
   - tests/kernels/test_awq_int4_to_int8.py
   - tests/kernels/quantization/test_cpu_fp8_scaled_mm.py
   - tests/kernels/mamba/cpu/test_cpu_gdn_ops.py
-  - tests/kernels/mamba/test_cpu_short_conv.py
   commands:
     - |
       bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 30m "
       pytest -x -v -s tests/kernels/attention/test_cpu_attn.py
       pytest -x -v -s tests/kernels/moe/test_cpu_fused_moe.py
       pytest -x -v -s tests/kernels/moe/test_cpu_quant_fused_moe.py
-      pytest -x -v -s tests/kernels/mamba/test_cpu_short_conv.py
       pytest -x -v -s tests/kernels/test_onednn.py
       pytest -x -v -s tests/kernels/test_awq_int4_to_int8.py
       pytest -x -v -s tests/kernels/quantization/test_cpu_fp8_scaled_mm.py
@@ -55,7 +53,7 @@ steps:
   - tests/models/language/pooling/
   commands:
     - |
-      bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 50m "
+      bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 40m "
       pytest -x -v -s tests/models/language/generation -m cpu_model
       pytest -x -v -s tests/models/language/pooling -m cpu_model"
 
@@ -70,15 +68,13 @@ steps:
   - vllm/v1/sample/ops/topk_topp_triton.py
   - vllm/v1/sample/ops/topk_topp_sampler.py
   - tests/v1/sample/test_topk_topp_sampler.py
-  - tests/v1/e2e/test_cpu_linear_attn_chunked_prefix.py
   commands:
     - |
       bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 45m "
       uv pip install git+https://github.com/triton-lang/triton-cpu.git@270e696d
       VLLM_USE_V2_MODEL_RUNNER=1 pytest -x -v -s tests/models/language/generation/test_granite.py -m cpu_model
       # TODO: move to CPU-Kernel Tests once triton-cpu has a pre-built wheel
-      pytest -x -v -s tests/v1/sample/test_topk_topp_sampler.py::TestTritonTopkTopp
-      pytest -x -v -s tests/v1/e2e/test_cpu_linear_attn_chunked_prefix.py"
+      pytest -x -v -s tests/v1/sample/test_topk_topp_sampler.py::TestTritonTopkTopp"
 
 - label: CPU-Quantization Model Tests
   depends_on: []
@@ -93,13 +89,11 @@ steps:
   - vllm/model_executor/layers/fused_moe/experts/cpu_moe.py
   - tests/quantization/test_compressed_tensors.py
   - tests/quantization/test_cpu_wna16.py
-  - tests/quantization/test_cpu_w8a8.py
   commands:
     - |
       bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 45m "
       pytest -x -v -s tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_logprobs
-      pytest -x -v -s tests/quantization/test_cpu_wna16.py
-      pytest -x -v -s tests/quantization/test_cpu_w8a8.py"
+      pytest -x -v -s tests/quantization/test_cpu_wna16.py"
       
 - label: CPU-Distributed Tests (PP+TP)
   depends_on: []
@@ -142,7 +136,7 @@ steps:
     - |
       bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 45m "
       pytest -x -v -s tests/models/multimodal/generation --ignore=tests/models/multimodal/generation/test_pixtral.py -m cpu_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
-  parallelism: 4
+  parallelism: 3
 
 - label: "Arm CPU Test"
   depends_on: []

.buildkite/hardware_tests/intel_xpu_ci/test-intel.yaml

@@ -45,7 +45,7 @@ steps:
     agent_tags:
       label: production
       gpu: 1+
-      mem: 24+
+      mem: 16+
     no_plugin: true
     env:
       REGISTRY: "public.ecr.aws/q9t5s3a7"

.buildkite/image_build/image_build_arm64.sh

@@ -21,13 +21,12 @@ else
   exit 0
 fi
 
-# build for arm64 GPU targets: Grace/GH200 (sm_90) and DGX Spark/GB10
-# (sm_121, family-covered by 12.0 under CUDA 13)
+# build (Grace/GH200 is the arm64 GPU target; sm_90)
 docker build --file docker/Dockerfile \
   --platform linux/arm64 \
   --build-arg max_jobs=16 \
   --build-arg nvcc_threads=4 \
-  --build-arg torch_cuda_arch_list="9.0 12.0" \
+  --build-arg torch_cuda_arch_list="9.0" \
   --build-arg USE_SCCACHE=1 \
   --build-arg buildkite_commit="$BUILDKITE_COMMIT" \
   --tag "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64 \

.buildkite/intel_jobs/basic_correctness.yaml

@@ -23,5 +23,4 @@ steps:
       bash .buildkite/scripts/hardware_ci/run-intel-test.sh
       'cd tests &&
       export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
-      pytest -v -s basic_correctness/test_cpu_offload.py &&
       pytest -v -s basic_correctness/test_mem.py::test_end_to_end'

.buildkite/intel_jobs/expert_parallelism_intel.yaml

@@ -1,5 +1,5 @@
 group: Expert Parallelism
-depends_on:
+depends_on: 
   - image-build-xpu
 steps:
 - label: EPLB Algorithm

.buildkite/intel_jobs/lora_intel.yaml

@@ -81,9 +81,7 @@ steps:
       'cd tests &&
       export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
       set -o pipefail &&
-      pytest -v -s lora/test_punica_ops.py::test_kernels &&
-      pytest -v -s lora/test_punica_ops.py::test_kernels_hidden_size &&
-      pytest -v -s lora/test_punica_ops.py::test_add_lora_fused_moe_early_exit'
+      pytest -v -s lora/test_punica_ops.py --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[expand-0-xpu:0-dtype0-3-43264-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype1-1-2049-64-128-16]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-128-1-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-256-1-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-256-8-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels[expand-0-xpu:0-dtype0-3-2049-128-8-16]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-128-8-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels[expand-0-xpu:0-dtype1-1-2049-256-128-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype0-3-64256-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype1-2-29696-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype1-3-49408-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype0-2-16384-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[expand-0-xpu:0-dtype0-2-51328-32-4-4]"'
 
 - label: LoRA Punica FP8/XPU Ops
   timeout_in_minutes: 45
@@ -130,10 +128,10 @@ steps:
       bash .buildkite/scripts/hardware_ci/run-intel-test.sh
       'cd tests &&
       export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
+      (pytest -v -s lora/test_mixtral.py --deselect="tests/lora/test_mixtral.py::test_mixtral_lora[4]" || true) &&
       pytest -v -s lora/test_quant_model.py --deselect="tests/lora/test_quant_model.py::test_quant_model_lora[model0]" --deselect="tests/lora/test_quant_model.py::test_quant_model_lora[model1]" --deselect="tests/lora/test_quant_model.py::test_quant_model_tp_equality[model0]" &&
       pytest -v -s lora/test_transformers_model.py &&
       pytest -v -s lora/test_chatglm3_tp.py &&
-      pytest -v -s lora/test_llama_tp.py::test_llama_lora &&
       pytest -s -v lora/test_minicpmv_tp.py'
 
 - label: LoRA Multimodal

.buildkite/intel_jobs/misc_intel.yaml

@@ -38,7 +38,7 @@ steps:
   agent_tags:
     label: production
     gpu: 1+
-    mem: 24+
+    mem: 16+
   no_plugin: true
   working_dir: "."
   env:
@@ -76,30 +76,6 @@ steps:
       pytest -v -s v1/sample/test_logprobs.py &&
       pytest -v -s v1/sample/test_logprobs_e2e.py'
 
-- label: Basic Models Tests (Initialization)
-  timeout_in_minutes: 60
-  device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 16+
-  no_plugin: true
-  working_dir: "."
-  env:
-    REGISTRY: "public.ecr.aws/q9t5s3a7"
-    REPO: "vllm-ci-test-repo"
-    VLLM_TEST_DEVICE: "xpu"
-  source_file_dependencies:
-    - vllm/
-    - tests/models/test_initialization.py
-    - tests/models/registry.py
-  commands:
-    - >-
-      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-      'export VLLM_XPU_FUSED_MOE_USE_REF=1 &&
-      cd tests &&
-      pytest -v -s models/test_initialization.py::test_can_initialize_large_subset[Eagle3MiniMaxM2ForCausalLM]'
-
 - label: XPU CPU Offload
   timeout_in_minutes: 60
   device: intel_gpu
@@ -127,31 +103,6 @@ steps:
       pytest -v -s v1/kv_offload &&
       pytest -v -s v1/kv_connector/unit/test_offloading_connector.py'
 
-- label: NixlConnector PD accuracy (2 GPUs)
-  timeout_in_minutes: 60
-  num_devices: 2
-  device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 2+
-    mem: 16+
-  no_plugin: true
-  working_dir: "."
-  env:
-    REGISTRY: "public.ecr.aws/q9t5s3a7"
-    REPO: "vllm-ci-test-repo"
-    VLLM_TEST_DEVICE: "xpu"
-  source_file_dependencies:
-    - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
-    - vllm/v1/worker/kv_connector_model_runner_mixin.py
-    - tests/v1/kv_connector/nixl_integration/
-    - vllm/platforms/xpu.py
-  commands:
-    - >-
-      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-      'cd tests &&
-      bash v1/kv_connector/nixl_integration/run_xpu_disagg_accuracy_test.sh'
-
 - label: Regression
   key: regression
   timeout_in_minutes: 30
@@ -182,7 +133,7 @@ steps:
   commands:
   - >-
       bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-      'pip install modelscope\<1.38 &&
+      'pip install modelscope &&
       cd tests &&
       pytest -v -s test_regression.py'
 

.buildkite/intel_jobs/models_distributed_intel.yaml

@@ -1,27 +0,0 @@
-group: Models - Distributed
-depends_on: 
-  - image-build-xpu
-steps:
-- label: Distributed Model Tests (2 GPUs)
-  key: distributed-model-tests-2-gpus
-  timeout_in_minutes: 50
-  device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 2+
-    mem: 16+
-  no_plugin: true
-  working_dir: "."
-  env:
-    REGISTRY: "public.ecr.aws/q9t5s3a7"
-    REPO: "vllm-ci-test-repo"
-    VLLM_TEST_DEVICE: "xpu"
-  source_file_dependencies:
-  - vllm/model_executor/model_loader/sharded_state_loader.py
-  - vllm/model_executor/models/
-  - tests/model_executor/model_loader/test_sharded_state_loader.py
-  commands:
-    - >-
-      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-      'cd tests &&
-      pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m "not slow_test"'

.buildkite/intel_jobs/models_multimodal_intel.yaml

@@ -1,5 +1,5 @@
 group: Models - Multimodal
-depends_on:
+depends_on: 
   - image-build-xpu
 steps:
 - label: "Multi-Modal Models (Standard) 1: qwen2"
@@ -9,7 +9,7 @@ steps:
   agent_tags:
     label: production
     gpu: 1+
-    mem: 24+
+    mem: 16+
   no_plugin: true
   working_dir: "."
   env:
@@ -22,7 +22,7 @@ steps:
   commands:
     - >-
       bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-      'pip install av &&
+      'pip install av git+https://github.com/TIGER-AI-Lab/Mantis.git &&
       cd tests &&
       pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2" &&
       pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model'
@@ -47,7 +47,8 @@ steps:
   commands:
     - >-
       bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-      'cd tests &&
+      'pip install git+https://github.com/TIGER-AI-Lab/Mantis.git &&
+      cd tests &&
       pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model'
 
 - label: "Multi-Modal Models (Standard) 3: llava + qwen2_vl"
@@ -70,7 +71,8 @@ steps:
   commands:
     - >-
       bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-      'cd tests &&
+      'pip install git+https://github.com/TIGER-AI-Lab/Mantis.git &&
+      cd tests &&
       pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma" &&
       pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model'
 
@@ -81,7 +83,7 @@ steps:
   agent_tags:
     label: production
     gpu: 1+
-    mem: 24+
+    mem: 16+
   no_plugin: true
   working_dir: "."
   env:
@@ -94,7 +96,7 @@ steps:
   commands:
     - >-
       bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-      'pip install av &&
+      'pip install av git+https://github.com/TIGER-AI-Lab/Mantis.git &&
       cd tests &&
       pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py  --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/processing'
 
@@ -105,7 +107,7 @@ steps:
   agent_tags:
     label: production
     gpu: 1+
-    mem: 24+
+    mem: 16+
   no_plugin: true
   working_dir: "."
   env:
@@ -119,9 +121,11 @@ steps:
   commands:
     - >-
       bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-      'pip install av matplotlib ftfy &&
+      'pip install av matplotlib ftfy git+https://github.com/TIGER-AI-Lab/Mantis.git &&
       pip install open-clip-torch --no-deps &&
       cd tests &&
       pytest -v -s models/multimodal/processing/test_tensor_schema.py
+      --deselect "tests/models/multimodal/processing/test_tensor_schema.py::test_model_tensor_schema[mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4]"
+      --deselect "tests/models/multimodal/processing/test_tensor_schema.py::test_model_tensor_schema[Qwen/Qwen2.5-Omni-7B-AWQ]"
       --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB'
   parallelism: 4

.buildkite/intel_jobs/quantization.yaml

@@ -1,28 +0,0 @@
-group: Quantization
-depends_on:
-  - image-build-xpu
-steps:
-- label: Quantization
-  key: quantization
-  timeout_in_minutes: 30
-  env:
-    REGISTRY: "public.ecr.aws/q9t5s3a7"
-    REPO: "vllm-ci-test-repo"
-    VLLM_TEST_DEVICE: "xpu"
-  no_plugin: true
-  working_dir: "."
-  device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 16+
-  source_file_dependencies:
-  - csrc/
-  - vllm/model_executor/layers/quantization
-  - tests/quantization
-  commands:
-  # - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py
-  - >-
-    bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-    'VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s tests/quantization/test_per_token_kv_cache.py --deselect="tests/quantization/test_per_token_kv_cache.py::test_triton_unified_attention_per_token_head_scale[int4-16-128-num_heads0-seq_lens1]"'
-

.buildkite/intel_jobs/test-intel.yaml

@@ -42,37 +42,12 @@ steps:
         python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --kv-cache-dtype fp8 &&
         python3 examples/basic/offline_inference/generate.py --model nvidia/Llama-3.1-8B-Instruct-FP8 --block-size 64 --enforce-eager --quantization modelopt --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --max-model-len 4096 &&
         python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --block-size 64 --enforce-eager --max-model-len 8192 &&
-        python3 examples/basic/offline_inference/generate.py --model TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ --block-size 64 --enforce-eager &&
         python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 &&
         python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 --enable-expert-parallel &&
         python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --max-model-len 8192 &&
         VLLM_XPU_FUSED_MOE_USE_REF=1 python3 examples/basic/offline_inference/generate.py --model Qwen/Qwen3-30B-A3B-Instruct-2507-FP8 --enforce-eager -tp 2 --max-model-len 8192 &&
         python3 examples/basic/offline_inference/generate.py --model INCModel/Qwen3-30B-A3B-Instruct-2507-MXFP4-LLMC --enforce-eager -tp 2 --max-model-len 8192
         '
-  - label: "XPU W8A8 FP8 Linear Examples"
-    depends_on:
-      - image-build-xpu
-    timeout_in_minutes: 60
-    device: intel_gpu
-    agent_tags:
-      label: production
-      gpu: 1+
-      mem: 24+
-    no_plugin: true
-    env:
-      REGISTRY: "public.ecr.aws/q9t5s3a7"
-      REPO: "vllm-ci-test-repo"
-      VLLM_TEST_DEVICE: "xpu"
-    source_file_dependencies:
-      - vllm/
-      - .buildkite/intel_jobs/test-intel.yaml
-    commands:
-      - >-
-        bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-        'python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8 --enforce-eager --max-model-len 4096 &&
-        python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model neuralmagic/Llama-3.2-1B-Instruct-FP8-dynamic --enforce-eager --max-model-len 4096 &&
-        python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8 --enforce-eager --max-model-len 4096
-        '
   - label: "XPU V1 test"
     depends_on:
       - image-build-xpu
@@ -81,7 +56,7 @@ steps:
     agent_tags:
       label: production
       gpu: 1+
-      mem: 24+
+      mem: 16+
     no_plugin: true
     env:
       REGISTRY: "public.ecr.aws/q9t5s3a7"
@@ -93,6 +68,7 @@ steps:
       - >-
         bash .buildkite/scripts/hardware_ci/run-intel-test.sh
         'cd tests &&
+        bash v1/kv_connector/nixl_integration/run_xpu_disagg_accuracy_test.sh &&
         pytest -v -s v1/core --ignore=v1/core/test_reset_prefix_cache_e2e.py --ignore=v1/core/test_scheduler_e2e.py &&
         pytest -v -s v1/engine --ignore=v1/engine/test_output_processor.py &&
         pytest -v -s v1/sample --ignore=v1/sample/test_logprobs.py --ignore=v1/sample/test_logprobs_e2e.py -k "not test_topk_only and not test_topp_only and not test_topk_and_topp" &&
@@ -144,27 +120,4 @@ steps:
       - >-
         bash .buildkite/scripts/hardware_ci/run-intel-test.sh
         'cd tests &&
-        pytest -v -s quantization/test_auto_round.py'
-  - label: "XPU compressed tensors FP8 test"
-    depends_on:
-      - image-build-xpu
-    timeout_in_minutes: 60
-    device: intel_gpu
-    agent_tags:
-      label: production
-      gpu: 1+
-      mem: 16+
-    no_plugin: true
-    env:
-      REGISTRY: "public.ecr.aws/q9t5s3a7"
-      REPO: "vllm-ci-test-repo"
-      VLLM_TEST_DEVICE: "xpu"
-    source_file_dependencies:
-      - vllm/
-      - tests/quantization/test_compressed_tensors.py
-      - .buildkite/intel_jobs/test-intel.yaml
-    commands:
-      - >-
-        bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-        'cd tests &&
-        pytest -v -s quantization/test_compressed_tensors.py::test_compressed_tensors_fp8'
+        pytest -v -s quantization/test_auto_round.py'

.buildkite/scripts/ci-bake-rocm.sh

@@ -15,10 +15,9 @@ set -euo pipefail
 
 DEFAULT_REPO_SLUG="vllm-project/vllm"
 DEFAULT_CI_HCL_SOURCE="docker/ci-rocm.hcl"
-DEFAULT_CI_BASE_CONTENT_FILES="requirements/common.txt requirements/rocm.txt requirements/test/rocm.txt docker/Dockerfile.rocm_base docker/ci-rocm.hcl docker/docker-bake-rocm.hcl tools/install_torchcodec_rocm.sh tests/vllm_test_utils .buildkite/scripts/ci-bake-rocm.sh"
+DEFAULT_CI_BASE_CONTENT_FILES="requirements/common.txt requirements/rocm.txt requirements/test/rocm.txt docker/Dockerfile.rocm_base tools/install_torchcodec_rocm.sh tests/vllm_test_utils"
 DEFAULT_CI_BASE_DOCKERFILE="docker/Dockerfile.rocm"
 DEFAULT_CI_BASE_DOCKERFILE_STAGES="base build_rixl build_rocshmem build_deepep mori_base ci_base"
-DEFAULT_CI_BASE_METADATA_VERSION="1"
 IMAGE_EXISTED_BEFORE_BUILD=0
 
 TARGET=""
@@ -526,22 +525,6 @@ get_remote_image_label_with_retry() {
     return 0
 }
 
-remote_ci_base_metadata_is_current() {
-    local image_ref="$1"
-    local metadata_version=""
-
-    metadata_version=$(get_remote_image_label "${image_ref}" "vllm.ci_base.metadata_version")
-    [[ "${metadata_version}" == "${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}" ]]
-}
-
-remote_ci_base_metadata_is_current_with_retry() {
-    local image_ref="$1"
-    local metadata_version=""
-
-    metadata_version=$(get_remote_image_label_with_retry "${image_ref}" "vllm.ci_base.metadata_version")
-    [[ "${metadata_version}" == "${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}" ]]
-}
-
 remote_image_exists() {
     local image_ref="$1"
     docker manifest inspect "${image_ref}" >/dev/null 2>&1
@@ -598,7 +581,6 @@ init_config() {
     CI_BASE_CONTENT_FILES="${CI_BASE_CONTENT_FILES:-${DEFAULT_CI_BASE_CONTENT_FILES}}"
     CI_BASE_DOCKERFILE="${CI_BASE_DOCKERFILE:-${DEFAULT_CI_BASE_DOCKERFILE}}"
     CI_BASE_DOCKERFILE_STAGES="${CI_BASE_DOCKERFILE_STAGES:-${DEFAULT_CI_BASE_DOCKERFILE_STAGES}}"
-    CI_BASE_METADATA_VERSION="${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}"
     CI_BASE_IMAGE_TAG="${CI_BASE_IMAGE_TAG:-rocm/vllm-dev:ci_base}"
     export PYTORCH_ROCM_ARCH
 
@@ -653,10 +635,6 @@ load_ci_hcl() {
     echo "Copied ${CI_HCL_SOURCE} to ${CI_HCL_PATH}"
 }
 
-init_bake_files() {
-    BAKE_FILES=(-f "${VLLM_BAKE_FILE}" -f "${CI_HCL_PATH}")
-}
-
 compute_ci_base_hash_if_needed() {
     if [[ -z "${CI_BASE_CONTENT_FILES:-}" ]]; then
         return 0
@@ -698,14 +676,12 @@ configure_ci_base_image_refs() {
     fi
 
     content_tag=$(ci_base_tag_with_suffix "${stable_tag}" "${CI_BASE_CONTENT_HASH}")
-    CI_BASE_IMAGE_TAG_CONTENT_REF="${content_tag}"
     if [[ -n "${BUILDKITE_COMMIT:-}" ]]; then
         commit_tag=$(ci_base_tag_with_suffix "${stable_tag}" "${BUILDKITE_COMMIT}")
+        CI_BASE_IMAGE_TAG_COMMIT="${commit_tag}"
+        export CI_BASE_IMAGE_TAG_COMMIT
     fi
-    CI_BASE_IMAGE_TAG_COMMIT_REF="${commit_tag}"
 
-    # *_REF is the logical tag recorded in metadata. *_EXTRA is only passed to
-    # bake when that tag is not already the primary tag, avoiding duplicates.
     if should_push_stable_ci_base_tag; then
         primary_tag="${content_tag}"
         CI_BASE_IMAGE_TAG_STABLE="${stable_tag}"
@@ -715,33 +691,19 @@ configure_ci_base_image_refs() {
     fi
     CI_BASE_IMAGE_TAG="${primary_tag}"
     if [[ "${primary_tag}" == "${content_tag}" ]]; then
-        CI_BASE_IMAGE_TAG_CONTENT_EXTRA=""
+        CI_BASE_IMAGE_TAG_CONTENT=""
     else
-        CI_BASE_IMAGE_TAG_CONTENT_EXTRA="${content_tag}"
+        CI_BASE_IMAGE_TAG_CONTENT="${content_tag}"
     fi
-    if [[ -n "${commit_tag}" && "${commit_tag}" != "${primary_tag}" ]]; then
-        CI_BASE_IMAGE_TAG_COMMIT_EXTRA="${commit_tag}"
-    else
-        CI_BASE_IMAGE_TAG_COMMIT_EXTRA=""
-    fi
-    export CI_BASE_IMAGE_TAG
-    export CI_BASE_IMAGE_TAG_COMMIT_EXTRA
-    export CI_BASE_IMAGE_TAG_CONTENT_EXTRA
-    export CI_BASE_IMAGE_TAG_CONTENT_REF
-    export CI_BASE_IMAGE_TAG_COMMIT_REF
-    export CI_BASE_IMAGE_TAG_STABLE
+    export CI_BASE_IMAGE_TAG CI_BASE_IMAGE_TAG_CONTENT CI_BASE_IMAGE_TAG_STABLE
 
     if is_ci_base_target; then
         IMAGE_TAG="${primary_tag}"
         export IMAGE_TAG
 
         echo "ci_base primary image tag: ${CI_BASE_IMAGE_TAG}"
-        if [[ -n "${commit_tag}" ]]; then
-            if [[ "${commit_tag}" == "${primary_tag}" ]]; then
-                echo "ci_base commit image tag: ${commit_tag} (primary)"
-            else
-                echo "ci_base commit image tag: ${commit_tag}"
-            fi
+        if [[ -n "${CI_BASE_IMAGE_TAG_COMMIT:-}" ]]; then
+            echo "ci_base commit image tag: ${CI_BASE_IMAGE_TAG_COMMIT}"
         fi
         echo "ci_base content image tag: ${content_tag}"
         if [[ -n "${CI_BASE_IMAGE_TAG_STABLE}" ]]; then
@@ -766,8 +728,8 @@ ci_base_candidate_refs() {
     printf '%s\n' \
         "${IMAGE_TAG:-}" \
         "${CI_BASE_IMAGE_TAG:-}" \
-        "${CI_BASE_IMAGE_TAG_COMMIT_EXTRA:-}" \
-        "${CI_BASE_IMAGE_TAG_CONTENT_EXTRA:-}" \
+        "${CI_BASE_IMAGE_TAG_COMMIT:-}" \
+        "${CI_BASE_IMAGE_TAG_CONTENT:-}" \
         "${CI_BASE_IMAGE_TAG_STABLE:-}" \
         | awk 'NF && !seen[$0]++'
 }
@@ -781,10 +743,6 @@ find_matching_ci_base_ref() {
         remote_image_exists "${candidate}" || continue
         candidate_hash=$(get_remote_image_label "${candidate}" "vllm.ci_base.content_hash")
         if [[ "${candidate_hash}" == "${CI_BASE_CONTENT_HASH}" ]]; then
-            if ! remote_ci_base_metadata_is_current "${candidate}"; then
-                echo "Found matching ci_base content hash but stale metadata: ${candidate}" >&2
-                continue
-            fi
             printf '%s\n' "${candidate}"
             return 0
         fi
@@ -859,10 +817,6 @@ maybe_skip_existing_image() {
         if [[ -n "${remote_hash}" ]]; then
             echo "Remote ci_base content hash: ${remote_hash:0:16}..."
             if [[ "${remote_hash}" == "${CI_BASE_CONTENT_HASH}" ]]; then
-                if ! remote_ci_base_metadata_is_current "${IMAGE_TAG}"; then
-                    echo "Content hashes match but ci_base metadata is stale; rebuilding to refresh metadata"
-                    return 0
-                fi
                 if ! refresh_ci_base_tags_from_ref "${IMAGE_TAG}"; then
                     echo "ci_base tag refresh failed; rebuilding to push expected tags"
                     return 0
@@ -1044,104 +998,12 @@ prepare_git_cache_metadata() {
     fi
 }
 
-ci_base_metadata_pairs() {
-    local dockerfile="${CI_BASE_DOCKERFILE:-${DEFAULT_CI_BASE_DOCKERFILE}}"
-    local stages="${CI_BASE_DOCKERFILE_STAGES:-${DEFAULT_CI_BASE_DOCKERFILE_STAGES}}"
-    local content_files="${CI_BASE_CONTENT_FILES:-${DEFAULT_CI_BASE_CONTENT_FILES}}"
-    local content_files_hash=""
-    local base_image=""
-    local base_image_digest=""
-    local git_branch=""
-    local -a content_paths=()
-    local -a content_args=()
-
-    read -r -a content_paths <<< "${content_files}"
-    if [[ ${#content_paths[@]} -gt 0 ]]; then
-        content_files_hash=$(compute_content_hash "${content_paths[@]}")
-    fi
-    mapfile -t content_args < <(
-        get_content_arg_names "${dockerfile}" "${stages}" "${CI_BASE_CONTENT_ARGS:-}"
-    )
-
-    base_image=$(resolve_dockerfile_arg_value "${dockerfile}" "BASE_IMAGE")
-    if [[ -n "${base_image}" ]]; then
-        base_image_digest=$(resolve_image_digest "${base_image}")
-    fi
-    git_branch="${BUILDKITE_BRANCH:-${VLLM_BRANCH:-}}"
-
-    metadata_pair "vllm.ci_base.metadata_version" "${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}"
-    metadata_pair "vllm.ci_base.content_hash" "${CI_BASE_CONTENT_HASH:-}"
-    metadata_pair "vllm.ci_base.content_files_hash" "${content_files_hash}"
-    metadata_pair "vllm.ci_base.content_files" "${content_files}"
-    metadata_pair "vllm.ci_base.content_args" "$(join_words "${content_args[@]}")"
-    metadata_pair "vllm.ci_base.dockerfile" "${dockerfile}"
-    metadata_pair "vllm.ci_base.dockerfile_stages" "${stages}"
-    metadata_pair "vllm.ci_base.image.primary" "${CI_BASE_IMAGE_TAG:-}"
-    metadata_pair "vllm.ci_base.image.content" "${CI_BASE_IMAGE_TAG_CONTENT_REF:-${CI_BASE_IMAGE_TAG_CONTENT_EXTRA:-}}"
-    metadata_pair "vllm.ci_base.image.commit" "${CI_BASE_IMAGE_TAG_COMMIT_REF:-${CI_BASE_IMAGE_TAG_COMMIT_EXTRA:-}}"
-    metadata_pair "vllm.ci_base.image.stable" "${CI_BASE_IMAGE_TAG_STABLE:-}"
-    metadata_pair "vllm.ci_base.git_commit" "${BUILDKITE_COMMIT:-}"
-    metadata_pair "vllm.ci_base.git_branch" "${git_branch}"
-    metadata_pair "vllm.ci_base.vllm_branch" "${VLLM_BRANCH:-}"
-    metadata_pair "vllm.ci_base.stable_branch" "${CI_BASE_STABLE_BRANCH:-main}"
-
-    metadata_pair "vllm.rocm.base_image" "${base_image}"
-    metadata_pair "vllm.rocm.base_image_digest" "${base_image_digest}"
-    metadata_pair "vllm.rocm.pytorch_rocm_arch" "${PYTORCH_ROCM_ARCH:-}"
-    metadata_pair "vllm.rocm.nic_backend" "$(resolve_dockerfile_arg_value "${dockerfile}" "NIC_BACKEND")"
-    metadata_pair "vllm.rocm.ainic_version" "$(resolve_dockerfile_arg_value "${dockerfile}" "AINIC_VERSION")"
-    metadata_pair "vllm.rocm.ubuntu_codename" "$(resolve_dockerfile_arg_value "${dockerfile}" "UBUNTU_CODENAME")"
-    metadata_pair "vllm.rocm.rixl_repo" "$(resolve_dockerfile_arg_value "${dockerfile}" "RIXL_REPO")"
-    metadata_pair "vllm.rocm.rixl_commit" "${RIXL_BRANCH:-$(resolve_dockerfile_arg_value "${dockerfile}" "RIXL_BRANCH")}"
-    metadata_pair "vllm.rocm.ucx_repo" "$(resolve_dockerfile_arg_value "${dockerfile}" "UCX_REPO")"
-    metadata_pair "vllm.rocm.ucx_commit" "${UCX_BRANCH:-$(resolve_dockerfile_arg_value "${dockerfile}" "UCX_BRANCH")}"
-    metadata_pair "vllm.rocm.rocshmem_repo" "$(resolve_dockerfile_arg_value "${dockerfile}" "ROCSHMEM_REPO")"
-    metadata_pair "vllm.rocm.rocshmem_commit" "${ROCSHMEM_BRANCH:-$(resolve_dockerfile_arg_value "${dockerfile}" "ROCSHMEM_BRANCH")}"
-    metadata_pair "vllm.rocm.deepep_repo" "$(resolve_dockerfile_arg_value "${dockerfile}" "DEEPEP_REPO")"
-    metadata_pair "vllm.rocm.deepep_commit" "${DEEPEP_BRANCH:-$(resolve_dockerfile_arg_value "${dockerfile}" "DEEPEP_BRANCH")}"
-    metadata_pair "vllm.rocm.deepep_nic" "$(resolve_dockerfile_arg_value "${dockerfile}" "DEEPEP_NIC")"
-    metadata_pair "vllm.rocm.deepep_rocm_arch" "$(resolve_dockerfile_arg_value "${dockerfile}" "DEEPEP_ROCM_ARCH")"
-    metadata_pair "vllm.rocm.rixl_cache_key" "${RIXL_CACHE_KEY:-}"
-    metadata_pair "vllm.rocm.rocshmem_cache_key" "${ROCSHMEM_CACHE_KEY:-}"
-    metadata_pair "vllm.rocm.deepep_cache_key" "${DEEPEP_CACHE_KEY:-}"
-
-    metadata_pair "vllm.buildkite.build_number" "${BUILDKITE_BUILD_NUMBER:-}"
-    metadata_pair "vllm.buildkite.build_id" "${BUILDKITE_BUILD_ID:-}"
-}
-
-write_ci_base_metadata_annotations() {
-    local metadata="$1"
-    local key=""
-    local value=""
-    local annotation=""
-
-    [[ -n "${metadata}" ]] || return 0
-    while IFS=$'\t' read -r key value; do
-        [[ -n "${key}" && -n "${value}" ]] || continue
-        annotation="manifest:${key}=${value}"
-        printf '    "%s",\n' "$(hcl_escape_string "${annotation}")"
-    done <<< "${metadata}"
-}
-
-write_ci_base_metadata_labels() {
-    local metadata="$1"
-    local key=""
-    local value=""
-
-    [[ -n "${metadata}" ]] || return 0
-    while IFS=$'\t' read -r key value; do
-        [[ -n "${key}" && -n "${value}" ]] || continue
-        printf '    "%s" = "%s"\n' \
-            "$(hcl_escape_string "${key}")" \
-            "$(hcl_escape_string "${value}")"
-    done <<< "${metadata}"
-}
-
 write_ci_base_label_override() {
     local target_name=""
-    local metadata=""
     local -a ci_base_targets=()
 
+    BAKE_FILES=(-f "${VLLM_BAKE_FILE}" -f "${CI_HCL_PATH}")
+
     if [[ -z "${CI_BASE_CONTENT_HASH:-}" ]]; then
         return 0
     fi
@@ -1157,23 +1019,16 @@ write_ci_base_label_override() {
         return 0
     fi
 
-    metadata=$(ci_base_metadata_pairs)
-
     : > "${CI_BASE_LABEL_OVERRIDE_PATH}"
     for target_name in "${ci_base_targets[@]}"; do
         cat >> "${CI_BASE_LABEL_OVERRIDE_PATH}" <<EOF
 target "${target_name}" {
   annotations = [
     "manifest:org.opencontainers.image.revision=",
-EOF
-        write_ci_base_metadata_annotations "${metadata}" >> "${CI_BASE_LABEL_OVERRIDE_PATH}"
-        cat >> "${CI_BASE_LABEL_OVERRIDE_PATH}" <<EOF
   ]
   labels = {
     "org.opencontainers.image.revision" = ""
-EOF
-        write_ci_base_metadata_labels "${metadata}" >> "${CI_BASE_LABEL_OVERRIDE_PATH}"
-        cat >> "${CI_BASE_LABEL_OVERRIDE_PATH}" <<EOF
+    "vllm.ci_base.content_hash" = "${CI_BASE_CONTENT_HASH}"
   }
 }
 
@@ -1181,7 +1036,7 @@ EOF
     done
 
     BAKE_FILES+=(-f "${CI_BASE_LABEL_OVERRIDE_PATH}")
-    echo "Appended ci_base metadata label override for targets: ${ci_base_targets[*]}"
+    echo "Appended ci_base content-hash label override for targets: ${ci_base_targets[*]}"
 }
 
 uses_rocm_csrc_cache() {
@@ -1264,18 +1119,6 @@ hcl_escape_string() {
     printf '%s' "${value}"
 }
 
-join_words() {
-    local IFS=" "
-    printf '%s' "$*"
-}
-
-metadata_pair() {
-    local key="$1"
-    local value="${2:-}"
-
-    printf '%s\t%s\n' "${key}" "${value}"
-}
-
 write_hcl_string_list() {
     local indent="$1"
     shift
@@ -1698,13 +1541,7 @@ confirm_remote_image_push() {
 
         remote_hash=$(get_remote_image_label_with_retry "${image_ref}" "vllm.ci_base.content_hash")
         if [[ -n "${remote_hash}" && "${remote_hash}" == "${CI_BASE_CONTENT_HASH}" ]]; then
-            if remote_ci_base_metadata_is_current_with_retry "${image_ref}"; then
-                return 0
-            fi
-
-            echo "Remote image exists with the expected ci_base content hash but stale metadata."
-            echo "  expected metadata version: ${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}"
-            return 1
+            return 0
         fi
 
         echo "Remote image exists but does not have the expected ci_base content hash."
@@ -1887,16 +1724,15 @@ main() {
     print_header
     validate_inputs
     load_ci_hcl
-    init_bake_files
     compute_ci_base_hash_if_needed
     configure_ci_base_image_refs
     maybe_skip_existing_image
     setup_builder
     prepare_git_cache_metadata
+    write_ci_base_label_override
     extract_dependency_pins
     write_rocm_build_arg_override
     compute_dependency_cache_keys
-    write_ci_base_label_override
     compute_rocm_csrc_content_hash_if_needed
     write_rocm_cache_override
     resolve_ci_base_dependency_targets

.buildkite/scripts/hardware_ci/run-amd-test.sh

@@ -367,20 +367,6 @@ remove_docker_container() {
 }
 trap remove_docker_container EXIT
 
-# python_only_compile.sh runs `python setup.py develop` and needs the full repo tree
-# under /vllm-workspace (Dockerfile.rocm test stage: mkdir src && mv vllm).
-# The ROCm wheel artifact tarball only ships a thin tree (tests, etc.), so
-# artifact images cannot satisfy that test — use the full rocm/vllm-ci image.
-_cmd_probe="${VLLM_TEST_COMMANDS:-}"
-if [[ -z "${_cmd_probe}" ]]; then
-  _cmd_probe="$*"
-fi
-if [[ "${VLLM_CI_USE_ARTIFACTS:-0}" == "1" && "${_cmd_probe}" == *python_only_compile.sh* ]]; then
-  echo "INFO: disabling VLLM_CI_USE_ARTIFACTS for python_only_compile (requires full /vllm-workspace tree)"
-  export VLLM_CI_USE_ARTIFACTS=0
-fi
-unset -v _cmd_probe
-
 if ! prepare_artifact_image; then
   echo "Using full ROCm CI image: ${image_name}"
   docker pull "${image_name}" || exit 1
@@ -440,26 +426,6 @@ fi
 
 echo "Final commands: $commands"
 
-# The ROCm test image often ships /vllm-workspace without .git (artifact tarball unpack).
-# tests/standalone_tests/python_only_compile.sh uses merge-base(HEAD, origin/main) for
-# wheels.vllm.ai; compute on the agent (full git checkout) and pass into the container.
-vllm_standalone_merge_base=""
-checkout="${BUILDKITE_BUILD_CHECKOUT_PATH:-}"
-if [[ -z "${checkout}" || ! -d "${checkout}" ]]; then
-  checkout="."
-fi
-# Pass safe.directory per-command (-c) because buildkite runs will always fail
-# the next check on git 2.35.2+ due to mixed uses of root and buildkite-agent/uids.
-if git -c "safe.directory=${checkout}" -C "${checkout}" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
-  vllm_standalone_merge_base="$(
-    git -c "safe.directory=${checkout}" -C "${checkout}" merge-base HEAD origin/main 2>/dev/null || true
-  )"
-fi
-if [[ -z "${vllm_standalone_merge_base}" ]]; then
-  vllm_standalone_merge_base="${BUILDKITE_COMMIT:-}"
-fi
-echo "INFO: passing VLLM_STANDALONE_MERGE_BASE into container: ${vllm_standalone_merge_base}"
-
 MYPYTHONPATH="/vllm-workspace"
 
 container_job_id="${BUILDKITE_JOB_ID:-${BUILDKITE_PARALLEL_JOB:-0}}"
@@ -536,29 +502,13 @@ else
   echo "--- Single-node job"
   echo "Render devices: $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES"
 
-  ulimit_core_hard=$(ulimit -H -c)
-  if [[ "$ulimit_core_hard" == "unlimited" ]]; then
-    # docker run can't pass "unlimited" to --ulimit
-    ulimit_core_hard="-1"
-  fi
-   # Disable core dumps in the ROCm test container unless the ROCm debug agent is enabled
-  coredump_flags="--ulimit core=0:$ulimit_core_hard"
-  if [[ "$commands" == *"ROCm debug agent enabled"* ]]; then
-    # Works around https://github.com/rocm/rocm-systems/issues/6206
-    coredump_flags='-e HSA_COREDUMP_PATTERN="/tmp/gpucore.%p"'
-  else
-    echo "ROCm debug agent not enabled, coredumps are disabled in the test container."
-  fi
-
   docker run \
-    -t -i \
     --device /dev/kfd $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES \
     $RDMA_FLAGS \
     --network=host \
     --shm-size=16gb \
     --group-add "$render_gid" \
     --rm \
-    $coredump_flags \
     -e HF_TOKEN \
     -e "HF_HUB_DOWNLOAD_TIMEOUT=${HF_HUB_DOWNLOAD_TIMEOUT}" \
     -e "HF_HUB_ETAG_TIMEOUT=${HF_HUB_ETAG_TIMEOUT}" \
@@ -575,7 +525,6 @@ else
     -e "VLLM_CACHE_ROOT=${CONTAINER_CACHE_ROOT}/vllm" \
     -e "XDG_CACHE_HOME=${CONTAINER_CACHE_ROOT}/xdg" \
     -e "PYTORCH_ROCM_ARCH=" \
-    -e "VLLM_STANDALONE_MERGE_BASE=${vllm_standalone_merge_base}" \
     --name "${container_name}" \
     "${image_name}" \
     /bin/bash -c "${CONTAINER_PREFLIGHT} && ${commands}"

.buildkite/scripts/hardware_ci/run-cpu-test-arm.sh

@@ -8,7 +8,7 @@ set -ex
 CORE_RANGE=${CORE_RANGE:-0-31}
 OMP_CORE_RANGE=${OMP_CORE_RANGE:-0-31}
 
-export CMAKE_BUILD_PARALLEL_LEVEL=32
+export CMAKE_BUILD_PARALLEL_LEVEL=16
 
 # Setup cleanup
 remove_docker_container() {
@@ -37,10 +37,8 @@ function cpu_tests() {
     pytest -x -v -s tests/kernels/test_onednn.py
     pytest -x -v -s tests/kernels/attention/test_cpu_attn.py
     pytest -x -v -s tests/kernels/core/test_cpu_activation.py
-    pytest -x -v -s tests/kernels/moe/test_cpu_fused_moe.py
-    pytest -x -v -s tests/kernels/mamba/cpu/test_cpu_gdn_ops.py
-    pytest -x -v -s tests/kernels/moe/test_cpu_int4_moe.py
-    pytest -x -v -s tests/kernels/mamba/test_cpu_short_conv.py"
+    pytest -x -v -s tests/kernels/moe/test_moe.py -k test_cpu_fused_moe_basic
+    pytest -x -v -s tests/kernels/mamba/cpu/test_cpu_gdn_ops.py"
 
   # skip tests requiring model downloads if HF_TOKEN is not set
   # due to rate-limits
@@ -64,6 +62,7 @@ function cpu_tests() {
     set -e
     pytest -x -v -s tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_logprobs"
 
+
   # basic online serving
   docker exec cpu-test bash -c '
     set -e

.buildkite/scripts/hardware_ci/run-cpu-test.sh

@@ -7,49 +7,10 @@ set -euox pipefail
 # allow to bind to different cores
 CORE_RANGE=${CORE_RANGE:-48-95}
 NUMA_NODE=${NUMA_NODE:-1}
-AGENT_SLOT=${AGENT_SLOT:-}
-IMAGE_NAME="cpu-test-${NUMA_NODE}${AGENT_SLOT:+-${AGENT_SLOT}}"
+IMAGE_NAME="cpu-test-$NUMA_NODE"
 TIMEOUT_VAL=$1
 TEST_COMMAND=$2
 
-# Disk hygiene knobs. Reclaim space only once the Docker root filesystem crosses
-# DISK_USAGE_THRESHOLD percent, and cap the shared BuildKit cache at
-# BUILDKIT_CACHE_MAX so subsequent builds keep reusing the hottest layers.
-DISK_USAGE_THRESHOLD=${DISK_USAGE_THRESHOLD:-70}
-BUILDKIT_CACHE_MAX=${BUILDKIT_CACHE_MAX:-80GB}
-
-# Reclaim disk only when the host is under pressure. We trim (not purge) the
-# shared BuildKit cache so cross-job/cross-agent reuse stays intact, and only
-# touch dangling images; other agents' uniquely tagged images are left alone.
-prune_if_disk_pressure() {
-    local docker_root disk_usage
-    docker_root=$(docker info -f '{{.DockerRootDir}}' 2>/dev/null || true)
-    if [ -z "$docker_root" ]; then
-        return 0
-    fi
-    disk_usage=$(df "$docker_root" 2>/dev/null | tail -1 | awk '{print $5}' | tr -d '%')
-    if [ "${disk_usage:-0}" -gt "$DISK_USAGE_THRESHOLD" ]; then
-        echo "--- :broom: Disk usage ${disk_usage}% exceeds ${DISK_USAGE_THRESHOLD}%, reclaiming space"
-        docker image prune -f || true
-        docker builder prune -f --keep-storage="$BUILDKIT_CACHE_MAX" || true
-    else
-        echo "Disk usage ${disk_usage:-unknown}% within ${DISK_USAGE_THRESHOLD}% threshold; skipping prune"
-    fi
-}
-
-# Always drop this agent's image once the job ends (the default builder never
-# uses it as a cache source, so removing it costs no rebuild speed), then
-# reclaim space if needed. Guard every docker call with `|| true` so the trap
-# never overrides the test's exit code.
-cleanup() {
-    docker image rm -f "$IMAGE_NAME" || true
-    prune_if_disk_pressure
-}
-trap cleanup EXIT
-
-# Free space up front so a nearly-full host doesn't fail the build.
-prune_if_disk_pressure
-
 # building the docker image
 echo "--- :docker: Building Docker image"
 docker build --progress plain --tag "$IMAGE_NAME" --target vllm-test -f docker/Dockerfile.cpu .

.buildkite/scripts/hardware_ci/run-intel-ci-test.sh

@@ -21,7 +21,6 @@ case "${test_suite}" in
     python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --kv-cache-dtype fp8
     python3 examples/basic/offline_inference/generate.py --model nvidia/Llama-3.1-8B-Instruct-FP8 --block-size 64 --enforce-eager --quantization modelopt --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --max-model-len 4096
     python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --block-size 64 --enforce-eager --max-model-len 8192
-    python3 examples/basic/offline_inference/generate.py --model TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ --block-size 64 --enforce-eager 
     python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2
     python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 --enable-expert-parallel
     python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --max-model-len 8192

.buildkite/scripts/hardware_ci/run-intel-test.sh

@@ -360,7 +360,7 @@ export HF_TOKEN ZE_AFFINITY_MASK
     --ipc=host \
     --privileged \
     -v /dev/dri/by-path:/dev/dri/by-path \
-    -v "/data/huggingface:/root/.cache/huggingface" \
+    -v "${HOME}/.cache/huggingface:/root/.cache/huggingface" \
     --entrypoint='' \
     -e HF_TOKEN \
     -e ZE_AFFINITY_MASK \
@@ -369,7 +369,7 @@ export HF_TOKEN ZE_AFFINITY_MASK
     -e CMDS \
     --name "${container_name}" \
     "${IMAGE}" \
-    bash -c 'set -e; source /opt/intel/oneapi/setvars.sh --force; source /opt/intel/oneapi/ccl/2021.15/env/vars.sh --force; echo "ZE_AFFINITY_MASK is ${ZE_AFFINITY_MASK:-}"; eval "$CMDS"' \
+    bash -c 'set -e; echo "ZE_AFFINITY_MASK is ${ZE_AFFINITY_MASK:-}"; eval "$CMDS"' \
     >/dev/null
 } 9>/tmp/docker-pull.lock
 

.buildkite/scripts/hardware_ci/run-npu-test.sh

@@ -85,7 +85,7 @@ RUN pip config set global.index-url http://cache-service-vllm.nginx-pypi-cache.s
 
 # Install for pytest to make the docker build cache layer always valid
 RUN --mount=type=cache,target=/root/.cache/pip \
-    pip install pytest>=6.0  'modelscope<1.38'
+    pip install pytest>=6.0  modelscope
 
 WORKDIR /workspace/vllm
 

.buildkite/scripts/run-multi-node-test.sh

@@ -109,9 +109,7 @@ run_nodes() {
         if [ "$node" -ne 0 ]; then
             docker exec -d "node$node" /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}"
         else
-            # Allocate a TTY (-t -i) for the foreground head node so its output
-            # keeps ANSI color in the Buildkite log (see run-amd-test.sh).
-            docker exec -t -i "node$node" /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}"
+            docker exec "node$node" /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}"
         fi
     done
 }

.buildkite/scripts/run-rust-frontend-cargo-ci.sh

@@ -8,12 +8,7 @@ if [[ "$MODE" != "style-clippy" && "$MODE" != "test" ]]; then
   exit 2
 fi
 
-if ROOT_DIR="$(git rev-parse --show-toplevel 2>/dev/null)"; then
-  :
-else
-  SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
-  ROOT_DIR="$(cd -- "${SCRIPT_DIR}/../.." && pwd -P)"
-fi
+ROOT_DIR="$(git rev-parse --show-toplevel)"
 cd "$ROOT_DIR"
 
 export CARGO_TERM_COLOR="${CARGO_TERM_COLOR:-always}"
@@ -95,16 +90,6 @@ install_cargo_sort() {
   cargo binstall --no-confirm cargo-sort
 }
 
-install_cargo_deny() {
-  if command -v cargo-deny >/dev/null 2>&1; then
-    return
-  fi
-
-  log_section "Installing cargo-deny"
-  install_cargo_binstall
-  cargo binstall --no-confirm cargo-deny
-}
-
 install_cargo_nextest() {
   if command -v cargo-nextest >/dev/null 2>&1; then
     return
@@ -157,7 +142,6 @@ PY
 
 run_style_clippy() {
   install_cargo_sort
-  install_cargo_deny
 
   log_section "Checking Rust formatting"
   cargo fmt --manifest-path rust/Cargo.toml --all -- --check
@@ -165,13 +149,6 @@ run_style_clippy() {
   log_section "Checking Cargo.toml ordering"
   cargo sort --workspace --check rust
 
-  log_section "Checking Rust dependency bans"
-  cargo deny \
-    --manifest-path rust/Cargo.toml \
-    check \
-    --config rust/deny.toml \
-    bans
-
   log_section "Running clippy"
   cargo clippy \
     --manifest-path rust/Cargo.toml \

.buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_prefetch_offload.sh

@@ -33,14 +33,6 @@ if [[ -n "${ATTENTION_BACKEND:-}" ]]; then
   EXTRA_ARGS+=(--attention-backend "${ATTENTION_BACKEND}")
 fi
 
-# ROCm: run eager to avoid intermittent HIP-graph decode corruption.
-# See https://github.com/ROCm/clr/issues/279
-# TODO(aarushjain29): Revert after TheRock 7.14
-if command -v rocm-smi &> /dev/null || command -v amd-smi &> /dev/null || [[ -d /opt/rocm ]] || [[ -n "${ROCM_PATH:-}" ]]; then
-  echo "ROCm platform detected: adding --enforce-eager to avoid HIP-graph decode corruption"
-  EXTRA_ARGS+=(--enforce-eager)
-fi
-
 cleanup() {
   if [[ -n "${SERVER_PID:-}" ]] && kill -0 "${SERVER_PID}" 2>/dev/null; then
     kill "${SERVER_PID}" 2>/dev/null || true

.buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_dp4_async_eplb.sh

@@ -18,10 +18,6 @@ wait_for_server() {
 
 MODEL="Qwen/Qwen3-30B-A3B-FP8"
 BACK="allgather_reducescatter"
-if command -v rocm-smi &> /dev/null || [[ -d /opt/rocm ]] || [[ -n "${ROCM_PATH:-}" ]]; then
-  # Disable MOE padding for ROCm since it is causing eplb to fail.
-  export VLLM_ROCM_MOE_PADDING=0
-fi
 
 cleanup() {
   if [[ -n "${SERVER_PID:-}" ]] && kill -0 "${SERVER_PID}" 2>/dev/null; then

.buildkite/test_areas/basic_correctness.yaml

@@ -12,13 +12,6 @@ steps:
   - tests/basic_correctness/test_cpu_offload
   - tests/basic_correctness/test_mem.py
   commands:
-  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-  - pytest -v -s basic_correctness/test_mem.py
-  - pytest -v -s basic_correctness/test_basic_correctness.py
-  - pytest -v -s basic_correctness/test_cpu_offload.py
-  mirror:
-    amd:
-      device: mi325_1
-      timeout_in_minutes: 50
-      depends_on:
-      - image-build-amd
+  - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s basic_correctness/test_mem.py
+  - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s basic_correctness/test_basic_correctness.py
+  - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s basic_correctness/test_cpu_offload.py

.buildkite/test_areas/benchmarks.yaml

@@ -11,11 +11,6 @@ steps:
   - tests/benchmarks/
   commands:
   - pytest -v -s benchmarks/
-  mirror:
-    amd:
-      device: mi300_1
-      depends_on:
-      - image-build-amd
 
 - label: Attention Benchmarks Smoke Test (B200)
   key: attention-benchmarks-smoke-test-b200

.buildkite/test_areas/compile.yaml

@@ -14,8 +14,7 @@ steps:
   - vllm/v1/cudagraph_dispatcher.py
   - tests/compile/correctness_e2e/test_sequence_parallel.py
   commands:
-  - export VLLM_TEST_CLEAN_GPU_MEMORY=1
-  - pytest -v -s tests/compile/correctness_e2e/test_sequence_parallel.py
+  - VLLM_TEST_CLEAN_GPU_MEMORY=1 pytest -v -s tests/compile/correctness_e2e/test_sequence_parallel.py
 
 - label: Sequence Parallel Correctness Tests (2xH100)
   key: sequence-parallel-correctness-tests-2xh100
@@ -25,8 +24,7 @@ steps:
   optional: true
   num_devices: 2
   commands:
-  - export VLLM_TEST_CLEAN_GPU_MEMORY=1
-  - pytest -v -s tests/compile/correctness_e2e/test_sequence_parallel.py
+  - VLLM_TEST_CLEAN_GPU_MEMORY=1 pytest -v -s tests/compile/correctness_e2e/test_sequence_parallel.py
 
 - label: AsyncTP Correctness Tests (2xH100)
   key: asynctp-correctness-tests-2xh100
@@ -36,8 +34,7 @@ steps:
   optional: true
   num_devices: 2
   commands:
-  - export VLLM_TEST_CLEAN_GPU_MEMORY=1
-  - pytest -v -s tests/compile/correctness_e2e/test_async_tp.py
+  - VLLM_TEST_CLEAN_GPU_MEMORY=1 pytest -v -s tests/compile/correctness_e2e/test_async_tp.py
 
 - label: AsyncTP Correctness Tests (B200)
   key: asynctp-correctness-tests-b200
@@ -47,8 +44,7 @@ steps:
   optional: true
   num_devices: 2
   commands:
-  - export VLLM_TEST_CLEAN_GPU_MEMORY=1
-  - pytest -v -s tests/compile/correctness_e2e/test_async_tp.py
+  - VLLM_TEST_CLEAN_GPU_MEMORY=1 pytest -v -s tests/compile/correctness_e2e/test_async_tp.py
 
 - label: Distributed Compile Unit Tests (2xH100)
   key: distributed-compile-unit-tests-2xh100
@@ -61,8 +57,7 @@ steps:
   - vllm/model_executor/layers
   - tests/compile/passes/distributed/
   commands:
-  - export VLLM_TEST_CLEAN_GPU_MEMORY=1
-  - pytest -s -v tests/compile/passes/distributed
+  - VLLM_TEST_CLEAN_GPU_MEMORY=1 pytest -s -v tests/compile/passes/distributed
 
 - label: Fusion and Compile Unit Tests (2xB200)
   key: fusion-and-compile-unit-tests-2xb200

.buildkite/test_areas/cuda.yaml

@@ -2,8 +2,8 @@ group: CUDA
 depends_on: 
   - image-build
 steps:
-- label: Platform Tests
-  key: platform-tests
+- label: Platform Tests (CUDA)
+  key: platform-tests-cuda
   timeout_in_minutes: 15
   device: h200_18gb
   source_file_dependencies:

.buildkite/test_areas/disaggregated.yaml

@@ -13,20 +13,6 @@ steps:
   commands:
     - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
     - bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
-  mirror:
-    amd:
-      device: mi300_4
-      timeout_in_minutes: 110
-      depends_on:
-        - image-build-amd
-      source_file_dependencies:
-        - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
-        - tests/v1/kv_connector/nixl_integration/
-        - vllm/platforms/rocm.py
-      commands:
-        - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
-        - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
-
 - label: Distributed FlashInfer NixlConnector PD accuracy (4 GPUs)
   key: distributed-flashinfer-nixlconnector-pd-accuracy-4-gpus
   timeout_in_minutes: 30
@@ -50,19 +36,6 @@ steps:
   commands:
     - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
     - DP_EP=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
-  mirror:
-    amd:
-      device: mi300_4
-      timeout_in_minutes: 50
-      depends_on:
-        - image-build-amd
-      source_file_dependencies:
-        - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
-        - tests/v1/kv_connector/nixl_integration/
-        - vllm/platforms/rocm.py
-      commands:
-        - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
-        - DP_EP=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
 
 - label: CrossLayer KV layout Distributed NixlConnector PD accuracy tests (4 GPUs)
   key: crosslayer-kv-layout-distributed-nixlconnector-pd-accuracy-tests-4-gpus
@@ -75,19 +48,6 @@ steps:
   commands:
     - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
     - CROSS_LAYERS_BLOCKS=True bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
-  mirror:
-    amd:
-      device: mi300_4
-      timeout_in_minutes: 110
-      depends_on:
-        - image-build-amd
-      source_file_dependencies:
-        - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
-        - tests/v1/kv_connector/nixl_integration/
-        - vllm/platforms/rocm.py
-      commands:
-        - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
-        - CROSS_LAYERS_BLOCKS=True ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
 
 - label: Hybrid SSM NixlConnector PD accuracy tests (4 GPUs)
   key: hybrid-ssm-nixlconnector-pd-accuracy-tests-4-gpus
@@ -100,19 +60,6 @@ steps:
   commands:
     - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
     - HYBRID_SSM=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
-  mirror:
-    amd:
-      device: mi300_4
-      timeout_in_minutes: 60
-      depends_on:
-        - image-build-amd
-      source_file_dependencies:
-        - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
-        - tests/v1/kv_connector/nixl_integration/
-        - vllm/platforms/rocm.py
-      commands:
-        - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
-        - HYBRID_SSM=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
 
 - label: Hybrid SSM NixlConnector PD prefix cache test (2 GPUs)
   key: hybrid-ssm-nixlconnector-pd-prefix-cache-2-gpus
@@ -156,20 +103,6 @@ steps:
   commands:
     - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
     - bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh
-  mirror:
-    amd:
-      device: mi300_2
-      timeout_in_minutes: 60
-      depends_on:
-        - image-build-amd
-      source_file_dependencies:
-        - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
-        - vllm/v1/worker/kv_connector_model_runner_mixin.py
-        - tests/v1/kv_connector/nixl_integration/
-        - vllm/platforms/rocm.py
-      commands:
-        - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
-        - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh
 
 - label: MultiConnector (Nixl+Offloading) PD edge cases (2 GPUs)
   key: multiconnector-nixl-offloading-pd-edge-cases-2-gpus

.buildkite/test_areas/distributed.yaml

@@ -32,26 +32,10 @@ steps:
   - tests/entrypoints/openai/test_multi_api_servers.py
   commands:
   # https://github.com/NVIDIA/nccl/issues/1838
-  - export NCCL_CUMEM_HOST_ENABLE=0
-  - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py
-  - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
-  - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py
-  - DP_SIZE=2 pytest -v -s entrypoints/openai/test_multi_api_servers.py
-  mirror:
-    amd:
-      device: mi300_2
-      depends_on:
-      - image-build-amd
-      source_file_dependencies:
-      - vllm/distributed/
-      - vllm/engine/
-      - vllm/executor/
-      - vllm/worker/worker_base.py
-      - vllm/v1/engine/
-      - vllm/v1/worker/
-      - tests/v1/distributed
-      - tests/entrypoints/openai/test_multi_api_servers.py
-      - vllm/platforms/rocm.py
+  - NCCL_CUMEM_HOST_ENABLE=0 TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py
+  - NCCL_CUMEM_HOST_ENABLE=0 TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
+  - NCCL_CUMEM_HOST_ENABLE=0 TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py
+  - NCCL_CUMEM_HOST_ENABLE=0 DP_SIZE=2 pytest -v -s entrypoints/openai/test_multi_api_servers.py
 
 - label: Distributed Compile + RPC Tests (2 GPUs)
   key: distributed-compile-rpc-tests-2-gpus
@@ -71,10 +55,9 @@ steps:
   - tests/entrypoints/llm/test_collective_rpc.py
   commands:
   # https://github.com/NVIDIA/nccl/issues/1838
-  - export NCCL_CUMEM_HOST_ENABLE=0
-  - pytest -v -s entrypoints/llm/test_collective_rpc.py
-  - pytest -v -s ./compile/fullgraph/test_basic_correctness.py
-  - pytest -v -s ./compile/test_wrapper.py
+  - NCCL_CUMEM_HOST_ENABLE=0 pytest -v -s entrypoints/llm/test_collective_rpc.py
+  - NCCL_CUMEM_HOST_ENABLE=0 pytest -v -s ./compile/fullgraph/test_basic_correctness.py
+  - NCCL_CUMEM_HOST_ENABLE=0 pytest -v -s ./compile/test_wrapper.py
 
 - label: Distributed Torchrun + Shutdown Tests (2 GPUs)
   key: distributed-torchrun-shutdown-tests-2-gpus
@@ -93,11 +76,10 @@ steps:
   - tests/v1/worker/test_worker_memory_snapshot.py
   commands:
   # https://github.com/NVIDIA/nccl/issues/1838
-  - export NCCL_CUMEM_HOST_ENABLE=0
-  - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed'
-  - VLLM_TEST_SAME_HOST=1 VLLM_TEST_WITH_DEFAULT_DEVICE_SET=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed'
-  - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s v1/shutdown
-  - pytest -v -s v1/worker/test_worker_memory_snapshot.py
+  - NCCL_CUMEM_HOST_ENABLE=0 VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed'
+  - NCCL_CUMEM_HOST_ENABLE=0 VLLM_TEST_SAME_HOST=1 VLLM_TEST_WITH_DEFAULT_DEVICE_SET=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed'
+  - NCCL_CUMEM_HOST_ENABLE=0 CUDA_VISIBLE_DEVICES=0,1 pytest -v -s v1/shutdown
+  - NCCL_CUMEM_HOST_ENABLE=0 pytest -v -s v1/worker/test_worker_memory_snapshot.py
 
 - label: Distributed Torchrun + Examples (4 GPUs)
   key: distributed-torchrun-examples-4-gpus
@@ -112,24 +94,23 @@ steps:
   - tests/examples/features/data_parallel/data_parallel_offline.py
   commands:
   # https://github.com/NVIDIA/nccl/issues/1838
-  - export NCCL_CUMEM_HOST_ENABLE=0
   # test with torchrun tp=2 and external_dp=2
-  - torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example.py
+  - NCCL_CUMEM_HOST_ENABLE=0 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example.py
   # test with torchrun tp=2 and pp=2
-  - PP_SIZE=2 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example.py
+  - NCCL_CUMEM_HOST_ENABLE=0 PP_SIZE=2 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example.py
   # test with torchrun tp=4 and dp=1
-  - TP_SIZE=4 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
+  - NCCL_CUMEM_HOST_ENABLE=0 TP_SIZE=4 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
   # test with torchrun tp=2, pp=2 and dp=1
-  - PP_SIZE=2 TP_SIZE=2 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
+  - NCCL_CUMEM_HOST_ENABLE=0 PP_SIZE=2 TP_SIZE=2 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
   # test with torchrun tp=1 and dp=4 with ep
-  - DP_SIZE=4 ENABLE_EP=1 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
+  - NCCL_CUMEM_HOST_ENABLE=0 DP_SIZE=4 ENABLE_EP=1 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
   # test with torchrun tp=2 and dp=2 with ep
-  - TP_SIZE=2 DP_SIZE=2 ENABLE_EP=1 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
+  - NCCL_CUMEM_HOST_ENABLE=0 TP_SIZE=2 DP_SIZE=2 ENABLE_EP=1 torchrun --nproc-per-node=4 tests/distributed/test_torchrun_example_moe.py
   # test with internal dp
-  - python3 examples/features/data_parallel/data_parallel_offline.py --enforce-eager
+  - NCCL_CUMEM_HOST_ENABLE=0 python3 examples/features/data_parallel/data_parallel_offline.py --enforce-eager
   # rlhf examples
-  - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_nccl.py
-  - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_ipc.py
+  - NCCL_CUMEM_HOST_ENABLE=0 VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_nccl.py
+  - NCCL_CUMEM_HOST_ENABLE=0 VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_ipc.py
 
 - label: Distributed DP Tests (4 GPUs)
   key: distributed-dp-tests-4-gpus
@@ -143,14 +124,13 @@ steps:
   - tests/distributed/test_utils
   commands:
   # https://github.com/NVIDIA/nccl/issues/1838
-  - export NCCL_CUMEM_HOST_ENABLE=0
-  - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py
-  - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
-  - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py
-  - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_internal_lb_dp.py
-  - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_hybrid_lb_dp.py
-  - pytest -v -s v1/engine/test_engine_core_client.py::test_kv_cache_events_dp
-  - pytest -v -s distributed/test_utils.py
+  - NCCL_CUMEM_HOST_ENABLE=0 TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py
+  - NCCL_CUMEM_HOST_ENABLE=0 TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
+  - NCCL_CUMEM_HOST_ENABLE=0 TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py
+  - NCCL_CUMEM_HOST_ENABLE=0 TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_internal_lb_dp.py
+  - NCCL_CUMEM_HOST_ENABLE=0 TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_hybrid_lb_dp.py
+  - NCCL_CUMEM_HOST_ENABLE=0 pytest -v -s v1/engine/test_engine_core_client.py::test_kv_cache_events_dp
+  - NCCL_CUMEM_HOST_ENABLE=0 pytest -v -s distributed/test_utils.py
 
 - label: Distributed Compile + Comm (4 GPUs)
   key: distributed-compile-comm-4-gpus
@@ -166,16 +146,15 @@ steps:
   - tests/distributed/test_multiproc_executor.py
   commands:
   # https://github.com/NVIDIA/nccl/issues/1838
-  - export NCCL_CUMEM_HOST_ENABLE=0
-  - pytest -v -s compile/fullgraph/test_basic_correctness.py
-  - pytest -v -s distributed/test_pynccl.py
-  - pytest -v -s distributed/test_events.py
-  - pytest -v -s distributed/test_symm_mem_allreduce.py
+  - NCCL_CUMEM_HOST_ENABLE=0 pytest -v -s compile/fullgraph/test_basic_correctness.py
+  - NCCL_CUMEM_HOST_ENABLE=0 pytest -v -s distributed/test_pynccl.py
+  - NCCL_CUMEM_HOST_ENABLE=0 pytest -v -s distributed/test_events.py
+  - NCCL_CUMEM_HOST_ENABLE=0 pytest -v -s distributed/test_symm_mem_allreduce.py
   # test multi-node TP with multiproc executor (simulated on single node)
-  - pytest -v -s distributed/test_multiproc_executor.py::test_multiproc_executor_multi_node
+  - NCCL_CUMEM_HOST_ENABLE=0 pytest -v -s distributed/test_multiproc_executor.py::test_multiproc_executor_multi_node
 
-- label: Distributed Tests (8xH100)
-  key: distributed-tests-8xh100
+- label: Distributed Tests (8 GPUs)(H100)
+  key: distributed-tests-8-gpus-h100
   timeout_in_minutes: 10
   device: h100
   num_devices: 8
@@ -191,12 +170,11 @@ steps:
 
   commands:
   # https://github.com/NVIDIA/nccl/issues/1838
-  - export NCCL_CUMEM_HOST_ENABLE=0
   # test with torchrun tp=2 and dp=4 with ep
-  - torchrun --nproc-per-node=8 ../examples/features/torchrun/torchrun_dp_example_offline.py --tp-size=2 --pp-size=1 --dp-size=4 --enable-ep
+  - NCCL_CUMEM_HOST_ENABLE=0 torchrun --nproc-per-node=8 ../examples/features/torchrun/torchrun_dp_example_offline.py --tp-size=2 --pp-size=1 --dp-size=4 --enable-ep
 
-- label: Distributed Tests (4xA100)
-  key: distributed-tests-4xa100
+- label: Distributed Tests (4 GPUs)(A100)
+  key: distributed-tests-4-gpus-a100
   device: a100
   optional: true
   num_devices: 4
@@ -210,8 +188,8 @@ steps:
   - TARGET_TEST_SUITE=A100 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
   - pytest -v -s -x lora/test_mixtral.py
 
-- label: Distributed Tests (2xH100-2xMI300)
-  key: distributed-tests-2xh100-2xmi300
+- label: Distributed Tests (2 GPUs)(H100)
+  key: distributed-tests-2-gpus-h100
   timeout_in_minutes: 15
   device: h100
   optional: true
@@ -225,15 +203,15 @@ steps:
     - VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer.py
     - pytest -v -s tests/distributed/test_packed_tensor.py
 
-- label: Distributed Tests (2xB200)
-  key: distributed-tests-2xb200
+- label: Distributed Tests (2 GPUs)(B200)
+  key: distributed-tests-2-gpus-b200
   device: b200-k8s
   optional: true
   working_dir: "/vllm-workspace/"
   num_devices: 2
   commands:
     - pytest -v -s tests/distributed/test_context_parallel.py
-    - pytest -v -s tests/distributed/test_nccl_symm_mem.py
+    - pytest -v -s tests/distributed/test_nccl_symm_mem_allreduce.py
     - pytest -v -s tests/v1/distributed/test_dbo.py
     - pytest -v -s tests/distributed/test_mnnvl_alltoall.py
 
@@ -286,9 +264,7 @@ steps:
   - tests/distributed/test_pipeline_parallel.py
   - tests/basic_correctness/test_basic_correctness.py
   commands:
-  - export VLLM_USE_RAY_V2_EXECUTOR_BACKEND=1
-  - export NCCL_CUMEM_HOST_ENABLE=0
-  - pytest -v -s distributed/test_ray_v2_executor.py
-  - pytest -v -s distributed/test_ray_v2_executor_e2e.py
-  - pytest -v -s distributed/test_pipeline_parallel.py -k "ray"
-  - TARGET_TEST_SUITE=L4 pytest -v -s basic_correctness/test_basic_correctness.py -k "ray"
+  - VLLM_USE_RAY_V2_EXECUTOR_BACKEND=1 NCCL_CUMEM_HOST_ENABLE=0 pytest -v -s distributed/test_ray_v2_executor.py
+  - VLLM_USE_RAY_V2_EXECUTOR_BACKEND=1 NCCL_CUMEM_HOST_ENABLE=0 pytest -v -s distributed/test_ray_v2_executor_e2e.py
+  - VLLM_USE_RAY_V2_EXECUTOR_BACKEND=1 NCCL_CUMEM_HOST_ENABLE=0 pytest -v -s distributed/test_pipeline_parallel.py -k "ray"
+  - VLLM_USE_RAY_V2_EXECUTOR_BACKEND=1 NCCL_CUMEM_HOST_ENABLE=0 TARGET_TEST_SUITE=L4 pytest -v -s basic_correctness/test_basic_correctness.py -k "ray"

.buildkite/test_areas/e2e_integration.yaml

@@ -2,8 +2,8 @@ group: E2E Integration
 depends_on: 
   - image-build
 steps:
-- label: DeepSeek V2-Lite Sync EPLB Accuracy (4xH100)
-  key: deepseek-v2-lite-sync-eplb-accuracy-4xh100
+- label: DeepSeek V2-Lite Sync EPLB Accuracy
+  key: deepseek-v2-lite-sync-eplb-accuracy
   timeout_in_minutes: 60
   device: h100
   optional: true
@@ -12,8 +12,8 @@ steps:
   commands:
   - bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh 0.25 200 8010
 
-- label: Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy (4xH100)
-  key: qwen3-30b-a3b-fp8-block-sync-eplb-accuracy-4xh100
+- label: Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy
+  key: qwen3-30b-a3b-fp8-block-sync-eplb-accuracy
   timeout_in_minutes: 60
   device: h100
   optional: true
@@ -22,8 +22,8 @@ steps:
   commands:
   - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020
 
-- label: Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy (2xB200)
-  key: qwen3-30b-a3b-fp8-block-sync-eplb-accuracy-2xb200
+- label: Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy (B200)
+  key: qwen3-30b-a3b-fp8-block-sync-eplb-accuracy-b200
   timeout_in_minutes: 60
   device: b200-k8s
   optional: true

.buildkite/test_areas/engine.yaml

@@ -112,11 +112,6 @@ steps:
   commands:
     # Only run tests that need exactly 2 GPUs
     - pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "tensor_parallelism"
-  mirror:
-    amd:
-      device: mi300_2
-      depends_on:
-      - image-build-amd
 
 - label: V1 e2e (4 GPUs)
   key: v1-e2e-4-gpus

.buildkite/test_areas/entrypoints.yaml

@@ -22,15 +22,12 @@ steps:
   - vllm/
   - tests/entrypoints/llm
   commands:
-  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-  - pytest -v -s entrypoints/llm --ignore=entrypoints/llm/test_generate.py --ignore=entrypoints/llm/test_collective_rpc.py --ignore=entrypoints/llm/offline_mode
-  - pytest -v -s entrypoints/llm/test_generate.py # it needs a clean process
-  - pytest -v -s entrypoints/llm/offline_mode # Needs to avoid interference with other tests
+  - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s entrypoints/llm --ignore=entrypoints/llm/test_generate.py --ignore=entrypoints/llm/test_collective_rpc.py --ignore=entrypoints/llm/offline_mode
+  - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s entrypoints/llm/test_generate.py # it needs a clean process
+  - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s entrypoints/llm/offline_mode # Needs to avoid interference with other tests
   mirror:
     amd:
       device: mi325_1
-      # TODO(akaratza): Test after Torch >= 2.12 bump
-      soft_fail: true
       depends_on:
       - image-build-amd
 
@@ -42,12 +39,9 @@ steps:
   source_file_dependencies:
   - vllm/
   - tests/entrypoints/serve
-  - tests/entrypoints/scale_out
   commands:
-  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-  - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc
-  - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc
-  - pytest -v -s entrypoints/scale_out
+  - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc
+  - VLLM_WORKER_MULTIPROC_METHOD=spawn PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc
   mirror:
     amd:
       device: mi325_1
@@ -63,8 +57,7 @@ steps:
   - tests/entrypoints/openai
   - tests/entrypoints/test_chat_utils
   commands:
-  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-  - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/correctness
+  - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s entrypoints/openai --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/correctness
   mirror:
     amd:
       device: mi325_1
@@ -81,9 +74,8 @@ steps:
   - tests/entrypoints/openai
   - tests/entrypoints/test_chat_utils
   commands:
-  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-  - pytest -v -s entrypoints/openai/chat_completion
-  - pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py
+  - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s entrypoints/openai/chat_completion
+  - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py
   mirror:
     amd:
       device: mi325_1
@@ -132,8 +124,7 @@ steps:
   - vllm/
   - tests/entrypoints/speech_to_text
   commands:
-  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-  - pytest -v -s entrypoints/speech_to_text
+  - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s entrypoints/speech_to_text
 
 - label: Entrypoints Integration (Multimodal)
   device: h200_35gb
@@ -144,8 +135,7 @@ steps:
   - vllm/
   - tests/entrypoints/multimodal
   commands:
-  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-  - pytest -v -s entrypoints/multimodal
+  - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s entrypoints/multimodal
 
 - label: Entrypoints Integration (Pooling)
   key: entrypoints-integration-pooling
@@ -155,8 +145,7 @@ steps:
   - vllm/
   - tests/entrypoints/pooling
   commands:
-  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-  - pytest -v -s entrypoints/pooling
+  - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s entrypoints/pooling
 
 - label: OpenAI API Correctness
   key: openai-api-correctness

.buildkite/test_areas/expert_parallelism.yaml

@@ -14,16 +14,6 @@ steps:
   commands:
   - pytest -v -s distributed/test_eplb_algo.py
   - pytest -v -s distributed/test_eplb_utils.py
-  mirror:
-    amd:
-      device: mi300_1
-      depends_on:
-      - image-build-amd
-      source_file_dependencies:
-      - vllm/distributed/eplb
-      - tests/distributed/test_eplb_algo.py
-      - tests/distributed/test_eplb_utils.py
-      - vllm/platforms/rocm.py
 
 - label: EPLB Execution # 17min
   key: eplb-execution

.buildkite/test_areas/kernels.yaml

@@ -47,10 +47,8 @@ steps:
   - csrc/fused_deepseek_v4_qnorm_rope_kv_insert_kernel.cu
   - vllm/models/deepseek_v4/common/ops/
   - tests/kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py
-  - tests/kernels/test_top_k_per_row.py # it runs on Blackwell too - some kernels have arch-specific optimizations
   commands:
     - pytest -v -s kernels/test_fused_deepseek_v4_*.py
-    - pytest -v -s kernels/test_top_k_per_row.py
 
 - label: Deepseek V4 Kernel Test (B200)
   key: deepseek-v4-kernel-test-b200
@@ -274,8 +272,8 @@ steps:
     - pytest -v -s kernels/helion/
 
  
-- label: Kernels FP8 MoE Test (1xH100)
-  key: kernels-fp8-moe-test-1xh100
+- label: Kernels FP8 MoE Test (1 H100)
+  key: kernels-fp8-moe-test-1-h100
   timeout_in_minutes: 90
   device: h100
   num_devices: 1
@@ -291,8 +289,8 @@ steps:
     - pytest -v -s kernels/moe/test_triton_moe_no_act_mul.py
     - pytest -v -s kernels/moe/test_triton_moe_ptpc_fp8.py
 
-- label: Kernels FP8 MoE Test (2xH100)
-  key: kernels-fp8-moe-test-2xh100
+- label: Kernels FP8 MoE Test (2 H100s)
+  key: kernels-fp8-moe-test-2-h100s
   timeout_in_minutes: 90
   device: h100
   num_devices: 2

.buildkite/test_areas/lm_eval.yaml

@@ -28,8 +28,7 @@ steps:
       - vllm/_aiter_ops.py
       - vllm/platforms/rocm.py
 
-# - label: LM Eval Large Models (4xA100)
-#   key: lm-eval-large-models-4xa100
+# - label: LM Eval Large Models (4 GPUs)(A100)
 #   device: a100
 #   optional: true
 #   num_devices: 4
@@ -41,8 +40,8 @@ steps:
 #   - export VLLM_WORKER_MULTIPROC_METHOD=spawn
 #   - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large.txt --tp-size=4
 
-- label: LM Eval Large Models (4xH100)
-  key: lm-eval-large-models-4xh100
+- label: LM Eval Large Models (4 GPUs)(H100)
+  key: lm-eval-large-models-4-gpus-h100
   device: h100
   optional: true
   num_devices: 4
@@ -51,11 +50,10 @@ steps:
   - csrc/
   - vllm/model_executor/layers/quantization
   commands:
-    - export VLLM_USE_DEEP_GEMM=0  # We found Triton is faster than DeepGEMM for H100
-    - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-hopper.txt --tp-size=4
+    - VLLM_USE_DEEP_GEMM=0 pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-hopper.txt --tp-size=4  # Triton is faster than DeepGEMM for H100
 
-- label: LM Eval Small Models (1xB200)
-  key: lm-eval-small-models-1xb200
+- label: LM Eval Small Models (B200)
+  key: lm-eval-small-models-b200
   timeout_in_minutes: 120
   device: b200-k8s
   optional: true
@@ -65,21 +63,8 @@ steps:
   commands:
   - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell.txt
 
-- label: LM Eval Small Models Distributed (2xB200)
-  key: lm-eval-small-models-distributed-2xb200
-  timeout_in_minutes: 120
-  device: b200-k8s
-  num_devices: 2
-  optional: true
-  source_file_dependencies:
-  - csrc/
-  - vllm/model_executor/layers/quantization
-  autorun_on_main: true
-  commands:
-  - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small-tp.txt
-
-- label: LM Eval Large Models EP (2xB200)
-  key: lm-eval-large-models-ep-2xb200
+- label: LM Eval Large Models (B200, EP)
+  key: lm-eval-large-models-b200-ep
   timeout_in_minutes: 120
   device: b200-k8s
   optional: true
@@ -90,8 +75,8 @@ steps:
   commands:
   - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell-ep.txt
 
-- label: LM Eval Qwen3.5 Models (2xB200)
-  key: lm-eval-qwen3-5-models-2xb200
+- label: LM Eval Qwen3.5 Models (B200)
+  key: lm-eval-qwen3-5-models-b200
   timeout_in_minutes: 120
   device: b200-k8s
   optional: true
@@ -107,8 +92,8 @@ steps:
   commands:
   - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-qwen35-blackwell.txt
 
-- label: LM Eval Large Models (8xH200)
-  key: lm-eval-large-models-8xh200
+- label: LM Eval Large Models (H200)
+  key: lm-eval-large-models-h200
   timeout_in_minutes: 60
   device: h200
   optional: true
@@ -122,9 +107,7 @@ steps:
       depends_on:
       - image-build-amd
       commands:
-      - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-      - export PYTORCH_ROCM_ARCH=gfx942 # Limit Quark compilation to save time
-      - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx.txt
+      - VLLM_WORKER_MULTIPROC_METHOD=spawn PYTORCH_ROCM_ARCH=gfx942 pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx.txt  # Limit Quark compilation to save time
 
 - label: MoE Refactor Integration Test (H100 - TEMPORARY)
   key: moe-refactor-integration-test-h100-temporary
@@ -150,97 +133,6 @@ steps:
   commands:
     - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor-dp-ep/config-b200.txt
 
-- label: LM Eval Humming f16 (A100 - TEMPORARY)
-  key: lm-eval-humming-f16-a100
-  timeout_in_minutes: 120
-  device: a100
-  optional: true
-  num_devices: 1
-  source_file_dependencies:
-  - vllm/model_executor/layers/quantization/humming.py
-  - vllm/model_executor/layers/quantization/utils/humming_utils.py
-  - vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
-  - vllm/model_executor/layers/fused_moe/oracle/
-  - vllm/model_executor/kernels/linear/
-  commands:
-  - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config.txt
-
-- label: LM Eval Humming Act int8 (A100 - TEMPORARY)
-  key: lm-eval-humming-act-a100
-  timeout_in_minutes: 120
-  device: a100
-  optional: true
-  num_devices: 1
-  source_file_dependencies:
-  - vllm/model_executor/layers/quantization/humming.py
-  - vllm/model_executor/layers/quantization/utils/humming_utils.py
-  - vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
-  - vllm/model_executor/layers/fused_moe/oracle/
-  - vllm/model_executor/kernels/linear/
-  commands:
-  - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt
-
-- label: LM Eval Humming f16 (H100 - TEMPORARY)
-  key: lm-eval-humming-f16-h100
-  timeout_in_minutes: 120
-  device: h100
-  optional: true
-  num_devices: 1
-  source_file_dependencies:
-  - vllm/model_executor/layers/quantization/humming.py
-  - vllm/model_executor/layers/quantization/utils/humming_utils.py
-  - vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
-  - vllm/model_executor/layers/fused_moe/oracle/
-  - vllm/model_executor/kernels/linear/
-  commands:
-  - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config.txt
-
-- label: LM Eval Humming Act fp8/int8 (H100 - TEMPORARY)
-  key: lm-eval-humming-act-h100
-  timeout_in_minutes: 120
-  device: h100
-  optional: true
-  num_devices: 1
-  source_file_dependencies:
-  - vllm/model_executor/layers/quantization/humming.py
-  - vllm/model_executor/layers/quantization/utils/humming_utils.py
-  - vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
-  - vllm/model_executor/layers/fused_moe/oracle/
-  - vllm/model_executor/kernels/linear/
-  commands:
-  - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-fp8.txt
-  - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt
-
-- label: LM Eval Humming f16 (B200 - TEMPORARY)
-  key: lm-eval-humming-f16-b200
-  timeout_in_minutes: 120
-  device: b200-k8s
-  optional: true
-  num_devices: 1
-  source_file_dependencies:
-  - vllm/model_executor/layers/quantization/humming.py
-  - vllm/model_executor/layers/quantization/utils/humming_utils.py
-  - vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
-  - vllm/model_executor/layers/fused_moe/oracle/
-  - vllm/model_executor/kernels/linear/
-  commands:
-  - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config.txt
-
-- label: LM Eval Humming Act fp8/int8 (B200 - TEMPORARY)
-  key: lm-eval-humming-act-b200
-  timeout_in_minutes: 120
-  device: b200-k8s
-  optional: true
-  num_devices: 1
-  source_file_dependencies:
-  - vllm/model_executor/layers/quantization/humming.py
-  - vllm/model_executor/layers/quantization/utils/humming_utils.py
-  - vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
-  - vllm/model_executor/layers/fused_moe/oracle/
-  - vllm/model_executor/kernels/linear/
-  commands:
-  - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-fp8.txt
-  - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt
 
 - label: LM Eval TurboQuant KV Cache
   key: lm-eval-turboquant-kv-cache
@@ -254,8 +146,8 @@ steps:
   commands:
   - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant.txt
 
-- label: GPQA Eval (GPT-OSS) (2xH100)
-  key: gpqa-eval-gpt-oss-2xh100
+- label: GPQA Eval (GPT-OSS) (H100)
+  key: gpqa-eval-gpt-oss-h100
   timeout_in_minutes: 120
   device: h100
   optional: true
@@ -268,8 +160,8 @@ steps:
     - uv pip install --system 'gpt-oss[eval]==0.0.5'
     - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-h100.txt
 
-- label: GPQA Eval (GPT-OSS) (2xB200)
-  key: gpqa-eval-gpt-oss-2xb200
+- label: GPQA Eval (GPT-OSS) (B200)
+  key: gpqa-eval-gpt-oss-b200
   timeout_in_minutes: 120
   device: b200-k8s
   optional: true
@@ -282,22 +174,6 @@ steps:
     - uv pip install --system 'gpt-oss[eval]==0.0.5'
     - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-b200.txt
 
-- label: GPQA Eval (GPT-OSS) (DGX Spark)
-  key: gpqa-eval-gpt-oss-spark
-  timeout_in_minutes: 120
-  device: dgx-spark
-  optional: true
-  num_devices: 1
-  depends_on:
-  - arm64-image-build
-  source_file_dependencies:
-  - csrc/
-  - vllm/model_executor/layers/quantization
-  - tests/evals/gpt_oss/
-  commands:
-    - uv pip install --system 'gpt-oss[eval]==0.0.5'
-    - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-spark.txt
-
 - label: MRCR Eval Small Models
   device: h200_35gb
   timeout_in_minutes: 30

.buildkite/test_areas/lora.yaml

@@ -36,14 +36,14 @@ steps:
   commands:
     # FIXIT: find out which code initialize cuda before running the test
     # before the fix, we need to use spawn to test it
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+    #
     # Alot of these tests are on the edge of OOMing
-    - export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+    #
     # There is some Tensor Parallelism related processing logic in LoRA that
     # requires multi-GPU testing for validation.
-    - pytest -v -s -x lora/test_chatglm3_tp.py
-    - pytest -v -s -x lora/test_llama_tp.py
-    - pytest -v -s -x lora/test_qwen3_with_multi_loras.py
-    - pytest -v -s -x lora/test_olmoe_tp.py
-    - pytest -v -s -x lora/test_gptoss_tp.py
-    - pytest -v -s -x lora/test_qwen35_densemodel_lora.py
+    - VLLM_WORKER_MULTIPROC_METHOD=spawn PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True pytest -v -s -x lora/test_chatglm3_tp.py
+    - VLLM_WORKER_MULTIPROC_METHOD=spawn PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True pytest -v -s -x lora/test_llama_tp.py
+    - VLLM_WORKER_MULTIPROC_METHOD=spawn PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True pytest -v -s -x lora/test_qwen3_with_multi_loras.py
+    - VLLM_WORKER_MULTIPROC_METHOD=spawn PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True pytest -v -s -x lora/test_olmoe_tp.py
+    - VLLM_WORKER_MULTIPROC_METHOD=spawn PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True pytest -v -s -x lora/test_gptoss_tp.py
+    - VLLM_WORKER_MULTIPROC_METHOD=spawn PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True pytest -v -s -x lora/test_qwen35_densemodel_lora.py

.buildkite/test_areas/misc.yaml

@@ -18,9 +18,8 @@ steps:
     - vllm/v1/
     - tests/v1/spec_decode
   commands:
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
     # TODO: create another `optional` test group for slow tests
-    - pytest -v -s -m 'not slow_test' v1/spec_decode
+    - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s -m 'not slow_test' v1/spec_decode
   mirror:
     amd:
       device: mi300_1
@@ -50,12 +49,11 @@ steps:
     - tests/v1/test_request.py
     - tests/v1/test_outputs.py
   commands:
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -v -s v1/sample
-    - pytest -v -s v1/logits_processors
-    - pytest -v -s v1/test_oracle.py
-    - pytest -v -s v1/test_request.py
-    - pytest -v -s v1/test_outputs.py
+    - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s v1/sample
+    - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s v1/logits_processors
+    - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s v1/test_oracle.py
+    - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s v1/test_request.py
+    - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s v1/test_outputs.py
   mirror:
     amd:
       device: mi325_1
@@ -93,18 +91,17 @@ steps:
     - tests/entrypoints/openai/correctness/test_lmeval.py
   commands:
     - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
     # split the test to avoid interference
-    - pytest -v -s -m 'not cpu_test' v1/core
-    - pytest -v -s v1/executor
-    - pytest -v -s v1/kv_offload
-    - pytest -v -s v1/simple_kv_offload
-    - pytest -v -s v1/worker
-    - pytest -v -s -m 'not cpu_test' v1/kv_connector/unit
-    - pytest -v -s -m 'not cpu_test' v1/metrics
+    - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s -m 'not cpu_test' v1/core
+    - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s v1/executor
+    - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s v1/kv_offload
+    - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s v1/simple_kv_offload
+    - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s v1/worker
+    - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s -m 'not cpu_test' v1/kv_connector/unit
+    - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s -m 'not cpu_test' v1/metrics
     # Integration test for streaming correctness (requires special branch).
-    - pip install -U git+https://github.com/vllm-project/lm-evaluation-harness.git@streaming-api
-    - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
+    - pip install -U git+https://github.com/robertgshaw2-redhat/lm-evaluation-harness.git@streaming-api
+    - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
   mirror:
     amd:
       device: mi325_1
@@ -153,8 +150,7 @@ steps:
     - vllm/distributed/kv_transfer/kv_connector/v1/example_hidden_states_connector.py
     - tests/v1/kv_connector/extract_hidden_states_integration
   commands:
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -v -s v1/kv_connector/extract_hidden_states_integration
+    - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s v1/kv_connector/extract_hidden_states_integration
 
 - label: Extract Hidden States Integration (2 GPUs)
   key: extract-hidden-states-integration-2-gpus
@@ -167,8 +163,7 @@ steps:
     - vllm/distributed/kv_transfer/kv_connector/v1/example_hidden_states_connector.py
     - tests/v1/kv_connector/extract_hidden_states_integration
   commands:
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -v -s -m 'distributed' v1/kv_connector/extract_hidden_states_integration
+    - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s -m 'distributed' v1/kv_connector/extract_hidden_states_integration
 
 - label: Regression
   key: regression
@@ -188,7 +183,7 @@ steps:
   - vllm/v1/
   - tests/test_regression
   commands:
-  - pip install 'modelscope<1.38'
+  - pip install modelscope
   - pytest -v -s test_regression.py
   working_dir: "/vllm-workspace/tests" # optional
 
@@ -224,16 +219,6 @@ steps:
     - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048
     # https://github.com/vllm-project/vllm/pull/26682 uses slightly more memory in PyTorch 2.9+ causing this test to OOM in 1xL4 GPU
     - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536
-  mirror:
-    amd:
-      device: mi325_1
-      source_file_dependencies:
-      - vllm/entrypoints
-      - vllm/multimodal
-      - examples/
-      - vllm/platforms/rocm.py
-      depends_on:
-      - image-build-amd
 
 - label: Metrics, Tracing (2 GPUs)
   key: metrics-tracing-2-gpus
@@ -260,12 +245,6 @@ steps:
       'opentelemetry-exporter-otlp>=1.26.0' \
       'opentelemetry-semantic-conventions-ai>=0.4.1'"
   - pytest -v -s v1/tracing
-  mirror:
-    amd:
-      device: mi325_2
-      depends_on:
-      - image-build-amd
-      optional: true
 
 - label: Python-only Installation
   key: python-only-installation
@@ -278,16 +257,6 @@ steps:
   - setup.py
   commands:
   - bash standalone_tests/python_only_compile.sh
-  mirror:
-    amd:
-      device: mi325_1
-      timeout_in_minutes: 20
-      depends_on:
-      - image-build-amd
-      source_file_dependencies:
-      - tests/standalone_tests/python_only_compile.sh
-      - setup.py
-      - vllm/platforms/rocm.py
 
 - label: Async Engine, Inputs, Utils, Worker
   device: h200_35gb
@@ -370,7 +339,7 @@ steps:
   - pytest -v -s test_ray_env.py
   - pytest -v -s -m 'cpu_test' multimodal
   - pytest -v -s renderers
-  - pytest -v -s reasoning
+  - pytest -v -s reasoning --ignore=reasoning/test_seedoss_reasoning_parser.py --ignore=reasoning/test_glm4_moe_reasoning_parser.py
   - pytest -v -s tool_parsers
   - pytest -v -s tokenizers_
   - pytest -v -s parser
@@ -386,10 +355,9 @@ steps:
     - vllm/model_executor/layers
     - tests/v1/determinism/
   commands:
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
     - pip install pytest-timeout pytest-forked
-    - pytest -v -s v1/determinism/test_batch_invariance.py
-    - VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle[TRITON_MLA]
+    - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s v1/determinism/test_batch_invariance.py
+    - VLLM_WORKER_MULTIPROC_METHOD=spawn VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle[TRITON_MLA]
 
 - label: Batch Invariance (H100)
   key: batch-invariance-h100
@@ -400,12 +368,11 @@ steps:
     - vllm/model_executor/layers
     - tests/v1/determinism/
   commands:
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
     - pip install pytest-timeout pytest-forked
-    - pytest -v -s v1/determinism/test_batch_invariance.py
-    - pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py
-    - VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle[TRITON_MLA]
-    - VLLM_TEST_MODEL=Qwen/Qwen3-30B-A3B-Thinking-2507-FP8 pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle[FLASH_ATTN]
+    - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s v1/determinism/test_batch_invariance.py
+    - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py
+    - VLLM_WORKER_MULTIPROC_METHOD=spawn VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle[TRITON_MLA]
+    - VLLM_WORKER_MULTIPROC_METHOD=spawn VLLM_TEST_MODEL=Qwen/Qwen3-30B-A3B-Thinking-2507-FP8 pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle[FLASH_ATTN]
 
 - label: Batch Invariance (B200)
   key: batch-invariance-b200
@@ -416,14 +383,13 @@ steps:
     - vllm/model_executor/layers
     - tests/v1/determinism/
   commands:
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
     - pip install pytest-timeout pytest-forked
-    - pytest -v -s v1/determinism/test_batch_invariance.py
-    - pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py
-    - VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle[TRITON_MLA]
-    - VLLM_TEST_MODEL=Qwen/Qwen3-30B-A3B-Thinking-2507-FP8 pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle[FLASH_ATTN]
-    - pytest -v -s v1/determinism/test_nvfp4_batch_invariant.py
-    - pytest -v -s v1/determinism/test_nvfp4_batch_invariant_scaled_mm.py
+    - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s v1/determinism/test_batch_invariance.py
+    - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py
+    - VLLM_WORKER_MULTIPROC_METHOD=spawn VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle[TRITON_MLA]
+    - VLLM_WORKER_MULTIPROC_METHOD=spawn VLLM_TEST_MODEL=Qwen/Qwen3-30B-A3B-Thinking-2507-FP8 pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle[FLASH_ATTN]
+    - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s v1/determinism/test_nvfp4_batch_invariant.py
+    - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s v1/determinism/test_nvfp4_batch_invariant_scaled_mm.py
   
 - label: Acceptance Length Test (Large Models) # optional
   device: h200_35gb
@@ -438,5 +404,4 @@ steps:
   - vllm/model_executor/models/mlp_speculator.py
   - tests/v1/spec_decode/test_acceptance_length.py
   commands:
-    - export VLLM_ALLOW_INSECURE_SERIALIZATION=1
-    - pytest -v -s v1/spec_decode/test_acceptance_length.py -m slow_test
+    - VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s v1/spec_decode/test_acceptance_length.py -m slow_test

.buildkite/test_areas/model_executor.yaml

@@ -13,26 +13,12 @@ steps:
   - tests/entrypoints/openai/completion/test_tensorizer_entrypoint.py
   commands:
     - apt-get update && apt-get install -y curl libsodium23
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    # Dump tracebacks of all threads if a test hangs, so a wedged GPU/CUDA
-    # init surfaces a stack instead of silently stalling.
-    - export PYTHONFAULTHANDLER=1
     # Per-test watchdog: a single hung test (e.g. stuck during engine/CUDA
     # init) fails fast with a traceback instead of running until the global
     # build timeout. The `thread` method also handles hangs inside C/CUDA
     # calls that the signal method cannot interrupt.
-    - pytest -v -s model_executor -m '(not slow_test)' --timeout=900 --timeout-method=thread
-    - pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py --timeout=900 --timeout-method=thread
-  mirror:
-    amd:
-      device: mi300_1
-      depends_on:
-      - image-build-amd
-      source_file_dependencies:
-      - vllm/engine/arg_utils.py
-      - vllm/config/model.py
-      - vllm/model_executor
-      - tests/model_executor
-      - tests/entrypoints/openai/completion/test_tensorizer_entrypoint.py
-      - vllm/_aiter_ops.py
-      - vllm/platforms/rocm.py
+    #
+    # Env vars are inlined because CONTINUE_ON_FAILURE wraps each command
+    # in a subshell, so a standalone `export` would be lost.
+    - VLLM_WORKER_MULTIPROC_METHOD=spawn PYTHONFAULTHANDLER=1 pytest -v -s model_executor -m '(not slow_test)' --timeout=900 --timeout-method=thread
+    - VLLM_WORKER_MULTIPROC_METHOD=spawn PYTHONFAULTHANDLER=1 pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py --timeout=900 --timeout-method=thread

.buildkite/test_areas/model_runner_v2.yaml

@@ -16,13 +16,14 @@ steps:
   - tests/entrypoints/llm/test_struct_output_generate.py
   commands:
   - set -x
-  - export VLLM_USE_V2_MODEL_RUNNER=1
-  - pytest -v -s v1/engine/test_llm_engine.py -k "not test_engine_metrics"
-  - pytest -v -s v1/e2e/general/test_async_scheduling.py -k "not ngram"
-  - pytest -v -s v1/e2e/general/test_context_length.py
-  - pytest -v -s v1/e2e/general/test_min_tokens.py
+  - VLLM_USE_V2_MODEL_RUNNER=1 pytest -v -s v1/engine/test_llm_engine.py -k "not test_engine_metrics"
+  # This requires eager until we sort out CG correctness issues.
+  # TODO: remove ENFORCE_EAGER here after https://github.com/vllm-project/vllm/pull/32936 is merged.
+  - VLLM_USE_V2_MODEL_RUNNER=1 ENFORCE_EAGER=1 pytest -v -s v1/e2e/general/test_async_scheduling.py -k "not ngram"
+  - VLLM_USE_V2_MODEL_RUNNER=1 pytest -v -s v1/e2e/general/test_context_length.py
+  - VLLM_USE_V2_MODEL_RUNNER=1 pytest -v -s v1/e2e/general/test_min_tokens.py
   # Temporary hack filter to exclude ngram spec decoding based tests.
-  - pytest -v -s entrypoints/llm/test_struct_output_generate.py -k "xgrammar and not speculative_config6 and not speculative_config7 and not speculative_config8 and not speculative_config0"
+  - VLLM_USE_V2_MODEL_RUNNER=1 pytest -v -s entrypoints/llm/test_struct_output_generate.py -k "xgrammar and not speculative_config6 and not speculative_config7 and not speculative_config8 and not speculative_config0"
 
 - label: Model Runner V2 Examples
   device: h200_35gb
@@ -40,26 +41,25 @@ steps:
     - examples/features/tensorize_vllm_model.py
   commands:
     - set -x
-    - export VLLM_USE_V2_MODEL_RUNNER=1
     - pip install tensorizer # for tensorizer test
-    - python3 basic/offline_inference/chat.py # for basic
-    - python3 basic/offline_inference/generate.py --model facebook/opt-125m
-    #- python3 basic/offline_inference/generate.py --model meta-llama/Llama-2-13b-chat-hf --cpu-offload-gb 10  # TODO
-    #- python3 basic/offline_inference/embed.py   # TODO
+    - VLLM_USE_V2_MODEL_RUNNER=1 python3 basic/offline_inference/chat.py # for basic
+    - VLLM_USE_V2_MODEL_RUNNER=1 python3 basic/offline_inference/generate.py --model facebook/opt-125m
+    #- VLLM_USE_V2_MODEL_RUNNER=1 python3 basic/offline_inference/generate.py --model meta-llama/Llama-2-13b-chat-hf --cpu-offload-gb 10  # TODO
+    #- VLLM_USE_V2_MODEL_RUNNER=1 python3 basic/offline_inference/embed.py   # TODO
     # for multi-modal models
-    - python3 generate/multimodal/audio_language_offline.py --seed 0
-    - python3 generate/multimodal/vision_language_offline.py --seed 0
-    - python3 generate/multimodal/vision_language_multi_image_offline.py --seed 0
-    - python3 generate/multimodal/encoder_decoder_multimodal_offline.py --model-type whisper --seed 0
+    - VLLM_USE_V2_MODEL_RUNNER=1 python3 generate/multimodal/audio_language_offline.py --seed 0
+    - VLLM_USE_V2_MODEL_RUNNER=1 python3 generate/multimodal/vision_language_offline.py --seed 0
+    - VLLM_USE_V2_MODEL_RUNNER=1 python3 generate/multimodal/vision_language_multi_image_offline.py --seed 0
+    - VLLM_USE_V2_MODEL_RUNNER=1 python3 generate/multimodal/encoder_decoder_multimodal_offline.py --model-type whisper --seed 0
     # for pooling models
-    - python3 pooling/embed/vision_embedding_offline.py --seed 0
+    - VLLM_USE_V2_MODEL_RUNNER=1 python3 pooling/embed/vision_embedding_offline.py --seed 0
     # for features demo
-    - python3 features/automatic_prefix_caching/prefix_caching_offline.py
-    - python3 deployment/llm_engine_example.py
-    - python3 features/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && python3 features/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
-    - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048
+    - VLLM_USE_V2_MODEL_RUNNER=1 python3 features/automatic_prefix_caching/prefix_caching_offline.py
+    - VLLM_USE_V2_MODEL_RUNNER=1 python3 deployment/llm_engine_example.py
+    - VLLM_USE_V2_MODEL_RUNNER=1 python3 features/tensorize_vllm_model.py --model facebook/opt-125m serialize --serialized-directory /tmp/ --suffix v1 && VLLM_USE_V2_MODEL_RUNNER=1 python3 features/tensorize_vllm_model.py --model facebook/opt-125m deserialize --path-to-tensors /tmp/vllm/facebook/opt-125m/v1/model.tensors
+    - VLLM_USE_V2_MODEL_RUNNER=1 python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048
     # https://github.com/vllm-project/vllm/pull/26682 uses slightly more memory in PyTorch 2.9+ causing this test to OOM in 1xL4 GPU
-    - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536
+    - VLLM_USE_V2_MODEL_RUNNER=1 python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536
 
 - label: Model Runner V2 Distributed (2 GPUs)
   key: model-runner-v2-distributed-2-gpus
@@ -74,13 +74,11 @@ steps:
     - tests/v1/distributed/test_eagle_dp.py
   commands:
     - set -x
-    - export VLLM_USE_V2_MODEL_RUNNER=1
     # The "and not True" here is a hacky way to exclude the prompt_embeds cases which aren't yet supported.
-    - TARGET_TEST_SUITE=L4 pytest -v -s basic_correctness/test_basic_correctness.py -m 'distributed(num_gpus=2)' -k "not ray and not True"
+    - VLLM_USE_V2_MODEL_RUNNER=1 TARGET_TEST_SUITE=L4 pytest -v -s basic_correctness/test_basic_correctness.py -m 'distributed(num_gpus=2)' -k "not ray and not True"
     # https://github.com/NVIDIA/nccl/issues/1838
-    - export NCCL_CUMEM_HOST_ENABLE=0
-    - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py -k "not ray"
-    - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
+    - VLLM_USE_V2_MODEL_RUNNER=1 NCCL_CUMEM_HOST_ENABLE=0 TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py -k "not ray"
+    - VLLM_USE_V2_MODEL_RUNNER=1 NCCL_CUMEM_HOST_ENABLE=0 TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
 
 - label: Model Runner V2 Pipeline Parallelism (4 GPUs)
   key: model-runner-v2-pipeline-parallelism-4-gpus
@@ -95,10 +93,9 @@ steps:
     - tests/v1/distributed/test_pp_dp_v2.py
   commands:
     - set -x
-    - export VLLM_USE_V2_MODEL_RUNNER=1
-    - pytest -v -s distributed/test_pipeline_parallel.py -k "not ray and not Jamba"
-    - pytest -v -s distributed/test_pp_cudagraph.py -k "not ray"
-    - pytest -v -s v1/distributed/test_pp_dp_v2.py
+    - VLLM_USE_V2_MODEL_RUNNER=1 pytest -v -s distributed/test_pipeline_parallel.py -k "not ray and not Jamba"
+    - VLLM_USE_V2_MODEL_RUNNER=1 pytest -v -s distributed/test_pp_cudagraph.py -k "not ray"
+    - VLLM_USE_V2_MODEL_RUNNER=1 pytest -v -s v1/distributed/test_pp_dp_v2.py
 
 - label: Model Runner V2 Spec Decode
   device: h200_35gb
@@ -113,8 +110,7 @@ steps:
   - tests/v1/e2e/spec_decode/test_spec_decode.py
   commands:
   - set -x
-  - export VLLM_USE_V2_MODEL_RUNNER=1
-  - pytest -v -s v1/spec_decode/test_max_len.py -k "eagle or mtp"
-  - pytest -v -s v1/spec_decode/test_rejection_sampler_utils.py
-  - pytest -v -s v1/spec_decode/test_synthetic_rejection_sampler_utils.py
-  - pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "eagle or mtp"
+  - VLLM_USE_V2_MODEL_RUNNER=1 pytest -v -s v1/spec_decode/test_max_len.py -k "eagle or mtp"
+  - VLLM_USE_V2_MODEL_RUNNER=1 pytest -v -s v1/spec_decode/test_rejection_sampler_utils.py
+  - VLLM_USE_V2_MODEL_RUNNER=1 pytest -v -s v1/spec_decode/test_synthetic_rejection_sampler_utils.py
+  - VLLM_USE_V2_MODEL_RUNNER=1 pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "eagle or mtp"

.buildkite/test_areas/models_basic.yaml

@@ -6,6 +6,7 @@ steps:
   key: basic-models-tests-initialization
   timeout_in_minutes: 45
   device: h200_18gb
+  torch_nightly: true
   source_file_dependencies:
   - vllm/
   - tests/models/test_initialization.py
@@ -13,6 +14,8 @@ steps:
   commands:
     # Run a subset of model initialization tests
     - pytest -v -s models/test_initialization.py::test_can_initialize_small_subset
+  mirror:
+    torch_nightly: {}
 
 - label: Basic Models Tests (Extra Initialization) %N
   device: h200_35gb
@@ -28,6 +31,8 @@ steps:
     # test.) Also run if model initialization test file is modified
     - pytest -v -s models/test_initialization.py -k 'not test_can_initialize_small_subset' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
   parallelism: 2
+  mirror:
+    torch_nightly: {}
 
 - label: Basic Models Tests (Other)
   device: h200_35gb
@@ -36,15 +41,10 @@ steps:
   source_file_dependencies:
   - vllm/
   - tests/models/test_terratorch.py
-  - tests/models/transformers/test_backend.py
+  - tests/models/test_transformers.py
   - tests/models/test_registry.py
   commands:
-    - pytest -v -s models/test_terratorch.py models/transformers/test_backend.py models/test_registry.py
-  mirror:
-    amd:
-      device: mi325_1
-      depends_on:
-      - image-build-amd
+    - pytest -v -s models/test_terratorch.py models/test_transformers.py models/test_registry.py
 
 - label: Basic Models Test (Other CPU) # 5min
   key: basic-models-test-other-cpu
@@ -55,7 +55,6 @@ steps:
   - vllm/
   - tests/models/test_utils.py
   - tests/models/test_vision.py
-  - tests/models/transformers/fusers/
   device: cpu-small
   commands:
-    - pytest -v -s models/test_utils.py models/test_vision.py models/transformers/fusers/
+    - pytest -v -s models/test_utils.py models/test_vision.py

.buildkite/test_areas/models_distributed.yaml

@@ -17,7 +17,7 @@ steps:
   - TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
   - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)'
   # Avoid importing model tests that cause CUDA reinitialization error
-  - pytest models/transformers/test_backend.py -v -s -m 'distributed(num_gpus=2)'
+  - pytest models/test_transformers.py -v -s -m 'distributed(num_gpus=2)'
   - pytest models/language -v -s -m 'distributed(num_gpus=2)'
   - pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)'
   - pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py

.buildkite/test_areas/models_language.yaml

@@ -14,10 +14,7 @@ steps:
     - pip freeze | grep -E 'torch'
     - pytest -v -s models/language -m 'core_model and (not slow_test)'
   mirror:
-    amd:
-      device: mi300_1
-      depends_on:
-      - image-build-amd
+    torch_nightly: {}
 
 - label: Language Models Tests (Extra Standard) %N
   key: language-models-tests-extra-standard
@@ -34,21 +31,7 @@ steps:
     - pytest -v -s models/language -m 'core_model and slow_test' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
   parallelism: 2
   mirror:
-    amd:
-      device: mi300_1
-      depends_on:
-      - image-build-amd
-      source_file_dependencies:
-      - vllm/model_executor/models/
-      - vllm/model_executor/model_loader/
-      - vllm/model_executor/layers/
-      - vllm/v1/attention/backends/
-      - vllm/v1/attention/selector.py
-      - tests/models/language/pooling/test_embedding.py
-      - tests/models/language/generation/test_common.py
-      - tests/models/language/pooling/test_classification.py
-      - vllm/_aiter_ops.py
-      - vllm/platforms/rocm.py
+    torch_nightly: {}
 
 - label: Language Models Tests (Hybrid) %N
   key: language-models-tests-hybrid
@@ -65,6 +48,7 @@ steps:
     - pytest -v -s models/language/generation -m hybrid_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
   parallelism: 2
   mirror:
+    torch_nightly: {}
     amd:
       device: mi325_1
       timeout_in_minutes: 90

.buildkite/test_areas/models_multimodal.yaml

@@ -10,6 +10,7 @@ steps:
   - vllm/
   - tests/models/multimodal
   commands:
+    - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
     - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2"
     - pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model
   mirror:
@@ -26,8 +27,8 @@ steps:
   - vllm/
   - tests/models/multimodal
   commands:
+    - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
     - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen3 or gemma"
-    - pytest -v -s models/multimodal/generation/test_mm_prefix_lm.py -m core_model
     - pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model
   mirror:
     amd:
@@ -43,6 +44,7 @@ steps:
   - vllm/
   - tests/models/multimodal
   commands:
+    - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
     - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma"
     - pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model
   mirror:
@@ -59,7 +61,8 @@ steps:
   - vllm/
   - tests/models/multimodal
   commands:
-    - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_mm_prefix_lm.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/generation/test_vit_cudagraph.py --ignore models/multimodal/processing
+    - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
+    - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py  --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/generation/test_vit_cudagraph.py --ignore models/multimodal/processing
     - pytest -v -s models/multimodal/generation/test_vit_cudagraph.py -m core_model
     - pytest models/multimodal/generation/test_memory_leak.py -m core_model
     - cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model  # Otherwise, mp_method="spawn" doesn't work
@@ -80,6 +83,7 @@ steps:
   - tests/models/registry.py
   device: cpu-medium
   commands:
+    - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
     - pytest -v -s models/multimodal/processing --ignore models/multimodal/processing/test_tensor_schema.py
 
 - label: Multi-Modal Processor # 44min
@@ -91,6 +95,7 @@ steps:
   - tests/models/multimodal
   - tests/models/registry.py
   commands:
+    - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
     - pytest -v -s models/multimodal/processing/test_tensor_schema.py
 
 - label: Multi-Modal Accuracy Eval (Small Models) # 50min
@@ -104,17 +109,6 @@ steps:
   - vllm/v1/core/
   commands:
   - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-mm-small.txt --tp-size=1
-  mirror:
-    amd:
-      device: mi300_1
-      depends_on:
-      - image-build-amd
-      source_file_dependencies:
-      - vllm/multimodal/
-      - vllm/inputs/
-      - vllm/v1/core/
-      - vllm/platforms/rocm.py
-      - vllm/model_executor/model_loader/
 
 - label: Multi-Modal Models (Extended Generation 1)
   key: multi-modal-models-extended-generation-1
@@ -124,6 +118,7 @@ steps:
   - tests/models/multimodal/generation
   - tests/models/multimodal/test_mapping.py
   commands:
+    - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
     - pytest -v -s models/multimodal/generation -m 'not core_model' --ignore models/multimodal/generation/test_common.py
     - pytest -v -s models/multimodal/test_mapping.py
   mirror:
@@ -140,6 +135,7 @@ steps:
   - vllm/
   - tests/models/multimodal/generation
   commands:
+    - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
     - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=0) and not core_model'
 
 - label: Multi-Modal Models (Extended Generation 3)
@@ -150,6 +146,7 @@ steps:
   - vllm/
   - tests/models/multimodal/generation
   commands:
+    - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
     - pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model'
 
 - label: Multi-Modal Models (Extended Pooling)

.buildkite/test_areas/pytorch.yaml

@@ -195,11 +195,3 @@ steps:
   - requirements/test/nightly-torch.txt
   commands:
   - bash standalone_tests/pytorch_nightly_dependency.sh
-  mirror:
-    amd:
-      device: mi300_1
-      depends_on:
-      - image-build-amd
-      source_file_dependencies:
-      - requirements/test/nightly-torch.txt
-      - vllm/platforms/rocm.py

.buildkite/test_areas/rust_frontend.yaml

@@ -23,17 +23,15 @@ steps:
   # - tests/entrypoints/openai/test_uds.py
   - tests/v1/sample/test_logprobs_e2e.py
   commands:
-  - export VLLM_USE_RUST_FRONTEND=1
-  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-  - pytest -v -s benchmarks/test_serve_cli.py -k "not insecure and not (test_bench_serve and not test_bench_serve_chat)"
-  - pytest -v -s entrypoints/openai/chat_completion/test_chat_completion.py -k "not test_invalid_json_schema and not test_invalid_regex"
-  # - pytest -v -s entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py -k "not invalid"
+  - VLLM_USE_RUST_FRONTEND=1 VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s benchmarks/test_serve_cli.py -k "not insecure and not (test_bench_serve and not test_bench_serve_chat)"
+  - VLLM_USE_RUST_FRONTEND=1 VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s entrypoints/openai/chat_completion/test_chat_completion.py
+  # - VLLM_USE_RUST_FRONTEND=1 VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py -k "not invalid"
 
-  # - pytest -v -s entrypoints/openai/completion/test_prompt_validation.py -k "not prompt_embeds"
-  - pytest -v -s entrypoints/openai/completion/test_shutdown.py -k "not engine_failure and not test_abort_timeout_exits_quickly"
-  # - pytest -v -s entrypoints/openai/test_return_token_ids.py
-  # - pytest -v -s entrypoints/openai/test_uds.py
-  - pytest -v -s v1/sample/test_logprobs_e2e.py -k "test_prompt_logprobs_e2e_server"
+  # - VLLM_USE_RUST_FRONTEND=1 VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s entrypoints/openai/completion/test_prompt_validation.py -k "not prompt_embeds"
+  - VLLM_USE_RUST_FRONTEND=1 VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s entrypoints/openai/completion/test_shutdown.py -k "not engine_failure and not test_abort_timeout_exits_quickly"
+  # - VLLM_USE_RUST_FRONTEND=1 VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s entrypoints/openai/test_return_token_ids.py
+  # - VLLM_USE_RUST_FRONTEND=1 VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s entrypoints/openai/test_uds.py
+  - VLLM_USE_RUST_FRONTEND=1 VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s v1/sample/test_logprobs_e2e.py -k "test_prompt_logprobs_e2e_server"
 
 - label: Rust Frontend Serve/Admin Coverage
   timeout_in_minutes: 60
@@ -46,18 +44,16 @@ steps:
   - vllm/v1/engine/
   - tests/utils.py
   # - tests/entrypoints/serve/dev/rpc/test_collective_rpc.py
-  - tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py
+  - tests/entrypoints/serve/disagg/test_serving_tokens.py
   - tests/entrypoints/serve/instrumentator/test_basic.py
   - tests/entrypoints/serve/instrumentator/test_metrics.py
   # - tests/entrypoints/serve/dev/test_sleep.py
   commands:
-  - export VLLM_USE_RUST_FRONTEND=1
-  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-  # - pytest -v -s entrypoints/serve/dev/rpc/test_collective_rpc.py
-  - pytest -v -s entrypoints/serve/instrumentator/test_basic.py -k "not show_version and not server_load"
-  - pytest -v -s entrypoints/scale_out/token_in_token_out/test_serving_tokens.py -k "not stream and not lora and not test_generate_logprobs and not stop_string_workflow"
-  - pytest -v -s entrypoints/serve/instrumentator/test_metrics.py -k "text and not show and not run_batch and not test_metrics_counts and not test_metrics_exist"
-  # - pytest -v -s entrypoints/serve/dev/test_sleep.py
+  # - VLLM_USE_RUST_FRONTEND=1 VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s entrypoints/serve/dev/rpc/test_collective_rpc.py
+  - VLLM_USE_RUST_FRONTEND=1 VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s entrypoints/serve/instrumentator/test_basic.py -k "not show_version and not server_load"
+  - VLLM_USE_RUST_FRONTEND=1 VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s entrypoints/serve/disagg/test_serving_tokens.py -k "not stream and not lora and not test_generate_logprobs and not stop_string_workflow"
+  - VLLM_USE_RUST_FRONTEND=1 VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s entrypoints/serve/instrumentator/test_metrics.py -k "text and not show and not run_batch and not test_metrics_counts and not test_metrics_exist"
+  # - VLLM_USE_RUST_FRONTEND=1 VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s entrypoints/serve/dev/test_sleep.py
 
 - label: Rust Frontend Core Correctness
   timeout_in_minutes: 30
@@ -69,9 +65,7 @@ steps:
   - tests/utils.py
   - tests/entrypoints/openai/correctness/test_lmeval.py
   commands:
-  - export VLLM_USE_RUST_FRONTEND=1
-  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-  - pytest -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
+  - VLLM_USE_RUST_FRONTEND=1 VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
 
 - label: Rust Frontend Tool Use
   timeout_in_minutes: 60
@@ -83,9 +77,7 @@ steps:
   - tests/utils.py
   - tests/tool_use/
   commands:
-  - export VLLM_USE_RUST_FRONTEND=1
-  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-  - pytest -v -s tool_use --ignore=tool_use/mistral --models llama3.2 -k "not test_response_format_with_tool_choice_required and not test_parallel_tool_calls_false and not test_tool_call_and_choice"
+  - VLLM_USE_RUST_FRONTEND=1 VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tool_use --ignore=tool_use/mistral --models llama3.2 -k "not test_response_format_with_tool_choice_required and not test_parallel_tool_calls_false and not test_tool_call_and_choice"
 
 - label: Rust Frontend Distributed
   timeout_in_minutes: 30
@@ -103,9 +95,6 @@ steps:
   - tests/v1/distributed/test_hybrid_lb_dp.py
   - tests/v1/distributed/test_internal_lb_dp.py
   commands:
-  - export VLLM_USE_RUST_FRONTEND=1
-  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-  - export NCCL_CUMEM_HOST_ENABLE=0
-  - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_internal_lb_dp.py -k "not 4 and not server_info"
-  - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py -k "not 4 and not server_info"
-  - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_hybrid_lb_dp.py -k "not 4 and not server_info"
+  - VLLM_USE_RUST_FRONTEND=1 VLLM_WORKER_MULTIPROC_METHOD=spawn NCCL_CUMEM_HOST_ENABLE=0 TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_internal_lb_dp.py -k "not 4 and not server_info"
+  - VLLM_USE_RUST_FRONTEND=1 VLLM_WORKER_MULTIPROC_METHOD=spawn NCCL_CUMEM_HOST_ENABLE=0 TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py -k "not 4 and not server_info"
+  - VLLM_USE_RUST_FRONTEND=1 VLLM_WORKER_MULTIPROC_METHOD=spawn NCCL_CUMEM_HOST_ENABLE=0 TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_hybrid_lb_dp.py -k "not 4 and not server_info"

.buildkite/test_areas/spec_decode.yaml

@@ -94,8 +94,6 @@ steps:
     amd:
       device: mi325_1
       timeout_in_minutes: 65
-      # TODO(akaratza): Test after Torch >= 2.12 bump
-      soft_fail: true
       depends_on:
       - image-build-amd
       source_file_dependencies:
@@ -155,8 +153,7 @@ steps:
     - vllm/model_executor/models/qwen3_dflash.py
     - tests/v1/spec_decode/test_speculators_correctness.py
   commands:
-    - export VLLM_ALLOW_INSECURE_SERIALIZATION=1
-    - pytest -v -s v1/spec_decode/test_speculators_correctness.py -m slow_test
+    - VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s v1/spec_decode/test_speculators_correctness.py -m slow_test
 
 - label: Spec Decode MTP hybrid (B200)
   timeout_in_minutes: 30

.buildkite/test_areas/weight_loading.yaml

@@ -13,13 +13,6 @@ steps:
   - tests/weight_loading
   commands:
     - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models.txt
-  mirror:
-    amd:
-      device: mi300_2
-      depends_on:
-      - image-build-amd
-      commands:
-      - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-amd.txt
 
 # - label: Weight Loading Multiple GPU - Large Models # optional
 #   working_dir: "/vllm-workspace/tests"

.github/CODEOWNERS

@@ -3,7 +3,7 @@
 
 # This lists cover the "core" components of vLLM that require careful review
 /vllm/compilation @zou3519 @youkaichao @ProExpertProg @BoyuanFeng
-/vllm/distributed/kv_transfer @NickLucche @ApostaC @orozery @xuechendi @ivanium
+/vllm/distributed/kv_transfer @NickLucche @ApostaC @orozery @xuechendi
 /vllm/lora @jeejeelee
 /vllm/model_executor/layers/attention @LucasWilkinson @MatthewBonanni
 /vllm/model_executor/layers/fused_moe @mgoin @pavanimajety @zyongye
@@ -11,7 +11,7 @@
 /vllm/model_executor/layers/mamba @tdoublep @tomeras91
 /vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py @tdoublep @ZJY0516 @vadiklyutiy
 /vllm/model_executor/model_loader @22quinn
-/vllm/model_executor/layers/batch_invariant.py @yewentao256
+/vllm/model_executor/layers/batch_invariant.py @yewentao256 
 /vllm/ir @ProExpertProg
 /vllm/kernels/ @ProExpertProg @tjtanaa
 /vllm/kernels/helion @ProExpertProg @zou3519
@@ -23,7 +23,7 @@
 # Any change to the VllmConfig changes can have a large user-facing impact,
 # so spam a lot of people
 /vllm/config @WoosukKwon @youkaichao @robertgshaw2-redhat @mgoin @tlrmchlsmth @houseroad @yewentao256 @ProExpertProg
-/vllm/config/cache.py @heheda12345 @ivanium
+/vllm/config/cache.py @heheda12345
 
 # Config utils
 /vllm/config/utils.py @hmellor
@@ -67,17 +67,16 @@
 /vllm/v1/attention/backends/flashinfer.py @mgoin @pavanimajety @vadiklyutiy
 /vllm/v1/attention/backends/triton_attn.py @tdoublep
 /vllm/v1/attention/backends/gdn_attn.py @ZJY0516 @vadiklyutiy
-/vllm/v1/core @WoosukKwon @robertgshaw2-redhat @njhill @ywang96 @alexm-redhat @heheda12345 @ApostaC @orozery @ivanium
+/vllm/v1/core @WoosukKwon @robertgshaw2-redhat @njhill @ywang96 @alexm-redhat @heheda12345 @ApostaC @orozery
 /vllm/v1/sample @22quinn @houseroad @njhill
 /vllm/v1/spec_decode @benchislett @luccafong @MatthewBonanni
 /vllm/v1/structured_output @mgoin @russellb @aarnphm @benchislett
-/vllm/v1/kv_cache_interface.py @heheda12345 @ivanium
+/vllm/v1/kv_cache_interface.py @heheda12345
 /vllm/v1/kv_offload @ApostaC @orozery
-/vllm/v1/simple_kv_offload @ivanium
 /vllm/v1/engine @njhill
 /vllm/v1/executor @njhill
 /vllm/v1/worker @njhill
-/vllm/v1/worker/kv_connector_model_runner_mixin.py @orozery @NickLucche @ivanium
+/vllm/v1/worker/kv_connector_model_runner_mixin.py @orozery @NickLucche
 
 # Model runner V2
 /vllm/v1/worker/gpu @WoosukKwon @njhill @yewentao256
@@ -104,14 +103,13 @@
 /tests/test_inputs.py @DarkLight1337 @ywang96
 /tests/entrypoints/llm/test_struct_output_generate.py @mgoin @russellb @aarnphm
 /tests/v1/structured_output @mgoin @russellb @aarnphm
-/tests/v1/core @WoosukKwon @robertgshaw2-redhat @njhill @ywang96 @alexm-redhat @heheda12345 @ApostaC @orozery @ivanium
+/tests/v1/core @WoosukKwon @robertgshaw2-redhat @njhill @ywang96 @alexm-redhat @heheda12345 @ApostaC @orozery
 /tests/weight_loading @mgoin @youkaichao @yewentao256
 /tests/lora @jeejeelee
 /tests/models/language/generation/test_hybrid.py @tdoublep @tomeras91
 /tests/v1/kv_connector/nixl_integration @NickLucche
-/tests/v1/kv_connector @ApostaC @orozery @ivanium
+/tests/v1/kv_connector @ApostaC @orozery
 /tests/v1/kv_offload @ApostaC @orozery
-/tests/v1/simple_kv_offload @ivanium
 /tests/v1/determinism @yewentao256
 /tests/reasoning @aarnphm @chaunceyjiang @sfeng33 @bbrowning
 /tests/tool_parsers @aarnphm @chaunceyjiang @sfeng33 @bbrowning
@@ -119,7 +117,7 @@
 
 # Transformers modeling backend
 /vllm/model_executor/models/transformers @hmellor
-/tests/models/transformers @hmellor
+/tests/models/test_transformers.py @hmellor
 
 # Docs
 /docs/mkdocs @hmellor

.github/actionlint.yaml

@@ -3,5 +3,3 @@
 self-hosted-runner:
   labels:
     - vllm-runners
-    # Not yet in actionlint's known-label set.
-    - macos-26

.github/workflows/issue_autolabel.yml

@@ -327,7 +327,7 @@ jobs:
                 message: 'CC {users} for ROCm-related issue',
               },
               mistral: {
-                users: ['patrickvonplaten', 'juliendenize', 'andylolu2', 'NickLucche'],
+                users: ['patrickvonplaten', 'juliendenize', 'andylolu2'],
                 message: 'CC {users} for Mistral-related issue',
               },
               // Add more label -> user mappings here

.github/workflows/macos-smoke-test.yml

@@ -11,25 +11,13 @@ permissions:
 
 jobs:
   macos-m1-smoke-test:
-    # macos-26 (the supported target) is still a preview runner, so gate on GA
-    # macos-15 and keep macos-26 non-blocking.
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - os: macos-15
-            required: true
-          - os: macos-26
-            required: false
-    name: macos-m1-smoke-test (${{ matrix.os }})
-    runs-on: ${{ matrix.os }}
-    continue-on-error: ${{ !matrix.required }}
+    runs-on: macos-latest
     timeout-minutes: 30
 
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+      - uses: actions/checkout@v6.0.1
 
-      - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+      - uses: astral-sh/setup-uv@v7
         with:
           enable-cache: true
           cache-dependency-glob: |
@@ -84,11 +72,14 @@ jobs:
           # Test health endpoint
           curl -f http://localhost:8000/health
 
-          # Long prompt: hits the split-KV path that short prompts skip (#46769).
-          PAYLOAD=$(python -c "import json; print(json.dumps({'model': 'Qwen/Qwen3-0.6B', 'prompt': 'The quick brown fox jumps over the lazy dog. ' * 24, 'max_tokens': 16}))")
-          curl -f --max-time 120 http://localhost:8000/v1/completions \
+          # Test completion
+          curl -f http://localhost:8000/v1/completions \
             -H "Content-Type: application/json" \
-            -d "$PAYLOAD"
+            -d '{
+              "model": "Qwen/Qwen3-0.6B",
+              "prompt": "Hello",
+              "max_tokens": 5
+            }'
 
           # Cleanup
           kill "$SERVER_PID"

.github/workflows/pre-commit.yml

@@ -48,7 +48,7 @@ jobs:
     if: always() && (needs.pre-run-check.result == 'success' || needs.pre-run-check.result == 'skipped')
     runs-on: [self-hosted, linux, x64, vllm-runners]
     steps:
-    - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+    - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
     - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
       with:
         python-version: "3.12"

.pre-commit-config.yaml

@@ -131,19 +131,6 @@ repos:
         --python-version, "3.12",
       ]
       files: ^requirements/(common|xpu|test/xpu)\.(in|txt)$
-    - id: pip-compile
-      alias: pip-compile-cpu
-      name: pip-compile-cpu
-      args: [
-        requirements/test/cuda.in,
-        -o, requirements/test/cpu.txt,
-        --index-strategy, unsafe-best-match,
-        --torch-backend, cpu,
-        --python-platform, x86_64-manylinux_2_28,
-        --python-version, "3.12",
-      ]
-      files: ^requirements/(common|cpu|test/(cuda|cpu))\.(in|txt)$
-      exclude: ^requirements/test/cuda\.txt$
     - id: pip-compile
       alias: pip-compile-docs
       name: pip-compile-docs

CMakeLists.txt

@@ -140,21 +140,6 @@ if(Python_VERSION VERSION_GREATER_EQUAL "3.11")
     WITH_SOABI)
 endif()
 
-#
-# fs_io extension (pure CXX; must stay above the non-CUDA device branch
-# so CPU builds define the target before the early return).
-# GIL-releasing filesystem helpers for FileSystemTierManager.
-#
-if(Python_VERSION VERSION_GREATER_EQUAL "3.11")
-  define_extension_target(
-    fs_io_C
-    DESTINATION vllm
-    LANGUAGE CXX
-    SOURCES csrc/fs_io.cpp
-    USE_SABI 3.11
-    WITH_SOABI)
-endif()
-
 #
 # Forward the non-CUDA device extensions to external CMake scripts.
 #
@@ -285,16 +270,6 @@ if(VLLM_GPU_LANG STREQUAL "HIP")
   #
   set(CMAKE_${VLLM_GPU_LANG}_FLAGS "${CMAKE_${VLLM_GPU_LANG}_FLAGS} -Wno-unused-result -Wno-unused-value")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-result -Wno-unused-value")
-
-  # When using LTO then *.cpp files must be compiled with same compiler as used linker
-  # So if HIP uses clang linker we also must use it
-  # Otherwise symbols will be missing from .so
-  if (CMAKE_CXX_FLAGS MATCHES "\-flto")
-    if(NOT CMAKE_CXX_COMPILER_ID STREQUAL CMAKE_HIP_COMPILER_ID)
-      message(FATAL_ERROR "LTO is enabled for ROCm build, but the C++ compiler (${CMAKE_CXX_COMPILER_ID}) and HIP compiler (${CMAKE_HIP_COMPILER_ID}) are different which is not supported. "
-                          "Please ensure they are same by setting CXX=${CMAKE_HIP_COMPILER} environment variable. Or alternatively disable LTO.")
-    endif()
-   endif()
 endif()
 
 #
@@ -350,7 +325,9 @@ endif()
 if(VLLM_GPU_LANG STREQUAL "HIP")
   set(VLLM_EXT_SRC
     "csrc/torch_bindings.cpp"
-    "csrc/custom_quickreduce.cu")
+    "csrc/custom_quickreduce.cu"
+    "csrc/cuda_view.cu"
+    "csrc/libtorch_stable/cuda_utils_kernels.cu")
 
   message(STATUS "Enabling C extension.")
   define_extension_target(
@@ -378,8 +355,6 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
   #
   set(VLLM_STABLE_EXT_SRC
     "csrc/libtorch_stable/torch_bindings.cpp"
-    "csrc/libtorch_stable/cuda_view.cu"
-    "csrc/libtorch_stable/cuda_utils_kernels.cu"
     "csrc/libtorch_stable/activation_kernels.cu"
     "csrc/libtorch_stable/quantization/activation_kernels.cu"
     "csrc/libtorch_stable/quantization/w8a8/int8/scaled_quant.cu"
@@ -399,29 +374,14 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
     "csrc/libtorch_stable/sampler.cu"
     "csrc/libtorch_stable/topk.cu"
     "csrc/libtorch_stable/mamba/selective_scan_fwd.cu"
+    "csrc/libtorch_stable/attention/paged_attention_v1.cu"
+    "csrc/libtorch_stable/attention/paged_attention_v2.cu"
+    "csrc/libtorch_stable/cache_kernels.cu"
     "csrc/libtorch_stable/cache_kernels.cu"
     "csrc/libtorch_stable/cache_kernels_fused.cu"
     "csrc/libtorch_stable/custom_all_reduce.cu"
     "csrc/libtorch_stable/fused_deepseek_v4_qnorm_rope_kv_insert_kernel.cu")
 
-  if(VLLM_GPU_LANG STREQUAL "CUDA" AND
-     DEFINED CMAKE_CUDA_COMPILER_VERSION AND
-     CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 12.0)
-
-    if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0)
-      cuda_archs_loose_intersection(COOPERATIVE_TOPK_ARCHS
-        "9.0a;10.0f;10.1f;10.3f;11.0f;12.0f;12.1f" "${CUDA_ARCHS}")
-    else()
-      cuda_archs_loose_intersection(COOPERATIVE_TOPK_ARCHS
-        "9.0a;10.0a;10.1a;10.3a;12.0a;12.1a" "${CUDA_ARCHS}")
-    endif()
-
-    if(COOPERATIVE_TOPK_ARCHS)
-      list(APPEND VLLM_GPU_FLAGS "-DVLLM_ENABLE_COOPERATIVE_TOPK=1")
-
-    endif()
-  endif()
-
   if(VLLM_GPU_LANG STREQUAL "CUDA")
     SET(CUTLASS_ENABLE_HEADERS_ONLY ON CACHE BOOL "Enable only the header library")
 
@@ -456,6 +416,8 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
     FetchContent_MakeAvailable(cutlass)
 
     list(APPEND VLLM_STABLE_EXT_SRC
+      "csrc/libtorch_stable/cuda_view.cu"
+      "csrc/libtorch_stable/cuda_utils_kernels.cu"
       "csrc/libtorch_stable/cutlass_extensions/common.cpp"
       "csrc/libtorch_stable/quantization/w8a8/cutlass/scaled_mm_entry.cu"
       "csrc/libtorch_stable/quantization/fp4/nvfp4_quant_entry.cu"
@@ -536,14 +498,6 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
       SRCS "${VLLM_STABLE_EXT_SRC}"
       CUDA_ARCHS "${CUDA_ARCHS}")
 
-    if(COOPERATIVE_TOPK_ARCHS)
-      list(APPEND VLLM_STABLE_EXT_SRC
-        "csrc/libtorch_stable/cooperative_topk.cu")
-      set_gencode_flags_for_srcs(
-        SRCS "csrc/libtorch_stable/cooperative_topk.cu"
-        CUDA_ARCHS "${COOPERATIVE_TOPK_ARCHS}")
-    endif()
-
     # Only build Marlin kernels if we are building for at least some compatible archs.
     # Keep building Marlin for 9.0 as there are some group sizes and shapes that
     # are not supported by Machete yet.
@@ -889,9 +843,9 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
   endif()
 
   if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0)
-    cuda_archs_loose_intersection(SCALED_MM_ARCHS "10.0f;11.0f" "${CUDA_ARCHS}")
+    cuda_archs_loose_intersection(SCALED_MM_ARCHS "10.0f" "${CUDA_ARCHS}")
   else()
-    cuda_archs_loose_intersection(SCALED_MM_ARCHS "10.0a;10.1a;10.3a" "${CUDA_ARCHS}")
+    cuda_archs_loose_intersection(SCALED_MM_ARCHS "10.0a;10.3a" "${CUDA_ARCHS}")
   endif()
   if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8 AND SCALED_MM_ARCHS)
     set(CUTLASS_MOE_SM100_SRCS "csrc/libtorch_stable/quantization/w8a8/cutlass/moe/grouped_mm_c3x_sm100.cu")
@@ -1086,24 +1040,25 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
     USE_SABI 3
     WITH_SOABI)
 
-  # Set TORCH_TARGET_VERSION for stable ABI compatibility.
-  # This ensures we only use C-shim APIs available in PyTorch 2.11.
-  # _C_stable_libtorch is abi compatible with PyTorch >= TORCH_TARGET_VERSION
-  # which is currently set to 2.11.
-  target_compile_definitions(_C_stable_libtorch PRIVATE
-    TORCH_TARGET_VERSION=0x020B000000000000ULL)
-
   # Needed to use cuda/hip APIs from C-shim
   if(VLLM_GPU_LANG STREQUAL "CUDA")
+    # Set TORCH_TARGET_VERSION for stable ABI compatibility.
+    # This ensures we only use C-shim APIs available in PyTorch 2.11.
+    # _C_stable_libtorch is abi compatible with PyTorch >= TORCH_TARGET_VERSION
+    # which is currently set to 2.11.
+    target_compile_definitions(_C_stable_libtorch PRIVATE
+      TORCH_TARGET_VERSION=0x020B000000000000ULL)
     target_compile_definitions(_C_stable_libtorch PRIVATE USE_CUDA)
-    if(COOPERATIVE_TOPK_ARCHS)
-      target_compile_definitions(_C_stable_libtorch PRIVATE
-        VLLM_ENABLE_COOPERATIVE_TOPK=1)
-    endif()
     # Needed by CUTLASS kernels
     target_compile_definitions(_C_stable_libtorch PRIVATE
       CUTLASS_ENABLE_DIRECT_CUDA_DRIVER_CALL=1)
   elseif(VLLM_GPU_LANG STREQUAL "HIP")
+    # Set TORCH_TARGET_VERSION for stable ABI compatibility.
+    # This ensures we only use C-shim APIs available in PyTorch 2.10.
+    # _C_stable_libtorch is abi compatible with PyTorch >= TORCH_TARGET_VERSION
+    # which is currently set to 2.10.
+    target_compile_definitions(_C_stable_libtorch PRIVATE
+      TORCH_TARGET_VERSION=0x020A000000000000ULL)
     target_compile_definitions(_C_stable_libtorch PRIVATE USE_ROCM)
   endif()
 
@@ -1311,20 +1266,25 @@ define_extension_target(
   USE_SABI 3
   WITH_SOABI)
 
-# Set TORCH_TARGET_VERSION for stable ABI compatibility.
-# This ensures we only use C-shim APIs available in PyTorch 2.11.
-# _moe_C_stable_libtorch is abi compatible with PyTorch >= TORCH_TARGET_VERSION
-# which is currently set to 2.11.
-target_compile_definitions(_moe_C_stable_libtorch PRIVATE
-  TORCH_TARGET_VERSION=0x020B000000000000ULL)
-
 # Needed to use cuda/hip APIs from C-shim
 if(VLLM_GPU_LANG STREQUAL "CUDA")
+  # Set TORCH_TARGET_VERSION for stable ABI compatibility.
+  # This ensures we only use C-shim APIs available in PyTorch 2.11.
+  # _moe_C_stable_libtorch is abi compatible with PyTorch >= TORCH_TARGET_VERSION
+  # which is currently set to 2.11.
+  target_compile_definitions(_moe_C_stable_libtorch PRIVATE
+    TORCH_TARGET_VERSION=0x020B000000000000ULL)
   target_compile_definitions(_moe_C_stable_libtorch PRIVATE USE_CUDA)
   # Needed by CUTLASS kernels
   target_compile_definitions(_moe_C_stable_libtorch PRIVATE
     CUTLASS_ENABLE_DIRECT_CUDA_DRIVER_CALL=1)
 elseif(VLLM_GPU_LANG STREQUAL "HIP")
+  # Set TORCH_TARGET_VERSION for stable ABI compatibility.
+  # This ensures we only use C-shim APIs available in PyTorch 2.10.
+  # _moe_C_stable_libtorch is abi compatible with PyTorch >= TORCH_TARGET_VERSION
+  # which is currently set to 2.10.
+  target_compile_definitions(_moe_C_stable_libtorch PRIVATE
+    TORCH_TARGET_VERSION=0x020A000000000000ULL)
   target_compile_definitions(_moe_C_stable_libtorch PRIVATE USE_ROCM)
 endif()
 

benchmarks/kernels/benchmark_fused_collective.py

@@ -80,17 +80,13 @@ _FI_MAX_SIZES = {
     2: 64 * MiB,  # 64MB
     4: 64 * MiB,  # 64MB
     8: 64 * MiB,  # 64MB
-    16: 64 * MiB,  # 64MB (multi-node)
 }
 
 # Global workspace tensors for FlashInfer (keyed by backend name)
 _FI_WORKSPACES: dict = {}
 
-# Backends to benchmark. trtllm is single-node only and can hang cross-node, so
-# multi-node sweeps can restrict to mnnvl via FI_BACKENDS=mnnvl.
-FLASHINFER_BACKENDS = [
-    b for b in os.environ.get("FI_BACKENDS", "trtllm,mnnvl").split(",") if b
-]
+# Backends to benchmark
+FLASHINFER_BACKENDS = ["trtllm", "mnnvl"]
 
 
 def setup_flashinfer_workspace(
@@ -999,10 +995,7 @@ def main():
     rank = int(os.environ["RANK"])
     world_size = int(os.environ["WORLD_SIZE"])
 
-    # Use LOCAL_RANK for the device so multi-node runs (global rank >= GPUs per
-    # node) map to a valid local GPU; falls back to global rank single-node.
-    local_rank = int(os.environ.get("LOCAL_RANK", rank))
-    device = torch.device(f"cuda:{local_rank}")
+    device = torch.device(f"cuda:{rank}")
     torch.accelerator.set_device_index(device)
     torch.set_default_device(device)
 

benchmarks/kernels/benchmark_moe.py

@@ -391,19 +391,16 @@ def get_configs_compute_bound(use_fp16, block_quant_shape) -> list[dict[str, int
         config = dict(zip(keys, config_values))
         configs.append(config)
 
-    # Drop configs incompatible with fp8 block quantization. A tile must align
-    # to the quant-block scale grid, i.e. tile and block must divide one
-    # another. The kernel indexes scales per element (offs_bn // group_n,
-    # k_start // group_k), so a tile narrower than the block (e.g. N=64 with
-    # block_n=128) is valid -- and often faster at small batch. An exact
-    # multiple was required before, which dropped those smaller tiles entirely.
+    # Remove configs that are not compatible with fp8 block quantization
+    # BLOCK_SIZE_K must be a multiple of block_k
+    # BLOCK_SIZE_N must be a multiple of block_n
     if block_quant_shape is not None and not use_fp16:
         block_n, block_k = block_quant_shape[0], block_quant_shape[1]
         for config in configs[:]:
-            bn, bk = config["BLOCK_SIZE_N"], config["BLOCK_SIZE_K"]
-            n_aligned = bn % block_n == 0 or block_n % bn == 0
-            k_aligned = bk % block_k == 0 or block_k % bk == 0
-            if not (n_aligned and k_aligned):
+            if (
+                config["BLOCK_SIZE_K"] % block_k != 0
+                or config["BLOCK_SIZE_N"] % block_n != 0
+            ):
                 configs.remove(config)
     return configs
 

benchmarks/kernels/benchmark_paged_attention.py

@@ -19,11 +19,13 @@ from vllm.utils.torch_utils import (
 logger = init_logger(__name__)
 
 NUM_BLOCKS = 128 * 1024
+PARTITION_SIZE = 512
 PARTITION_SIZE_ROCM = 256
 
 
 @torch.inference_mode()
 def main(
+    version: str,
     num_seqs: int,
     seq_len: int,
     num_query_heads: int,
@@ -80,20 +82,27 @@ def main(
 
     # Prepare for the paged attention kernel.
     output = torch.empty_like(query)
-    num_partitions = (max_seq_len + PARTITION_SIZE_ROCM - 1) // PARTITION_SIZE_ROCM
-    tmp_output = torch.empty(
-        size=(num_seqs, num_query_heads, num_partitions, head_size),
-        dtype=output.dtype,
-        device=output.device,
-    )
-    exp_sums = torch.empty(
-        size=(num_seqs, num_query_heads, num_partitions),
-        dtype=torch.float32,
-        device=output.device,
-    )
-    max_logits = torch.empty_like(exp_sums)
+    if version == "v2":
+        if current_platform.is_rocm():
+            global PARTITION_SIZE
+            if not args.custom_paged_attn and not current_platform.is_navi():
+                PARTITION_SIZE = 1024
+            else:
+                PARTITION_SIZE = PARTITION_SIZE_ROCM
+        num_partitions = (max_seq_len + PARTITION_SIZE - 1) // PARTITION_SIZE
+        tmp_output = torch.empty(
+            size=(num_seqs, num_query_heads, num_partitions, head_size),
+            dtype=output.dtype,
+            device=output.device,
+        )
+        exp_sums = torch.empty(
+            size=(num_seqs, num_query_heads, num_partitions),
+            dtype=torch.float32,
+            device=output.device,
+        )
+        max_logits = torch.empty_like(exp_sums)
 
-    def run_benchmark(num_iters: int, profile: bool = False) -> float:
+    def run_cuda_benchmark(num_iters: int, profile: bool = False) -> float:
         torch.accelerator.synchronize()
         if profile:
             torch.cuda.cudart().cudaProfilerStart()
@@ -103,26 +112,67 @@ def main(
         k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
 
         for _ in range(num_iters):
-            ops.paged_attention_rocm(
-                output,
-                exp_sums,
-                max_logits,
-                tmp_output,
-                query,
-                key_cache,
-                value_cache,
-                num_kv_heads,
-                scale,
-                block_tables,
-                seq_lens,
-                None,
-                block_size,
-                max_seq_len,
-                alibi_slopes,
-                kv_cache_dtype,
-                k_scale,
-                v_scale,
-            )
+            if version == "v1":
+                ops.paged_attention_v1(
+                    output,
+                    query,
+                    key_cache,
+                    value_cache,
+                    num_kv_heads,
+                    scale,
+                    block_tables,
+                    seq_lens,
+                    block_size,
+                    max_seq_len,
+                    alibi_slopes,
+                    kv_cache_dtype,
+                    k_scale,
+                    v_scale,
+                )
+            elif version == "v2":
+                if not args.custom_paged_attn:
+                    ops.paged_attention_v2(
+                        output,
+                        exp_sums,
+                        max_logits,
+                        tmp_output,
+                        query,
+                        key_cache,
+                        value_cache,
+                        num_kv_heads,
+                        scale,
+                        block_tables,
+                        seq_lens,
+                        block_size,
+                        max_seq_len,
+                        alibi_slopes,
+                        kv_cache_dtype,
+                        k_scale,
+                        v_scale,
+                    )
+                else:
+                    ops.paged_attention_rocm(
+                        output,
+                        exp_sums,
+                        max_logits,
+                        tmp_output,
+                        query,
+                        key_cache,
+                        value_cache,
+                        num_kv_heads,
+                        scale,
+                        block_tables,
+                        seq_lens,
+                        None,
+                        block_size,
+                        max_seq_len,
+                        alibi_slopes,
+                        kv_cache_dtype,
+                        k_scale,
+                        v_scale,
+                    )
+            else:
+                raise ValueError(f"Invalid version: {version}")
         torch.accelerator.synchronize()
 
         end_time = time.perf_counter()
@@ -132,6 +182,7 @@ def main(
 
     # Warmup.
     print("Warming up...")
+    run_benchmark = run_cuda_benchmark
     run_benchmark(num_iters=3, profile=False)
 
     # Benchmark.
@@ -144,13 +195,12 @@ def main(
 
 if __name__ == "__main__":
     logger.warning(
-        "This script benchmarks the ROCm paged attention kernel. "
+        "This script benchmarks the paged attention kernel. "
         "By default this is no longer used in vLLM inference."
     )
-    if not current_platform.is_rocm():
-        raise RuntimeError("This benchmark requires the ROCm platform.")
 
     parser = FlexibleArgumentParser(description="Benchmark the paged attention kernel.")
+    parser.add_argument("--version", type=str, choices=["v1", "v2"], default="v2")
     parser.add_argument("--batch-size", type=int, default=8)
     parser.add_argument("--seq-len", type=int, default=4096)
     parser.add_argument("--num-query-heads", type=int, default=64)
@@ -158,7 +208,7 @@ if __name__ == "__main__":
     parser.add_argument(
         "--head-size",
         type=int,
-        choices=[64, 128],
+        choices=[64, 80, 96, 112, 120, 128, 192, 256],
         default=128,
     )
     parser.add_argument("--block-size", type=int, choices=[16, 32], default=16)
@@ -174,7 +224,11 @@ if __name__ == "__main__":
         choices=["auto", "fp8", "fp8_e5m2", "fp8_e4m3"],
         default="auto",
         help="Data type for kv cache storage. If 'auto', will use model "
-        "data type. ROCm (AMD GPU) supports fp8 (=fp8_e4m3)",
+        "data type. CUDA 11.8+ supports fp8 (=fp8_e4m3) and fp8_e5m2. "
+        "ROCm (AMD GPU) supports fp8 (=fp8_e4m3)",
+    )
+    parser.add_argument(
+        "--custom-paged-attn", action="store_true", help="Use custom paged attention"
     )
     args = parser.parse_args()
     print(args)
@@ -182,6 +236,7 @@ if __name__ == "__main__":
     if args.num_query_heads % args.num_kv_heads != 0:
         raise ValueError("num_query_heads must be divisible by num_kv_heads")
     main(
+        version=args.version,
         num_seqs=args.batch_size,
         seq_len=args.seq_len,
         num_query_heads=args.num_query_heads,

benchmarks/kernels/cpu/benchmark_cpu_fused_moe.py

@@ -7,7 +7,6 @@ import time
 import numpy as np
 import torch
 
-from vllm.platforms import CpuArchEnum, current_platform
 from vllm.utils.argparse_utils import FlexibleArgumentParser
 from vllm.utils.torch_utils import set_random_seed
 
@@ -15,15 +14,17 @@ from vllm.utils.torch_utils import set_random_seed
 try:
     from vllm._custom_ops import cpu_fused_moe, cpu_prepack_moe_weight
 except (ImportError, AttributeError) as e:
+    print("ERROR: CPU fused MoE operations are not available on this platform.")
+    print("This benchmark requires x86 CPU with proper vLLM CPU extensions compiled.")
+    print(
+        "The cpu_fused_moe kernel is typically available on Linux x86_64 "
+        "with AVX2/AVX512."
+    )
     print(f"Import error: {e}")
     sys.exit(1)
 
 # ISA selection following test_cpu_fused_moe.py pattern
-ISA_CHOICES = ["vec"]
-if torch.cpu._is_amx_tile_supported():
-    ISA_CHOICES.append("amx")
-if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
-    ISA_CHOICES.append("neon")
+ISA_CHOICES = ["amx", "vec"] if torch.cpu._is_amx_tile_supported() else ["vec"]
 
 
 @torch.inference_mode()
@@ -144,7 +145,7 @@ if __name__ == "__main__":
         "--isa",
         type=str,
         choices=ISA_CHOICES,
-        default="vec",
+        default=ISA_CHOICES[0],
         help=f"ISA to use (available: {ISA_CHOICES})",
     )
     parser.add_argument("--seed", type=int, default=0)

cmake/cpu_extension.cmake

@@ -15,7 +15,6 @@ endif()
 #
 set(ENABLE_X86_ISA $ENV{VLLM_CPU_X86})
 set(ENABLE_ARM_BF16 $ENV{VLLM_CPU_ARM_BF16})
-set(ENABLE_RVV_BF16 $ENV{VLLM_CPU_RVV_BF16})
 
 include_directories("${CMAKE_SOURCE_DIR}/csrc")
 
@@ -25,10 +24,7 @@ set (ENABLE_NUMA TRUE)
 # Check the compile flags
 #
 if(MACOSX_FOUND)
-    # Apple clang needs -Xpreprocessor to enable OpenMP. No runtime link is
-    # needed: _C is a dynamic_lookup bundle and resolves libomp from torch.
     list(APPEND CXX_COMPILE_FLAGS
-        "-Xpreprocessor" "-fopenmp"
         "-DVLLM_CPU_EXTENSION")
 else()
     list(APPEND CXX_COMPILE_FLAGS
@@ -111,13 +107,6 @@ else()
         set(ARM_BF16_FOUND ON)
         message(STATUS "ARM BF16 support enabled via VLLM_CPU_ARM_BF16 environment variable")
     endif()
-    # Some kernels (e.g. Bianbu on Spacemit X100) do not report zvfbfmin
-    # in /proc/cpuinfo despite hardware support. VLLM_CPU_RVV_BF16=1
-    # overrides the detection result.
-    if (ENABLE_RVV_BF16)
-        set(RVV_BF16_FOUND ON)
-        message(STATUS "RVV BF16 support enabled via VLLM_CPU_RVV_BF16 environment variable")
-    endif()
 endif()
 
 if (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|amd64" OR ENABLE_X86_ISA)
@@ -177,19 +166,15 @@ elseif (S390_FOUND)
         "-mtune=native")
 elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "riscv64")
     message(STATUS "RISC-V detected")
-    if(DEFINED VLLM_RVV_VLEN AND VLLM_RVV_VLEN LESS 0)
+    if(DEFINED VLLM_RVV_VLEN AND NOT VLLM_RVV_VLEN GREATER 0)
         message(FATAL_ERROR
-            "VLLM_RVV_VLEN must be zero or a positive integer; got '${VLLM_RVV_VLEN}'")
+            "VLLM_RVV_VLEN must be a positive integer; got '${VLLM_RVV_VLEN}'")
     endif()
     # VLLM_RVV_VLEN selects the target VLEN. Auto-detected from /proc/cpuinfo
-    # by default; set -DVLLM_RVV_VLEN=0 to force scalar RISC-V build.
-    # Override with -DVLLM_RVV_VLEN=128 or -DVLLM_RVV_VLEN=256 for RVV.
+    # by default; override with -DVLLM_RVV_VLEN=128 or -DVLLM_RVV_VLEN=256.
     if(NOT DEFINED VLLM_RVV_VLEN)
         # Auto-detect: find the largest zvl<N>b in /proc/cpuinfo isa line.
-        # Skip when cross-compiling — /proc/cpuinfo describes the build host.
-        if(CMAKE_CROSSCOMPILING)
-            message(STATUS "Cross-compiling: skipping VLEN auto-detection from /proc/cpuinfo")
-        elseif(EXISTS /proc/cpuinfo)
+        if(EXISTS /proc/cpuinfo)
             file(READ /proc/cpuinfo _cpuinfo)
             set(_best 0)
             foreach(_n IN ITEMS 128 256 512 1024)
@@ -197,13 +182,6 @@ elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "riscv64")
                     set(_best ${_n})
                 endif()
             endforeach()
-            # Only VLEN=128 and VLEN=256 are supported by the RVV kernels.
-            if(_best GREATER 256)
-                message(WARNING
-                    "Detected VLEN=${_best} but only 128/256 are supported; "
-                    "clamping to 256")
-                set(_best 256)
-            endif()
             if(_best GREATER 0)
                 set(VLLM_RVV_VLEN ${_best})
             endif()
@@ -213,9 +191,9 @@ elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "riscv64")
         if(NOT DEFINED VLLM_RVV_VLEN AND (RVV_FP16_FOUND OR RVV_BF16_FOUND))
             message(FATAL_ERROR
                 "RISC-V RVV is available but VLEN could not be auto-detected. "
-                "Please specify VLEN explicitly via CMAKE_ARGS:\n"
-                "  CMAKE_ARGS='-DVLLM_RVV_VLEN=128'   (for VLEN=128 hardware)\n"
-                "  CMAKE_ARGS='-DVLLM_RVV_VLEN=256'   (for VLEN=256 hardware, e.g. Spacemit X100)")
+                "Please specify VLEN explicitly:\n"
+                "  -DVLLM_RVV_VLEN=128   (for VLEN=128 hardware)\n"
+                "  -DVLLM_RVV_VLEN=256   (for VLEN=256 hardware, e.g. Spacemit X100)")
         endif()
     endif()
     if(VLLM_RVV_VLEN AND VLLM_RVV_VLEN GREATER 0)
@@ -227,7 +205,7 @@ elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "riscv64")
             message(STATUS "BF16 extension detected")
             set(MARCH_FLAGS -march=rv64gcv_zvfh_zfbfmin_zvfbfmin_zvl${VLLM_RVV_VLEN}b -mrvv-vector-bits=zvl -mabi=lp64d)
         elseif(RVV_FP16_FOUND)
-            message(WARNING "BF16 functionality is not available.")
+            message(WARNING "BF16 functionality is not available")
             set(MARCH_FLAGS -march=rv64gcv_zvfh_zvl${VLLM_RVV_VLEN}b -mrvv-vector-bits=zvl -mabi=lp64d)
         else()
             message(STATUS "compile riscv with scalar (no FP16/BF16)")
@@ -347,7 +325,7 @@ if (ENABLE_X86_ISA OR (ASIMD_FOUND AND NOT APPLE_SILICON_FOUND) OR POWER9_FOUND
     set(ONEDNN_ENABLE_PRIMITIVE "MATMUL;REORDER")
     set(ONEDNN_BUILD_GRAPH "OFF")
     set(ONEDNN_ENABLE_JIT_PROFILING "ON")
-    set(ONEDNN_ENABLE_ITT_TASKS "ON")
+    set(ONEDNN_ENABLE_ITT_TASKS "OFF")
     set(ONEDNN_ENABLE_MAX_CPU_ISA "ON")
     set(ONEDNN_ENABLE_CPU_ISA_HINTS "ON")
     set(ONEDNN_VERBOSE "ON")
@@ -445,8 +423,6 @@ if (ASIMD_FOUND AND NOT APPLE_SILICON_FOUND)
     set(VLLM_EXT_SRC
         "csrc/cpu/shm.cpp"
         "csrc/cpu/activation_lut_bf16.cpp"
-        "csrc/cpu/cpu_tanhf_neon.hpp"
-        "csrc/cpu/cpu_fused_moe.cpp"
         ${VLLM_EXT_SRC})
 endif()
 

cmake/external_projects/deepgemm.cmake

@@ -8,74 +8,43 @@ if (DEFINED ENV{DEEPGEMM_SRC_DIR})
   set(DEEPGEMM_SRC_DIR $ENV{DEEPGEMM_SRC_DIR})
 endif()
 
-# Local tree: set deepgemm_SOURCE_DIR directly (no FetchContent download).
-# Upstream git: use FetchContent_Populate with explicit options (CMP0169 NEW
-# disallows one-argument Populate(dep) after Declare; MakeAvailable would run
-# DeepGEMM's top-level CMakeLists.txt, which vLLM must not load).
 if(DEEPGEMM_SRC_DIR)
-  # cmake_path(ABSOLUTE_PATH <var> ...) reads the path from <var>; NORMALIZE is a
-  # flag (no trailing path argument). Resolve relative paths against vLLM root.
-  set(_deepgemm_user_src "${DEEPGEMM_SRC_DIR}")
-  cmake_path(ABSOLUTE_PATH _deepgemm_user_src
-    BASE_DIRECTORY "${CMAKE_SOURCE_DIR}"
-    NORMALIZE)
-  set(DEEPGEMM_SRC_DIR "${_deepgemm_user_src}")
-  if(NOT IS_DIRECTORY "${DEEPGEMM_SRC_DIR}")
-    message(FATAL_ERROR
-      "DEEPGEMM_SRC_DIR is not an existing directory: '${DEEPGEMM_SRC_DIR}'")
-  endif()
-  set(deepgemm_SOURCE_DIR "${DEEPGEMM_SRC_DIR}")
-  message(STATUS "DeepGEMM using local DEEPGEMM_SRC_DIR: ${deepgemm_SOURCE_DIR}")
+  FetchContent_Declare(
+    deepgemm
+    SOURCE_DIR ${DEEPGEMM_SRC_DIR}
+    CONFIGURE_COMMAND ""
+    BUILD_COMMAND ""
+  )
 else()
-  # Keep in sync with tools/install_deepgemm.sh
-  set(_DEEPGEMM_UPSTREAM_REPO "https://github.com/deepseek-ai/DeepGEMM.git")
-  # NOTE: This is currently targeting nv-dev branch due to sm120 support
-  set(_DEEPGEMM_UPSTREAM_TAG "a6b593d2826719dcf4892609af7b84ee23aaf32a")
-
-  set(_deepgemm_fc_root "${FETCHCONTENT_BASE_DIR}")
-  if(NOT _deepgemm_fc_root)
-    set(_deepgemm_fc_root "${CMAKE_BINARY_DIR}/_deps")
-  endif()
-  set(_deepgemm_src "${_deepgemm_fc_root}/deepgemm-src")
-  set(_deepgemm_bin "${_deepgemm_fc_root}/deepgemm-build")
-  set(_deepgemm_sub "${_deepgemm_fc_root}/deepgemm-subbuild")
-
-  if(EXISTS "${_deepgemm_src}/csrc/python_api.cpp")
-    set(deepgemm_SOURCE_DIR "${_deepgemm_src}")
-    set(deepgemm_BINARY_DIR "${_deepgemm_bin}")
-  else()
-    FetchContent_Populate(
-      deepgemm
-      SUBBUILD_DIR "${_deepgemm_sub}"
-      SOURCE_DIR "${_deepgemm_src}"
-      BINARY_DIR "${_deepgemm_bin}"
-      GIT_REPOSITORY "${_DEEPGEMM_UPSTREAM_REPO}"
-      GIT_TAG "${_DEEPGEMM_UPSTREAM_TAG}"
-      GIT_SUBMODULES "third-party/cutlass" "third-party/fmt"
-      GIT_PROGRESS TRUE
-    )
-  endif()
-  message(STATUS "DeepGEMM is available at ${deepgemm_SOURCE_DIR}")
+  # This ref should be kept in sync with tools/install_deepgemm.sh
+  FetchContent_Declare(
+    deepgemm
+    GIT_REPOSITORY https://github.com/deepseek-ai/DeepGEMM.git
+    GIT_TAG 891d57b4db1071624b5c8fa0d1e51cb317fa709f
+    GIT_SUBMODULES "third-party/cutlass" "third-party/fmt"
+    GIT_PROGRESS TRUE
+    CONFIGURE_COMMAND ""
+    BUILD_COMMAND ""
+  )
 endif()
 
-# DeepGEMM requires CUDA 12.3+ for SM90, 12.9+ for SM100 (official upstream),
-# and 12.8+ for SM120 / SM12x. CUDA 13+ can use the family-specific SM12x
-# arch; CUDA 12.x builds the arch-specific SM120/SM121 variants.
+# Use FetchContent_Populate (not MakeAvailable) to avoid processing
+# DeepGEMM's own CMakeLists.txt which has incompatible find_package calls.
+FetchContent_GetProperties(deepgemm)
+if(NOT deepgemm_POPULATED)
+  FetchContent_Populate(deepgemm)
+endif()
+message(STATUS "DeepGEMM is available at ${deepgemm_SOURCE_DIR}")
+
+# DeepGEMM requires CUDA 12.3+ for SM90, 12.9+ for SM100
 set(DEEPGEMM_SUPPORT_ARCHS)
 if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.3)
   list(APPEND DEEPGEMM_SUPPORT_ARCHS "9.0a")
 endif()
-if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8)
-  if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.9)
-    list(APPEND DEEPGEMM_SUPPORT_ARCHS "10.0f")
-  else()
-    list(APPEND DEEPGEMM_SUPPORT_ARCHS "10.0a")
-  endif()
-  if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0)
-    list(APPEND DEEPGEMM_SUPPORT_ARCHS "12.0f")
-  else()
-    list(APPEND DEEPGEMM_SUPPORT_ARCHS "12.0a" "12.1a")
-  endif()
+if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.9)
+  list(APPEND DEEPGEMM_SUPPORT_ARCHS "10.0f")
+elseif(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8)
+  list(APPEND DEEPGEMM_SUPPORT_ARCHS "10.0a")
 endif()
 
 cuda_archs_loose_intersection(DEEPGEMM_ARCHS

cmake/external_projects/fmha_sm100.cmake

@@ -17,7 +17,7 @@ else()
   FetchContent_Declare(
     fmha_sm100
     GIT_REPOSITORY https://github.com/vllm-project/MSA.git
-    GIT_TAG 2e63ec37a0fc29bc20f39cd1a52e0f5affc33a73
+    GIT_TAG 544eee5e09ae2dfa774d5b06739013f9b7402c57
     GIT_PROGRESS TRUE
     CONFIGURE_COMMAND ""
     BUILD_COMMAND ""
@@ -36,38 +36,13 @@ set(FMHA_SM100_PY_ROOT "${fmha_sm100_SOURCE_DIR}/python/fmha_sm100")
 
 install(FILES
   "${FMHA_SM100_PY_ROOT}/__init__.py"
-  "${FMHA_SM100_PY_ROOT}/api.py"
-  "${FMHA_SM100_PY_ROOT}/bench_utils.py"
-  "${FMHA_SM100_PY_ROOT}/jit.py"
   "${FMHA_SM100_PY_ROOT}/sparse.py"
-  "${FMHA_SM100_PY_ROOT}/sparse_fmha_adapter.py"
   DESTINATION vllm/third_party/fmha_sm100
   COMPONENT fmha_sm100)
 
-install(DIRECTORY "${FMHA_SM100_PY_ROOT}/csrc/"
-  DESTINATION vllm/third_party/fmha_sm100/csrc
-  COMPONENT fmha_sm100
-  PATTERN "__pycache__" EXCLUDE
-  PATTERN "*.pyc" EXCLUDE
-  PATTERN ".git*" EXCLUDE)
-
 install(DIRECTORY "${FMHA_SM100_PY_ROOT}/cute/"
   DESTINATION vllm/third_party/fmha_sm100/cute
   COMPONENT fmha_sm100
   PATTERN "__pycache__" EXCLUDE
   PATTERN "*.pyc" EXCLUDE
   PATTERN ".git*" EXCLUDE)
-
-install(DIRECTORY "${FMHA_SM100_PY_ROOT}/cutlass/include/"
-  DESTINATION vllm/third_party/fmha_sm100/cutlass/include
-  COMPONENT fmha_sm100
-  PATTERN "__pycache__" EXCLUDE
-  PATTERN "*.pyc" EXCLUDE
-  PATTERN ".git*" EXCLUDE)
-
-install(DIRECTORY "${FMHA_SM100_PY_ROOT}/cutlass/tools/util/include/"
-  DESTINATION vllm/third_party/fmha_sm100/cutlass/tools/util/include
-  COMPONENT fmha_sm100
-  PATTERN "__pycache__" EXCLUDE
-  PATTERN "*.pyc" EXCLUDE
-  PATTERN ".git*" EXCLUDE)

cmake/external_projects/qutlass.cmake

@@ -6,48 +6,26 @@ if(DEFINED ENV{QUTLASS_SRC_DIR})
   set(QUTLASS_SRC_DIR $ENV{QUTLASS_SRC_DIR})
 endif()
 
-# CMP0169 NEW: one-argument FetchContent_Populate(name) after Declare is invalid.
-# Use explicit Populate(...) for git, or set SOURCE_DIR for local trees.
 if(QUTLASS_SRC_DIR)
-  set(_qutlass_user_src "${QUTLASS_SRC_DIR}")
-  cmake_path(ABSOLUTE_PATH _qutlass_user_src
-    BASE_DIRECTORY "${CMAKE_SOURCE_DIR}"
-    NORMALIZE)
-  set(QUTLASS_SRC_DIR "${_qutlass_user_src}")
-  if(NOT IS_DIRECTORY "${QUTLASS_SRC_DIR}")
-    message(FATAL_ERROR
-      "[QUTLASS] QUTLASS_SRC_DIR is not an existing directory: '${QUTLASS_SRC_DIR}'")
-  endif()
-  set(qutlass_SOURCE_DIR "${QUTLASS_SRC_DIR}")
-  set(qutlass_BINARY_DIR "${CMAKE_BINARY_DIR}/qutlass-binary-dir-unused")
+  FetchContent_Declare(
+    qutlass
+    SOURCE_DIR ${QUTLASS_SRC_DIR}
+    CONFIGURE_COMMAND ""
+    BUILD_COMMAND ""
+  )
 else()
-  set(_QUTLASS_UPSTREAM_REPO "https://github.com/IST-DASLab/qutlass.git")
-  set(_QUTLASS_UPSTREAM_TAG "830d2c4537c7396e14a02a46fbddd18b5d107c65")
-
-  set(_qutlass_fc_root "${FETCHCONTENT_BASE_DIR}")
-  if(NOT _qutlass_fc_root)
-    set(_qutlass_fc_root "${CMAKE_BINARY_DIR}/_deps")
-  endif()
-  set(_qutlass_src "${_qutlass_fc_root}/qutlass-src")
-  set(_qutlass_bin "${_qutlass_fc_root}/qutlass-build")
-  set(_qutlass_sub "${_qutlass_fc_root}/qutlass-subbuild")
-
-  if(EXISTS "${_qutlass_src}/qutlass/csrc/bindings.cpp")
-    set(qutlass_SOURCE_DIR "${_qutlass_src}")
-    set(qutlass_BINARY_DIR "${_qutlass_bin}")
-  else()
-    FetchContent_Populate(
-      qutlass
-      SUBBUILD_DIR "${_qutlass_sub}"
-      SOURCE_DIR "${_qutlass_src}"
-      BINARY_DIR "${_qutlass_bin}"
-      GIT_REPOSITORY "${_QUTLASS_UPSTREAM_REPO}"
-      GIT_TAG "${_QUTLASS_UPSTREAM_TAG}"
-      GIT_PROGRESS TRUE
-    )
-  endif()
+  FetchContent_Declare(
+    qutlass
+    GIT_REPOSITORY https://github.com/IST-DASLab/qutlass.git
+    GIT_TAG 830d2c4537c7396e14a02a46fbddd18b5d107c65
+    GIT_PROGRESS TRUE
+    CONFIGURE_COMMAND ""
+    BUILD_COMMAND ""
+  )
 endif()
 
+FetchContent_Populate(qutlass)
+
 if(NOT qutlass_SOURCE_DIR)
   message(FATAL_ERROR "[QUTLASS] source directory could not be resolved.")
 endif()

cmake/external_projects/vllm_flash_attn.cmake

@@ -39,7 +39,7 @@ else()
   FetchContent_Declare(
           vllm-flash-attn
           GIT_REPOSITORY https://github.com/vllm-project/flash-attention.git
-          GIT_TAG 2c839c33742309ec41e620bf837495ec9926c56e
+          GIT_TAG 803020a8fa15407871341d41eba4919ade2ee1ee
           GIT_PROGRESS TRUE
           # Don't share the vllm-flash-attn build between build types
           BINARY_DIR ${CMAKE_BINARY_DIR}/vllm-flash-attn

csrc/cpu/activation.cpp

@@ -126,18 +126,6 @@ void gelu_tanh_and_mul(torch::Tensor& out,    // [..., d]
       });
 }
 
-void gelu_tanh(torch::Tensor& out, torch::Tensor& input) {
-  int num_tokens = input.numel() / input.size(-1);
-  int d = input.size(-1);
-
-  VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "gelu_tanh_impl", [&] {
-    CPU_KERNEL_GUARD_IN(gelu_tanh_impl)
-    activation_kernel<scalar_t, gelu_tanh_act, false>(
-        num_tokens, d, input.data_ptr<scalar_t>(), out.data_ptr<scalar_t>());
-    CPU_KERNEL_GUARD_OUT(gelu_tanh_impl)
-  });
-}
-
 void gelu_new(torch::Tensor& out, torch::Tensor& input) {
   int num_tokens = input.numel() / input.size(-1);
   int d = input.size(-1);

csrc/cpu/cpu_attn.cpp

@@ -13,8 +13,7 @@ static inline cpu_attention::Fp8KVCacheDataType parse_fp8_kv_dtype(
 
 bool cpu_attn_has_isa(const std::string& isa) {
   if (isa == "rvv") {
-#if defined(__riscv) && defined(__riscv_v_min_vlen) && \
-    (__riscv_v_min_vlen == 128 || __riscv_v_min_vlen == 256)
+#if defined(__riscv) && defined(__riscv_v_min_vlen) && __riscv_v_min_vlen == 128
     return true;
 #else
     return false;

csrc/cpu/cpu_attn_impl.hpp

@@ -124,7 +124,7 @@ struct AttentionMetadata {
         workitem_group_num(workitem_group_num),
         reduction_item_num(reduction_item_num),
         reduction_split_num(reduction_split_num),
-        thread_num(cpu_utils::get_max_threads()),
+        thread_num(omp_get_max_threads()),
         effective_thread_num(thread_num),
         split_kv_q_token_num_threshold(split_kv_q_token_num_threshold),
         attention_scratchpad_size_per_thread(0),
@@ -405,7 +405,7 @@ class AttentionScheduler {
   torch::Tensor schedule(const ScheduleInput& input) const {
     const bool causal = input.causal;
     const bool is_dynamic_causal = input.dynamic_causal != nullptr;
-    const int32_t thread_num = cpu_utils::get_max_threads();
+    const int32_t thread_num = omp_get_max_threads();
     const int64_t cache_size = cpu_utils::get_available_l2_size();
     const int32_t max_num_q_per_iter = input.max_num_q_per_iter;
     const int32_t kv_len_alignment = input.kv_block_alignment;
@@ -417,10 +417,8 @@ class AttentionScheduler {
       has_decode_request = has_decode_request || (q_token_num == 1);
       decode_only_batch = decode_only_batch && (q_token_num == 1);
     }
-    const int32_t original_q_head_per_kv =
-        input.num_heads_q / input.num_heads_kv;
-    int32_t q_head_per_kv = original_q_head_per_kv;
-    const bool supports_gqa = original_q_head_per_kv <= max_num_q_per_iter;
+    int32_t q_head_per_kv = input.num_heads_q / input.num_heads_kv;
+    const bool supports_gqa = q_head_per_kv <= max_num_q_per_iter;
     const bool use_gqa_fast_path = supports_gqa && decode_only_batch;
     const bool use_gqa_scratchpad = supports_gqa && has_decode_request;
     if (!use_gqa_scratchpad) {
@@ -673,62 +671,22 @@ class AttentionScheduler {
     metadata_ptr->effective_thread_num = effective_thread_num;
 
     {
+      // when q_tile_size = max_num_q_per_iter, requires max
+      // attention_scratchpad_size
       AttentionScratchPad sc(0, *metadata_ptr, 0x0);
-      int64_t max_attention_scratchpad_size = 0;
-
-      for (const AttentionWorkItemGroup& item : workitems) {
-        const bool curr_use_gqa =
-            use_gqa_fast_path || (supports_gqa && item.q_token_num == 1);
-        const int32_t curr_q_heads_per_kv =
-            curr_use_gqa ? original_q_head_per_kv : 1;
-        const int32_t curr_default_q_tile_token_num =
-            default_tile_size / curr_q_heads_per_kv;
-
-        for (int32_t q_token_offset = 0; q_token_offset < item.q_token_num;
-             q_token_offset += curr_default_q_tile_token_num) {
-          const int32_t actual_q_token_num = std::min(
-              curr_default_q_tile_token_num, item.q_token_num - q_token_offset);
-          const int32_t q_head_tile_size =
-              actual_q_token_num * curr_q_heads_per_kv;
-          const int32_t rounded_q_head_tile_size =
-              ((q_head_tile_size + max_num_q_per_iter - 1) /
-               max_num_q_per_iter) *
-              max_num_q_per_iter;
-
-          const int64_t n = AttentionScheduler::calcu_tile_size_with_constant_q(
-              cache_size, input.head_dim, input.elem_size,
-              input.q_buffer_elem_size, input.logits_buffer_elem_size,
-              input.output_buffer_elem_size, max_num_q_per_iter,
-              kv_len_alignment, rounded_q_head_tile_size,
-              rounded_q_head_tile_size <= max_num_q_per_iter);
-
-          sc.update(input.head_dim, input.q_buffer_elem_size,
-                    input.logits_buffer_elem_size,
-                    input.output_buffer_elem_size, max_num_q_per_iter,
-                    rounded_q_head_tile_size, n);
-
-          max_attention_scratchpad_size = std::max(
-              max_attention_scratchpad_size, sc.get_thread_scratchpad_size());
-        }
-      }
-
+      int64_t n = AttentionScheduler::calcu_tile_size_with_constant_q(
+          cache_size, input.head_dim, input.elem_size, input.q_buffer_elem_size,
+          input.logits_buffer_elem_size, input.output_buffer_elem_size,
+          max_num_q_per_iter, kv_len_alignment, max_num_q_per_iter, true);
+      sc.update(input.head_dim, input.q_buffer_elem_size,
+                input.logits_buffer_elem_size, input.output_buffer_elem_size,
+                max_num_q_per_iter, max_num_q_per_iter, n);
       metadata_ptr->attention_scratchpad_size_per_thread =
-          ((max_attention_scratchpad_size + 63) / 64) * 64;
-
-      int32_t max_reduction_q_head_tile_size = 0;
-      for (const ReductionWorkItemGroup& item : reduce_workitems) {
-        const bool curr_use_gqa =
-            use_gqa_fast_path || (supports_gqa && item.q_token_id_num == 1);
-        const int32_t curr_q_heads_per_kv =
-            curr_use_gqa ? original_q_head_per_kv : 1;
-
-        max_reduction_q_head_tile_size =
-            std::max(max_reduction_q_head_tile_size,
-                     item.q_token_id_num * curr_q_heads_per_kv);
-      }
+          ((sc.get_thread_scratchpad_size() + 63) / 64) * 64;
 
       sc.update(0, metadata_ptr->reduction_split_num, input.head_dim,
-                max_reduction_q_head_tile_size, input.output_buffer_elem_size);
+                q_head_per_kv * split_kv_q_token_num_threshold,
+                input.output_buffer_elem_size);
       metadata_ptr->reduction_scratchpad_size_per_kv_head =
           ((sc.get_reduction_scratchpad_size() + 63) / 64) * 64;
     }
@@ -929,10 +887,12 @@ struct VecTypeTrait<c10::BFloat16> {
   using vec_t = vec_op::BF16Vec16;
 };
 
+#if !defined(__powerpc__)
 template <>
 struct VecTypeTrait<c10::Half> {
   using vec_t = vec_op::FP16Vec16;
 };
+#endif
 
 template <typename T>
 void print_logits(const char* name, T* ptr, int32_t row, int32_t col,
@@ -1465,7 +1425,7 @@ class AttentionMainLoop {
 
  public:
   void operator()(const AttentionInput* input) {
-    const int thread_num = cpu_utils::get_max_threads();
+    const int thread_num = omp_get_max_threads();
     TORCH_CHECK_EQ(input->metadata->thread_num, thread_num);
     std::atomic<int32_t> guard_counter(0);
     std::atomic<int32_t>* guard_counter_ptr = &guard_counter;

csrc/cpu/cpu_attn_vsx.hpp

@@ -50,16 +50,7 @@ FORCE_INLINE void load_row8_B_as_f32<c10::BFloat16>(const c10::BFloat16* p,
   b1 = (__vector float)vec_mergel(zeros, raw);
 }
 
-// [3] Half (FP16) Specialization
-template <>
-FORCE_INLINE void load_row8_B_as_f32<c10::Half>(const c10::Half* p,
-                                                __vector float& b0,
-                                                __vector float& b1) {
-  vec_op::FP16Vec8 fp16_vec(p);
-  vec_op::FP32Vec8 fp32_vec(fp16_vec);
-  b0 = fp32_vec.reg.val[0];
-  b1 = fp32_vec.reg.val[1];
-}
+// Note: c10::Half (FP16) is not supported on PowerPC architecture
 
 template <int32_t M, typename kv_cache_t>
 FORCE_INLINE void gemm_micro_ppc64le_Mx8_Ku4(

csrc/cpu/cpu_fused_moe.cpp

@@ -1,3 +1,5 @@
+#include <sleef.h>
+
 #include "cpu/cpu_types.hpp"
 #include "cpu/utils.hpp"
 #include "cpu/micro_gemm/cpu_micro_gemm_vec.hpp"
@@ -14,18 +16,6 @@
   #define AMX_DISPATCH(...) case cpu_utils::ISA::AMX:
 #endif
 
-#if defined(ARM_BF16_SUPPORT)
-  #include "cpu/micro_gemm/cpu_micro_gemm_neon.hpp"
-  #define NEON_DISPATCH(...)                                         \
-    case cpu_utils::ISA::NEON: {                                     \
-      using gemm_t =                                                 \
-          cpu_micro_gemm::MicroGemm<cpu_utils::ISA::NEON, scalar_t>; \
-      return __VA_ARGS__();                                          \
-    }
-#else
-  #define NEON_DISPATCH(...) case cpu_utils::ISA::NEON:
-#endif
-
 #define CPU_ISA_DISPATCH_IMPL(ISA_TYPE, ...)                          \
   [&] {                                                               \
     switch (ISA_TYPE) {                                               \
@@ -35,7 +25,6 @@
             cpu_micro_gemm::MicroGemm<cpu_utils::ISA::VEC, scalar_t>; \
         return __VA_ARGS__();                                         \
       }                                                               \
-        NEON_DISPATCH(__VA_ARGS__)                                    \
       default: {                                                      \
         TORCH_CHECK(false, "Invalid CPU ISA type.");                  \
       }                                                               \
@@ -70,12 +59,10 @@ void swigluoai_and_mul(float* __restrict__ input, scalar_t* __restrict__ output,
                        const int32_t input_stride,
                        const int32_t output_stride) {
   using scalar_vec_t = typename cpu_utils::VecTypeTrait<scalar_t>::vec_t;
-#if !defined(__aarch64__)
   // For GPT-OSS interleaved gate-up weights
   alignas(64) static int32_t index[16] = {0,  2,  4,  6,  8,  10, 12, 14,
                                           16, 18, 20, 22, 24, 26, 28, 30};
   vec_op::INT32Vec16 index_vec(index);
-#endif
   vec_op::FP32Vec16 gate_up_max_vec(7.0);
   vec_op::FP32Vec16 up_min_vec(-7.0);
   vec_op::FP32Vec16 alpha_vec(1.702);
@@ -85,15 +72,8 @@ void swigluoai_and_mul(float* __restrict__ input, scalar_t* __restrict__ output,
 
   for (int32_t m = 0; m < m_size; ++m) {
     for (int32_t n = 0; n < n_size; n += 32) {
-      // Note: AdvSIMD does not support gather loads
-#if defined(__aarch64__)
-      vec_op::FP32Vec16 gate_vec(vec_op::uninit);
-      vec_op::FP32Vec16 up_vec(vec_op::uninit);
-      vec_op::FP32Vec16::load_even_odd(input + n, gate_vec, up_vec);
-#else
       vec_op::FP32Vec16 gate_vec(input + n, index_vec);
       vec_op::FP32Vec16 up_vec(input + n + 1, index_vec);
-#endif
       gate_vec = gate_vec.min(gate_up_max_vec);
       up_vec = up_vec.clamp(up_min_vec, gate_up_max_vec);
       auto sigmoid_vec = one_vec / (one_vec + fast_exp(-gate_vec * alpha_vec));
@@ -194,7 +174,7 @@ void gelu_tanh_and_mul(float* __restrict__ input, scalar_t* __restrict__ output,
       auto inner_vec = w1_vec * (gate_vec + w3_vec * gate_pow3_vec);
       // Note: can't use fast_exp form because diffusiongemma will generate
       // wrong results
-      auto tanh_vec = inner_vec.tanh();
+      vec_op::FP32Vec16 tanh_vec(Sleef_tanhf16_u10(inner_vec.reg));
       auto gelu_tanh = gate_vec * w2_vec * (one_vec + tanh_vec);
       auto gated_output_fp32 = up_vec * gelu_tanh;
       scalar_vec_t gated_output = scalar_vec_t(gated_output_fp32);
@@ -260,14 +240,13 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input,
   constexpr int32_t gemm_n_tile_size = gemm_t::NSize;
   constexpr int32_t gemm_m_tile_size = gemm_t::MaxMSize;
   constexpr int32_t min_w13_n_tile_size = 2 * gemm_n_tile_size;
-  constexpr bool pack_a = gemm_t::PackA;
   static_assert(gemm_n_tile_size % 16 == 0);
 
   TORCH_CHECK_EQ(output_size_13 % min_w13_n_tile_size, 0);
   TORCH_CHECK_EQ(output_size_2 % gemm_n_tile_size, 0);
   TORCH_CHECK_EQ(output_size_13 / 2, input_size_2);
 
-  const int32_t thread_num = cpu_utils::get_max_threads();
+  const int32_t thread_num = omp_get_max_threads();
 
   const int32_t w13_input_buffer_size = cpu_utils::round_up<64>(
       gemm_m_tile_size * input_size_13 * sizeof(scalar_t));
@@ -287,18 +266,12 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input,
 
   const int32_t w2_input_tile_size = cpu_utils::round_up<64>(
       gemm_m_tile_size * input_size_2 * sizeof(scalar_t));
-  // use w2 input buffer only when we need to pack input
-  const int32_t w2_input_buffer_size =
-      pack_a ? cpu_utils::round_up<64>(gemm_m_tile_size * input_size_2 *
-                                       sizeof(scalar_t))
-             : 0;
 
   const int32_t w2_n_tile_size = [&]() {
     const int64_t cache_size = cpu_utils::get_available_l2_size();
-    // input tile + optional packed input + weight
+    // input tile + weight
     const int32_t n_size_cache_limit =
-        (cache_size - (pack_a ? w2_input_buffer_size : w2_input_tile_size)) /
-        (input_size_2 * sizeof(scalar_t));
+        (cache_size - w2_input_tile_size) / (input_size_2 * sizeof(scalar_t));
     const int32_t n_size_thread_limit =
         output_size_2 / std::max(1, thread_num / topk_num);
     const int32_t n_size = cpu_utils::round_down<gemm_n_tile_size>(
@@ -351,9 +324,6 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input,
   const int32_t w13_output_buffer_offset = w13_thread_buffer_offset;
   w13_thread_buffer_offset += w13_output_buffer_size;
 
-  const int32_t w2_input_buffer_offset = w13_thread_buffer_offset;
-  w13_thread_buffer_offset += w2_input_buffer_size;
-
   // Weighted sum thread buffer
   const int32_t ws_output_buffer_size =
       cpu_utils::round_up<64>(output_size_2 * sizeof(float));
@@ -433,8 +403,7 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input,
       gemm_t gemm;
 
       const int32_t input_size_13_bytes = input_size_13 * sizeof(scalar_t);
-      const int32_t w13_n_group_stride =
-          gemm_t::WeightOCGroupSize * input_size_13;
+      const int32_t w13_n_group_stride = 16 * input_size_13;
       const int32_t w13_n_tile_stride = gemm_n_tile_size * input_size_13;
 
       for (;;) {
@@ -497,23 +466,8 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input,
              token_idx += gemm_m_tile_size) {
           const int32_t actual_token_num =
               std::min(gemm_m_tile_size, curr_token_num - token_idx);
-
-          scalar_t* __restrict__ curr_w13_gemm_input_buffer = nullptr;
-          if constexpr (pack_a) {
-            // copy and pack inputs
-            curr_w13_gemm_input_buffer = w13_input_buffer;
-            const scalar_t* w13_input_rows[gemm_m_tile_size];
-            for (int32_t i = 0; i < actual_token_num; ++i) {
-              w13_input_rows[i] =
-                  input + curr_expand_token_id_buffer[i] * input_size_13;
-            }
-            gemm_t::pack_input_from_rows(w13_input_rows,
-                                         curr_w13_gemm_input_buffer,
-                                         actual_token_num, input_size_13);
-            curr_expand_token_id_buffer += actual_token_num;
-          } else {
-            // copy inputs
-            curr_w13_gemm_input_buffer = curr_w13_input_buffer;
+          // copy inputs
+          {
             scalar_t* __restrict__ curr_w13_input_buffer_iter =
                 curr_w13_input_buffer;
             for (int32_t i = 0; i < actual_token_num; ++i) {
@@ -545,12 +499,14 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input,
             scalar_t* __restrict__ w13_weight_ptr_1_iter = w13_weight_ptr_1;
             scalar_t* __restrict__ w13_bias_ptr_0_iter = w13_bias_ptr_0;
             scalar_t* __restrict__ w13_bias_ptr_1_iter = w13_bias_ptr_1;
+            scalar_t* __restrict__ curr_w13_input_buffer_iter =
+                curr_w13_input_buffer;
             float* __restrict__ w13_output_buffer_0_iter = w13_output_buffer;
             float* __restrict__ w13_output_buffer_1_iter =
                 w13_output_buffer + actual_n_tile_size / 2;
             for (int32_t i = 0; i < actual_n_tile_size;
                  i += min_w13_n_tile_size) {
-              gemm.gemm(curr_w13_gemm_input_buffer, w13_weight_ptr_0_iter,
+              gemm.gemm(curr_w13_input_buffer_iter, w13_weight_ptr_0_iter,
                         w13_output_buffer_0_iter, actual_token_num,
                         input_size_13, input_size_13, w13_n_group_stride,
                         actual_n_tile_size, false);
@@ -563,7 +519,7 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input,
                 w13_bias_ptr_0_iter += gemm_n_tile_size;
               }
 
-              gemm.gemm(curr_w13_gemm_input_buffer, w13_weight_ptr_1_iter,
+              gemm.gemm(curr_w13_input_buffer_iter, w13_weight_ptr_1_iter,
                         w13_output_buffer_1_iter, actual_token_num,
                         input_size_13, input_size_13, w13_n_group_stride,
                         actual_n_tile_size, false);
@@ -616,8 +572,7 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input,
       gemm_t gemm;
 
       const int32_t w2_n_tile_stride = gemm_n_tile_size * input_size_2;
-      const int32_t w2_n_group_stride =
-          gemm_t::WeightOCGroupSize * input_size_2;
+      const int32_t w2_n_group_stride = 16 * input_size_2;
 
       for (;;) {
         int32_t task_id = counter_ptr->acquire_counter();
@@ -656,30 +611,13 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input,
              token_idx += gemm_m_tile_size) {
           const int32_t actual_token_num =
               std::min(gemm_m_tile_size, curr_token_num - token_idx);
-          scalar_t* __restrict__ curr_w2_gemm_input_buffer =
-              curr_w13_gemm_output_buffer;
-          if constexpr (pack_a) {
-            uint8_t* __restrict__ thread_buffer =
-                thread_buffer_start + thread_id * w13_thread_buffer_offset;
-            scalar_t* __restrict__ w2_input_buffer =
-                reinterpret_cast<scalar_t*>(thread_buffer +
-                                            w2_input_buffer_offset);
-            curr_w2_gemm_input_buffer = w2_input_buffer;
-            const scalar_t* w2_input_rows[gemm_m_tile_size];
-            for (int32_t i = 0; i < actual_token_num; ++i) {
-              w2_input_rows[i] = curr_w13_gemm_output_buffer + i * input_size_2;
-            }
-            gemm_t::pack_input_from_rows(w2_input_rows,
-                                         curr_w2_gemm_input_buffer,
-                                         actual_token_num, input_size_2);
-          }
 
           scalar_t* __restrict__ w2_weight_ptr_iter = w2_weight_ptr;
           scalar_t* __restrict__ w2_bias_ptr_iter = w2_bias_ptr;
           float* __restrict__ curr_w2_gemm_output_buffer_iter =
               curr_w2_gemm_output_buffer;
           for (int32_t i = 0; i < actual_n_tile_size; i += gemm_n_tile_size) {
-            gemm.gemm(curr_w2_gemm_input_buffer, w2_weight_ptr_iter,
+            gemm.gemm(curr_w13_gemm_output_buffer, w2_weight_ptr_iter,
                       curr_w2_gemm_output_buffer_iter, actual_token_num,
                       input_size_2, input_size_2, w2_n_group_stride,
                       output_size_2, false);

csrc/cpu/cpu_tanhf_neon.hpp

@@ -1,128 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-#ifndef CPU_TANHF_NEON_HPP
-#define CPU_TANHF_NEON_HPP
-
-#include <cstdint>
-#include <arm_neon.h>
-
-namespace vec_op {
-
-namespace {
-
-struct TanhfConstants {
-  float32x4_t special_bound;
-  float32x4_t two;
-  float32x4_t c0;
-  float32x4_t c2;
-  int32x4_t exponent_bias;
-  float c1;
-  float c3;
-  float two_over_ln2;
-  float c4;
-  float ln2_hi;
-  float ln2_lo;
-};
-
-const TanhfConstants kTanhfConstants = {
-    // 9.01, above which tanhf rounds to 1 (or -1 for  negative).
-    .special_bound = vdupq_n_f32(0x1.205966p+3f),
-    .two = vdupq_n_f32(0x1.0p+1f),
-    .c0 = vdupq_n_f32(0x1.fffffep-2f),
-    .c2 = vdupq_n_f32(0x1.555736p-5f),
-    .exponent_bias = vdupq_n_s32(0x3f800000),
-    .c1 = 0x1.5554aep-3f,
-    .c3 = 0x1.12287cp-7f,
-    .two_over_ln2 = 0x1.715476p+1f,
-    .c4 = 0x1.6b55a2p-10f,
-    .ln2_hi = 0x1.62e4p-1f,
-    .ln2_lo = 0x1.7f7d1cp-20f,
-};
-
-// Return the ptr but hide it's value from the compiler so accesses
-// through it can't be optimised based on contents.
-template <typename T>
-inline const T* ptr_barrier(const T* ptr) {
-  const T* opaque_ptr = ptr;
-  __asm__("" : "+r"(opaque_ptr));
-  return opaque_ptr;
-}
-
-// Check whether any lanes in the mask are set
-inline bool any_u32(uint32x4_t x) { return vmaxvq_u32(x) != 0; }
-
-// e^2x - 1 inline helper
-inline float32x4_t e2xm1f_inline(float32x4_t x, const TanhfConstants* d) {
-  float32x2_t ln2 = vld1_f32(&d->ln2_hi);
-  float32x4_t lane_consts = vld1q_f32(&d->c1);
-
-  // Reduce argument: f in [-ln2/2, ln2/2], i is exact.
-  float32x4_t j = vrndaq_f32(vmulq_laneq_f32(x, lane_consts, 2));
-  int32x4_t i = vcvtq_s32_f32(j);
-  float32x4_t f = vaddq_f32(x, x);
-  f = vfmsq_lane_f32(f, j, ln2, 0);
-  f = vfmsq_lane_f32(f, j, ln2, 1);
-
-  // Approximate expm1(f) with polynomial P, expm1(f) ~= f + f^2 * P(f)
-  float32x4_t f2 = vmulq_f32(f, f);
-  float32x4_t f4 = vmulq_f32(f2, f2);
-  float32x4_t p01 = vfmaq_laneq_f32(d->c0, f, lane_consts, 0);
-  float32x4_t p23 = vfmaq_laneq_f32(d->c2, f, lane_consts, 1);
-  float32x4_t poly = vfmaq_f32(p01, f2, p23);
-  poly = vfmaq_laneq_f32(poly, f4, lane_consts, 3);
-  poly = vfmaq_f32(f, f2, poly);
-
-  // scale = 2^i
-  int32x4_t u = vaddq_s32(vshlq_n_s32(i, 23), d->exponent_bias);
-  float32x4_t scale = vreinterpretq_f32_s32(u);
-  return vfmaq_f32(vsubq_f32(scale, vdupq_n_f32(1.0f)), poly, scale);
-}
-
-// Calculate the result tanh(x) = q / (q+2) and set special lanes to ±1
-inline float32x4_t special_case(float32x4_t x, float32x4_t q,
-                                uint32x4_t special) {
-  const TanhfConstants* d = ptr_barrier(&kTanhfConstants);
-
-  float32x4_t y = vdivq_f32(q, vaddq_f32(q, d->two));
-  uint32x4_t ix = vreinterpretq_u32_f32(x);
-  uint32x4_t one_bits = vreinterpretq_u32_s32(d->exponent_bias);
-  uint32x4_t sign_mask = vdupq_n_u32(0x80000000u);
-  uint32x4_t special_bits = vbslq_u32(sign_mask, ix, one_bits);
-  float32x4_t special_y = vreinterpretq_f32_u32(special_bits);
-  return vbslq_f32(special, special_y, y);
-}
-
-}  // namespace
-
-// Implementation of tanhf adapted from Arm Optimized Routines (tanhf
-// AdvSIMD)
-// https://github.com/ARM-software/optimized-routines/blob/master/math/aarch64/advsimd/tanhf.c
-//
-// Approximation for single-precision vector tanh(x), using a simplified
-// version of expm1f. The maximum error is 2.08 + 0.5 ULP:
-// _ZGVnN4v_tanhf (0x1.fa5eep-5) got 0x1.f9ba02p-5 want 0x1.f9ba08p-5.
-inline float32x4_t fast_tanhf_f32x4(float32x4_t x) {
-  const TanhfConstants* d = ptr_barrier(&kTanhfConstants);
-
-  // tanh(x) = (e^2x - 1) / (e^2x + 1)
-  // q = e^2x -1
-  float32x4_t q = e2xm1f_inline(x, d);
-
-  // Check for special cases
-  uint32x4_t special = vcagtq_f32(x, d->special_bound);
-
-  // Fall back to vectorised special case for any lanes which would cause
-  // expm1 to overflow
-  if (any_u32(special)) {
-    return special_case(x, q, special);
-  }
-
-  // Complete fast path if no special lanes
-  // tanh(x) = q / (q+2)
-  return vdivq_f32(q, vaddq_f32(q, d->two));
-}
-
-}  // namespace vec_op
-
-#endif  // CPU_TANHF_NEON_HPP

csrc/cpu/cpu_types.hpp

@@ -25,20 +25,4 @@
   #include <omp.h>
 #endif
 
-#include <c10/util/Exception.h>
-
-namespace cpu_utils {
-// Without OpenMP the omp pragmas compile to serial loops, so report 1: kernels
-// that barrier on the thread count would otherwise deadlock.
-inline int get_max_threads() {
-#ifdef _OPENMP
-  return omp_get_max_threads();
-#else
-  TORCH_WARN_ONCE(
-      "vLLM CPU was built without OpenMP; running single-threaded.");
-  return 1;
-#endif
-}
-}  // namespace cpu_utils
-
 #endif

csrc/cpu/cpu_types_arm.hpp

@@ -3,8 +3,6 @@
 
 #include <arm_neon.h>
 
-#include "cpu/cpu_tanhf_neon.hpp"
-
 #include <torch/all.h>
 #include <ATen/cpu/vec/functional.h>
 #include <ATen/cpu/vec/vec.h>
@@ -347,10 +345,6 @@ struct FP32Vec4 : public VectorizedRegWrapper<FP32Vec4, 1, float> {
   explicit FP32Vec4(float32x4_t data) : Base(VectorizedT(data)) {};
 
   explicit FP32Vec4(const FP32Vec4& data) : Base(data) {};
-
-  FORCE_INLINE FP32Vec4 tanh() const {
-    return FP32Vec4(fast_tanhf_f32x4(reg.val[0]));
-  }
 };
 
 struct FP32Vec8 : public VectorizedRegWrapper<FP32Vec8, 2, float> {
@@ -397,13 +391,6 @@ struct FP32Vec8 : public VectorizedRegWrapper<FP32Vec8, 2, float> {
     reg.val[1] = Vectorized<float>(data.val[1]);
   }
 
-  FORCE_INLINE FP32Vec8 tanh() const {
-    FP32Vec8 r(uninit);
-    r.reg.val[0] = Vectorized<float>(fast_tanhf_f32x4(reg.val[0]));
-    r.reg.val[1] = Vectorized<float>(fast_tanhf_f32x4(reg.val[1]));
-    return r;
-  }
-
   FORCE_INLINE float reduce_sum() const noexcept {
     float answer = 0;
     std::plus<VectorizedT> add;
@@ -510,35 +497,6 @@ struct FP32Vec16 : public VectorizedRegWrapper<FP32Vec16, 4, float> {
     reg.val[3] = Vectorized<float>(vcvt_f32_f16(vget_high_f16(v.reg.val[1])));
   };
 
-  FORCE_INLINE FP32Vec16 tanh() const {
-    FP32Vec16 r(uninit);
-    r.reg.val[0] = Vectorized<float>(fast_tanhf_f32x4(reg.val[0]));
-    r.reg.val[1] = Vectorized<float>(fast_tanhf_f32x4(reg.val[1]));
-    r.reg.val[2] = Vectorized<float>(fast_tanhf_f32x4(reg.val[2]));
-    r.reg.val[3] = Vectorized<float>(fast_tanhf_f32x4(reg.val[3]));
-    return r;
-  }
-
-  static FORCE_INLINE void load_even_odd(const float* ptr, FP32Vec16& even,
-                                         FP32Vec16& odd) noexcept {
-    const float32x4x2_t x01 = vuzpq_f32(vld1q_f32(ptr), vld1q_f32(ptr + 4));
-    const float32x4x2_t x23 =
-        vuzpq_f32(vld1q_f32(ptr + 8), vld1q_f32(ptr + 12));
-    const float32x4x2_t x45 =
-        vuzpq_f32(vld1q_f32(ptr + 16), vld1q_f32(ptr + 20));
-    const float32x4x2_t x67 =
-        vuzpq_f32(vld1q_f32(ptr + 24), vld1q_f32(ptr + 28));
-
-    even.reg.val[0] = VectorizedT(x01.val[0]);
-    even.reg.val[1] = VectorizedT(x23.val[0]);
-    even.reg.val[2] = VectorizedT(x45.val[0]);
-    even.reg.val[3] = VectorizedT(x67.val[0]);
-    odd.reg.val[0] = VectorizedT(x01.val[1]);
-    odd.reg.val[1] = VectorizedT(x23.val[1]);
-    odd.reg.val[2] = VectorizedT(x45.val[1]);
-    odd.reg.val[3] = VectorizedT(x67.val[1]);
-  }
-
   FORCE_INLINE FP32Vec16 operator+(const FP32Vec16& b) const noexcept {
     FP32Vec16 r(uninit);
     r.reg.val[0] = reg.val[0] + b.reg.val[0];
@@ -557,15 +515,6 @@ struct FP32Vec16 : public VectorizedRegWrapper<FP32Vec16, 4, float> {
     return r;
   }
 
-  FORCE_INLINE FP32Vec16 operator-() const noexcept {
-    FP32Vec16 r(uninit);
-    r.reg.val[0] = reg.val[0].neg();
-    r.reg.val[1] = reg.val[1].neg();
-    r.reg.val[2] = reg.val[2].neg();
-    r.reg.val[3] = reg.val[3].neg();
-    return r;
-  }
-
   FORCE_INLINE FP32Vec16 operator*(const FP32Vec16& b) const noexcept {
     FP32Vec16 r(uninit);
     r.reg.val[0] = reg.val[0] * b.reg.val[0];
@@ -984,4 +933,4 @@ inline void storeFP32<c10::BFloat16>(float v, c10::BFloat16* ptr) {
 
 inline void prefetch(const void* addr) { __builtin_prefetch(addr, 0, 1); };
 
-};  // namespace vec_op
+};  // namespace vec_op

csrc/cpu/cpu_types_riscv_defs.hpp

@@ -3,17 +3,13 @@
 
 // VLEN-to-LMUL mapping for RISC-V Vector extension.
 //
-// LMUL_<N> expands to the LMUL suffix giving N total bits of vector data.
-// LMUL_64 is used by 8-lane int8/uint8 vectors.
-//   VLEN=128:
-//     LMUL_64=mf2, LMUL_128=m1, LMUL_256=m2, LMUL_512=m4, LMUL_1024=m8
-//   VLEN=256:
-//     LMUL_64=mf4, LMUL_128=mf2, LMUL_256=m1, LMUL_512=m2, LMUL_1024=m4
+// LMUL_<N> expands to the LMUL suffix giving N total bits of vector data:
+//   VLEN=128: LMUL_128=m1,  LMUL_256=m2,  LMUL_512=m4,  LMUL_1024=m8
+//   VLEN=256: LMUL_128=mf2, LMUL_256=m1,  LMUL_512=m2,  LMUL_1024=m4
 
 #include <riscv_vector.h>
 
 #if __riscv_v_min_vlen == 128
-  #define LMUL_64 mf2
   #define LMUL_128 m1
   #define LMUL_256 m2
   #define LMUL_512 m4
@@ -21,7 +17,6 @@
   #define BOOL_256 b16
   #define BOOL_512 b8
 #elif __riscv_v_min_vlen == 256
-  #define LMUL_64 mf4
   #define LMUL_128 mf2
   #define LMUL_256 m1
   #define LMUL_512 m2
@@ -46,16 +41,6 @@
 
 // ---- Semantic fixed-vector typedefs (named by element count) ----
 
-// uint8 / int8
-typedef RVVTYPE(vuint8, LMUL_64, _t) fixed_u8x8_t
-    __attribute__((riscv_rvv_vector_bits(64)));
-typedef RVVTYPE(vint8, LMUL_64, _t) fixed_i8x8_t
-    __attribute__((riscv_rvv_vector_bits(64)));
-
-// int16
-typedef RVVTYPE(vint16, LMUL_128, _t) fixed_i16x8_t
-    __attribute__((riscv_rvv_vector_bits(128)));
-
 // float16
 typedef RVVTYPE(vfloat16, LMUL_128, _t) fixed_fp16x8_t
     __attribute__((riscv_rvv_vector_bits(128)));

csrc/cpu/cpu_types_riscv_impl.hpp

@@ -214,18 +214,11 @@ struct BF16Vec32 : public Vec<BF16Vec32> {
 
   explicit BF16Vec32(const BF16Vec8& v) {
     fixed_u16x8_t u16_val = bf16_to_u16(v.reg);
-    // Widen LMUL_128 → LMUL_256 so vslideup operands share a type.
-    // At VLEN=256 this is mf2→m1 (both integer); at VLEN=128 it is m1→m2.
-    fixed_u16x16_t ext =
-        RVVI4(__riscv_vlmul_ext_v_u16, LMUL_128, _u16, LMUL_256)(u16_val);
-    // Build 16-element half: place the 8 elements at offsets 0 and 8.
-    fixed_u16x16_t half = RVVI(__riscv_vmv_v_x_u16, LMUL_256)(0, 16);
-    half = RVVI(__riscv_vslideup_vx_u16, LMUL_256)(half, ext, 0, 8);
-    half = RVVI(__riscv_vslideup_vx_u16, LMUL_256)(half, ext, 8, 16);
-    // Double to LMUL_512 (m1→m2 at VLEN=256, m2→m4 at VLEN=128).
-    fixed_u16x32_t dst =
-        RVVI4(__riscv_vcreate_v_u16, LMUL_256, _u16, LMUL_512)(half, half);
-    reg = RVVI4(__riscv_vreinterpret_v_u16, LMUL_512, _bf16, LMUL_512)(dst);
+    fixed_u16x32_t u16_combined =
+        RVVI4(__riscv_vcreate_v_u16, LMUL_128, _u16, LMUL_512)(
+            u16_val, u16_val, u16_val, u16_val);
+    reg = RVVI4(__riscv_vreinterpret_v_u16, LMUL_512, _bf16,
+                LMUL_512)(u16_combined);
   };
 
   void save(void* ptr) const {
@@ -630,29 +623,17 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
             data.reg, data.reg)) {};
   explicit FP32Vec16(const FP32Vec16& data) : reg(data.reg) {};
   explicit FP32Vec16(int64_t value, const FP32Vec16& lut) {
-    // Split into two 32-bit halves to avoid u64 @ LMUL_1024 (m8 on
-    // VLEN=128 / m4 on VLEN=256), which causes heavy register spilling.
-    constexpr int HALF = VEC_ELEM_NUM / 2;
-    const auto q = static_cast<uint64_t>(value);
-    const uint32_t lo = static_cast<uint32_t>(q);
-    const uint32_t hi = static_cast<uint32_t>(q >> 32);
-
-    auto lane_ids = RVVI(__riscv_vid_v_u32, LMUL_256)(HALF);
-    auto shifts = RVVI(__riscv_vsll_vx_u32, LMUL_256)(lane_ids, 2, HALF);
-
-    auto packed_lo = RVVI(__riscv_vmv_v_x_u32, LMUL_256)(lo, HALF);
-    auto idx_lo = RVVI(__riscv_vand_vx_u32, LMUL_256)(
-        RVVI(__riscv_vsrl_vv_u32, LMUL_256)(packed_lo, shifts, HALF), 0xF,
-        HALF);
-
-    auto packed_hi = RVVI(__riscv_vmv_v_x_u32, LMUL_256)(hi, HALF);
-    auto idx_hi = RVVI(__riscv_vand_vx_u32, LMUL_256)(
-        RVVI(__riscv_vsrl_vv_u32, LMUL_256)(packed_hi, shifts, HALF), 0xF,
-        HALF);
-
-    auto idx =
-        RVVI4(__riscv_vcreate_v_u32, LMUL_256, _u32, LMUL_512)(idx_lo, idx_hi);
-    reg = RVVI(__riscv_vrgather_vv_f32, LMUL_512)(lut.reg, idx, VEC_ELEM_NUM);
+    const uint64_t q_values = static_cast<uint64_t>(value);
+    auto packed = RVVI(__riscv_vmv_v_x_u64, LMUL_1024)(q_values, VEC_ELEM_NUM);
+    auto lane_ids = RVVI(__riscv_vid_v_u64, LMUL_1024)(VEC_ELEM_NUM);
+    auto shifts =
+        RVVI(__riscv_vsll_vx_u64, LMUL_1024)(lane_ids, 2, VEC_ELEM_NUM);
+    auto shifted =
+        RVVI(__riscv_vsrl_vv_u64, LMUL_1024)(packed, shifts, VEC_ELEM_NUM);
+    auto idx64 =
+        RVVI(__riscv_vand_vx_u64, LMUL_1024)(shifted, 0xF, VEC_ELEM_NUM);
+    auto idx32 = RVVI(__riscv_vnsrl_wx_u32, LMUL_512)(idx64, 0, VEC_ELEM_NUM);
+    reg = RVVI(__riscv_vrgather_vv_f32, LMUL_512)(lut.reg, idx32, VEC_ELEM_NUM);
   }
   explicit FP32Vec16(const FP16Vec16& v);
 

csrc/cpu/cpu_types_scalar.hpp

@@ -363,13 +363,6 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
     return FP32Vec16(ret);
   }
 
-  FP32Vec16 tanh() const {
-    f32x16_t ret;
-    unroll_loop<int, VEC_ELEM_NUM>(
-        [&ret, this](int i) { ret.val[i] = std::tanh(reg.val[i]); });
-    return FP32Vec16(ret);
-  }
-
   float reduce_sum() const {
     float result = 0.0f;
     unroll_loop<int, VEC_ELEM_NUM>(

csrc/cpu/cpu_types_vsx.hpp

@@ -13,10 +13,10 @@ namespace vec_op {
 struct fp8_e4m3_tag {};
 struct fp8_e5m2_tag {};
 
-#define VLLM_DISPATCH_CASE_FLOATING_TYPES(...)            \
-  AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__)    \
-  AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__) \
-  AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__)
+// FIXME: FP16 is not fully supported in Torch-CPU
+#define VLLM_DISPATCH_CASE_FLOATING_TYPES(...)         \
+  AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
+  AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__)
 
 #define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
   AT_DISPATCH_SWITCH(TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__))
@@ -34,87 +34,6 @@ struct fp8_e5m2_tag {};
 #define FORCE_INLINE __attribute__((always_inline)) inline
 
 namespace {
-
-FORCE_INLINE __vector float fp16_to_fp32_bits(__vector unsigned int x) {
-  const __vector unsigned int mask_sign = {0x8000, 0x8000, 0x8000, 0x8000};
-  const __vector unsigned int mask_exp = {0x7C00, 0x7C00, 0x7C00, 0x7C00};
-  const __vector unsigned int mask_mant = {0x03FF, 0x03FF, 0x03FF, 0x03FF};
-  const __vector unsigned int bias_adj = {112, 112, 112, 112};
-  const __vector unsigned int exp_max_fp16 = {0x1F, 0x1F, 0x1F, 0x1F};
-  const __vector unsigned int exp_max_fp32 = {0xFF, 0xFF, 0xFF, 0xFF};
-
-  __vector unsigned int s = (x & mask_sign) << 16;
-  __vector unsigned int e = (x & mask_exp) >> 10;
-  __vector unsigned int m = (x & mask_mant) << 13;
-
-  __vector __bool int is_nan_inf = vec_cmpeq(e, exp_max_fp16);
-
-  __vector unsigned int e_normal = e + bias_adj;
-  e = vec_sel(e_normal, exp_max_fp32, is_nan_inf);
-
-  return (__vector float)(s | (e << 23) | m);
-}
-
-FORCE_INLINE __vector unsigned int fp32_to_fp16_bits(__vector float f_in) {
-  __vector unsigned int in = (__vector unsigned int)f_in;
-
-  const __vector unsigned int mask_sign_32 = {0x80000000, 0x80000000,
-                                              0x80000000, 0x80000000};
-  const __vector unsigned int mask_exp_32 = {0x7F800000, 0x7F800000, 0x7F800000,
-                                             0x7F800000};
-  const __vector unsigned int mask_mant_32 = {0x007FFFFF, 0x007FFFFF,
-                                              0x007FFFFF, 0x007FFFFF};
-
-  const __vector signed int bias_adj = {112, 112, 112, 112};
-  const __vector signed int zero = {0, 0, 0, 0};
-  const __vector signed int max_exp = {31, 31, 31, 31};
-  const __vector unsigned int exp_max_fp32 = {0xFF, 0xFF, 0xFF, 0xFF};
-  const __vector unsigned int exp_max_fp16 = {0x1F, 0x1F, 0x1F, 0x1F};
-
-  __vector unsigned int s = (in & mask_sign_32) >> 16;
-  __vector unsigned int e_u = (in & mask_exp_32) >> 23;
-
-  __vector __bool int is_nan_inf = vec_cmpeq(e_u, exp_max_fp32);
-
-  __vector signed int e_s = (__vector signed int)e_u;
-  e_s = vec_sub(e_s, bias_adj);
-  e_s = vec_max(e_s, zero);
-  e_s = vec_min(e_s, max_exp);
-  __vector unsigned int e_normal = (__vector unsigned int)e_s;
-
-  __vector unsigned int e_final = vec_sel(e_normal, exp_max_fp16, is_nan_inf);
-
-  const __vector unsigned int one_v = {1, 1, 1, 1};
-  const __vector unsigned int mask_sticky = {0xFFF, 0xFFF, 0xFFF, 0xFFF};
-
-  __vector unsigned int round_bit = (in >> 12) & one_v;
-  __vector unsigned int sticky = in & mask_sticky;
-  __vector unsigned int m = (in & mask_mant_32) >> 13;
-  __vector unsigned int lsb = m & one_v;
-
-  // Round up if: round_bit && (sticky || lsb)
-  __vector __bool int sticky_nonzero =
-      vec_cmpgt(sticky, (__vector unsigned int){0, 0, 0, 0});
-  __vector __bool int lsb_set = vec_cmpeq(lsb, one_v);
-  __vector __bool int round_up =
-      vec_and(vec_cmpeq(round_bit, one_v), vec_or(sticky_nonzero, lsb_set));
-
-  m = vec_sel(m, m + one_v, round_up);
-
-  const __vector unsigned int mant_mask = {0x3FF, 0x3FF, 0x3FF, 0x3FF};
-  const __vector unsigned int max_normal_exp = {0x1E, 0x1E, 0x1E, 0x1E};
-  __vector __bool int mant_overflows = vec_cmpgt(m, mant_mask);
-  __vector __bool int would_overflow_to_inf =
-      vec_and(mant_overflows, vec_cmpeq(e_final, max_normal_exp));
-  __vector unsigned int e_inc = vec_min(e_final + one_v, exp_max_fp16);
-  e_final = vec_sel(e_final, e_inc, mant_overflows);
-  m = vec_and(m, mant_mask);
-  e_final = vec_sel(e_final, max_normal_exp, would_overflow_to_inf);
-  m = vec_sel(m, mant_mask, would_overflow_to_inf);
-
-  return s | (e_final << 10) | m;
-}
-
 template <typename T, T... indexes, typename F>
 constexpr void unroll_loop_item(std::integer_sequence<T, indexes...>, F&& f) {
   (f(std::integral_constant<T, indexes>{}), ...);
@@ -170,19 +89,6 @@ struct BF16Vec8 : public Vec<BF16Vec8> {
   }
 };
 
-struct FP16Vec8 : public Vec<FP16Vec8> {
-  constexpr static int VEC_ELEM_NUM = 8;
-
-  __vector signed short reg;
-
-  explicit FP16Vec8(const void* ptr) : reg(*(__vector signed short*)ptr) {}
-  explicit FP16Vec8(const FP32Vec8&);
-
-  void save(void* ptr) const {
-    *reinterpret_cast<__vector signed short*>(ptr) = reg;
-  }
-};
-
 struct FP16Vec16 : public Vec<FP16Vec16> {
   constexpr static int VEC_ELEM_NUM = 16;
   ss16x8x2_t reg;
@@ -218,11 +124,13 @@ struct BF16Vec16 : public Vec<BF16Vec16> {
   ss16x8x2_t reg;
 
   explicit BF16Vec16(const void* ptr) {
+    // Load 256 bits in two parts
     reg.val[0] = (__vector signed short)vec_xl(0, (signed short*)ptr);
     reg.val[1] = (__vector signed short)vec_xl(16, (signed short*)ptr);
   }
 
   explicit BF16Vec16(bool, const void* ptr) : BF16Vec16(ptr) {}
+
   explicit BF16Vec16(const FP32Vec16&);
 
   void save(void* ptr) const {
@@ -234,16 +142,20 @@ struct BF16Vec16 : public Vec<BF16Vec16> {
   void save(void* ptr, const int elem_num) const {
     const int clamped_elem = std::max(0, std::min(elem_num, 16));
 
+    // Calculate elements to store in each 128-bit part (8 elements each)
     const int elements_val0 = std::min(clamped_elem, 8);
     const int elements_val1 = std::max(clamped_elem - 8, 0);
 
+    // Convert elements to bytes (2 bytes per element)
     const size_t bytes_val0 = elements_val0 * sizeof(signed short);
     const size_t bytes_val1 = elements_val1 * sizeof(signed short);
 
     signed short* dest = static_cast<signed short*>(ptr);
+    // Store the first part using vec_xst_len
     if (bytes_val0 > 0) {
       vec_xst_len(reg.val[0], dest, bytes_val0);
     }
+    // Store the second part if needed
     if (bytes_val1 > 0) {
       vec_xst_len(reg.val[1], dest + elements_val0, bytes_val1);
     }
@@ -326,15 +238,6 @@ struct FP32Vec8 : public Vec<FP32Vec8> {
     reg.val[1] = (__vector float)vec_mergel(zero, v.reg);
   }
 
-  explicit FP32Vec8(const FP16Vec8& v) {
-    __vector unsigned short raw_u = (__vector unsigned short)v.reg;
-    __vector unsigned int raw_hi =
-        (__vector unsigned int)vec_unpackh((__vector signed short)raw_u);
-    __vector unsigned int raw_lo =
-        (__vector unsigned int)vec_unpackl((__vector signed short)raw_u);
-    reg.val[0] = fp16_to_fp32_bits(raw_hi);
-    reg.val[1] = fp16_to_fp32_bits(raw_lo);
-  }
   float reduce_sum() const {
     AliasReg ar;
     ar.reg = reg;
@@ -507,9 +410,8 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
     reg.val[3] = vec_xl(48, ptr);
   }
 
-  explicit FP32Vec16(const c10::Half* ptr) : FP32Vec16(FP16Vec16(ptr)) {}
-  explicit FP32Vec16(const FP16Vec16&);
   explicit FP32Vec16(bool, const float* ptr) : FP32Vec16(ptr) {}
+
   explicit FP32Vec16(f32x4x4_t data) : reg(data) {}
 
   explicit FP32Vec16(const FP32Vec16& data) {
@@ -533,6 +435,7 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
     reg.val[3] = data.reg.val[1];
   }
 
+  explicit FP32Vec16(const FP16Vec16& v);
   explicit FP32Vec16(const BF16Vec16& v) {
     reg.val[0] = (__vector float)vec_mergeh(zero, v.reg.val[0]);
     reg.val[1] = (__vector float)vec_mergel(zero, v.reg.val[0]);
@@ -599,20 +502,28 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
   FP32Vec16 max(const FP32Vec16& b, int elem_num) const {
     FP32Vec16 result;
 
+    // Create a vector of element indices for each chunk
     __vector unsigned int indices = {0, 1, 2, 3};
     __vector unsigned int elem_num_vec =
         vec_splats(static_cast<unsigned int>(elem_num));
 
-    __vector unsigned int chunk_offset0 = {0, 0, 0, 0};
-    __vector unsigned int chunk_offset1 = {4, 4, 4, 4};
-    __vector unsigned int chunk_offset2 = {8, 8, 8, 8};
-    __vector unsigned int chunk_offset3 = {12, 12, 12, 12};
+    // Compute masks for each chunk
+    __vector unsigned int chunk_offset0 = {0, 0, 0,
+                                           0};  // Chunk 0: Elements 0-3
+    __vector unsigned int chunk_offset1 = {4, 4, 4,
+                                           4};  // Chunk 1: Elements 4-7
+    __vector unsigned int chunk_offset2 = {8, 8, 8,
+                                           8};  // Chunk 2: Elements 8-11
+    __vector unsigned int chunk_offset3 = {12, 12, 12,
+                                           12};  // Chunk 3: Elements 12-15
 
+    // Compute masks for each chunk
     __vector bool int mask0 = vec_cmplt(indices + chunk_offset0, elem_num_vec);
     __vector bool int mask1 = vec_cmplt(indices + chunk_offset1, elem_num_vec);
     __vector bool int mask2 = vec_cmplt(indices + chunk_offset2, elem_num_vec);
     __vector bool int mask3 = vec_cmplt(indices + chunk_offset3, elem_num_vec);
 
+    // Apply masks to compute the result for each chunk
     result.reg.val[0] = vec_sel(this->reg.val[0],
                                 vec_max(this->reg.val[0], b.reg.val[0]), mask0);
     result.reg.val[1] = vec_sel(this->reg.val[1],
@@ -715,16 +626,6 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
     vec_xst(reg.val[3], 48, ptr);
   }
 
-  void save(c10::Half* ptr) const {
-    FP16Vec16 fp16_vec(*this);
-    fp16_vec.save(ptr);
-  }
-
-  void save(c10::Half* ptr, const int elem_num) const {
-    FP16Vec16 fp16_vec(*this);
-    fp16_vec.save(ptr, elem_num);
-  }
-
   void save(float* ptr, const int elem_num) const {
     const int elements_in_chunk1 =
         (elem_num >= 0) ? ((elem_num >= 4) ? 4 : elem_num) : 0;
@@ -758,7 +659,7 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
 };
 
 struct INT8Vec16 : public Vec<INT8Vec16> {
-  constexpr static int VEC_NUM_ELEM = 16;
+  constexpr static int VEC_NUM_ELEM = 16;  // 128 bits / 8 bits = 16
 
   union AliasReg {
     __vector signed char reg;
@@ -806,11 +707,6 @@ struct VecType<c10::BFloat16> {
   using vec_type = BF16Vec8;
 };
 
-template <>
-struct VecType<c10::Half> {
-  using vec_type = FP16Vec8;
-};
-
 template <typename T>
 void storeFP32(float v, T* ptr) {
   *ptr = v;
@@ -827,15 +723,6 @@ inline void storeFP32<c10::BFloat16>(float v, c10::BFloat16* ptr) {
   *ptr = *(v_ptr + 1);
 }
 
-template <>
-inline void storeFP32<c10::Half>(float v, c10::Half* ptr) {
-  __vector float v_vec = {v, 0.0f, 0.0f, 0.0f};
-  __vector unsigned int fp16_bits = fp32_to_fp16_bits(v_vec);
-  unsigned short result =
-      (unsigned short)((__vector unsigned short)fp16_bits)[0];
-  *reinterpret_cast<unsigned short*>(ptr) = result;
-}
-
 #ifndef __VEC_CLASS_FP_NAN
   #define __VEC_CLASS_FP_NAN (1 << 6)
 #endif
@@ -882,39 +769,38 @@ inline BF16Vec8::BF16Vec8(const FP32Vec8& v) {
 #endif
 }
 
-inline FP16Vec8::FP16Vec8(const FP32Vec8& v) {
-  __vector unsigned int fp16_hi = fp32_to_fp16_bits(v.reg.val[0]);
-  __vector unsigned int fp16_lo = fp32_to_fp16_bits(v.reg.val[1]);
-  reg = (__vector signed short)vec_perm((__vector unsigned char)fp16_hi,
-                                        (__vector unsigned char)fp16_lo, omask);
-}
-
 inline FP16Vec16::FP16Vec16(const FP32Vec16& v) {
-  __vector unsigned int fp16_0 = fp32_to_fp16_bits(v.reg.val[0]);
-  __vector unsigned int fp16_1 = fp32_to_fp16_bits(v.reg.val[1]);
-  __vector unsigned int fp16_2 = fp32_to_fp16_bits(v.reg.val[2]);
-  __vector unsigned int fp16_3 = fp32_to_fp16_bits(v.reg.val[3]);
-  reg.val[0] = (__vector signed short)vec_perm(
-      (__vector unsigned char)fp16_0, (__vector unsigned char)fp16_1, omask);
-  reg.val[1] = (__vector signed short)vec_perm(
-      (__vector unsigned char)fp16_2, (__vector unsigned char)fp16_3, omask);
+  alignas(16) float temp_fp32[16];
+  alignas(16) c10::Half temp_fp16[16];
+
+  vec_xst(v.reg.val[0], 0, temp_fp32);
+  vec_xst(v.reg.val[1], 16, temp_fp32);
+  vec_xst(v.reg.val[2], 32, temp_fp32);
+  vec_xst(v.reg.val[3], 48, temp_fp32);
+
+  for (int i = 0; i < 16; i++) {
+    temp_fp16[i] = c10::Half(temp_fp32[i]);
+  }
+
+  reg.val[0] = (__vector signed short)vec_xl(0, (signed short*)temp_fp16);
+  reg.val[1] = (__vector signed short)vec_xl(16, (signed short*)temp_fp16);
 }
 
 inline FP32Vec16::FP32Vec16(const FP16Vec16& v) {
-  __vector unsigned short raw_u0 = (__vector unsigned short)v.reg.val[0];
-  __vector unsigned short raw_u1 = (__vector unsigned short)v.reg.val[1];
-  __vector unsigned int raw_hi0 =
-      (__vector unsigned int)vec_unpackh((__vector signed short)raw_u0);
-  __vector unsigned int raw_lo0 =
-      (__vector unsigned int)vec_unpackl((__vector signed short)raw_u0);
-  __vector unsigned int raw_hi1 =
-      (__vector unsigned int)vec_unpackh((__vector signed short)raw_u1);
-  __vector unsigned int raw_lo1 =
-      (__vector unsigned int)vec_unpackl((__vector signed short)raw_u1);
-  reg.val[0] = fp16_to_fp32_bits(raw_hi0);
-  reg.val[1] = fp16_to_fp32_bits(raw_lo0);
-  reg.val[2] = fp16_to_fp32_bits(raw_hi1);
-  reg.val[3] = fp16_to_fp32_bits(raw_lo1);
+  alignas(16) c10::Half temp_fp16[16];
+  alignas(16) float temp_fp32[16];
+
+  vec_xst(v.reg.val[0], 0, (signed short*)temp_fp16);
+  vec_xst(v.reg.val[1], 16, (signed short*)temp_fp16);
+
+  for (int i = 0; i < 16; i++) {
+    temp_fp32[i] = float(temp_fp16[i]);
+  }
+
+  reg.val[0] = vec_xl(0, temp_fp32);
+  reg.val[1] = vec_xl(16, temp_fp32);
+  reg.val[2] = vec_xl(32, temp_fp32);
+  reg.val[3] = vec_xl(48, temp_fp32);
 }
 
 inline BF16Vec16::BF16Vec16(const FP32Vec16& v) {
@@ -978,6 +864,7 @@ inline void prefetch(const void* addr) {
 
 struct INT8Vec64 {
   __vector signed char data[4];
+
   INT8Vec64() = default;
 
   explicit INT8Vec64(const int8_t* ptr) {
@@ -1013,4 +900,5 @@ struct INT8Vec64 {
   void nt_save(int8_t* ptr) const { save(ptr); }
 };
 }  // namespace vec_op
+
 #endif

csrc/cpu/cpu_types_x86.hpp

@@ -3,7 +3,6 @@
 #define CPU_TYPES_X86_HPP
 
 #include <immintrin.h>
-#include <sleef.h>
 #include <torch/all.h>
 
 #ifndef __AVX2__
@@ -593,8 +592,6 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
 
   FP32Vec16 abs() const { return FP32Vec16(_mm512_abs_ps(reg)); }
 
-  FP32Vec16 tanh() const { return FP32Vec16(Sleef_tanhf16_u10(reg)); }
-
   float reduce_sum() const { return _mm512_reduce_add_ps(reg); }
 
   float reduce_max() const { return _mm512_reduce_max_ps(reg); }
@@ -792,12 +789,6 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
                      _mm256_andnot_ps(sign_mask, reg_high));
   }
 
-  FP32Vec16 tanh() const {
-    FP32Vec8 low(reg_low);
-    FP32Vec8 high(reg_high);
-    return FP32Vec16(low.tanh().reg, high.tanh().reg);
-  }
-
   FP32Vec16 min(const FP32Vec16& b) const {
     return FP32Vec16(_mm256_min_ps(reg_low, b.reg_low),
                      _mm256_min_ps(reg_high, b.reg_high));

csrc/cpu/cpu_wna16.cpp

@@ -155,7 +155,7 @@ void cpu_gemm_wna16_impl(
   constexpr int32_t gemm_m_tile_size = gemm_t::MaxMSize;
   constexpr int32_t n_block_size = 16;
   static_assert(gemm_n_tile_size % n_block_size == 0);
-  const int32_t thread_num = cpu_utils::get_max_threads();
+  const int32_t thread_num = omp_get_max_threads();
 
   // a simple schedule policy, just to hold more B tiles in L2 and make sure
   // each thread has tasks

csrc/cpu/dnnl_kernels.cpp

@@ -202,7 +202,7 @@ void dynamic_quant_epilogue(const float* input, scalar_t* output,
   using cvt_vec_t = typename KernelVecType<scalar_t>::cvt_vec_type;
   constexpr int vec_elem_num = load_vec_t::VEC_ELEM_NUM;
 
-  const int64_t thread_num = cpu_utils::get_max_threads();
+  const int64_t thread_num = omp_get_max_threads();
   if (num_tokens > thread_num) {
 #pragma omp parallel for
     for (int64_t i = 0; i < num_tokens; ++i) {

csrc/cpu/micro_gemm/cpu_micro_gemm_amx.hpp

@@ -213,8 +213,6 @@ class MicroGemm<cpu_utils::ISA::AMX, scalar_t> {
  public:
   static constexpr int32_t MaxMSize = 32;
   static constexpr int32_t NSize = 32;
-  static constexpr int32_t WeightOCGroupSize = 16;
-  static constexpr bool PackA = false;
 
  public:
   MicroGemm() : curr_m_(-1) {

csrc/cpu/micro_gemm/cpu_micro_gemm_impl.hpp

@@ -21,9 +21,6 @@ class MicroGemm {
  public:
   static constexpr int32_t MaxMSize = 16;
   static constexpr int32_t NSize = 16;
-  static constexpr int32_t WeightOCGroupSize = 16;
-  // callers must pack A matrix before GEMM
-  static constexpr bool PackA = false;
 
  public:
   void gemm(DEFINE_CPU_MICRO_GEMM_PARAMS) {

csrc/cpu/micro_gemm/cpu_micro_gemm_neon.hpp

@@ -1,503 +0,0 @@
-#ifndef CPU_MICRO_GEMM_NEON_HPP
-#define CPU_MICRO_GEMM_NEON_HPP
-
-#include <algorithm>
-#include <cstdint>
-
-#include "cpu/micro_gemm/cpu_micro_gemm_impl.hpp"
-
-#include <arm_bf16.h>
-#include <arm_neon.h>
-
-namespace cpu_micro_gemm {
-
-namespace {
-
-constexpr int32_t K = 4;
-constexpr int32_t Cols = 2;
-constexpr int32_t TileSize = K * Cols;
-constexpr int32_t Mr = 8;
-constexpr int32_t Nr = 8;
-constexpr int32_t Nr_gemv = 16;
-
-// a = [a0, a1, a2, a3], b = [b0, b1, b2, b3] -> [a0, a1, b0, b1]
-FORCE_INLINE float32x4_t zip1_f32x4(const float32x4_t a, const float32x4_t b) {
-  return vreinterpretq_f32_f64(
-      vzip1q_f64(vreinterpretq_f64_f32(a), vreinterpretq_f64_f32(b)));
-}
-
-// a = [a0, a1, a2, a3], b = [b0, b1, b2, b3] -> [a2, a3, b2, b3]
-FORCE_INLINE float32x4_t zip2_f32x4(const float32x4_t a, const float32x4_t b) {
-  return vreinterpretq_f32_f64(
-      vzip2q_f64(vreinterpretq_f64_f32(a), vreinterpretq_f64_f32(b)));
-}
-
-FORCE_INLINE void init_acc_rowpair(float32x4_t& acc01, float32x4_t& acc23,
-                                   float32x4_t& acc45, float32x4_t& acc67,
-                                   const float* __restrict__ c_ptr,
-                                   const int64_t ldc, const int32_t m_rows,
-                                   const bool accum_c) {
-  if (!accum_c || m_rows == 0) {
-    acc01 = vdupq_n_f32(0.0f);
-    acc23 = vdupq_n_f32(0.0f);
-    acc45 = vdupq_n_f32(0.0f);
-    acc67 = vdupq_n_f32(0.0f);
-    return;
-  }
-
-  const float32x4_t row0_0123 = vld1q_f32(c_ptr);
-  const float32x4_t row0_4567 = vld1q_f32(c_ptr + 4);
-  const float32x4_t row1_0123 =
-      (m_rows == 2) ? vld1q_f32(c_ptr + ldc) : vdupq_n_f32(0.0f);
-  const float32x4_t row1_4567 =
-      (m_rows == 2) ? vld1q_f32(c_ptr + ldc + 4) : vdupq_n_f32(0.0f);
-
-  acc01 = zip1_f32x4(row0_0123, row1_0123);
-  acc23 = zip2_f32x4(row0_0123, row1_0123);
-  acc45 = zip1_f32x4(row0_4567, row1_4567);
-  acc67 = zip2_f32x4(row0_4567, row1_4567);
-}
-
-FORCE_INLINE void store_acc_rowpair(const float32x4_t acc01,
-                                    const float32x4_t acc23,
-                                    const float32x4_t acc45,
-                                    const float32x4_t acc67,
-                                    float* __restrict__ c_ptr,
-                                    const int64_t ldc, const int32_t m_rows) {
-  if (m_rows == 0) {
-    return;
-  }
-
-  vst1q_f32(c_ptr, zip1_f32x4(acc01, acc23));
-  vst1q_f32(c_ptr + 4, zip1_f32x4(acc45, acc67));
-
-  if (m_rows == 2) {
-    vst1q_f32(c_ptr + ldc, zip2_f32x4(acc01, acc23));
-    vst1q_f32(c_ptr + ldc + 4, zip2_f32x4(acc45, acc67));
-  }
-}
-
-FORCE_INLINE void gemm_micro_bfmmla_8x8_packed_a(
-    const bfloat16_t* __restrict__ a_packed,
-    const bfloat16_t* __restrict__ b_packed, float* __restrict__ c_ptr,
-    const int32_t m, const int32_t k_size, const int64_t ldc,
-    const bool accum_c) {
-  float32x4_t acc0101, acc0123, acc0145, acc0167;
-  float32x4_t acc2301, acc2323, acc2345, acc2367;
-  float32x4_t acc4501, acc4523, acc4545, acc4567;
-  float32x4_t acc6701, acc6723, acc6745, acc6767;
-
-  init_acc_rowpair(acc0101, acc0123, acc0145, acc0167, c_ptr, ldc,
-                   std::min(2, m), accum_c);
-  init_acc_rowpair(acc2301, acc2323, acc2345, acc2367, c_ptr + 2 * ldc, ldc,
-                   std::min(2, std::max(0, m - 2)), accum_c);
-  init_acc_rowpair(acc4501, acc4523, acc4545, acc4567, c_ptr + 4 * ldc, ldc,
-                   std::min(2, std::max(0, m - 4)), accum_c);
-  init_acc_rowpair(acc6701, acc6723, acc6745, acc6767, c_ptr + 6 * ldc, ldc,
-                   std::min(2, std::max(0, m - 6)), accum_c);
-
-  const bfloat16_t* __restrict__ a_tile = a_packed;
-  const bfloat16_t* __restrict__ b_tile = b_packed;
-
-#pragma GCC unroll 8
-  for (int32_t k_idx = 0; k_idx < k_size; k_idx += K) {
-    const bfloat16x8_t a_tile01 = vld1q_bf16(a_tile);
-    const bfloat16x8_t a_tile23 = vld1q_bf16(a_tile + TileSize);
-    const bfloat16x8_t a_tile45 = vld1q_bf16(a_tile + 2 * TileSize);
-    const bfloat16x8_t a_tile67 = vld1q_bf16(a_tile + 3 * TileSize);
-
-    const bfloat16x8_t b_tile01 = vld1q_bf16(b_tile);
-    const bfloat16x8_t b_tile23 = vld1q_bf16(b_tile + TileSize);
-    const bfloat16x8_t b_tile45 = vld1q_bf16(b_tile + 2 * TileSize);
-    const bfloat16x8_t b_tile67 = vld1q_bf16(b_tile + 3 * TileSize);
-
-    acc0101 = vbfmmlaq_f32(acc0101, a_tile01, b_tile01);
-    acc2301 = vbfmmlaq_f32(acc2301, a_tile23, b_tile01);
-    acc4501 = vbfmmlaq_f32(acc4501, a_tile45, b_tile01);
-    acc6701 = vbfmmlaq_f32(acc6701, a_tile67, b_tile01);
-
-    acc0123 = vbfmmlaq_f32(acc0123, a_tile01, b_tile23);
-    acc2323 = vbfmmlaq_f32(acc2323, a_tile23, b_tile23);
-    acc4523 = vbfmmlaq_f32(acc4523, a_tile45, b_tile23);
-    acc6723 = vbfmmlaq_f32(acc6723, a_tile67, b_tile23);
-
-    acc0145 = vbfmmlaq_f32(acc0145, a_tile01, b_tile45);
-    acc2345 = vbfmmlaq_f32(acc2345, a_tile23, b_tile45);
-    acc4545 = vbfmmlaq_f32(acc4545, a_tile45, b_tile45);
-    acc6745 = vbfmmlaq_f32(acc6745, a_tile67, b_tile45);
-
-    acc0167 = vbfmmlaq_f32(acc0167, a_tile01, b_tile67);
-    acc2367 = vbfmmlaq_f32(acc2367, a_tile23, b_tile67);
-    acc4567 = vbfmmlaq_f32(acc4567, a_tile45, b_tile67);
-    acc6767 = vbfmmlaq_f32(acc6767, a_tile67, b_tile67);
-
-    a_tile += 4 * TileSize;
-    b_tile += Nr * K;
-  }
-
-  store_acc_rowpair(acc0101, acc0123, acc0145, acc0167, c_ptr, ldc,
-                    std::min(2, m));
-  store_acc_rowpair(acc2301, acc2323, acc2345, acc2367, c_ptr + 2 * ldc, ldc,
-                    std::min(2, std::max(0, m - 2)));
-  store_acc_rowpair(acc4501, acc4523, acc4545, acc4567, c_ptr + 4 * ldc, ldc,
-                    std::min(2, std::max(0, m - 4)));
-  store_acc_rowpair(acc6701, acc6723, acc6745, acc6767, c_ptr + 6 * ldc, ldc,
-                    std::min(2, std::max(0, m - 6)));
-}
-
-FORCE_INLINE void gemm_micro_bfmmla_4x16_packed_a(
-    const bfloat16_t* __restrict__ a_packed,
-    const bfloat16_t* __restrict__ b_packed, float* __restrict__ c_ptr,
-    const int32_t m, const int32_t k_size, const int64_t b_n_group_stride,
-    const int64_t ldc, const bool accum_c) {
-  const int32_t m_rows_01 = std::min(2, m);
-  const int32_t m_rows_23 = std::min(2, std::max(0, m - 2));
-
-  float32x4_t acc0101, acc0123, acc0145, acc0167;
-  float32x4_t acc2301, acc2323, acc2345, acc2367;
-  float32x4_t acc0189, acc011011, acc011213, acc011415;
-  float32x4_t acc2389, acc231011, acc231213, acc231415;
-
-  init_acc_rowpair(acc0101, acc0123, acc0145, acc0167, c_ptr, ldc, m_rows_01,
-                   accum_c);
-  init_acc_rowpair(acc2301, acc2323, acc2345, acc2367, c_ptr + 2 * ldc, ldc,
-                   m_rows_23, accum_c);
-  init_acc_rowpair(acc0189, acc011011, acc011213, acc011415, c_ptr + 8, ldc,
-                   m_rows_01, accum_c);
-  init_acc_rowpair(acc2389, acc231011, acc231213, acc231415,
-                   c_ptr + 2 * ldc + 8, ldc, m_rows_23, accum_c);
-
-  const bfloat16_t* __restrict__ a_tile = a_packed;
-  const bfloat16_t* __restrict__ b_tile0 = b_packed;
-  const bfloat16_t* __restrict__ b_tile1 = b_packed + b_n_group_stride;
-
-#pragma GCC unroll 8
-  for (int32_t k_idx = 0; k_idx < k_size; k_idx += K) {
-    const bfloat16x8_t a_tile01 = vld1q_bf16(a_tile);
-    const bfloat16x8_t a_tile23 = vld1q_bf16(a_tile + TileSize);
-    const bfloat16x8_t b_tile01 = vld1q_bf16(b_tile0);
-    const bfloat16x8_t b_tile23 = vld1q_bf16(b_tile0 + TileSize);
-    const bfloat16x8_t b_tile45 = vld1q_bf16(b_tile0 + 2 * TileSize);
-    const bfloat16x8_t b_tile67 = vld1q_bf16(b_tile0 + 3 * TileSize);
-    const bfloat16x8_t b_tile89 = vld1q_bf16(b_tile1);
-    const bfloat16x8_t b_tile1011 = vld1q_bf16(b_tile1 + TileSize);
-    const bfloat16x8_t b_tile1213 = vld1q_bf16(b_tile1 + 2 * TileSize);
-    const bfloat16x8_t b_tile1415 = vld1q_bf16(b_tile1 + 3 * TileSize);
-
-    acc0101 = vbfmmlaq_f32(acc0101, a_tile01, b_tile01);
-    acc2301 = vbfmmlaq_f32(acc2301, a_tile23, b_tile01);
-    acc0123 = vbfmmlaq_f32(acc0123, a_tile01, b_tile23);
-    acc2323 = vbfmmlaq_f32(acc2323, a_tile23, b_tile23);
-
-    acc0145 = vbfmmlaq_f32(acc0145, a_tile01, b_tile45);
-    acc2345 = vbfmmlaq_f32(acc2345, a_tile23, b_tile45);
-    acc0167 = vbfmmlaq_f32(acc0167, a_tile01, b_tile67);
-    acc2367 = vbfmmlaq_f32(acc2367, a_tile23, b_tile67);
-
-    acc0189 = vbfmmlaq_f32(acc0189, a_tile01, b_tile89);
-    acc2389 = vbfmmlaq_f32(acc2389, a_tile23, b_tile89);
-    acc011011 = vbfmmlaq_f32(acc011011, a_tile01, b_tile1011);
-    acc231011 = vbfmmlaq_f32(acc231011, a_tile23, b_tile1011);
-
-    acc011213 = vbfmmlaq_f32(acc011213, a_tile01, b_tile1213);
-    acc231213 = vbfmmlaq_f32(acc231213, a_tile23, b_tile1213);
-    acc011415 = vbfmmlaq_f32(acc011415, a_tile01, b_tile1415);
-    acc231415 = vbfmmlaq_f32(acc231415, a_tile23, b_tile1415);
-
-    a_tile += 2 * TileSize;
-    b_tile0 += Nr * K;
-    b_tile1 += Nr * K;
-  }
-
-  store_acc_rowpair(acc0101, acc0123, acc0145, acc0167, c_ptr, ldc, m_rows_01);
-  store_acc_rowpair(acc2301, acc2323, acc2345, acc2367, c_ptr + 2 * ldc, ldc,
-                    m_rows_23);
-  store_acc_rowpair(acc0189, acc011011, acc011213, acc011415, c_ptr + 8, ldc,
-                    m_rows_01);
-  store_acc_rowpair(acc2389, acc231011, acc231213, acc231415,
-                    c_ptr + 2 * ldc + 8, ldc, m_rows_23);
-}
-
-}  // namespace
-
-template <typename scalar_t>
-class MicroGemm<cpu_utils::ISA::NEON, scalar_t> {
- public:
-  static constexpr int32_t MaxMSize = 8;
-  static constexpr int32_t NSize = 32;
-  static constexpr int32_t WeightOCGroupSize = Nr;
-  static constexpr bool PackA = false;
-
- public:
-  void gemm(DEFINE_CPU_MICRO_GEMM_PARAMS) {
-    TORCH_CHECK(false, "NEON BFMMLA MicroGemm only supports bfloat16.");
-  }
-
-  static void pack_weight(const scalar_t* __restrict__ /*weight*/,
-                          scalar_t* __restrict__ /*packed_weight*/,
-                          const int32_t /*output_size*/,
-                          const int32_t /*input_size*/) {
-    TORCH_CHECK(false, "NEON BFMMLA MicroGemm only supports bfloat16.");
-  }
-};
-
-template <>
-class MicroGemm<cpu_utils::ISA::NEON, c10::BFloat16> {
- public:
-  using scalar_t = c10::BFloat16;
-
-  static constexpr int32_t MaxMSize = 8;
-  static constexpr int32_t NSize = 32;
-  static constexpr int32_t WeightOCGroupSize = Nr;
-  static constexpr bool PackA = true;
-
- public:
-  // physical layout [
-  //  M / 8; Mr is 8
-  //  K / 4; K for bfmmla is 4
-  //  4,   ; 4 row-pairs for each 8 rows
-  //  2,   ; row-pair is 2 rows
-  //  4    ; 4 elements per row
-  // ]
-
-  static void pack_input_from_rows(const scalar_t* const* __restrict__ rows,
-                                   scalar_t* __restrict__ a_packed,
-                                   const int32_t m, const int32_t k) {
-    TORCH_CHECK(m > 0 && m <= MaxMSize);
-    TORCH_CHECK_EQ(k % K, 0);
-
-    auto* __restrict__ out = reinterpret_cast<bfloat16_t*>(a_packed);
-    const bfloat16x8_t zero_q = vdupq_n_bf16(bfloat16_t{});
-    const bfloat16x4_t zero = vget_low_bf16(zero_q);
-
-    for (int32_t row_base = 0; row_base < m; row_base += Mr) {
-      const int32_t actual_m = std::min(Mr, m - row_base);
-      const bfloat16_t* __restrict__ row[Mr];
-      for (int32_t i = 0; i < actual_m; ++i) {
-        row[i] = reinterpret_cast<const bfloat16_t*>(rows[row_base + i]);
-      }
-
-      if (actual_m == 8) {
-        int32_t k_idx = 0;
-        for (; k_idx + 8 <= k; k_idx += 8) {
-          bfloat16_t* __restrict__ block0 = out;
-          bfloat16_t* __restrict__ block1 = out + 4 * TileSize;
-
-          bfloat16x8_t a0 = vld1q_bf16(row[0] + k_idx);
-          bfloat16x8_t a1 = vld1q_bf16(row[1] + k_idx);
-          vst1q_bf16(block0,
-                     vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1)));
-          vst1q_bf16(block1,
-                     vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1)));
-
-          a0 = vld1q_bf16(row[2] + k_idx);
-          a1 = vld1q_bf16(row[3] + k_idx);
-          vst1q_bf16(block0 + TileSize,
-                     vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1)));
-          vst1q_bf16(block1 + TileSize,
-                     vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1)));
-
-          a0 = vld1q_bf16(row[4] + k_idx);
-          a1 = vld1q_bf16(row[5] + k_idx);
-          vst1q_bf16(block0 + 2 * TileSize,
-                     vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1)));
-          vst1q_bf16(block1 + 2 * TileSize,
-                     vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1)));
-
-          a0 = vld1q_bf16(row[6] + k_idx);
-          a1 = vld1q_bf16(row[7] + k_idx);
-          vst1q_bf16(block0 + 3 * TileSize,
-                     vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1)));
-          vst1q_bf16(block1 + 3 * TileSize,
-                     vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1)));
-
-          out += 8 * TileSize;
-        }
-
-        for (; k_idx < k; k_idx += K) {
-          bfloat16x4_t a0 = vld1_bf16(row[0] + k_idx);
-          bfloat16x4_t a1 = vld1_bf16(row[1] + k_idx);
-          vst1q_bf16(out, vcombine_bf16(a0, a1));
-
-          a0 = vld1_bf16(row[2] + k_idx);
-          a1 = vld1_bf16(row[3] + k_idx);
-          vst1q_bf16(out + TileSize, vcombine_bf16(a0, a1));
-
-          a0 = vld1_bf16(row[4] + k_idx);
-          a1 = vld1_bf16(row[5] + k_idx);
-          vst1q_bf16(out + 2 * TileSize, vcombine_bf16(a0, a1));
-
-          a0 = vld1_bf16(row[6] + k_idx);
-          a1 = vld1_bf16(row[7] + k_idx);
-          vst1q_bf16(out + 3 * TileSize, vcombine_bf16(a0, a1));
-
-          out += 4 * TileSize;
-        }
-        continue;
-      }
-
-      if (actual_m == 4) {
-        int32_t k_idx = 0;
-        for (; k_idx + 8 <= k; k_idx += 8) {
-          bfloat16_t* __restrict__ block0 = out;
-          bfloat16_t* __restrict__ block1 = out + 2 * TileSize;
-
-          bfloat16x8_t a0 = vld1q_bf16(row[0] + k_idx);
-          bfloat16x8_t a1 = vld1q_bf16(row[1] + k_idx);
-          vst1q_bf16(block0,
-                     vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1)));
-          vst1q_bf16(block1,
-                     vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1)));
-
-          a0 = vld1q_bf16(row[2] + k_idx);
-          a1 = vld1q_bf16(row[3] + k_idx);
-          vst1q_bf16(block0 + TileSize,
-                     vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1)));
-          vst1q_bf16(block1 + TileSize,
-                     vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1)));
-
-          out += 4 * TileSize;
-        }
-
-        for (; k_idx < k; k_idx += K) {
-          bfloat16x4_t a0 = vld1_bf16(row[0] + k_idx);
-          bfloat16x4_t a1 = vld1_bf16(row[1] + k_idx);
-          vst1q_bf16(out, vcombine_bf16(a0, a1));
-
-          a0 = vld1_bf16(row[2] + k_idx);
-          a1 = vld1_bf16(row[3] + k_idx);
-          vst1q_bf16(out + TileSize, vcombine_bf16(a0, a1));
-
-          out += 2 * TileSize;
-        }
-        continue;
-      }
-
-      const int32_t row_pair_count = (actual_m <= 4) ? 2 : Mr / 2;
-
-      int32_t k_idx = 0;
-      for (; k_idx + 8 <= k; k_idx += 8) {
-        bfloat16_t* __restrict__ block0 = out;
-        bfloat16_t* __restrict__ block1 = out + row_pair_count * TileSize;
-
-        bfloat16x8_t a0 = vld1q_bf16(row[0] + k_idx);
-        bfloat16x8_t a1 = (actual_m > 1) ? vld1q_bf16(row[1] + k_idx) : zero_q;
-        vst1q_bf16(block0, vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1)));
-        vst1q_bf16(block1,
-                   vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1)));
-
-        a0 = (actual_m > 2) ? vld1q_bf16(row[2] + k_idx) : zero_q;
-        a1 = (actual_m > 3) ? vld1q_bf16(row[3] + k_idx) : zero_q;
-        vst1q_bf16(block0 + TileSize,
-                   vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1)));
-        vst1q_bf16(block1 + TileSize,
-                   vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1)));
-
-        if (actual_m > 4) {
-          a0 = vld1q_bf16(row[4] + k_idx);
-          a1 = (actual_m > 5) ? vld1q_bf16(row[5] + k_idx) : zero_q;
-          vst1q_bf16(block0 + 2 * TileSize,
-                     vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1)));
-          vst1q_bf16(block1 + 2 * TileSize,
-                     vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1)));
-
-          a0 = (actual_m > 6) ? vld1q_bf16(row[6] + k_idx) : zero_q;
-          a1 = (actual_m > 7) ? vld1q_bf16(row[7] + k_idx) : zero_q;
-          vst1q_bf16(block0 + 3 * TileSize,
-                     vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1)));
-          vst1q_bf16(block1 + 3 * TileSize,
-                     vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1)));
-        }
-
-        out += 2 * row_pair_count * TileSize;
-      }
-
-      for (; k_idx < k; k_idx += K) {
-        bfloat16x4_t a0 = vld1_bf16(row[0] + k_idx);
-        bfloat16x4_t a1 = (actual_m > 1) ? vld1_bf16(row[1] + k_idx) : zero;
-        vst1q_bf16(out, vcombine_bf16(a0, a1));
-
-        a0 = (actual_m > 2) ? vld1_bf16(row[2] + k_idx) : zero;
-        a1 = (actual_m > 3) ? vld1_bf16(row[3] + k_idx) : zero;
-        vst1q_bf16(out + TileSize, vcombine_bf16(a0, a1));
-
-        if (actual_m > 4) {
-          a0 = vld1_bf16(row[4] + k_idx);
-          a1 = (actual_m > 5) ? vld1_bf16(row[5] + k_idx) : zero;
-          vst1q_bf16(out + 2 * TileSize, vcombine_bf16(a0, a1));
-
-          a0 = (actual_m > 6) ? vld1_bf16(row[6] + k_idx) : zero;
-          a1 = (actual_m > 7) ? vld1_bf16(row[7] + k_idx) : zero;
-          vst1q_bf16(out + 3 * TileSize, vcombine_bf16(a0, a1));
-        }
-        out += row_pair_count * TileSize;
-      }
-    }
-  }
-
-  void gemm(DEFINE_CPU_MICRO_GEMM_PARAMS) {
-    (void)lda;  // A is packed, so lda is not needed
-    TORCH_CHECK_EQ(k % K, 0);
-
-    for (int32_t n_idx = 0; n_idx < NSize; n_idx += Nr_gemv) {
-      const bfloat16_t* __restrict__ b_panel =
-          reinterpret_cast<const bfloat16_t*>(b_ptr) + n_idx * k;
-
-      for (int32_t row_base = 0; row_base < m; row_base += Mr) {
-        const int32_t panel_m = std::min(Mr, m - row_base);
-        const bfloat16_t* __restrict__ a_panel =
-            reinterpret_cast<const bfloat16_t*>(a_ptr) + row_base * k;
-        float* __restrict__ c_panel = c_ptr + row_base * ldc + n_idx;
-
-        if (panel_m <= 4) {
-          gemm_micro_bfmmla_4x16_packed_a(a_panel, b_panel, c_panel, panel_m, k,
-                                          b_n_group_stride, ldc, accum_c);
-        } else {
-          gemm_micro_bfmmla_8x8_packed_a(a_panel, b_panel, c_panel, panel_m, k,
-                                         ldc, accum_c);
-          gemm_micro_bfmmla_8x8_packed_a(a_panel, b_panel + b_n_group_stride,
-                                         c_panel + Nr, panel_m, k, ldc,
-                                         accum_c);
-        }
-      }
-    }
-  }
-
-  // physical layout [
-  //  N / 8; Nr is 8
-  //  K / 4; K for bfmmla is 4
-  //  4,   ; 4 col-pairs for each 8 cols
-  //  2,   ; col-pair is 2 cols
-  //  4    ; 4 elements per col
-  // ]
-  static void pack_weight(const c10::BFloat16* __restrict__ weight,
-                          c10::BFloat16* __restrict__ packed_weight,
-                          const int32_t output_size, const int32_t input_size) {
-    TORCH_CHECK_EQ(output_size % NSize, 0);
-    TORCH_CHECK_EQ(input_size % K, 0);
-
-    for (int32_t o_idx = 0; o_idx < output_size; o_idx += Nr) {
-      c10::BFloat16* __restrict__ dst = packed_weight + o_idx * input_size;
-      for (int32_t k_idx = 0; k_idx < input_size; k_idx += K) {
-        for (int32_t pair_idx = 0; pair_idx < Nr; pair_idx += Cols) {
-          const c10::BFloat16* __restrict__ row0 =
-              weight + (o_idx + pair_idx) * input_size;
-          const c10::BFloat16* __restrict__ row1 = row0 + input_size;
-          dst[0] = row0[k_idx + 0];
-          dst[1] = row0[k_idx + 1];
-          dst[2] = row0[k_idx + 2];
-          dst[3] = row0[k_idx + 3];
-          dst[4] = row1[k_idx + 0];
-          dst[5] = row1[k_idx + 1];
-          dst[6] = row1[k_idx + 2];
-          dst[7] = row1[k_idx + 3];
-          dst += TileSize;
-        }
-      }
-    }
-  }
-};
-
-}  // namespace cpu_micro_gemm
-
-#endif

csrc/cpu/micro_gemm/cpu_micro_gemm_vec.hpp

@@ -104,8 +104,6 @@ class MicroGemm<cpu_utils::ISA::VEC, scalar_t> {
  public:
   static constexpr int32_t MaxMSize = 8;
   static constexpr int32_t NSize = 32;
-  static constexpr int32_t WeightOCGroupSize = 16;
-  static constexpr bool PackA = false;
 
  public:
   void gemm(DEFINE_CPU_MICRO_GEMM_PARAMS) {

csrc/cpu/mla_decode.cpp

@@ -18,9 +18,17 @@ struct KernelVecType<float> {
 
 template <>
 struct KernelVecType<c10::Half> {
+#if defined(__powerpc64__)
+  // Power specific vector types
+  using qk_load_vec_type = vec_op::FP32Vec16;
+  using qk_vec_type = vec_op::FP32Vec16;
+  using v_load_vec_type = vec_op::FP32Vec16;
+#else
+  // Fallback for other architectures, including x86
   using qk_load_vec_type = vec_op::FP16Vec16;
   using qk_vec_type = vec_op::FP32Vec16;
   using v_load_vec_type = vec_op::FP16Vec16;
+#endif
 };
 
 #ifdef __AVX512BF16__
@@ -251,7 +259,7 @@ void mla_decode_kvcache_cpu_impl(
   constexpr int QK_NUM_ELEM = qk_vec_type::VEC_ELEM_NUM;
 
   // shared across threads
-  const int max_threads = cpu_utils::get_max_threads();
+  const int max_threads = omp_get_max_threads();
   const int acc_out_nbytes =
       max_threads * num_heads * V_HEAD_DIM * sizeof(float);
   float* acc_out = static_cast<float*>(std::aligned_alloc(64, acc_out_nbytes));

csrc/cpu/pos_encoding.cpp

@@ -1,3 +1,4 @@
+
 #include "cpu_types.hpp"
 
 namespace {
@@ -96,91 +97,6 @@ void rotary_embedding_impl(
   }
 }
 
-template <>
-void rotary_embedding_impl<c10::Half>(
-    const int64_t* __restrict__ positions, c10::Half* __restrict__ query,
-    c10::Half* __restrict__ key, const c10::Half* __restrict__ cos_sin_cache,
-    const int rot_dim, const int64_t query_stride, const int64_t key_stride,
-    const int num_heads, const int num_kv_heads, const int head_size,
-    const int num_tokens) {
-  using scalar_vec_t = vec_op::FP16Vec8;
-  constexpr int VEC_ELEM_NUM = scalar_vec_t::get_elem_num();
-
-  const int embed_dim = rot_dim / 2;
-  bool flag = (embed_dim % VEC_ELEM_NUM == 0);
-  const int loop_upper = flag ? embed_dim : embed_dim - VEC_ELEM_NUM;
-
-  auto compute_loop = [&](const int64_t token_head, const c10::Half* cache_ptr,
-                          c10::Half* qk) {
-    int j = 0;
-    for (; j < loop_upper; j += VEC_ELEM_NUM) {
-      const int rot_offset = j;
-      const int x_index = rot_offset;
-      const int y_index = embed_dim + rot_offset;
-
-      const int64_t out_x = token_head + x_index;
-      const int64_t out_y = token_head + y_index;
-
-      const vec_op::FP16Vec8 cos_fp16(cache_ptr + x_index);
-      const vec_op::FP16Vec8 sin_fp16(cache_ptr + y_index);
-      const vec_op::FP16Vec8 q_x_fp16(qk + out_x);
-      const vec_op::FP16Vec8 q_y_fp16(qk + out_y);
-
-      const vec_op::FP32Vec8 fp32_cos(cos_fp16);
-      const vec_op::FP32Vec8 fp32_sin(sin_fp16);
-      const vec_op::FP32Vec8 fp32_q_x(q_x_fp16);
-      const vec_op::FP32Vec8 fp32_q_y(q_y_fp16);
-
-      auto out1 = fp32_q_x * fp32_cos - fp32_q_y * fp32_sin;
-      auto out2 = fp32_q_y * fp32_cos + fp32_q_x * fp32_sin;
-
-      vec_op::FP16Vec8(out1).save(qk + out_x);
-      vec_op::FP16Vec8(out2).save(qk + out_y);
-    }
-    if (!flag) {
-      for (; j < embed_dim; ++j) {
-        const int x_index = j;
-        const int y_index = embed_dim + j;
-
-        const int64_t out_x = token_head + x_index;
-        const int64_t out_y = token_head + y_index;
-
-        const float fp32_cos = static_cast<float>(cache_ptr[x_index]);
-        const float fp32_sin = static_cast<float>(cache_ptr[y_index]);
-        const float fp32_q_x = static_cast<float>(qk[out_x]);
-        const float fp32_q_y = static_cast<float>(qk[out_y]);
-
-        qk[out_x] =
-            static_cast<c10::Half>(fp32_q_x * fp32_cos - fp32_q_y * fp32_sin);
-        qk[out_y] =
-            static_cast<c10::Half>(fp32_q_y * fp32_cos + fp32_q_x * fp32_sin);
-      }
-    }
-  };
-
-#pragma omp parallel for
-  for (int token_idx = 0; token_idx < num_tokens; ++token_idx) {
-    int64_t pos = positions[token_idx];
-    const c10::Half* cache_ptr = cos_sin_cache + pos * rot_dim;
-
-    for (int i = 0; i < num_heads; ++i) {
-      const int head_idx = i;
-      const int64_t token_head =
-          token_idx * query_stride + head_idx * head_size;
-      compute_loop(token_head, cache_ptr, query);
-    }
-
-    if (key != nullptr) {
-      for (int i = 0; i < num_kv_heads; ++i) {
-        const int head_idx = i;
-        const int64_t token_head =
-            token_idx * key_stride + head_idx * head_size;
-        compute_loop(token_head, cache_ptr, key);
-      }
-    }
-  }
-}
-
 template <typename scalar_t>
 void rotary_embedding_gptj_impl(
     const int64_t* __restrict__ positions,  // [batch_size, seq_len] or
@@ -258,75 +174,6 @@ void rotary_embedding_gptj_impl(
     }
   }
 }
-
-template <>
-void rotary_embedding_gptj_impl<c10::Half>(
-    const int64_t* __restrict__ positions, c10::Half* __restrict__ query,
-    c10::Half* __restrict__ key, const c10::Half* __restrict__ cos_sin_cache,
-    const int rot_dim, const int64_t query_stride, const int64_t key_stride,
-    const int num_heads, const int num_kv_heads, const int head_size,
-    const int num_tokens) {
-  const int embed_dim = rot_dim / 2;
-
-#pragma omp parallel for collapse(2)
-  for (int token_idx = 0; token_idx < num_tokens; ++token_idx) {
-    for (int i = 0; i < num_heads; ++i) {
-      int64_t pos = positions[token_idx];
-      const c10::Half* cache_ptr = cos_sin_cache + pos * rot_dim;
-      const c10::Half* cos_cache_ptr = cache_ptr;
-      const c10::Half* sin_cache_ptr = cache_ptr + embed_dim;
-      const int head_idx = i;
-      const int64_t token_head =
-          token_idx * query_stride + head_idx * head_size;
-      c10::Half* head_query = token_head + query;
-      for (int j = 0; j < embed_dim; j += 1) {
-        const int rot_offset = j;
-        const int x_index = 2 * rot_offset;
-        const int y_index = 2 * rot_offset + 1;
-
-        const float cos = static_cast<float>(cos_cache_ptr[rot_offset]);
-        const float sin = static_cast<float>(sin_cache_ptr[rot_offset]);
-
-        const float x = static_cast<float>(head_query[x_index]);
-        const float y = static_cast<float>(head_query[y_index]);
-
-        head_query[x_index] = static_cast<c10::Half>(x * cos - y * sin);
-        head_query[y_index] = static_cast<c10::Half>(y * cos + x * sin);
-      }
-    }
-  }
-
-  if (key == nullptr) {
-    return;
-  }
-
-#pragma omp parallel for collapse(2)
-  for (int token_idx = 0; token_idx < num_tokens; ++token_idx) {
-    for (int i = 0; i < num_kv_heads; ++i) {
-      int64_t pos = positions[token_idx];
-      const c10::Half* cache_ptr = cos_sin_cache + pos * rot_dim;
-      const c10::Half* cos_cache_ptr = cache_ptr;
-      const c10::Half* sin_cache_ptr = cache_ptr + embed_dim;
-      const int head_idx = i;
-      const int64_t token_head = token_idx * key_stride + head_idx * head_size;
-      c10::Half* head_key = key + token_head;
-      for (int j = 0; j < embed_dim; j += 1) {
-        const int rot_offset = j;
-        const int x_index = 2 * rot_offset;
-        const int y_index = 2 * rot_offset + 1;
-
-        const float cos = static_cast<float>(cos_cache_ptr[rot_offset]);
-        const float sin = static_cast<float>(sin_cache_ptr[rot_offset]);
-
-        const float x = static_cast<float>(head_key[x_index]);
-        const float y = static_cast<float>(head_key[y_index]);
-
-        head_key[x_index] = static_cast<c10::Half>(x * cos - y * sin);
-        head_key[y_index] = static_cast<c10::Half>(y * cos + x * sin);
-      }
-    }
-  }
-}
 };  // namespace
 
 void rotary_embedding(torch::Tensor& positions, torch::Tensor& query,

csrc/cpu/sgl-kernels/conv.cpp

@@ -289,18 +289,19 @@ void causal_conv1d_fwd_kernel_impl(
   }
 }
 
-#define LAUNCH_TINYGEMM_VARLEN_KERNEL(K, NB_SIZE)                                                   \
-  tinygemm_kernel<scalar_t, K, NB_SIZE, has_bias, has_silu>::apply(                                 \
-      input + batch_offset * dim + mb_start * dim + nb_start,                                       \
-      weight + nb_start * width,                                                                    \
-      out + batch_offset * dim + mb_start * dim + nb_start,                                         \
-      has_bias ? bias + nb_start : nullptr,                                                         \
-      has_conv_states ? conv_states + conv_state_index * conv_state_slot_stride + nb_start : nullptr, \
-      has_initial_states_value,                                                                     \
-      mb_size,                                                                                      \
-      dim,                                                                                          \
+#define LAUNCH_TINYGEMM_VARLEN_KERNEL(K, NB_SIZE)                   \
+  tinygemm_kernel<scalar_t, K, NB_SIZE, has_bias, has_silu>::apply( \
+      input + batch_offset * dim + mb_start * dim + nb_start,       \
+      weight + nb_start * width,                                    \
+      out + batch_offset * dim + mb_start * dim + nb_start,         \
+      has_bias ? bias + nb_start : nullptr,                         \
+      nullptr,                                                      \
+      false,                                                        \
+      mb_size,                                                      \
+      dim,                                                          \
       mb_start == 0);
 
+// TODO: add `has_initial_state` support for varlen kernel
 template <typename scalar_t>
 void causal_conv1d_fwd_varlen_kernel_impl(
     scalar_t* __restrict__ out,
@@ -342,9 +343,6 @@ void causal_conv1d_fwd_varlen_kernel_impl(
         int64_t nb_start = nb * BLOCK_N;
         int64_t nb_size = std::min(dim - nb_start, BLOCK_N);
 
-        const bool has_initial_states_value = has_conv_states ? has_initial_state[bs] : false;
-        int32_t conv_state_index = has_conv_indices ? conv_indices[bs] : bs;
-
         switch (width << 4 | nb_size >> 4) {
           case 0x42:
             LAUNCH_TINYGEMM_VARLEN_KERNEL(4, 32);
@@ -375,7 +373,7 @@ void causal_conv1d_fwd_varlen_kernel_impl(
             width,
             dim,
             seqlen,
-            has_initial_state[bs]);
+            /* has_initial_state */ false);
       }
     });
   }

csrc/cpu/sgl-kernels/gemm_int4.cpp

@@ -285,125 +285,6 @@ inline int32_t load_uint4_vnni(const uint8_t* __restrict__ B, int64_t k, int64_t
   return (n_group % 2 == 0) ? (packed & 0x0f) : ((packed >> 4) & 0x0f);
 }
 
-#if defined(CPU_CAPABILITY_RVV)
-template <int64_t N, int64_t ldb, int group>
-inline fixed_i8x8_t load_uint4_as_int8_rvv(const uint8_t* __restrict__ B, int64_t k) {
-  constexpr int64_t n_group_size = 8;
-  constexpr int64_t vnni_size = 4;
-  static_assert(N == 32);
-  static_assert(ldb == N / 2);
-  static_assert(group >= 0 && group < N / n_group_size);
-
-  // Unpack: gather 8 packed int4 values from the VNNI4 layout.
-  const int64_t ki = k % vnni_size;
-  const int64_t k_base = k - ki;
-  constexpr int64_t packed_group = group / 2;
-  const uint8_t* packed_ptr = B + k_base * ldb + packed_group * n_group_size * vnni_size + ki;
-
-  fixed_u8x8_t packed = RVVI(__riscv_vlse8_v_u8, LMUL_64)(packed_ptr, vnni_size, n_group_size);
-  if constexpr (group % 2 == 1) {
-    packed = RVVI(__riscv_vsrl_vx_u8, LMUL_64)(packed, 4, n_group_size);
-  }
-  fixed_u8x8_t nibbles = RVVI(__riscv_vand_vx_u8, LMUL_64)(packed, 0x0f, n_group_size);
-  return RVVI4(__riscv_vreinterpret_v_u8, LMUL_64, _i8, LMUL_64)(nibbles);
-}
-
-inline fixed_i32x8_t gemm_accum_uint8_int8_rvv(fixed_i32x8_t acc, uint8_t a, fixed_i8x8_t b) {
-  constexpr int64_t vl = 8;
-  fixed_i16x8_t b_i16 = RVVI(__riscv_vsext_vf2_i16, LMUL_128)(b, vl);
-  return RVVI(__riscv_vwmacc_vx_i32, LMUL_256)(acc, static_cast<int16_t>(a), b_i16, vl);
-}
-
-template <int64_t N, int64_t ldb, int group>
-inline fixed_i32x8_t gemm_accum_uint4_rvv(
-    fixed_i32x8_t acc,
-    const uint8_t* __restrict__ B,
-    const int8_t* __restrict__ qzeros_b,
-    uint8_t a,
-    int64_t k) {
-  constexpr int64_t n_group_size = 8;
-  fixed_i8x8_t b = load_uint4_as_int8_rvv<N, ldb, group>(B, k);
-  fixed_i8x8_t qzeros =
-      RVVI(__riscv_vle8_v_i8, LMUL_64)(qzeros_b + group * n_group_size, n_group_size);
-  b = RVVI(__riscv_vsub_vv_i8, LMUL_64)(b, qzeros, n_group_size);
-  return gemm_accum_uint8_int8_rvv(acc, a, b);
-}
-
-template <int group>
-inline void _dequant_and_store_rvv(
-    float* __restrict__ C,
-    fixed_i32x8_t acc,
-    const float* __restrict__ scales_a,
-    const int32_t* __restrict__ qzeros_a,
-    const float* __restrict__ scales_b,
-    const int32_t* __restrict__ compensation,
-    int64_t m,
-    int64_t ldc) {
-  constexpr int64_t n_group_size = 8;
-  constexpr int64_t n = group * n_group_size;
-  constexpr int64_t vl = n_group_size;
-
-  // Dequant compensation: remove activation zero-point contribution.
-  fixed_i32x8_t comp = RVVI(__riscv_vle32_v_i32, LMUL_256)(compensation + n, vl);
-  fixed_i32x8_t zp_comp = RVVI(__riscv_vmul_vx_i32, LMUL_256)(comp, qzeros_a[m], vl);
-  acc = RVVI(__riscv_vsub_vv_i32, LMUL_256)(acc, zp_comp, vl);
-
-  // Scale: convert int32 accumulators to fp32 and apply activation/weight scales.
-  fixed_fp32x8_t acc_f = RVVI(__riscv_vfcvt_f_x_v_f32, LMUL_256)(acc, vl);
-  acc_f = RVVI(__riscv_vfmul_vf_f32, LMUL_256)(acc_f, scales_a[m], vl);
-  fixed_fp32x8_t scale_b = RVVI(__riscv_vle32_v_f32, LMUL_256)(scales_b + n, vl);
-  acc_f = RVVI(__riscv_vfmul_vv_f32, LMUL_256)(acc_f, scale_b, vl);
-
-  // Store: accumulate into the float scratch buffer that already holds bias/zero.
-  float* c_ptr = C + m * ldc + n;
-  fixed_fp32x8_t c_old = RVVI(__riscv_vle32_v_f32, LMUL_256)(c_ptr, vl);
-  fixed_fp32x8_t c_new = RVVI(__riscv_vfadd_vv_f32, LMUL_256)(c_old, acc_f, vl);
-  RVVI(__riscv_vse32_v_f32, LMUL_256)(c_ptr, c_new, vl);
-}
-
-template <int64_t N, int64_t ldb>
-void _dequant_gemm_accum_rvv(
-    float* __restrict__ C,
-    const uint8_t* __restrict__ A,
-    const float* __restrict__ scales_a,
-    const int32_t* __restrict__ qzeros_a,
-    const uint8_t* __restrict__ B,
-    const float* __restrict__ scales_b,
-    const int8_t* __restrict__ qzeros_b,
-    const int32_t* __restrict__ compensation,
-    int64_t M,
-    int64_t K,
-    int64_t lda,
-    int64_t ldc) {
-  static_assert(N == 32);
-  static_assert(ldb == N / 2);
-  constexpr int64_t vl = 8;
-
-  // Accumulate one C row over the 32-column block.
-  for (int64_t m = 0; m < M; ++m) {
-    fixed_i32x8_t acc0 = RVVI(__riscv_vmv_v_x_i32, LMUL_256)(0, vl);
-    fixed_i32x8_t acc1 = RVVI(__riscv_vmv_v_x_i32, LMUL_256)(0, vl);
-    fixed_i32x8_t acc2 = RVVI(__riscv_vmv_v_x_i32, LMUL_256)(0, vl);
-    fixed_i32x8_t acc3 = RVVI(__riscv_vmv_v_x_i32, LMUL_256)(0, vl);
-    // A[m][k] @ B[k][0:32] -> acc[m][0:32]
-    for (int64_t k = 0; k < K; ++k) {
-      // GEMM K step: one scalar activation updates four 8-column RVV tiles.
-      const uint8_t a = A[m * lda + k];
-      acc0 = gemm_accum_uint4_rvv<N, ldb, 0>(acc0, B, qzeros_b, a, k);
-      acc1 = gemm_accum_uint4_rvv<N, ldb, 1>(acc1, B, qzeros_b, a, k);
-      acc2 = gemm_accum_uint4_rvv<N, ldb, 2>(acc2, B, qzeros_b, a, k);
-      acc3 = gemm_accum_uint4_rvv<N, ldb, 3>(acc3, B, qzeros_b, a, k);
-    }
-
-    // Dequant/scale/store each 8-column group back into C.
-    _dequant_and_store_rvv<0>(C, acc0, scales_a, qzeros_a, scales_b, compensation, m, ldc);
-    _dequant_and_store_rvv<1>(C, acc1, scales_a, qzeros_a, scales_b, compensation, m, ldc);
-    _dequant_and_store_rvv<2>(C, acc2, scales_a, qzeros_a, scales_b, compensation, m, ldc);
-    _dequant_and_store_rvv<3>(C, acc3, scales_a, qzeros_a, scales_b, compensation, m, ldc);
-  }
-}
-#endif
-
 template <int64_t N, int64_t ldb, bool sym_quant_act>
 void _dequant_gemm_accum(
     float* C,
@@ -455,11 +336,6 @@ void _dequant_gemm_accum(
     _dequant_and_store<true, N, sym_quant_act>(
         C, C_i32, scales_a, qzeros_a, scales_b, compensation, M, N /*ldi*/, ldc, 1 /*ldsa*/);
   } else
-#elif defined(CPU_CAPABILITY_RVV)
-  if constexpr (!sym_quant_act && N == BLOCK_N && ldb == BLOCK_N / 2) {
-    _dequant_gemm_accum_rvv<N, ldb>(C, A, scales_a, qzeros_a, B, scales_b, qzeros_b, compensation, M, K, lda, ldc);
-    return;
-  } else
 #endif
   {
     for (int64_t m = 0; m < M; ++m) {

csrc/cpu/sgl-kernels/vec.h

@@ -9,19 +9,11 @@
 #define CPU_CAPABILITY_AVX512
 #endif
 
-#if defined(__riscv_v_min_vlen) && (__riscv_v_min_vlen == 128 || __riscv_v_min_vlen == 256)
-#define CPU_CAPABILITY_RVV
-#endif
-
 #include <ATen/cpu/vec/functional.h>
 #include <ATen/cpu/vec/vec.h>
 #if defined(CPU_CAPABILITY_AVX512)
 #include <immintrin.h>
 #endif
-
-#if defined(CPU_CAPABILITY_RVV)
-#include "../cpu_types_riscv_defs.hpp"
-#endif
 namespace {
 
 using namespace at::vec;

csrc/cpu/spec_decode_utils.cpp

@@ -208,89 +208,6 @@ void copy_and_expand_eagle_inputs_kernel_impl(
   }
 }
 
-void copy_and_expand_dflash_inputs_kernel_impl(
-    const torch::Tensor& next_token_ids, const torch::Tensor& target_positions,
-    torch::Tensor& out_input_ids, torch::Tensor& out_context_positions,
-    torch::Tensor& out_query_positions, torch::Tensor& out_context_slot_mapping,
-    torch::Tensor& out_query_slot_mapping, torch::Tensor& out_token_indices,
-    const torch::Tensor& block_table, const torch::Tensor& query_start_loc,
-    const std::optional<torch::Tensor>& num_rejected_tokens,
-    const int64_t parallel_drafting_token_id, const int64_t block_size,
-    const int64_t num_query_per_req, const int64_t num_speculative_tokens,
-    const int64_t total_input_tokens, const bool has_num_rejected) {
-  const int64_t num_reqs = query_start_loc.size(0) - 1;
-
-  const int64_t* next_ids_ptr = next_token_ids.data_ptr<int64_t>();
-  const int64_t* target_pos_ptr = target_positions.data_ptr<int64_t>();
-  const int32_t* block_table_ptr = block_table.data_ptr<int32_t>();
-  const int32_t* query_start_ptr = query_start_loc.data_ptr<int32_t>();
-  const int64_t* rejected_ptr =
-      has_num_rejected && num_rejected_tokens.has_value()
-          ? num_rejected_tokens.value().data_ptr<int64_t>()
-          : nullptr;
-
-  int64_t* out_ids_ptr = out_input_ids.data_ptr<int64_t>();
-  int64_t* out_ctx_pos_ptr = out_context_positions.data_ptr<int64_t>();
-  int64_t* out_query_pos_ptr = out_query_positions.data_ptr<int64_t>();
-  int64_t* out_ctx_slot_ptr = out_context_slot_mapping.data_ptr<int64_t>();
-  int64_t* out_query_slot_ptr = out_query_slot_mapping.data_ptr<int64_t>();
-  int32_t* out_token_idx_ptr = out_token_indices.data_ptr<int32_t>();
-
-  const int64_t block_table_stride = block_table.stride(0);
-
-#pragma omp parallel for
-  for (int64_t req_idx = 0; req_idx < num_reqs; ++req_idx) {
-    int32_t ctx_start = query_start_ptr[req_idx];
-    int32_t ctx_end = query_start_ptr[req_idx + 1];
-    int64_t num_ctx = ctx_end - ctx_start;
-    int64_t valid_ctx_end = ctx_end;
-    if (rejected_ptr != nullptr) {
-      valid_ctx_end -= rejected_ptr[req_idx];
-    }
-    // Guard against out-of-bounds: ensure valid_ctx_end > ctx_start so that
-    // valid_ctx_end - 1 never reads before the request's context range.
-    valid_ctx_end =
-        std::max(valid_ctx_end, static_cast<int64_t>(ctx_start + 1));
-
-    int64_t last_pos = target_pos_ptr[valid_ctx_end - 1];
-
-    for (int64_t j = 0; j < num_ctx; ++j) {
-      int64_t ctx_idx = ctx_start + j;
-      int64_t ctx_pos_idx = std::min(ctx_idx, total_input_tokens - 1);
-      int64_t position = target_pos_ptr[ctx_pos_idx];
-      int64_t block_num = position / block_size;
-      block_num = std::min(block_num, block_table_stride - 1);
-      int32_t block_id =
-          block_table_ptr[req_idx * block_table_stride + block_num];
-      int64_t slot = block_id * block_size + (position % block_size);
-
-      out_ctx_pos_ptr[ctx_idx] = position;
-      out_ctx_slot_ptr[ctx_idx] = slot;
-    }
-
-    for (int64_t query_off = 0; query_off < num_query_per_req; ++query_off) {
-      int64_t query_out = req_idx * num_query_per_req + query_off;
-      int64_t position = last_pos + 1 + query_off;
-      int64_t block_num = position / block_size;
-      block_num = std::min(block_num, block_table_stride - 1);
-      int32_t block_id =
-          block_table_ptr[req_idx * block_table_stride + block_num];
-      int64_t slot = block_id * block_size + (position % block_size);
-
-      out_query_pos_ptr[query_out] = position;
-      out_query_slot_ptr[query_out] = slot;
-      out_ids_ptr[query_out] =
-          query_off == 0 ? next_ids_ptr[req_idx] : parallel_drafting_token_id;
-
-      if (query_off > 0) {
-        int64_t sample_out_idx =
-            req_idx * num_speculative_tokens + (query_off - 1);
-        out_token_idx_ptr[sample_out_idx] = query_out;
-      }
-    }
-  }
-}
-
 void rejection_greedy_sample_kernel_impl(
     torch::Tensor& output_token_ids, const torch::Tensor& cu_num_draft_tokens,
     const torch::Tensor& draft_token_ids, const torch::Tensor& target_argmax,

csrc/cpu/torch_bindings.cpp

@@ -237,16 +237,6 @@ void copy_and_expand_eagle_inputs_kernel_impl(
     const int64_t padding_token_id, const int64_t parallel_drafting_token_id,
     const int64_t total_input_tokens,
     const int64_t num_padding_slots_per_request, const bool shift_input_ids);
-void copy_and_expand_dflash_inputs_kernel_impl(
-    const torch::Tensor& next_token_ids, const torch::Tensor& target_positions,
-    torch::Tensor& out_input_ids, torch::Tensor& out_context_positions,
-    torch::Tensor& out_query_positions, torch::Tensor& out_context_slot_mapping,
-    torch::Tensor& out_query_slot_mapping, torch::Tensor& out_token_indices,
-    const torch::Tensor& block_table, const torch::Tensor& query_start_loc,
-    const std::optional<torch::Tensor>& num_rejected_tokens,
-    const int64_t parallel_drafting_token_id, const int64_t block_size,
-    const int64_t num_query_per_req, const int64_t num_speculative_tokens,
-    const int64_t total_input_tokens, const bool has_num_rejected);
 void rejection_greedy_sample_kernel_impl(
     torch::Tensor& output_token_ids, const torch::Tensor& cu_num_draft_tokens,
     const torch::Tensor& draft_token_ids, const torch::Tensor& target_argmax,
@@ -278,8 +268,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
   ops.def(
       "dynamic_4bit_int_moe("
       "Tensor x, Tensor topk_ids, Tensor topk_weights,"
-      "Tensor w13_packed, Tensor w2_packed,"
-      "int hidden_size, int intermediate_size,"
+      "Tensor w13_packed, Tensor w2_packed, int H, int I, int I2,"
       "int group_size, bool apply_router_weight_on_input, int activation_kind"
       ") -> Tensor");
 
@@ -299,10 +288,6 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
   ops.def("gelu_tanh_and_mul(Tensor! out, Tensor input) -> ()");
   ops.impl("gelu_tanh_and_mul", torch::kCPU, &gelu_tanh_and_mul);
 
-  // GELU tanh implementation.
-  ops.def("gelu_tanh(Tensor! out, Tensor input) -> ()");
-  ops.impl("gelu_tanh", torch::kCPU, &gelu_tanh);
-
   // GELU implementation used in GPT-2.
   ops.def("gelu_new(Tensor! out, Tensor input) -> ()");
   ops.impl("gelu_new", torch::kCPU, &gelu_new);
@@ -553,7 +538,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
 #endif
 
   // fused moe
-#if defined(__AVX512F__) || (defined(ARM_BF16_SUPPORT))
+#if defined(__AVX512F__)
   ops.def(
       "prepack_moe_weight(Tensor weight, Tensor(a1!) packed_weight, str isa) "
       "-> ()");
@@ -564,7 +549,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
       "bool skip_weighted, "
       "str act, str isa) -> ()");
   ops.impl("cpu_fused_moe", torch::kCPU, &cpu_fused_moe);
-#endif  // #if defined(__AVX512F__) || (defined(ARM_BF16_SUPPORT))
+#endif
   ops.def(
       "mla_decode_kvcache("
       "   Tensor! out, Tensor query, Tensor kv_cache,"
@@ -614,19 +599,6 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
       "SymInt total_input_tokens, SymInt num_padding_slots_per_request, "
       "bool shift_input_ids) -> ()",
       &cpu_utils::copy_and_expand_eagle_inputs_kernel_impl);
-  ops.def(
-      "copy_and_expand_dflash_inputs_kernel_impl("
-      "Tensor next_token_ids, Tensor target_positions, "
-      "Tensor(a2!) out_input_ids, Tensor(a3!) out_context_positions, "
-      "Tensor(a4!) out_query_positions, "
-      "Tensor(a5!) out_context_slot_mapping, "
-      "Tensor(a6!) out_query_slot_mapping, "
-      "Tensor(a7!) out_token_indices, Tensor block_table, "
-      "Tensor query_start_loc, Tensor? num_rejected_tokens, "
-      "SymInt parallel_drafting_token_id, SymInt block_size, "
-      "SymInt num_query_per_req, SymInt num_speculative_tokens, "
-      "SymInt total_input_tokens, bool has_num_rejected) -> ()",
-      &cpu_utils::copy_and_expand_dflash_inputs_kernel_impl);
   ops.def(
       "rejection_greedy_sample_kernel_impl("
       "Tensor(a0!) output_token_ids, Tensor cu_num_draft_tokens, "

csrc/cpu/utils.hpp

@@ -2,14 +2,13 @@
 #define UTILS_HPP
 
 #include <atomic>
-#include <string>
 #include <unistd.h>
 #include <ATen/cpu/Utils.h>
 
 #include "cpu/cpu_types.hpp"
 
 namespace cpu_utils {
-enum class ISA { AMX, VEC, RVV, NEON };
+enum class ISA { AMX, VEC, RVV };
 
 inline ISA get_isa(const std::string& isa) {
   if (isa == "amx") {
@@ -18,8 +17,6 @@ inline ISA get_isa(const std::string& isa) {
     return ISA::VEC;
   } else if (isa == "rvv") {
     return ISA::RVV;
-  } else if (isa == "neon") {
-    return ISA::NEON;
   } else {
     TORCH_CHECK(false, "Invalid isa type: " + isa);
   }

csrc/cuda_view.cu

@@ -0,0 +1,60 @@
+// TODO: Remove this once ROCm upgrade to torch 2.11.
+#include <torch/all.h>
+#include <torch/cuda.h>
+#include <cuda_runtime.h>
+
+// This function assumes that `cpu_tensor` is a CPU tensor,
+// and that UVA (Unified Virtual Addressing) is enabled.
+torch::Tensor get_cuda_view_from_cpu_tensor(torch::Tensor& cpu_tensor) {
+  TORCH_CHECK(cpu_tensor.device().is_cpu(), "Input tensor must be on CPU");
+
+  // handle empty tensor
+  if (cpu_tensor.numel() == 0) {
+    return torch::empty(cpu_tensor.sizes(),
+                        cpu_tensor.options().device(torch::kCUDA));
+  }
+
+  if (cpu_tensor.is_pinned()) {
+    // If CPU tensor is pinned, directly get the device pointer.
+    void* host_ptr = const_cast<void*>(cpu_tensor.data_ptr());
+    void* device_ptr = nullptr;
+    cudaError_t err = cudaHostGetDevicePointer(&device_ptr, host_ptr, 0);
+    TORCH_CHECK(err == cudaSuccess,
+                "cudaHostGetDevicePointer failed: ", cudaGetErrorString(err));
+
+    return torch::from_blob(
+        device_ptr, cpu_tensor.sizes(), cpu_tensor.strides(),
+        [base = cpu_tensor](void*) {},  // keep cpu tensor alive
+        cpu_tensor.options().device(torch::kCUDA));
+  }
+
+  // If CPU tensor is not pinned, allocate a new pinned memory buffer.
+  torch::Tensor contiguous_cpu = cpu_tensor.contiguous();
+  size_t nbytes = contiguous_cpu.nbytes();
+
+  void* host_ptr = nullptr;
+  cudaError_t err = cudaHostAlloc(&host_ptr, nbytes, cudaHostAllocMapped);
+  if (err != cudaSuccess) {
+    AT_ERROR("cudaHostAlloc failed: ", cudaGetErrorString(err));
+  }
+
+  err = cudaMemcpy(host_ptr, contiguous_cpu.data_ptr(), nbytes,
+                   cudaMemcpyDefault);
+  if (err != cudaSuccess) {
+    cudaFreeHost(host_ptr);
+    AT_ERROR("cudaMemcpy failed: ", cudaGetErrorString(err));
+  }
+
+  void* device_ptr = nullptr;
+  err = cudaHostGetDevicePointer(&device_ptr, host_ptr, 0);
+  if (err != cudaSuccess) {
+    cudaFreeHost(host_ptr);
+    AT_ERROR("cudaHostGetDevicePointer failed: ", cudaGetErrorString(err));
+  }
+
+  auto deleter = [host_ptr](void*) { cudaFreeHost(host_ptr); };
+
+  return torch::from_blob(device_ptr, contiguous_cpu.sizes(),
+                          contiguous_cpu.strides(), deleter,
+                          contiguous_cpu.options().device(torch::kCUDA));
+}

csrc/cumem_allocator.cpp

@@ -48,8 +48,8 @@ static inline unsigned long long my_min(unsigned long long a,
 }
 
 static CUresult reserve_rocm_address(CUdeviceptr* d_mem, size_t size,
-                                     size_t alignment, CUdeviceptr addr = 0) {
-  CUresult status = cuMemAddressReserve(d_mem, size, alignment, addr, 0);
+                                     size_t alignment) {
+  CUresult status = cuMemAddressReserve(d_mem, size, alignment, 0, 0);
   if (status == CUresult(0) || alignment == 0) {
     return status;
   }
@@ -58,7 +58,7 @@ static CUresult reserve_rocm_address(CUdeviceptr* d_mem, size_t size,
   // alignment even when physical VRAM is free. Let HIP choose the default
   // alignment, then verify that the returned address still satisfies the
   // requested alignment before accepting it.
-  status = cuMemAddressReserve(d_mem, size, 0, addr, 0);
+  status = cuMemAddressReserve(d_mem, size, 0, 0, 0);
   if (status != CUresult(0)) {
     return status;
   }
@@ -535,14 +535,7 @@ void my_free(void* ptr, ssize_t size, int device, CUstream stream) {
   Py_DECREF(py_result);
   PyGILState_Release(gstate);
 
-  // An empty chunk list means this allocation is asleep: its physical chunks
-  // were already unmapped and released by sleep(), but the virtual address is
-  // still held as a placeholder reservation. Skip unmap/release (freeing the
-  // placeholder address happens below).
-  if (num_chunks > 0) {
-    unmap_and_release(device, size, d_mem, p_memHandle, chunk_sizes,
-                      num_chunks);
-  }
+  unmap_and_release(device, size, d_mem, p_memHandle, chunk_sizes, num_chunks);
 #else
   // Non-ROCm path: simple integer handle already extracted; drop temporary
   // Python refs while still holding the GIL, then release it.
@@ -555,13 +548,11 @@ void my_free(void* ptr, ssize_t size, int device, CUstream stream) {
   unmap_and_release(device, size, d_mem, p_memHandle);
 #endif
 
-  // Free the virtual address. On ROCm this also covers an asleep allocation,
-  // whose placeholder reservation made by sleep() is still held here.
+  // free address and the handle
   CUDA_CHECK(cuMemAddressFree(d_mem, size));
 #ifndef USE_ROCM
   free(p_memHandle);
 #else
-  // Only awake allocations have per-chunk handles to free.
   for (auto i = 0; i < num_chunks; ++i) {
     free(p_memHandle[i]);
   }
@@ -681,29 +672,6 @@ static PyObject* python_unmap_and_release(PyObject* self, PyObject* args) {
   unmap_and_release(recv_device, recv_size, d_mem_ptr, p_memHandle, chunk_sizes,
                     num_chunks);
 
-  // On ROCm/Linux, physical VRAM is only reclaimed once the virtual address
-  // range is freed; hipMemUnmap + hipMemRelease alone leave the memory
-  // resident (see ROCm#6021). Free the address to release physical memory,
-  // then immediately re-reserve the SAME address as an empty placeholder so
-  // the regular allocator cannot hand it out while we sleep. wake_up remaps
-  // physical chunks into this placeholder.
-  if (error_code == no_error) {
-    CUDA_CHECK(cuMemAddressFree(d_mem_ptr, recv_size));
-    if (error_code == no_error) {
-      CUdeviceptr reserved = 0;
-      CUDA_CHECK(reserve_rocm_address(&reserved, recv_size, /*alignment=*/0,
-                                      d_mem_ptr));
-      if (error_code == no_error && reserved != d_mem_ptr) {
-        (void)cuMemAddressFree(reserved, recv_size);
-        snprintf(error_msg, sizeof(error_msg),
-                 "failed to re-reserve placeholder address on sleep "
-                 "(requested %#llx, got %#llx)",
-                 (unsigned long long)d_mem_ptr, (unsigned long long)reserved);
-        error_code = CUresult(1);
-      }
-    }
-  }
-
   free(p_memHandle);
   free(chunk_sizes);
 #endif
@@ -768,7 +736,6 @@ static PyObject* python_create_and_map(PyObject* self, PyObject* args) {
     chunk_sizes[i] = PyLong_AsUnsignedLongLong(size_py);
   }
 
-  // Address already reserved as a placeholder by sleep(); just remap chunks.
   create_and_map(recv_device, recv_size, d_mem_ptr, p_memHandle, chunk_sizes,
                  num_chunks);
 

csrc/custom_all_reduce_test.cu

@@ -0,0 +1,361 @@
+/**
+ * This is a standalone test for custom allreduce.
+ * To compile, make sure you have MPI and NCCL installed in your system.
+ * export MPI_HOME=XXX
+ * nvcc -O2 -arch=native -std=c++17 custom_all_reduce_test.cu -o
+ * custom_all_reduce_test -lnccl -I${MPI_HOME}/include -lmpi
+ *
+ * Warning: this C++ test is not designed to be very readable and was used
+ * during the rapid prototyping process.
+ *
+ * To run:
+ * mpirun --allow-run-as-root -np 8 ./custom_all_reduce_test
+ */
+#include <cuda.h>
+#include <curand_kernel.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <limits>
+#include <vector>
+
+#include "cuda_profiler_api.h"
+#include "custom_all_reduce.cuh"
+#include "mpi.h"
+#ifdef USE_ROCM
+  #include <hip/hip_bf16.h>
+typedef __hip_bfloat16 nv_bfloat16;
+  #include "rccl/rccl.h"
+  #include "custom_all_reduce_hip.cuh"
+#else
+  #include "nccl.h"
+  #include "custom_all_reduce.cuh"
+#endif
+
+#define MPICHECK(cmd)                                                  \
+  do {                                                                 \
+    int e = cmd;                                                       \
+    if (e != MPI_SUCCESS) {                                            \
+      printf("Failed: MPI error %s:%d '%d'\n", __FILE__, __LINE__, e); \
+      exit(EXIT_FAILURE);                                              \
+    }                                                                  \
+  } while (0)
+
+#define NCCLCHECK(cmd)                                              \
+  do {                                                              \
+    ncclResult_t r = cmd;                                           \
+    if (r != ncclSuccess) {                                         \
+      printf("Failed, NCCL error %s:%d '%s'\n", __FILE__, __LINE__, \
+             ncclGetErrorString(r));                                \
+      exit(EXIT_FAILURE);                                           \
+    }                                                               \
+  } while (0)
+
+#ifdef USE_ROCM
+__global__ void dummy_kernel() {
+  for (int i = 0; i < 100; i++) {
+    uint64_t start = wall_clock64();
+    uint64_t cycles_elapsed;
+    do {
+      cycles_elapsed = wall_clock64() - start;
+    } while (cycles_elapsed < 100);
+  }
+  for (int i = 0; i < 100; i++) __nanosleep(1000000);  // 100ms
+}
+#else
+__global__ void dummy_kernel() {
+  #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 700
+  for (int i = 0; i < 100; i++) __nanosleep(1000000);  // 100ms
+  #else
+  for (int i = 0; i < 100; i++) {
+    long long int start = clock64();
+    while (clock64() - start < 150000000);  // approximately 98.4ms on P40
+  }
+  #endif
+}
+#endif
+
+template <typename T>
+__global__ void set_data(T* data, int size, int myRank) {
+  for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < size;
+       idx += gridDim.x * blockDim.x) {
+    data[idx] = myRank * 0.11f;
+  }
+}
+
+template <typename T>
+__global__ void convert_data(const T* data1, const T* data2, double* fdata1,
+                             double* fdata2, int size) {
+  for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < size;
+       idx += gridDim.x * blockDim.x) {
+    fdata1[idx] = data1[idx];
+    fdata2[idx] = data2[idx];
+  }
+}
+
+__global__ void init_rand(curandState_t* state, int size, int nRanks) {
+  for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < size;
+       idx += gridDim.x * blockDim.x) {
+    for (int i = 0; i < nRanks; i++) {
+      curand_init(i + 1, idx, 0, &state[idx * nRanks + i]);
+    }
+  }
+}
+
+template <typename T>
+__global__ void gen_data(curandState_t* state, T* data, double* ground_truth,
+                         int myRank, int nRanks, int size) {
+  for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < size;
+       idx += gridDim.x * blockDim.x) {
+    double sum = 0.0;
+    for (int i = 0; i < nRanks; i++) {
+      double val = curand_uniform_double(&state[idx * nRanks + i]) * 4;
+      T hval = val;  // downcast first
+      sum += static_cast<double>(hval);
+      if (i == myRank) data[idx] = hval;
+    }
+    ground_truth[idx] = sum;
+  }
+}
+
+template <typename T>
+void run(int myRank, int nRanks, ncclComm_t& comm, int threads, int block_limit,
+         int data_size, bool performance_test) {
+  T* result;
+  cudaStream_t stream;
+  CUDACHECK(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking));
+  CUDACHECK(cudaMalloc(&result, data_size * sizeof(T)));
+  CUDACHECK(cudaMemset(result, 0, data_size * sizeof(T)));
+
+  cudaIpcMemHandle_t self_data_handle;
+  cudaIpcMemHandle_t data_handles[8];
+  vllm::Signal* buffer;
+  T* self_data_copy;
+  /**
+   * Allocate IPC buffer
+   *
+   * The first section is a temporary buffer for storing intermediate allreduce
+   * results, if a particular algorithm requires it. The second section is for
+   * the input to the allreduce. The actual API takes the input pointer as an
+   * argument (that is, they can and usually should be allocated separately).
+   * But since the input pointers and the temporary buffer all require IPC
+   * registration, they are allocated and registered together in the test for
+   * convenience.
+   */
+#ifdef USE_ROCM
+  CUDACHECK(hipExtMallocWithFlags(
+      (void**)&buffer, 2 * data_size * sizeof(T) + sizeof(vllm::Signal),
+      hipDeviceMallocUncached));
+#else
+  CUDACHECK(
+      cudaMalloc(&buffer, 2 * data_size * sizeof(T) + sizeof(vllm::Signal)));
+#endif
+  CUDACHECK(
+      cudaMemset(buffer, 0, 2 * data_size * sizeof(T) + sizeof(vllm::Signal)));
+  CUDACHECK(cudaMalloc(&self_data_copy, data_size * sizeof(T)));
+  CUDACHECK(cudaIpcGetMemHandle(&self_data_handle, buffer));
+
+  MPICHECK(MPI_Allgather(&self_data_handle, sizeof(cudaIpcMemHandle_t),
+                         MPI_BYTE, data_handles, sizeof(cudaIpcMemHandle_t),
+                         MPI_BYTE, MPI_COMM_WORLD));
+
+  void* rank_data;
+  size_t rank_data_sz = 16 * 1024 * 1024;
+  CUDACHECK(cudaMalloc(&rank_data, rank_data_sz));
+  vllm::Signal* ipc_ptrs[8];
+  for (int i = 0; i < nRanks; i++) {
+    if (i == myRank)
+      ipc_ptrs[i] = buffer;
+    else
+      CUDACHECK(cudaIpcOpenMemHandle((void**)&ipc_ptrs[i], data_handles[i],
+                                     cudaIpcMemLazyEnablePeerAccess));
+  }
+  vllm::CustomAllreduce fa(ipc_ptrs, rank_data, rank_data_sz, myRank, nRanks);
+  auto* self_data =
+      reinterpret_cast<T*>(reinterpret_cast<char*>(buffer) +
+                           sizeof(vllm::Signal) + data_size * sizeof(T));
+  // hack buffer registration
+  {
+    void* data[8];
+    for (int i = 0; i < nRanks; i++) {
+      data[i] =
+          ((char*)ipc_ptrs[i]) + sizeof(vllm::Signal) + data_size * sizeof(T);
+    }
+    fa.register_buffer(data);
+  }
+
+  double* ground_truth;
+  CUDACHECK(cudaMallocHost(&ground_truth, data_size * sizeof(double)));
+  curandState_t* states;
+  CUDACHECK(cudaMalloc(&states, sizeof(curandState_t) * nRanks * data_size));
+  init_rand<<<108, 1024, 0, stream>>>(states, data_size, nRanks);
+  gen_data<T><<<108, 1024, 0, stream>>>(states, self_data, ground_truth, myRank,
+                                        nRanks, data_size);
+  CUDACHECK(cudaMemcpyAsync(self_data_copy, self_data, data_size * sizeof(T),
+                            cudaMemcpyDeviceToDevice, stream));
+  cudaEvent_t start, stop;
+  CUDACHECK(cudaEventCreate(&start));
+  CUDACHECK(cudaEventCreate(&stop));
+
+  ncclDataType_t ncclDtype;
+  if (std::is_same<T, half>::value) {
+    ncclDtype = ncclFloat16;
+  } else if (std::is_same<T, nv_bfloat16>::value) {
+    ncclDtype = ncclBfloat16;
+  } else {
+    ncclDtype = ncclFloat;
+  }
+  double *nccl_result, *my_result;
+  CUDACHECK(cudaMallocHost(&nccl_result, data_size * sizeof(double)));
+  CUDACHECK(cudaMallocHost(&my_result, data_size * sizeof(double)));
+  if (performance_test) {
+    dummy_kernel<<<1, 1, 0, stream>>>();
+    constexpr int warmup_iters = 5;
+    constexpr int num_iters = 100;
+    // warmup
+    for (int i = 0; i < warmup_iters; i++) {
+      NCCLCHECK(ncclAllReduce(result, result, data_size, ncclDtype, ncclSum,
+                              comm, stream));
+    }
+    CUDACHECK(cudaEventRecord(start, stream));
+    for (int i = 0; i < num_iters; i++) {
+      NCCLCHECK(ncclAllReduce(result, result, data_size, ncclDtype, ncclSum,
+                              comm, stream));
+    }
+    CUDACHECK(cudaEventRecord(stop, stream));
+    CUDACHECK(cudaStreamSynchronize(stream));
+    float allreduce_ms = 0;
+    cudaEventElapsedTime(&allreduce_ms, start, stop);
+
+    dummy_kernel<<<1, 1, 0, stream>>>();
+    // warm up
+    for (int i = 0; i < warmup_iters; i++) {
+      fa.allreduce<T>(stream, self_data, result, data_size, threads,
+                      block_limit);
+    }
+    CUDACHECK(cudaEventRecord(start, stream));
+    for (int i = 0; i < num_iters; i++) {
+      fa.allreduce<T>(stream, self_data, result, data_size, threads,
+                      block_limit);
+    }
+    CUDACHECK(cudaEventRecord(stop, stream));
+    CUDACHECK(cudaStreamSynchronize(stream));
+
+    float duration_ms = 0;
+    cudaEventElapsedTime(&duration_ms, start, stop);
+    if (myRank == 0)
+      printf(
+          "Rank %d done, nGPUs:%d, sz (kb): %d, %d, %d, my time:%.2fus, nccl "
+          "time:%.2fus\n",
+          myRank, nRanks, data_size * sizeof(T) / 1024, threads, block_limit,
+          duration_ms * 1e3 / num_iters, allreduce_ms * 1e3 / num_iters);
+
+    // And wait for all the queued up work to complete
+    CUDACHECK(cudaStreamSynchronize(stream));
+
+    NCCLCHECK(ncclAllReduce(self_data_copy, self_data, data_size, ncclDtype,
+                            ncclSum, comm, stream));
+
+    convert_data<T><<<108, 1024, 0, stream>>>(self_data, result, nccl_result,
+                                              my_result, data_size);
+    CUDACHECK(cudaStreamSynchronize(stream));
+
+    for (unsigned long j = 0; j < data_size; j++) {
+      auto diff = abs(nccl_result[j] - my_result[j]);
+      if (diff >= 4e-2) {
+        printf("Rank %d: Verification mismatch at %lld: %f != (my) %f, gt=%f\n",
+               myRank, j, nccl_result[j], my_result[j], ground_truth[j]);
+        break;
+      }
+    }
+    long double nccl_diffs = 0.0;
+    long double my_diffs = 0.0;
+    for (int j = 0; j < data_size; j++) {
+      nccl_diffs += abs(nccl_result[j] - ground_truth[j]);
+      my_diffs += abs(my_result[j] - ground_truth[j]);
+    }
+    if (myRank == 0)
+      std::cout << "average abs diffs: nccl: " << nccl_diffs / data_size
+                << " me: " << my_diffs / data_size << std::endl;
+  } else {
+    for (int i = 0; i < 100; i++) {
+      fa.allreduce<T>(stream, self_data, result, data_size, threads,
+                      block_limit);
+      CUDACHECK(cudaStreamSynchronize(stream));
+      NCCLCHECK(ncclAllReduce(self_data, self_data_copy, data_size, ncclDtype,
+                              ncclSum, comm, stream));
+      convert_data<T><<<108, 1024, 0, stream>>>(
+          self_data_copy, result, nccl_result, my_result, data_size);
+      CUDACHECK(cudaStreamSynchronize(stream));
+
+      for (unsigned long j = 0; j < data_size; j++) {
+        auto diff = abs(nccl_result[j] - my_result[j]);
+        if (diff >= 4e-2) {
+          printf(
+              "Rank %d: Verification mismatch at %lld: %f != (my) %f, gt=%f\n",
+              myRank, j, nccl_result[j], my_result[j], ground_truth[j]);
+          break;
+        }
+      }
+    }
+    if (myRank == 0)
+      printf("Test passed: nGPUs:%d, sz (kb): %d, %d, %d\n", nRanks,
+             data_size * sizeof(T) / 1024, threads, block_limit);
+    // long double nccl_diffs = 0.0;
+    // long double my_diffs = 0.0;
+    // for (int j = 0; j < data_size; j++) {
+    //   nccl_diffs += abs(nccl_result[j] - ground_truth[j]);
+    //   my_diffs += abs(my_result[j] - ground_truth[j]);
+    // }
+    // if (myRank == 0)
+    //   std::cout << "average abs diffs: nccl: " << nccl_diffs / data_size
+    //             << " me: " << my_diffs / data_size << std::endl;
+  }
+
+  CUDACHECK(cudaFree(result));
+  CUDACHECK(cudaFree(self_data_copy));
+  CUDACHECK(cudaFree(rank_data));
+  CUDACHECK(cudaFree(buffer));
+  CUDACHECK(cudaFree(states));
+  CUDACHECK(cudaFreeHost(ground_truth));
+  CUDACHECK(cudaFreeHost(nccl_result));
+  CUDACHECK(cudaFreeHost(my_result));
+  CUDACHECK(cudaStreamDestroy(stream));
+}
+
+int main(int argc, char** argv) {
+  int nRanks, myRank;
+  MPICHECK(MPI_Init(&argc, &argv));
+  MPICHECK(MPI_Comm_rank(MPI_COMM_WORLD, &myRank));
+  MPICHECK(MPI_Comm_size(MPI_COMM_WORLD, &nRanks));
+  CUDACHECK(cudaSetDevice(myRank));
+  ncclUniqueId id;
+  ncclComm_t comm;
+  if (myRank == 0) ncclGetUniqueId(&id);
+  MPICHECK(MPI_Bcast(static_cast<void*>(&id), sizeof(id), MPI_BYTE, 0,
+                     MPI_COMM_WORLD));
+  NCCLCHECK(ncclCommInitRank(&comm, nRanks, id, myRank));
+
+  bool performance_test = true;
+  cudaProfilerStart();
+// Uncomment to scan through different block size configs.
+// for (int threads : {256, 512, 1024}) {
+//   for (int block_limit = 16; block_limit < 112; block_limit += 4) {
+//     run<half>(myRank, nRanks, comm, threads, block_limit, 1024 * 1024,
+//     performance_test);
+//   }
+// }
+#ifdef USE_ROCM
+  const int block_limit = 16;
+#else
+  const int block_limit = 36;
+#endif
+  // Scan through different sizes to test performance.
+  for (int sz = 512; sz <= (8 << 20); sz *= 2) {
+    run<half>(myRank, nRanks, comm, 512, 36, sz + 8 * 47, performance_test);
+  }
+
+  cudaProfilerStop();
+  MPICHECK(MPI_Finalize());
+  return EXIT_SUCCESS;
+}

csrc/custom_quickreduce.cu

@@ -97,28 +97,18 @@ int64_t qr_max_size() {
                                                   cast_bf2half>;  \
     template struct quickreduce::AllReduceTwoshot<T, Codec<T, 8>, cast_bf2half>;
 
-  // INT3 (CodecQ3) is restricted to TP2 only, so we only instantiate the
-  // world_size == 2 kernel for it.
-  #define INSTANTIATE_FOR_WORLDSIZE_TP2_ONLY(T, Codec, cast_bf2half) \
-    template struct quickreduce::AllReduceTwoshot<T, Codec<T, 2>, cast_bf2half>;
-
 INSTANTIATE_FOR_WORLDSIZE(quickreduce::nv_bfloat16, quickreduce::CodecFP, false)
 INSTANTIATE_FOR_WORLDSIZE(quickreduce::nv_bfloat16, quickreduce::CodecQ4, false)
 INSTANTIATE_FOR_WORLDSIZE(quickreduce::nv_bfloat16, quickreduce::CodecQ6, false)
 INSTANTIATE_FOR_WORLDSIZE(quickreduce::nv_bfloat16, quickreduce::CodecQ8, false)
-INSTANTIATE_FOR_WORLDSIZE_TP2_ONLY(quickreduce::nv_bfloat16,
-                                   quickreduce::CodecQ3, false)
 INSTANTIATE_FOR_WORLDSIZE(quickreduce::nv_bfloat16, quickreduce::CodecFP, true)
 INSTANTIATE_FOR_WORLDSIZE(quickreduce::nv_bfloat16, quickreduce::CodecQ4, true)
 INSTANTIATE_FOR_WORLDSIZE(quickreduce::nv_bfloat16, quickreduce::CodecQ6, true)
 INSTANTIATE_FOR_WORLDSIZE(quickreduce::nv_bfloat16, quickreduce::CodecQ8, true)
-INSTANTIATE_FOR_WORLDSIZE_TP2_ONLY(quickreduce::nv_bfloat16,
-                                   quickreduce::CodecQ3, true)
 
 INSTANTIATE_FOR_WORLDSIZE(half, quickreduce::CodecFP, false)
 INSTANTIATE_FOR_WORLDSIZE(half, quickreduce::CodecQ4, false)
 INSTANTIATE_FOR_WORLDSIZE(half, quickreduce::CodecQ6, false)
 INSTANTIATE_FOR_WORLDSIZE(half, quickreduce::CodecQ8, false)
-INSTANTIATE_FOR_WORLDSIZE_TP2_ONLY(half, quickreduce::CodecQ3, false)
 
 #endif  // USE_ROCM