[Bugfix][Frontend] Cap fastapi < 0.137 to avoid prometheus-fastapi-instrumentator crash on serve startup

A fresh pip install pulls fastapi 0.137, whose new _IncludedRouter route type
(a BaseRoute with no .path) breaks prometheus-fastapi-instrumentator's route
iteration -> 'vllm serve' dies at startup with:
  AttributeError: '_IncludedRouter' object has no attribute 'path'

requirements/common.txt pinned fastapi[standard]>=0.115.0 with no upper bound.
Cap to <0.137; lift once the instrumentator handles _IncludedRouter
(https://github.com/trallnag/prometheus-fastapi-instrumentator/issues/370).

Signed-off-by: avesed <trevor.lijunlin@gmail.com>

.buildkite/ci_config_intel.yaml

@@ -2,16 +2,17 @@ name: vllm_intel_ci
 job_dirs:
   - ".buildkite/intel_jobs"
 run_all_patterns:
-  - ".buildkite/ci_config_intel.yaml"
   - "docker/Dockerfile"
-  - "docker/Dockerfile.xpu"
   - "CMakeLists.txt"
   - "requirements/common.txt"
   - "requirements/xpu.txt"
+  - "requirements/build/cuda.txt"
+  - "requirements/test/cuda.txt"
   - "setup.py"
   - "csrc/"
   - "cmake/"
 run_all_exclude_patterns:
+  - "docker/Dockerfile."
   - "csrc/cpu/"
   - "csrc/rocm/"
   - "cmake/hipify.py"

.buildkite/hardware_tests/amd.yaml

@@ -6,7 +6,6 @@ steps:
   # differ ci_base is rebuilt and pushed automatically.
   - label: "AMD: :docker: ensure ci_base"
     key: ensure-ci-base-amd
-    soft_fail: false
     depends_on: []
     device: amd_cpu
     no_plugin: true
@@ -27,7 +26,6 @@ steps:
 
   - label: "AMD: :docker: build test image and artifacts"
     key: image-build-amd
-    soft_fail: false
     depends_on:
       - ensure-ci-base-amd
     device: amd_cpu

.buildkite/hardware_tests/cpu.yaml

@@ -53,7 +53,7 @@ steps:
   - tests/models/language/pooling/
   commands:
     - |
-      bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 50m "
+      bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 40m "
       pytest -x -v -s tests/models/language/generation -m cpu_model
       pytest -x -v -s tests/models/language/pooling -m cpu_model"
 
@@ -68,15 +68,13 @@ steps:
   - vllm/v1/sample/ops/topk_topp_triton.py
   - vllm/v1/sample/ops/topk_topp_sampler.py
   - tests/v1/sample/test_topk_topp_sampler.py
-  - tests/v1/e2e/test_cpu_linear_attn_chunked_prefix.py
   commands:
     - |
       bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 45m "
       uv pip install git+https://github.com/triton-lang/triton-cpu.git@270e696d
       VLLM_USE_V2_MODEL_RUNNER=1 pytest -x -v -s tests/models/language/generation/test_granite.py -m cpu_model
       # TODO: move to CPU-Kernel Tests once triton-cpu has a pre-built wheel
-      pytest -x -v -s tests/v1/sample/test_topk_topp_sampler.py::TestTritonTopkTopp
+      pytest -x -v -s tests/v1/sample/test_topk_topp_sampler.py::TestTritonTopkTopp"
-      pytest -x -v -s tests/v1/e2e/test_cpu_linear_attn_chunked_prefix.py"
 
 - label: CPU-Quantization Model Tests
   depends_on: []
@@ -91,13 +89,11 @@ steps:
   - vllm/model_executor/layers/fused_moe/experts/cpu_moe.py
   - tests/quantization/test_compressed_tensors.py
   - tests/quantization/test_cpu_wna16.py
-  - tests/quantization/test_cpu_w8a8.py
   commands:
     - |
-      bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 45m "
+      bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 30m "
       pytest -x -v -s tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_logprobs
-      pytest -x -v -s tests/quantization/test_cpu_wna16.py
+      pytest -x -v -s tests/quantization/test_cpu_wna16.py"
-      pytest -x -v -s tests/quantization/test_cpu_w8a8.py"
       
 - label: CPU-Distributed Tests (PP+TP)
   depends_on: []
@@ -140,7 +136,7 @@ steps:
     - |
       bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 45m "
       pytest -x -v -s tests/models/multimodal/generation --ignore=tests/models/multimodal/generation/test_pixtral.py -m cpu_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
-  parallelism: 4
+  parallelism: 3
 
 - label: "Arm CPU Test"
   depends_on: []

.buildkite/hardware_tests/intel_xpu_ci/test-intel.yaml

@@ -1,80 +0,0 @@
-group: Intel
-steps:
-  - label: ":docker: Build XPU image"
-    soft_fail: true
-    optional: true
-    depends_on: []
-    key: image-build-xpu
-    commands:
-      - bash -lc '.buildkite/image_build/image_build_xpu.sh "public.ecr.aws/q9t5s3a7" "vllm-ci-test-repo" "$BUILDKITE_COMMIT"'
-    env:
-      DOCKER_BUILDKIT: "1"
-    retry:
-      automatic:
-        - exit_status: -1  # Agent was lost
-          limit: 2
-        - exit_status: -10  # Agent was lost
-          limit: 2
-  - label: "XPU example Test"
-    depends_on:
-      - image-build-xpu
-    timeout_in_minutes: 30
-    optional: true
-    device: intel_gpu
-    agent_tags:
-      label: production
-      gpu: 2+
-      mem: 24+
-    no_plugin: true
-    env:
-      REGISTRY: "public.ecr.aws/q9t5s3a7"
-      REPO: "vllm-ci-test-repo"
-    source_file_dependencies:
-      - .buildkite/hardware_tests/intel_xpu_ci/test-intel.yaml
-      - .buildkite/scripts/hardware_ci/run-intel-ci-test.sh
-    commands:
-      - >-
-        bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-        'bash .buildkite/scripts/hardware_ci/run-intel-ci-test.sh example'
-  - label: "XPU V1 test"
-    depends_on:
-      - image-build-xpu
-    timeout_in_minutes: 30
-    optional: true
-    device: intel_gpu
-    agent_tags:
-      label: production
-      gpu: 1+
-      mem: 16+
-    no_plugin: true
-    env:
-      REGISTRY: "public.ecr.aws/q9t5s3a7"
-      REPO: "vllm-ci-test-repo"
-    source_file_dependencies:
-      - .buildkite/hardware_tests/intel_xpu_ci/test-intel.yaml
-      - .buildkite/scripts/hardware_ci/run-intel-ci-test.sh
-    commands:
-      - >-
-        bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-        'bash .buildkite/scripts/hardware_ci/run-intel-ci-test.sh v1'
-  - label: "XPU server test"
-    depends_on:
-      - image-build-xpu
-    timeout_in_minutes: 30
-    optional: true
-    device: intel_gpu
-    agent_tags:
-      label: production
-      gpu: 1+
-      mem: 16+
-    no_plugin: true
-    env:
-      REGISTRY: "public.ecr.aws/q9t5s3a7"
-      REPO: "vllm-ci-test-repo"
-    source_file_dependencies:
-      - .buildkite/hardware_tests/intel_xpu_ci/test-intel.yaml
-      - .buildkite/scripts/hardware_ci/run-intel-ci-test.sh
-    commands:
-      - >-
-        bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-        'bash .buildkite/scripts/hardware_ci/run-intel-ci-test.sh server'

.buildkite/image_build/image_build_arm64.sh

@@ -21,13 +21,12 @@ else
   exit 0
 fi
 
-# build for arm64 GPU targets: Grace/GH200 (sm_90) and DGX Spark/GB10
+# build (Grace/GH200 is the arm64 GPU target; sm_90)
-# (sm_121, family-covered by 12.0 under CUDA 13)
 docker build --file docker/Dockerfile \
   --platform linux/arm64 \
   --build-arg max_jobs=16 \
   --build-arg nvcc_threads=4 \
-  --build-arg torch_cuda_arch_list="9.0 12.0" \
+  --build-arg torch_cuda_arch_list="9.0" \
   --build-arg USE_SCCACHE=1 \
   --build-arg buildkite_commit="$BUILDKITE_COMMIT" \
   --tag "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64 \

.buildkite/intel_jobs/basic_correctness.yaml

@@ -5,10 +5,6 @@ steps:
 - label: XPU Sleep Mode
   timeout_in_minutes: 30
   device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 16+
   no_plugin: true
   working_dir: "."
   env:
@@ -23,5 +19,4 @@ steps:
       bash .buildkite/scripts/hardware_ci/run-intel-test.sh
       'cd tests &&
       export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
-      pytest -v -s basic_correctness/test_cpu_offload.py &&
       pytest -v -s basic_correctness/test_mem.py::test_end_to_end'

.buildkite/intel_jobs/engine_intel.yaml

@@ -5,10 +5,6 @@ steps:
 - label: Engine (1 GPU)
   timeout_in_minutes: 30
   device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 16+
   no_plugin: true
   working_dir: "."
   env:

.buildkite/intel_jobs/expert_parallelism_intel.yaml

@@ -1,27 +0,0 @@
-group: Expert Parallelism
-depends_on:
-  - image-build-xpu
-steps:
-- label: EPLB Algorithm
-  key: eplb-algorithm
-  timeout_in_minutes: 45
-  device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 16+
-  no_plugin: true
-  working_dir: "."
-  env:
-    REGISTRY: "public.ecr.aws/q9t5s3a7"
-    REPO: "vllm-ci-test-repo"
-    VLLM_TEST_DEVICE: "xpu"
-  source_file_dependencies:
-  - vllm/distributed/eplb
-  - tests/distributed/test_eplb_algo.py
-  - tests/distributed/test_eplb_utils.py
-  commands:
-    - >-
-      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-      'cd tests &&
-      pytest -v -s distributed/test_eplb_algo.py'

.buildkite/intel_jobs/kernels_intel.yaml

@@ -5,10 +5,6 @@ steps:
 - label: vLLM IR Tests
   timeout_in_minutes: 30
   device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 16+
   no_plugin: true
   working_dir: "."
   env:

.buildkite/intel_jobs/lora_intel.yaml

@@ -5,10 +5,6 @@ steps:
 - label: LoRA Runtime + Utils
   timeout_in_minutes: 45
   device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 24+
   no_plugin: true
   working_dir: "."
   env:
@@ -38,10 +34,6 @@ steps:
 - label: LoRA Fused/MoE Kernels
   timeout_in_minutes: 45
   device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 16+
   no_plugin: true
   working_dir: "."
   env:
@@ -62,10 +54,6 @@ steps:
 - label: LoRA Punica Kernels
   timeout_in_minutes: 45
   device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 16+
   no_plugin: true
   working_dir: "."
   env:
@@ -86,10 +74,6 @@ steps:
 - label: LoRA Punica FP8/XPU Ops
   timeout_in_minutes: 45
   device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 16+
   no_plugin: true
   working_dir: "."
   env:
@@ -110,10 +94,6 @@ steps:
 - label: LoRA Models
   timeout_in_minutes: 45
   device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 2+
-    mem: 24+
   no_plugin: true
   working_dir: "."
   env:
@@ -128,19 +108,15 @@ steps:
       bash .buildkite/scripts/hardware_ci/run-intel-test.sh
       'cd tests &&
       export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
+      (pytest -v -s lora/test_mixtral.py --deselect="tests/lora/test_mixtral.py::test_mixtral_lora[4]" || true) &&
       pytest -v -s lora/test_quant_model.py --deselect="tests/lora/test_quant_model.py::test_quant_model_lora[model0]" --deselect="tests/lora/test_quant_model.py::test_quant_model_lora[model1]" --deselect="tests/lora/test_quant_model.py::test_quant_model_tp_equality[model0]" &&
       pytest -v -s lora/test_transformers_model.py &&
       pytest -v -s lora/test_chatglm3_tp.py &&
-      pytest -v -s lora/test_llama_tp.py::test_llama_lora &&
       pytest -s -v lora/test_minicpmv_tp.py'
 
 - label: LoRA Multimodal
   timeout_in_minutes: 45
   device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 16+
   no_plugin: true
   working_dir: "."
   env:

.buildkite/intel_jobs/misc_intel.yaml

@@ -5,10 +5,6 @@ steps:
 - label: V1 Core + KV + Metrics
   timeout_in_minutes: 30
   device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 16+
   no_plugin: true
   working_dir: "."
   env:
@@ -35,10 +31,6 @@ steps:
 - label: V1 Sample + Logits
   timeout_in_minutes: 30
   device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 16+
   no_plugin: true
   working_dir: "."
   env:
@@ -46,17 +38,7 @@ steps:
     REPO: "vllm-ci-test-repo"
     VLLM_TEST_DEVICE: "xpu"
   source_file_dependencies:
-    - vllm/config/
+    - vllm/
-    - vllm/distributed/
-    - vllm/engine/
-    - vllm/inputs/
-    - vllm/logger.py
-    - vllm/model_executor/
-    - vllm/platforms/
-    - vllm/sampling_params.py
-    - vllm/transformers_utils/
-    - vllm/utils/
-    - vllm/v1/
     - tests/v1/sample
     - tests/v1/logits_processors
     - tests/v1/test_oracle.py
@@ -65,24 +47,16 @@ steps:
   commands:
     - >-
       bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-      'pip install lm_eval[api]>=0.4.12 &&
+      'export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
-      export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
       cd tests &&
       pytest -v -s v1/logits_processors --ignore=v1/logits_processors/test_custom_online.py --ignore=v1/logits_processors/test_custom_offline.py &&
       pytest -v -s v1/test_oracle.py &&
       pytest -v -s v1/test_request.py &&
-      pytest -v -s v1/test_outputs.py &&
+      pytest -v -s v1/test_outputs.py'
-      pytest -v -s v1/sample/test_topk_topp_sampler.py &&
-      pytest -v -s v1/sample/test_logprobs.py &&
-      pytest -v -s v1/sample/test_logprobs_e2e.py'
 
 - label: XPU CPU Offload
   timeout_in_minutes: 60
   device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 16+
   no_plugin: true
   working_dir: "."
   env:
@@ -102,113 +76,3 @@ steps:
       cd tests &&
       pytest -v -s v1/kv_offload &&
       pytest -v -s v1/kv_connector/unit/test_offloading_connector.py'
-
-- label: Regression
-  key: regression
-  timeout_in_minutes: 30
-  device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 16+
-  no_plugin: true
-  working_dir: "."
-  env:
-    REGISTRY: "public.ecr.aws/q9t5s3a7"
-    REPO: "vllm-ci-test-repo"
-    VLLM_TEST_DEVICE: "xpu"
-  source_file_dependencies:
-  - vllm/config/
-  - vllm/distributed/
-  - vllm/engine/
-  - vllm/inputs/
-  - vllm/model_executor/
-  - vllm/multimodal/
-  - vllm/platforms/
-  - vllm/sampling_params.py
-  - vllm/transformers_utils/
-  - vllm/utils/
-  - vllm/v1/
-  - tests/test_regression
-  commands:
-  - >-
-      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-      'pip install modelscope &&
-      cd tests &&
-      pytest -v -s test_regression.py'
-
-- label: Metrics, Tracing (2 GPUs)
-  key: metrics-tracing-2-gpus
-  timeout_in_minutes: 30
-  num_devices: 2
-  device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 2+
-    mem: 16+
-  no_plugin: true
-  working_dir: "."
-  env:
-    REGISTRY: "public.ecr.aws/q9t5s3a7"
-    REPO: "vllm-ci-test-repo"
-    VLLM_TEST_DEVICE: "xpu"
-  source_file_dependencies:
-  - vllm/config/
-  - vllm/distributed/
-  - vllm/engine/
-  - vllm/inputs/
-  - vllm/model_executor/
-  - vllm/multimodal/
-  - vllm/platforms/
-  - vllm/sampling_params.py
-  - vllm/tracing/
-  - vllm/transformers_utils/
-  - vllm/utils/
-  - vllm/v1/
-  - tests/v1/tracing
-  commands:
-  - >-
-    bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-    'pip install opentelemetry-sdk\>=1.26.0 opentelemetry-api\>=1.26.0 opentelemetry-exporter-otlp\>=1.26.0 opentelemetry-semantic-conventions-ai\>=0.4.1 &&
-    cd tests &&
-    pytest -v -s v1/tracing'
-
-- label: Async Engine, Inputs, Utils, Worker
-  key: async-engine-inputs-utils-worker
-  timeout_in_minutes: 30
-  device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 24+
-  no_plugin: true
-  working_dir: "."
-  env:
-    REGISTRY: "public.ecr.aws/q9t5s3a7"
-    REPO: "vllm-ci-test-repo"
-    VLLM_TEST_DEVICE: "xpu"
-  source_file_dependencies:
-  - vllm/assets/
-  - vllm/config/
-  - vllm/distributed/
-  - vllm/engine/
-  - vllm/inputs/
-  - vllm/model_executor/
-  - vllm/multimodal/
-  - vllm/platforms/
-  - vllm/sampling_params.py
-  - vllm/tokenizers/
-  - vllm/transformers_utils/
-  - vllm/utils/
-  - vllm/v1/
-  - tests/detokenizer
-  - tests/multimodal
-  - tests/utils_
-  commands:
-  - >-
-    bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-    'cd tests &&
-    pip install av &&
-    pytest -v -s detokenizer &&
-    pytest -v -s -m "not cpu_test" ./multimodal &&
-    pytest -v -s utils_ --ignore=utils_/test_mem_utils.py'

.buildkite/intel_jobs/model_runner_v2_intel.yaml

@@ -1,62 +0,0 @@
-group: Model Runner V2 Intel
-depends_on:
-  - image-build-xpu
-steps:
-- label: Model Runner V2 Core Tests (Intel)
-  timeout_in_minutes: 45
-  device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 2+
-    mem: 16+
-  no_plugin: true
-  working_dir: "."
-  env:
-    REGISTRY: "public.ecr.aws/q9t5s3a7"
-    REPO: "vllm-ci-test-repo"
-    VLLM_TEST_DEVICE: "xpu"
-  source_file_dependencies:
-    - vllm/v1/worker/gpu/
-    - vllm/v1/worker/gpu_worker.py
-    - vllm/v1/core/sched/
-    - vllm/v1/attention/
-    - tests/v1/engine/test_llm_engine.py
-    - tests/v1/e2e/
-  commands:
-    - >-
-      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-      'export VLLM_USE_V2_MODEL_RUNNER=1 &&
-      cd tests &&
-      pytest -v -s v1/engine/test_llm_engine.py -k "not test_engine_metrics" &&
-      ENFORCE_EAGER=1 pytest -v -s v1/e2e/general/test_async_scheduling.py -k "not ngram" &&
-      pytest -v -s v1/e2e/general/test_min_tokens.py'
-
-- label: Model Runner V2 Examples (Intel)
-  timeout_in_minutes: 45
-  device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 24+
-  no_plugin: true
-  working_dir: "."
-  env:
-    REGISTRY: "public.ecr.aws/q9t5s3a7"
-    REPO: "vllm-ci-test-repo"
-    VLLM_TEST_DEVICE: "xpu"
-  source_file_dependencies:
-    - vllm/v1/worker/gpu/
-    - vllm/v1/core/sched/
-    - vllm/v1/worker/gpu_worker.py
-    - examples/basic/offline_inference/
-    - examples/generate/multimodal/
-    - examples/features/
-  commands:
-    - >-
-      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-      'export VLLM_USE_V2_MODEL_RUNNER=1 &&
-      cd examples &&
-      python3 basic/offline_inference/chat.py &&
-      python3 basic/offline_inference/generate.py --model facebook/opt-125m &&
-      python3 generate/multimodal/vision_language_offline.py --seed 0 &&
-      python3 features/automatic_prefix_caching/prefix_caching_offline.py'

.buildkite/intel_jobs/models_distributed_intel.yaml

@@ -1,27 +0,0 @@
-group: Models - Distributed
-depends_on: 
-  - image-build-xpu
-steps:
-- label: Distributed Model Tests (2 GPUs)
-  key: distributed-model-tests-2-gpus
-  timeout_in_minutes: 50
-  device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 2+
-    mem: 24+
-  no_plugin: true
-  working_dir: "."
-  env:
-    REGISTRY: "public.ecr.aws/q9t5s3a7"
-    REPO: "vllm-ci-test-repo"
-    VLLM_TEST_DEVICE: "xpu"
-  source_file_dependencies:
-  - vllm/model_executor/model_loader/sharded_state_loader.py
-  - vllm/model_executor/models/
-  - tests/model_executor/model_loader/test_sharded_state_loader.py
-  commands:
-    - >-
-      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-      'cd tests &&
-      pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m "not slow_test"'

.buildkite/intel_jobs/models_multimodal_intel.yaml

@@ -1,129 +0,0 @@
-group: Models - Multimodal
-depends_on:
-  - image-build-xpu
-steps:
-- label: "Multi-Modal Models (Standard) 1: qwen2"
-  key: multi-modal-models-standard-1-qwen2
-  timeout_in_minutes: 45
-  device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 16+
-  no_plugin: true
-  working_dir: "."
-  env:
-    REGISTRY: "public.ecr.aws/q9t5s3a7"
-    REPO: "vllm-ci-test-repo"
-    VLLM_TEST_DEVICE: "xpu"
-  source_file_dependencies:
-  - vllm/
-  - tests/models/multimodal
-  commands:
-    - >-
-      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-      'pip install av git+https://github.com/TIGER-AI-Lab/Mantis.git &&
-      cd tests &&
-      pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2" &&
-      pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model'
-
-- label: "Multi-Modal Models (Standard) 2: qwen3 + gemma"
-  key: multi-modal-models-standard-2-qwen3-gemma
-  timeout_in_minutes: 45
-  device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 16+
-  no_plugin: true
-  working_dir: "."
-  env:
-    REGISTRY: "public.ecr.aws/q9t5s3a7"
-    REPO: "vllm-ci-test-repo"
-    VLLM_TEST_DEVICE: "xpu"
-  source_file_dependencies:
-  - vllm/
-  - tests/models/multimodal
-  commands:
-    - >-
-      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-      'pip install git+https://github.com/TIGER-AI-Lab/Mantis.git &&
-      cd tests &&
-      pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model'
-
-- label: "Multi-Modal Models (Standard) 3: llava + qwen2_vl"
-  key: multi-modal-models-standard-3-llava-qwen2-vl
-  timeout_in_minutes: 45
-  device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 24+
-  no_plugin: true
-  working_dir: "."
-  env:
-    REGISTRY: "public.ecr.aws/q9t5s3a7"
-    REPO: "vllm-ci-test-repo"
-    VLLM_TEST_DEVICE: "xpu"
-  source_file_dependencies:
-  - vllm/
-  - tests/models/multimodal
-  commands:
-    - >-
-      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-      'pip install git+https://github.com/TIGER-AI-Lab/Mantis.git &&
-      cd tests &&
-      pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma" &&
-      pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model'
-
-- label: "Multi-Modal Models (Standard) 4: other + whisper"
-  key: multi-modal-models-standard-4-other-whisper
-  timeout_in_minutes: 45
-  device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 16+
-  no_plugin: true
-  working_dir: "."
-  env:
-    REGISTRY: "public.ecr.aws/q9t5s3a7"
-    REPO: "vllm-ci-test-repo"
-    VLLM_TEST_DEVICE: "xpu"
-  source_file_dependencies:
-  - vllm/
-  - tests/models/multimodal
-  commands:
-    - >-
-      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-      'pip install av git+https://github.com/TIGER-AI-Lab/Mantis.git &&
-      cd tests &&
-      pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py  --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/processing'
-
-- label: Multi-Modal Processor # 44min
-  key: multi-modal-processor
-  timeout_in_minutes: 45
-  device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 16+
-  no_plugin: true
-  working_dir: "."
-  env:
-    REGISTRY: "public.ecr.aws/q9t5s3a7"
-    REPO: "vllm-ci-test-repo"
-    VLLM_TEST_DEVICE: "xpu"
-  source_file_dependencies:
-  - vllm/
-  - tests/models/multimodal
-  - tests/models/registry.py
-  commands:
-    - >-
-      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-      'pip install av matplotlib ftfy git+https://github.com/TIGER-AI-Lab/Mantis.git &&
-      pip install open-clip-torch --no-deps &&
-      cd tests &&
-      pytest -v -s models/multimodal/processing/test_tensor_schema.py
-      --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB'
-  parallelism: 4

.buildkite/intel_jobs/test-intel.yaml

@@ -19,10 +19,6 @@ steps:
       - image-build-xpu
     timeout_in_minutes: 30
     device: intel_gpu
-    agent_tags:
-      label: production
-      gpu: 2+
-      mem: 24+
     no_plugin: true
     env:
       REGISTRY: "public.ecr.aws/q9t5s3a7"
@@ -53,10 +49,6 @@ steps:
       - image-build-xpu
     timeout_in_minutes: 30
     device: intel_gpu
-    agent_tags:
-      label: production
-      gpu: 1+
-      mem: 16+
     no_plugin: true
     env:
       REGISTRY: "public.ecr.aws/q9t5s3a7"
@@ -68,24 +60,19 @@ steps:
       - >-
         bash .buildkite/scripts/hardware_ci/run-intel-test.sh
         'cd tests &&
-        bash v1/kv_connector/nixl_integration/run_xpu_disagg_accuracy_test.sh &&
         pytest -v -s v1/core --ignore=v1/core/test_reset_prefix_cache_e2e.py --ignore=v1/core/test_scheduler_e2e.py &&
         pytest -v -s v1/engine --ignore=v1/engine/test_output_processor.py &&
         pytest -v -s v1/sample --ignore=v1/sample/test_logprobs.py --ignore=v1/sample/test_logprobs_e2e.py -k "not test_topk_only and not test_topp_only and not test_topk_and_topp" &&
         pytest -v -s v1/worker --ignore=v1/worker/test_gpu_model_runner.py --ignore=v1/worker/test_worker_memory_snapshot.py &&
         pytest -v -s v1/structured_output &&
         pytest -v -s v1/test_serial_utils.py &&
-        pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py --ignore=v1/spec_decode/test_speculators_correctness.py &&
+        pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py &&
         pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_multi_connector.py --ignore=v1/kv_connector/unit/test_example_connector.py --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py --ignore=v1/kv_connector/unit/test_offloading_connector.py'
   - label: "XPU server test"
     depends_on:
       - image-build-xpu
     timeout_in_minutes: 30
     device: intel_gpu
-    agent_tags:
-      label: production
-      gpu: 1+
-      mem: 16+
     no_plugin: true
     env:
       REGISTRY: "public.ecr.aws/q9t5s3a7"
@@ -98,26 +85,5 @@ steps:
         bash .buildkite/scripts/hardware_ci/run-intel-test.sh
         'pip install av &&
         cd tests &&
-        pytest -v -s entrypoints/multimodal/openai/chat_completion/test_audio_in_video.py &&
+        pytest -v -s entrypoints/openai/chat_completion/test_audio_in_video.py &&
         pytest -v -s benchmarks/test_serve_cli.py'
-  - label: "XPU quantization test"
-    depends_on:
-      - image-build-xpu
-    timeout_in_minutes: 30
-    device: intel_gpu
-    agent_tags:
-      label: production
-      gpu: 1+
-      mem: 16+
-    no_plugin: true
-    env:
-      REGISTRY: "public.ecr.aws/q9t5s3a7"
-      REPO: "vllm-ci-test-repo"
-    source_file_dependencies:
-      - vllm/
-      - .buildkite/intel_jobs/test-intel.yaml 
-    commands:
-      - >-
-        bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-        'cd tests &&
-        pytest -v -s quantization/test_auto_round.py'

.buildkite/lm-eval-harness/configs/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8.yaml

@@ -6,7 +6,9 @@ tasks:
     value: 0.7142
   - name: "exact_match,flexible-extract"
     value: 0.4579
-moe_backend: "flashinfer_cutlass"
+env_vars:
+  VLLM_USE_FLASHINFER_MOE_FP8: "1"
+  VLLM_FLASHINFER_MOE_BACKEND: "throughput"
 limit: 1319
 num_fewshot: 5
 max_model_len: 262144

.buildkite/lm-eval-harness/test_lm_eval_correctness.py

@@ -68,10 +68,6 @@ def launch_lm_eval(eval_config, tp_size):
     if current_platform.is_rocm() and "Nemotron-3" in eval_config["model_name"]:
         model_args += "attention_backend=TRITON_ATTN"
 
-    moe_backend = eval_config.get("moe_backend", None)
-    if moe_backend is not None:
-        model_args += f"moe_backend={moe_backend},"
-
     env_vars = eval_config.get("env_vars", None)
     with scoped_env_vars(env_vars):
         results = lm_eval.simple_evaluate(

.buildkite/release-pipeline.yaml

@@ -1,25 +1,12 @@
 # CUDA architecture lists — following PyTorch RELEASE.md
 # (https://github.com/pytorch/pytorch/blob/main/RELEASE.md)
 # SM86 included for broader Ampere coverage; SM89 for marlin fp8 support
-# These requested arches are filtered by CMake's CUDA_SUPPORTED_ARCHS before
-# per-kernel arch selection. Do not add +PTX here: top-level +PTX is stripped
-# during that filtering, so kernels that need PTX must request it locally.
 env:
-  # for CUDA >=13, sm_100+ targets have family specifiers (see CMakeLists.txt)
+  CUDA_ARCH_X86: "7.5 8.0 8.6 8.9 9.0 10.0 12.0+PTX"
-  # so targets like 10.3 and 12.1 are automatically supported with this list
+  # aarch64 only architectures: 8.7 for Orin, 11.0 for Thor (since CUDA 13)
-  CUDA_ARCH_X86: "7.5 8.0 8.6 8.9 9.0 10.0 12.0"
+  CUDA_ARCH_AARCH64: "8.0 8.7 8.9 9.0 10.0 11.0 12.0+PTX"
-  # aarch64-only targets: Orin (8.7), Thor (11.0, CUDA 13+)
-  CUDA_ARCH_AARCH64: "8.0 8.7 8.9 9.0 10.0 11.0 12.0"
-
-  # for CUDA <13, we need to specify all needed targets
-  # some targets (10.3, 12.1) are skipped to limit the wheel size (< 500MB)
-  # please use CUDA 13 wheels or compile yourself on these new devices
   CUDA_ARCH_X86_CU129: "7.5 8.0 8.6 8.9 9.0 10.0 12.0"
   CUDA_ARCH_AARCH64_CU129: "8.0 8.7 8.9 9.0 10.0 12.0"
-  
-  # pre-built mooncake wheels
-  # the manylinux_2_35 wheel has compatibility issue on Ubuntu 24.04
-  # so we use different wheels for the time being
   MOONCAKE_WHEEL_AARCH64_2_35: "https://vllm-wheels.s3.amazonaws.com/mooncake/mooncake_transfer_engine-0.3.10.post2-0da9dfea3-cp312-cp312-manylinux_2_35_aarch64.whl"
   MOONCAKE_WHEEL_AARCH64_2_39: "https://vllm-wheels.s3.amazonaws.com/mooncake/mooncake_transfer_engine-0.3.10.post2-0da9dfea3-cp312-cp312-manylinux_2_39_aarch64.whl"
   MOONCAKE_WHEEL_X86_64: "https://vllm-wheels.s3.amazonaws.com/mooncake/mooncake_transfer_engine-0.3.10.post2-0da9dfea3-cp312-cp312-manylinux_2_35_x86_64.whl"
@@ -859,6 +846,7 @@ steps:
         allow_failure: true
       - step: build-cpu-release-image-arm64
         allow_failure: true
+    if: build.env("NIGHTLY") != "1"
 
   - label: "Publish release images to DockerHub"
     depends_on:

.buildkite/scripts/ci-bake-rocm.sh

@@ -15,10 +15,9 @@ set -euo pipefail
 
 DEFAULT_REPO_SLUG="vllm-project/vllm"
 DEFAULT_CI_HCL_SOURCE="docker/ci-rocm.hcl"
-DEFAULT_CI_BASE_CONTENT_FILES="requirements/common.txt requirements/rocm.txt requirements/test/rocm.txt docker/Dockerfile.rocm_base docker/ci-rocm.hcl docker/docker-bake-rocm.hcl tools/install_torchcodec_rocm.sh tests/vllm_test_utils .buildkite/scripts/ci-bake-rocm.sh"
+DEFAULT_CI_BASE_CONTENT_FILES="requirements/common.txt requirements/rocm.txt requirements/test/rocm.txt docker/Dockerfile.rocm_base tools/install_torchcodec_rocm.sh tests/vllm_test_utils"
 DEFAULT_CI_BASE_DOCKERFILE="docker/Dockerfile.rocm"
 DEFAULT_CI_BASE_DOCKERFILE_STAGES="base build_rixl build_rocshmem build_deepep mori_base ci_base"
-DEFAULT_CI_BASE_METADATA_VERSION="1"
 IMAGE_EXISTED_BEFORE_BUILD=0
 
 TARGET=""
@@ -526,22 +525,6 @@ get_remote_image_label_with_retry() {
     return 0
 }
 
-remote_ci_base_metadata_is_current() {
-    local image_ref="$1"
-    local metadata_version=""
-
-    metadata_version=$(get_remote_image_label "${image_ref}" "vllm.ci_base.metadata_version")
-    [[ "${metadata_version}" == "${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}" ]]
-}
-
-remote_ci_base_metadata_is_current_with_retry() {
-    local image_ref="$1"
-    local metadata_version=""
-
-    metadata_version=$(get_remote_image_label_with_retry "${image_ref}" "vllm.ci_base.metadata_version")
-    [[ "${metadata_version}" == "${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}" ]]
-}
-
 remote_image_exists() {
     local image_ref="$1"
     docker manifest inspect "${image_ref}" >/dev/null 2>&1
@@ -598,7 +581,6 @@ init_config() {
     CI_BASE_CONTENT_FILES="${CI_BASE_CONTENT_FILES:-${DEFAULT_CI_BASE_CONTENT_FILES}}"
     CI_BASE_DOCKERFILE="${CI_BASE_DOCKERFILE:-${DEFAULT_CI_BASE_DOCKERFILE}}"
     CI_BASE_DOCKERFILE_STAGES="${CI_BASE_DOCKERFILE_STAGES:-${DEFAULT_CI_BASE_DOCKERFILE_STAGES}}"
-    CI_BASE_METADATA_VERSION="${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}"
     CI_BASE_IMAGE_TAG="${CI_BASE_IMAGE_TAG:-rocm/vllm-dev:ci_base}"
     export PYTORCH_ROCM_ARCH
 
@@ -653,10 +635,6 @@ load_ci_hcl() {
     echo "Copied ${CI_HCL_SOURCE} to ${CI_HCL_PATH}"
 }
 
-init_bake_files() {
-    BAKE_FILES=(-f "${VLLM_BAKE_FILE}" -f "${CI_HCL_PATH}")
-}
-
 compute_ci_base_hash_if_needed() {
     if [[ -z "${CI_BASE_CONTENT_FILES:-}" ]]; then
         return 0
@@ -698,14 +676,12 @@ configure_ci_base_image_refs() {
     fi
 
     content_tag=$(ci_base_tag_with_suffix "${stable_tag}" "${CI_BASE_CONTENT_HASH}")
-    CI_BASE_IMAGE_TAG_CONTENT_REF="${content_tag}"
     if [[ -n "${BUILDKITE_COMMIT:-}" ]]; then
         commit_tag=$(ci_base_tag_with_suffix "${stable_tag}" "${BUILDKITE_COMMIT}")
+        CI_BASE_IMAGE_TAG_COMMIT="${commit_tag}"
+        export CI_BASE_IMAGE_TAG_COMMIT
     fi
-    CI_BASE_IMAGE_TAG_COMMIT_REF="${commit_tag}"
 
-    # *_REF is the logical tag recorded in metadata. *_EXTRA is only passed to
-    # bake when that tag is not already the primary tag, avoiding duplicates.
     if should_push_stable_ci_base_tag; then
         primary_tag="${content_tag}"
         CI_BASE_IMAGE_TAG_STABLE="${stable_tag}"
@@ -715,33 +691,19 @@ configure_ci_base_image_refs() {
     fi
     CI_BASE_IMAGE_TAG="${primary_tag}"
     if [[ "${primary_tag}" == "${content_tag}" ]]; then
-        CI_BASE_IMAGE_TAG_CONTENT_EXTRA=""
+        CI_BASE_IMAGE_TAG_CONTENT=""
     else
-        CI_BASE_IMAGE_TAG_CONTENT_EXTRA="${content_tag}"
+        CI_BASE_IMAGE_TAG_CONTENT="${content_tag}"
     fi
-    if [[ -n "${commit_tag}" && "${commit_tag}" != "${primary_tag}" ]]; then
+    export CI_BASE_IMAGE_TAG CI_BASE_IMAGE_TAG_CONTENT CI_BASE_IMAGE_TAG_STABLE
-        CI_BASE_IMAGE_TAG_COMMIT_EXTRA="${commit_tag}"
-    else
-        CI_BASE_IMAGE_TAG_COMMIT_EXTRA=""
-    fi
-    export CI_BASE_IMAGE_TAG
-    export CI_BASE_IMAGE_TAG_COMMIT_EXTRA
-    export CI_BASE_IMAGE_TAG_CONTENT_EXTRA
-    export CI_BASE_IMAGE_TAG_CONTENT_REF
-    export CI_BASE_IMAGE_TAG_COMMIT_REF
-    export CI_BASE_IMAGE_TAG_STABLE
 
     if is_ci_base_target; then
         IMAGE_TAG="${primary_tag}"
         export IMAGE_TAG
 
         echo "ci_base primary image tag: ${CI_BASE_IMAGE_TAG}"
-        if [[ -n "${commit_tag}" ]]; then
+        if [[ -n "${CI_BASE_IMAGE_TAG_COMMIT:-}" ]]; then
-            if [[ "${commit_tag}" == "${primary_tag}" ]]; then
+            echo "ci_base commit image tag: ${CI_BASE_IMAGE_TAG_COMMIT}"
-                echo "ci_base commit image tag: ${commit_tag} (primary)"
-            else
-                echo "ci_base commit image tag: ${commit_tag}"
-            fi
         fi
         echo "ci_base content image tag: ${content_tag}"
         if [[ -n "${CI_BASE_IMAGE_TAG_STABLE}" ]]; then
@@ -766,8 +728,8 @@ ci_base_candidate_refs() {
     printf '%s\n' \
         "${IMAGE_TAG:-}" \
         "${CI_BASE_IMAGE_TAG:-}" \
-        "${CI_BASE_IMAGE_TAG_COMMIT_EXTRA:-}" \
+        "${CI_BASE_IMAGE_TAG_COMMIT:-}" \
-        "${CI_BASE_IMAGE_TAG_CONTENT_EXTRA:-}" \
+        "${CI_BASE_IMAGE_TAG_CONTENT:-}" \
         "${CI_BASE_IMAGE_TAG_STABLE:-}" \
         | awk 'NF && !seen[$0]++'
 }
@@ -781,10 +743,6 @@ find_matching_ci_base_ref() {
         remote_image_exists "${candidate}" || continue
         candidate_hash=$(get_remote_image_label "${candidate}" "vllm.ci_base.content_hash")
         if [[ "${candidate_hash}" == "${CI_BASE_CONTENT_HASH}" ]]; then
-            if ! remote_ci_base_metadata_is_current "${candidate}"; then
-                echo "Found matching ci_base content hash but stale metadata: ${candidate}" >&2
-                continue
-            fi
             printf '%s\n' "${candidate}"
             return 0
         fi
@@ -859,10 +817,6 @@ maybe_skip_existing_image() {
         if [[ -n "${remote_hash}" ]]; then
             echo "Remote ci_base content hash: ${remote_hash:0:16}..."
             if [[ "${remote_hash}" == "${CI_BASE_CONTENT_HASH}" ]]; then
-                if ! remote_ci_base_metadata_is_current "${IMAGE_TAG}"; then
-                    echo "Content hashes match but ci_base metadata is stale; rebuilding to refresh metadata"
-                    return 0
-                fi
                 if ! refresh_ci_base_tags_from_ref "${IMAGE_TAG}"; then
                     echo "ci_base tag refresh failed; rebuilding to push expected tags"
                     return 0
@@ -1044,104 +998,12 @@ prepare_git_cache_metadata() {
     fi
 }
 
-ci_base_metadata_pairs() {
-    local dockerfile="${CI_BASE_DOCKERFILE:-${DEFAULT_CI_BASE_DOCKERFILE}}"
-    local stages="${CI_BASE_DOCKERFILE_STAGES:-${DEFAULT_CI_BASE_DOCKERFILE_STAGES}}"
-    local content_files="${CI_BASE_CONTENT_FILES:-${DEFAULT_CI_BASE_CONTENT_FILES}}"
-    local content_files_hash=""
-    local base_image=""
-    local base_image_digest=""
-    local git_branch=""
-    local -a content_paths=()
-    local -a content_args=()
-
-    read -r -a content_paths <<< "${content_files}"
-    if [[ ${#content_paths[@]} -gt 0 ]]; then
-        content_files_hash=$(compute_content_hash "${content_paths[@]}")
-    fi
-    mapfile -t content_args < <(
-        get_content_arg_names "${dockerfile}" "${stages}" "${CI_BASE_CONTENT_ARGS:-}"
-    )
-
-    base_image=$(resolve_dockerfile_arg_value "${dockerfile}" "BASE_IMAGE")
-    if [[ -n "${base_image}" ]]; then
-        base_image_digest=$(resolve_image_digest "${base_image}")
-    fi
-    git_branch="${BUILDKITE_BRANCH:-${VLLM_BRANCH:-}}"
-
-    metadata_pair "vllm.ci_base.metadata_version" "${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}"
-    metadata_pair "vllm.ci_base.content_hash" "${CI_BASE_CONTENT_HASH:-}"
-    metadata_pair "vllm.ci_base.content_files_hash" "${content_files_hash}"
-    metadata_pair "vllm.ci_base.content_files" "${content_files}"
-    metadata_pair "vllm.ci_base.content_args" "$(join_words "${content_args[@]}")"
-    metadata_pair "vllm.ci_base.dockerfile" "${dockerfile}"
-    metadata_pair "vllm.ci_base.dockerfile_stages" "${stages}"
-    metadata_pair "vllm.ci_base.image.primary" "${CI_BASE_IMAGE_TAG:-}"
-    metadata_pair "vllm.ci_base.image.content" "${CI_BASE_IMAGE_TAG_CONTENT_REF:-${CI_BASE_IMAGE_TAG_CONTENT_EXTRA:-}}"
-    metadata_pair "vllm.ci_base.image.commit" "${CI_BASE_IMAGE_TAG_COMMIT_REF:-${CI_BASE_IMAGE_TAG_COMMIT_EXTRA:-}}"
-    metadata_pair "vllm.ci_base.image.stable" "${CI_BASE_IMAGE_TAG_STABLE:-}"
-    metadata_pair "vllm.ci_base.git_commit" "${BUILDKITE_COMMIT:-}"
-    metadata_pair "vllm.ci_base.git_branch" "${git_branch}"
-    metadata_pair "vllm.ci_base.vllm_branch" "${VLLM_BRANCH:-}"
-    metadata_pair "vllm.ci_base.stable_branch" "${CI_BASE_STABLE_BRANCH:-main}"
-
-    metadata_pair "vllm.rocm.base_image" "${base_image}"
-    metadata_pair "vllm.rocm.base_image_digest" "${base_image_digest}"
-    metadata_pair "vllm.rocm.pytorch_rocm_arch" "${PYTORCH_ROCM_ARCH:-}"
-    metadata_pair "vllm.rocm.nic_backend" "$(resolve_dockerfile_arg_value "${dockerfile}" "NIC_BACKEND")"
-    metadata_pair "vllm.rocm.ainic_version" "$(resolve_dockerfile_arg_value "${dockerfile}" "AINIC_VERSION")"
-    metadata_pair "vllm.rocm.ubuntu_codename" "$(resolve_dockerfile_arg_value "${dockerfile}" "UBUNTU_CODENAME")"
-    metadata_pair "vllm.rocm.rixl_repo" "$(resolve_dockerfile_arg_value "${dockerfile}" "RIXL_REPO")"
-    metadata_pair "vllm.rocm.rixl_commit" "${RIXL_BRANCH:-$(resolve_dockerfile_arg_value "${dockerfile}" "RIXL_BRANCH")}"
-    metadata_pair "vllm.rocm.ucx_repo" "$(resolve_dockerfile_arg_value "${dockerfile}" "UCX_REPO")"
-    metadata_pair "vllm.rocm.ucx_commit" "${UCX_BRANCH:-$(resolve_dockerfile_arg_value "${dockerfile}" "UCX_BRANCH")}"
-    metadata_pair "vllm.rocm.rocshmem_repo" "$(resolve_dockerfile_arg_value "${dockerfile}" "ROCSHMEM_REPO")"
-    metadata_pair "vllm.rocm.rocshmem_commit" "${ROCSHMEM_BRANCH:-$(resolve_dockerfile_arg_value "${dockerfile}" "ROCSHMEM_BRANCH")}"
-    metadata_pair "vllm.rocm.deepep_repo" "$(resolve_dockerfile_arg_value "${dockerfile}" "DEEPEP_REPO")"
-    metadata_pair "vllm.rocm.deepep_commit" "${DEEPEP_BRANCH:-$(resolve_dockerfile_arg_value "${dockerfile}" "DEEPEP_BRANCH")}"
-    metadata_pair "vllm.rocm.deepep_nic" "$(resolve_dockerfile_arg_value "${dockerfile}" "DEEPEP_NIC")"
-    metadata_pair "vllm.rocm.deepep_rocm_arch" "$(resolve_dockerfile_arg_value "${dockerfile}" "DEEPEP_ROCM_ARCH")"
-    metadata_pair "vllm.rocm.rixl_cache_key" "${RIXL_CACHE_KEY:-}"
-    metadata_pair "vllm.rocm.rocshmem_cache_key" "${ROCSHMEM_CACHE_KEY:-}"
-    metadata_pair "vllm.rocm.deepep_cache_key" "${DEEPEP_CACHE_KEY:-}"
-
-    metadata_pair "vllm.buildkite.build_number" "${BUILDKITE_BUILD_NUMBER:-}"
-    metadata_pair "vllm.buildkite.build_id" "${BUILDKITE_BUILD_ID:-}"
-}
-
-write_ci_base_metadata_annotations() {
-    local metadata="$1"
-    local key=""
-    local value=""
-    local annotation=""
-
-    [[ -n "${metadata}" ]] || return 0
-    while IFS=$'\t' read -r key value; do
-        [[ -n "${key}" && -n "${value}" ]] || continue
-        annotation="manifest:${key}=${value}"
-        printf '    "%s",\n' "$(hcl_escape_string "${annotation}")"
-    done <<< "${metadata}"
-}
-
-write_ci_base_metadata_labels() {
-    local metadata="$1"
-    local key=""
-    local value=""
-
-    [[ -n "${metadata}" ]] || return 0
-    while IFS=$'\t' read -r key value; do
-        [[ -n "${key}" && -n "${value}" ]] || continue
-        printf '    "%s" = "%s"\n' \
-            "$(hcl_escape_string "${key}")" \
-            "$(hcl_escape_string "${value}")"
-    done <<< "${metadata}"
-}
-
 write_ci_base_label_override() {
     local target_name=""
-    local metadata=""
     local -a ci_base_targets=()
 
+    BAKE_FILES=(-f "${VLLM_BAKE_FILE}" -f "${CI_HCL_PATH}")
+
     if [[ -z "${CI_BASE_CONTENT_HASH:-}" ]]; then
         return 0
     fi
@@ -1157,23 +1019,16 @@ write_ci_base_label_override() {
         return 0
     fi
 
-    metadata=$(ci_base_metadata_pairs)
-
     : > "${CI_BASE_LABEL_OVERRIDE_PATH}"
     for target_name in "${ci_base_targets[@]}"; do
         cat >> "${CI_BASE_LABEL_OVERRIDE_PATH}" <<EOF
 target "${target_name}" {
   annotations = [
     "manifest:org.opencontainers.image.revision=",
-EOF
-        write_ci_base_metadata_annotations "${metadata}" >> "${CI_BASE_LABEL_OVERRIDE_PATH}"
-        cat >> "${CI_BASE_LABEL_OVERRIDE_PATH}" <<EOF
   ]
   labels = {
     "org.opencontainers.image.revision" = ""
-EOF
+    "vllm.ci_base.content_hash" = "${CI_BASE_CONTENT_HASH}"
-        write_ci_base_metadata_labels "${metadata}" >> "${CI_BASE_LABEL_OVERRIDE_PATH}"
-        cat >> "${CI_BASE_LABEL_OVERRIDE_PATH}" <<EOF
   }
 }
 
@@ -1181,7 +1036,7 @@ EOF
     done
 
     BAKE_FILES+=(-f "${CI_BASE_LABEL_OVERRIDE_PATH}")
-    echo "Appended ci_base metadata label override for targets: ${ci_base_targets[*]}"
+    echo "Appended ci_base content-hash label override for targets: ${ci_base_targets[*]}"
 }
 
 uses_rocm_csrc_cache() {
@@ -1264,18 +1119,6 @@ hcl_escape_string() {
     printf '%s' "${value}"
 }
 
-join_words() {
-    local IFS=" "
-    printf '%s' "$*"
-}
-
-metadata_pair() {
-    local key="$1"
-    local value="${2:-}"
-
-    printf '%s\t%s\n' "${key}" "${value}"
-}
-
 write_hcl_string_list() {
     local indent="$1"
     shift
@@ -1698,13 +1541,7 @@ confirm_remote_image_push() {
 
         remote_hash=$(get_remote_image_label_with_retry "${image_ref}" "vllm.ci_base.content_hash")
         if [[ -n "${remote_hash}" && "${remote_hash}" == "${CI_BASE_CONTENT_HASH}" ]]; then
-            if remote_ci_base_metadata_is_current_with_retry "${image_ref}"; then
+            return 0
-                return 0
-            fi
-
-            echo "Remote image exists with the expected ci_base content hash but stale metadata."
-            echo "  expected metadata version: ${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}"
-            return 1
         fi
 
         echo "Remote image exists but does not have the expected ci_base content hash."
@@ -1887,16 +1724,15 @@ main() {
     print_header
     validate_inputs
     load_ci_hcl
-    init_bake_files
     compute_ci_base_hash_if_needed
     configure_ci_base_image_refs
     maybe_skip_existing_image
     setup_builder
     prepare_git_cache_metadata
+    write_ci_base_label_override
     extract_dependency_pins
     write_rocm_build_arg_override
     compute_dependency_cache_keys
-    write_ci_base_label_override
     compute_rocm_csrc_content_hash_if_needed
     write_rocm_cache_override
     resolve_ci_base_dependency_targets

.buildkite/scripts/ci-clean-log.sh

@@ -13,8 +13,5 @@ INPUT_FILE="$1"
 # Strip timestamps
 sed -i 's/^\[[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}T[0-9]\{2\}:[0-9]\{2\}:[0-9]\{2\}Z\] //' "$INPUT_FILE"
 
-# Strip Buildkite inline timestamp markers (ESC _bk;t=<ms> BEL)
-sed -i 's/\x1B_bk;t=[0-9]*\x07//g' "$INPUT_FILE"
-
 # Strip colorization
 sed -i -r 's/\x1B\[[0-9;]*[mK]//g' "$INPUT_FILE"

.buildkite/scripts/ci-fetch-log.sh

@@ -1,178 +1,74 @@
 #!/bin/bash
-# Fetch vLLM Buildkite CI logs (public; no login required).
+# Usage: ./ci-fetch-log.sh <buildkite_job_url> [output_file]
+#        ./ci-fetch-log.sh <build_number> <job_uuid> [output_file]
 #
-# Usage:
+# Downloads the raw log for a Buildkite job from the public, unauthenticated
-#   ci-fetch-log.sh [--soft|--all] --pr [<PR>]  failed jobs in the PR's latest
+# /organizations/<org>/pipelines/<pipeline>/builds/<n>/jobs/<uuid>/download
-#                                               build (current branch if omitted)
+# endpoint, then strips ANSI/timestamps via ci-clean-log.sh.
-#   ci-fetch-log.sh [--soft|--all] <build_url>  failed jobs in that build
-#   ci-fetch-log.sh <job_url> [output]          one job; both #<job_uuid> and
-#                                               ?sid=<id> URL forms work
-#   ci-fetch-log.sh <build> <job_uuid> [output]
 #
-# --soft also fetches soft-failed jobs; --all fetches every finished job.
+# Find <build_number> and <job_uuid> via:
-# Saves each log as ci-<build>-<job-name>.log (ANSI/timestamps stripped) and
+#   gh pr checks <PR> --repo vllm-project/vllm
-# prints "<file>\t<job name>" per job. [output] is single-job only; "-"
+# Each failing row's URL is .../builds/<build_number>#<job_uuid>.
-# streams to stdout. Existing files are kept; CI_FETCH_LOG_FORCE=1 refetches.
+#
+# Default output path: ci-<build>-<uuid_first_13_chars>.log (e.g.
+# ci-68478-019e6b07-daae.log). Jobs in the same build share the UUID's
+# first 8 chars, so the second segment is needed for uniqueness when
+# fetching multiple jobs in parallel. The script refuses to overwrite an
+# existing output file; pass an explicit path or set CI_FETCH_LOG_FORCE=1
+# to override.
 
 set -euo pipefail
 
 ORG="vllm"
 PIPELINE="ci"
-UA="vllm-ci-fetch-log"
-UUID_RE='[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'
 
 usage() {
-    sed -n '2,15p' "$0" | sed 's/^# \{0,1\}//'
+    echo "Usage: $0 <buildkite_job_url> [output_file]"
+    echo "       $0 <build_number> <job_uuid> [output_file]"
     exit 1
 }
 
-die() {
+if [ $# -lt 1 ]; then usage; fi
-    echo "$1" >&2
-    exit 1
-}
 
-BUILD="" JOB="" SID="" OUT=""
+if [[ "$1" == https://* ]]; then
-SCOPE="failed"
-
-while :; do
-    case "${1:-}" in
-    --soft) SCOPE="soft" ;;
-    --all) SCOPE="all" ;;
-    *) break ;;
-    esac
-    shift
-done
-
-case "${1:-}" in
---pr)
-    PR="${2:-}"
-    # gh pr checks exits non-zero when checks are failing; that is the
-    # expected case here.
-    URL=$(gh pr checks ${PR:+"$PR"} --repo vllm-project/vllm 2>/dev/null |
-        grep -oE "https://buildkite.com/${ORG}/${PIPELINE}/builds/[0-9]+" |
-        sort -t/ -k7 -n | tail -1 || true)
-    [ -n "$URL" ] || die "No Buildkite build found via: gh pr checks ${PR:-<current branch>}"
-    BUILD="${URL##*/}"
-    ;;
-https://*)
     BUILD=$(echo "$1" | sed -nE 's#.*/builds/([0-9]+).*#\1#p')
-    JOB=$(echo "$1" | grep -oE "#${UUID_RE}" | head -n 1 | cut -c2- || true)
+    JOB=$(echo "$1" | grep -oE '[0-9a-f]{8}-[0-9a-f-]+' | head -n 1)
-    SID=$(echo "$1" | grep -oE "[?&]sid=${UUID_RE}" | head -n 1 | sed 's/.*sid=//' || true)
     OUT="${2:-}"
-    [ -n "$BUILD" ] || die "Could not parse build number from: $1"
+else
-    ;;
+    if [ $# -lt 2 ]; then usage; fi
-[0-9]*)
-    [ $# -ge 2 ] || usage
     BUILD="$1"
     JOB="$2"
     OUT="${3:-}"
-    ;;
+fi
-*)
+
+if [ -z "$BUILD" ] || [ -z "$JOB" ]; then
+    echo "Could not parse build number or job UUID from: $1" >&2
     usage
-    ;;
+fi
-esac
+
+# Jobs in the same build share the UUID's first segment, so include the
+# second segment (chars 9-13, e.g. "019e6b07-daae") to keep default filenames
+# unique when fetching multiple jobs from one build in parallel.
+if [ -z "$OUT" ]; then
+    OUT="ci-${BUILD}-${JOB:0:13}.log"
+fi
+
+if [ -e "$OUT" ] && [ -z "${CI_FETCH_LOG_FORCE:-}" ]; then
+    echo "Refusing to overwrite existing $OUT (set CI_FETCH_LOG_FORCE=1 or pass an explicit output path)." >&2
+    exit 1
+fi
 
 COOKIES=$(mktemp)
-JOBS_TSV=$(mktemp)
+trap 'rm -f "$COOKIES"' EXIT
-trap 'rm -f "$COOKIES" "$JOBS_TSV"' EXIT
 
-# Buildkite issues a session cookie on first hit; later requests need it.
+# Buildkite issues a session cookie on first hit; subsequent /download needs it.
-curl -fsSL -c "$COOKIES" -A "$UA" \
+curl -fsSL -c "$COOKIES" -A "vllm-ci-fetch-log" \
     "https://buildkite.com/${ORG}/${PIPELINE}/builds/${BUILD}" -o /dev/null
 
-# The build's job list (id, step uuid, state, name) is served as JSON from
+curl -fsSL -b "$COOKIES" -A "vllm-ci-fetch-log" \
-# the user-facing /data/jobs endpoint. Flatten it to TSV for easy filtering:
+    "https://buildkite.com/organizations/${ORG}/pipelines/${PIPELINE}/builds/${BUILD}/jobs/${JOB}/download" \
-#   job_id  step_uuid  failed  soft_failed  finished  slug  name
+    -o "$OUT"
-curl -fsSL -b "$COOKIES" -A "$UA" \
-    "https://buildkite.com/${ORG}/${PIPELINE}/builds/${BUILD}/data/jobs" |
-    python3 -c '
-import json, re, sys
 
-data = json.load(sys.stdin)
+bash "$(dirname "$0")/ci-clean-log.sh" "$OUT"
-if data.get("has_next_page"):
-    print("warning: job list is paginated; some jobs not shown", file=sys.stderr)
-for r in data["records"]:
-    if r.get("type") != "script":
-        continue
-    name = (r.get("name") or "").replace("\t", " ").replace("\n", " ")
-    slug = re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-")[:60]
-    print("\t".join([
-        r["id"],
-        r.get("step_uuid") or "",
-        str(r.get("passed") is False),
-        str(bool(r.get("soft_failed"))),
-        str(bool(r.get("finished_at"))),
-        slug,
-        name,
-    ]))
-' >"$JOBS_TSV" || die "Could not list jobs for build ${BUILD}"
 
-if [ -n "$SID" ] && [ -z "$JOB" ]; then
+echo "$OUT"
-    # The ?sid= in builds/<N>/list URLs is the *step* uuid, not the job uuid.
-    JOB=$(awk -F'\t' -v s="$SID" '$1 == s || $2 == s {print $1; exit}' "$JOBS_TSV")
-    [ -n "$JOB" ] || die "No job matching sid=${SID} in build ${BUILD}"
-fi
-
-fetch_job() { # <job_uuid> <output_file>
-    curl -fsSL -b "$COOKIES" -A "$UA" \
-        "https://buildkite.com/organizations/${ORG}/pipelines/${PIPELINE}/builds/${BUILD}/jobs/$1/download" \
-        -o "$2"
-    bash "$(dirname "$0")/ci-clean-log.sh" "$2"
-}
-
-if [ -n "$JOB" ]; then
-    # Single-job mode.
-    NAME=$(awk -F'\t' -v j="$JOB" '$1 == j {print $7; exit}' "$JOBS_TSV")
-    SLUG=$(awk -F'\t' -v j="$JOB" '$1 == j {print $6; exit}' "$JOBS_TSV")
-    [ -n "$OUT" ] || OUT="ci-${BUILD}-${SLUG:-${JOB:0:13}}.log"
-    if [ "$OUT" = "-" ]; then
-        TMP=$(mktemp)
-        fetch_job "$JOB" "$TMP"
-        cat "$TMP"
-        rm -f "$TMP"
-        exit 0
-    fi
-    if [ -e "$OUT" ] && [ -z "${CI_FETCH_LOG_FORCE:-}" ]; then
-        die "Refusing to overwrite existing ${OUT} (set CI_FETCH_LOG_FORCE=1 or pass an output path)."
-    fi
-    fetch_job "$JOB" "$OUT"
-    printf '%s\t%s\n' "$OUT" "${NAME:-$JOB}"
-    exit 0
-fi
-
-# Build-wide mode: fetch finished jobs matching $SCOPE.
-[ -z "$OUT" ] || die "[output_file] is only valid when fetching a single job."
-
-case "$SCOPE" in
-failed) FILTER='$3 == "True" && $4 == "False" && $5 == "True"' ;;
-soft) FILTER='$3 == "True" && $5 == "True"' ;;
-all) FILTER='$5 == "True"' ;;
-esac
-
-if [ "$SCOPE" = "failed" ]; then
-    SOFT=$(awk -F'\t' '$3 == "True" && $4 == "True"' "$JOBS_TSV" | wc -l)
-    [ "$SOFT" -eq 0 ] || echo "Skipping ${SOFT} soft-failed job(s); use --soft to include them." >&2
-fi
-
-FOUND=0
-EMITTED=" "
-while IFS=$'\t' read -r job_id _ _ _ _ slug name; do
-    FOUND=$((FOUND + 1))
-    out="ci-${BUILD}-${slug:-${job_id:0:13}}.log"
-    # Retries share a name with the original job; disambiguate by uuid.
-    case "$EMITTED" in
-    *" $out "*) out="ci-${BUILD}-${slug:-job}-${job_id:0:13}.log" ;;
-    esac
-    EMITTED="${EMITTED}${out} "
-    if [ -e "$out" ] && [ -z "${CI_FETCH_LOG_FORCE:-}" ]; then
-        echo "Keeping existing ${out} (set CI_FETCH_LOG_FORCE=1 to refetch)." >&2
-    elif ! fetch_job "$job_id" "$out"; then
-        echo "Failed to download log for job ${job_id} (${name})." >&2
-        continue
-    fi
-    printf '%s\t%s\n' "$out" "$name"
-done < <(awk -F'\t' "$FILTER" "$JOBS_TSV")
-
-if [ "$FOUND" -eq 0 ]; then
-    echo "No matching jobs in build ${BUILD} (scope: ${SCOPE})." >&2
-fi

.buildkite/scripts/hardware_ci/run-amd-test.sh

@@ -367,20 +367,6 @@ remove_docker_container() {
 }
 trap remove_docker_container EXIT
 
-# python_only_compile.sh runs `python setup.py develop` and needs the full repo tree
-# under /vllm-workspace (Dockerfile.rocm test stage: mkdir src && mv vllm).
-# The ROCm wheel artifact tarball only ships a thin tree (tests, etc.), so
-# artifact images cannot satisfy that test — use the full rocm/vllm-ci image.
-_cmd_probe="${VLLM_TEST_COMMANDS:-}"
-if [[ -z "${_cmd_probe}" ]]; then
-  _cmd_probe="$*"
-fi
-if [[ "${VLLM_CI_USE_ARTIFACTS:-0}" == "1" && "${_cmd_probe}" == *python_only_compile.sh* ]]; then
-  echo "INFO: disabling VLLM_CI_USE_ARTIFACTS for python_only_compile (requires full /vllm-workspace tree)"
-  export VLLM_CI_USE_ARTIFACTS=0
-fi
-unset -v _cmd_probe
-
 if ! prepare_artifact_image; then
   echo "Using full ROCm CI image: ${image_name}"
   docker pull "${image_name}" || exit 1
@@ -440,24 +426,6 @@ fi
 
 echo "Final commands: $commands"
 
-# The ROCm test image often ships /vllm-workspace without .git (artifact tarball unpack).
-# tests/standalone_tests/python_only_compile.sh uses merge-base(HEAD, origin/main) for
-# wheels.vllm.ai; compute on the agent (full git checkout) and pass into the container.
-vllm_standalone_merge_base=""
-checkout="${BUILDKITE_BUILD_CHECKOUT_PATH:-}"
-if [[ -z "${checkout}" || ! -d "${checkout}" ]]; then
-  checkout="."
-fi
-if git -C "${checkout}" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
-  vllm_standalone_merge_base="$(
-    git -C "${checkout}" merge-base HEAD origin/main 2>/dev/null || true
-  )"
-fi
-if [[ -z "${vllm_standalone_merge_base}" ]]; then
-  vllm_standalone_merge_base="${BUILDKITE_COMMIT:-}"
-fi
-echo "INFO: passing VLLM_STANDALONE_MERGE_BASE into container: ${vllm_standalone_merge_base}"
-
 MYPYTHONPATH="/vllm-workspace"
 
 container_job_id="${BUILDKITE_JOB_ID:-${BUILDKITE_PARALLEL_JOB:-0}}"
@@ -557,7 +525,6 @@ else
     -e "VLLM_CACHE_ROOT=${CONTAINER_CACHE_ROOT}/vllm" \
     -e "XDG_CACHE_HOME=${CONTAINER_CACHE_ROOT}/xdg" \
     -e "PYTORCH_ROCM_ARCH=" \
-    -e "VLLM_STANDALONE_MERGE_BASE=${vllm_standalone_merge_base}" \
     --name "${container_name}" \
     "${image_name}" \
     /bin/bash -c "${CONTAINER_PREFLIGHT} && ${commands}"

.buildkite/scripts/hardware_ci/run-cpu-test-arm.sh

@@ -8,7 +8,7 @@ set -ex
 CORE_RANGE=${CORE_RANGE:-0-31}
 OMP_CORE_RANGE=${OMP_CORE_RANGE:-0-31}
 
-export CMAKE_BUILD_PARALLEL_LEVEL=32
+export CMAKE_BUILD_PARALLEL_LEVEL=16
 
 # Setup cleanup
 remove_docker_container() {
@@ -37,7 +37,7 @@ function cpu_tests() {
     pytest -x -v -s tests/kernels/test_onednn.py
     pytest -x -v -s tests/kernels/attention/test_cpu_attn.py
     pytest -x -v -s tests/kernels/core/test_cpu_activation.py
-    pytest -x -v -s tests/kernels/moe/test_cpu_fused_moe.py
+    pytest -x -v -s tests/kernels/moe/test_moe.py -k test_cpu_fused_moe_basic
     pytest -x -v -s tests/kernels/mamba/cpu/test_cpu_gdn_ops.py"
 
   # skip tests requiring model downloads if HF_TOKEN is not set

.buildkite/scripts/hardware_ci/run-cpu-test.sh

@@ -7,49 +7,10 @@ set -euox pipefail
 # allow to bind to different cores
 CORE_RANGE=${CORE_RANGE:-48-95}
 NUMA_NODE=${NUMA_NODE:-1}
-AGENT_SLOT=${AGENT_SLOT:-}
+IMAGE_NAME="cpu-test-$NUMA_NODE"
-IMAGE_NAME="cpu-test-${NUMA_NODE}${AGENT_SLOT:+-${AGENT_SLOT}}"
 TIMEOUT_VAL=$1
 TEST_COMMAND=$2
 
-# Disk hygiene knobs. Reclaim space only once the Docker root filesystem crosses
-# DISK_USAGE_THRESHOLD percent, and cap the shared BuildKit cache at
-# BUILDKIT_CACHE_MAX so subsequent builds keep reusing the hottest layers.
-DISK_USAGE_THRESHOLD=${DISK_USAGE_THRESHOLD:-70}
-BUILDKIT_CACHE_MAX=${BUILDKIT_CACHE_MAX:-80GB}
-
-# Reclaim disk only when the host is under pressure. We trim (not purge) the
-# shared BuildKit cache so cross-job/cross-agent reuse stays intact, and only
-# touch dangling images; other agents' uniquely tagged images are left alone.
-prune_if_disk_pressure() {
-    local docker_root disk_usage
-    docker_root=$(docker info -f '{{.DockerRootDir}}' 2>/dev/null || true)
-    if [ -z "$docker_root" ]; then
-        return 0
-    fi
-    disk_usage=$(df "$docker_root" 2>/dev/null | tail -1 | awk '{print $5}' | tr -d '%')
-    if [ "${disk_usage:-0}" -gt "$DISK_USAGE_THRESHOLD" ]; then
-        echo "--- :broom: Disk usage ${disk_usage}% exceeds ${DISK_USAGE_THRESHOLD}%, reclaiming space"
-        docker image prune -f || true
-        docker builder prune -f --keep-storage="$BUILDKIT_CACHE_MAX" || true
-    else
-        echo "Disk usage ${disk_usage:-unknown}% within ${DISK_USAGE_THRESHOLD}% threshold; skipping prune"
-    fi
-}
-
-# Always drop this agent's image once the job ends (the default builder never
-# uses it as a cache source, so removing it costs no rebuild speed), then
-# reclaim space if needed. Guard every docker call with `|| true` so the trap
-# never overrides the test's exit code.
-cleanup() {
-    docker image rm -f "$IMAGE_NAME" || true
-    prune_if_disk_pressure
-}
-trap cleanup EXIT
-
-# Free space up front so a nearly-full host doesn't fail the build.
-prune_if_disk_pressure
-
 # building the docker image
 echo "--- :docker: Building Docker image"
 docker build --progress plain --tag "$IMAGE_NAME" --target vllm-test -f docker/Dockerfile.cpu .

.buildkite/scripts/hardware_ci/run-intel-ci-test.sh

@@ -1,51 +0,0 @@
-#!/bin/bash
-
-set -euo pipefail
-
-test_suite="${1:-}"
-
-if [[ -z "${test_suite}" ]]; then
-  echo "Usage: $0 <example|v1|server>" >&2
-  exit 1
-fi
-
-case "${test_suite}" in
-  example)
-    pip install tblib==3.1.0
-
-    python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager
-    python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 -O3 -cc.cudagraph_mode=NONE
-    python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager -tp 2 --distributed-executor-backend mp
-    python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --attention-backend=TRITON_ATTN
-    python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --quantization fp8
-    python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --kv-cache-dtype fp8
-    python3 examples/basic/offline_inference/generate.py --model nvidia/Llama-3.1-8B-Instruct-FP8 --block-size 64 --enforce-eager --quantization modelopt --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --max-model-len 4096
-    python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --block-size 64 --enforce-eager --max-model-len 8192
-    python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2
-    python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 --enable-expert-parallel
-    python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --max-model-len 8192
-    ;;
-  v1)
-    cd tests
-
-    pytest -v -s v1/core --ignore=v1/core/test_reset_prefix_cache_e2e.py --ignore=v1/core/test_scheduler_e2e.py
-    pytest -v -s v1/engine --ignore=v1/engine/test_output_processor.py
-    pytest -v -s v1/sample --ignore=v1/sample/test_logprobs.py --ignore=v1/sample/test_logprobs_e2e.py -k "not test_topk_only and not test_topp_only and not test_topk_and_topp"
-    pytest -v -s v1/worker --ignore=v1/worker/test_gpu_model_runner.py --ignore=v1/worker/test_worker_memory_snapshot.py
-    pytest -v -s v1/structured_output
-    pytest -v -s v1/test_serial_utils.py
-    pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py --ignore=v1/spec_decode/test_speculators_correctness.py
-    pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_multi_connector.py --ignore=v1/kv_connector/unit/test_example_connector.py --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py --ignore=v1/kv_connector/unit/test_offloading_connector.py
-    ;;
-  server)
-    pip install av
-    cd tests
-
-    pytest -v -s entrypoints/multimodal/openai/chat_completion/test_audio_in_video.py
-    pytest -v -s benchmarks/test_serve_cli.py
-    ;;
-  *)
-    echo "Unknown Intel test suite: ${test_suite}" >&2
-    exit 1
-    ;;
-esac

.buildkite/scripts/hardware_ci/run-intel-test.sh

@@ -243,10 +243,8 @@ container_name="xpu_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head
 
 # ---- Command source selection ----
 commands=""
-commands_source=""
 if [[ -n "${VLLM_TEST_COMMANDS:-}" ]]; then
   commands="${VLLM_TEST_COMMANDS}"
-  commands_source="env"
   echo "Commands sourced from VLLM_TEST_COMMANDS (quoting preserved)"
 elif [[ $# -gt 0 ]]; then
   all_yaml=true
@@ -305,12 +303,8 @@ if [[ -z "$commands" ]]; then
 fi
 
 echo "Raw commands: $commands"
-if [[ "$commands_source" != "env" ]]; then
+commands=$(re_quote_pytest_markers "$commands")
-  commands=$(re_quote_pytest_markers "$commands")
+echo "After re-quoting: $commands"
-  echo "After re-quoting: $commands"
-else
-  echo "Skipping re-quoting for VLLM_TEST_COMMANDS input"
-fi
 commands=$(apply_intel_test_overrides "$commands")
 echo "Final commands: $commands"
 
@@ -330,6 +324,23 @@ IMAGE="${IMAGE_TAG_XPU:-${image_name}}"
 
 echo "Using image: ${IMAGE}"
 
+if docker image inspect "${IMAGE}" >/dev/null 2>&1; then
+  echo "Image already exists locally, skipping pull"
+else
+  echo "Image not found locally, waiting for lock..."
+
+  flock /tmp/docker-pull.lock bash -c "
+    if docker image inspect '${IMAGE}' >/dev/null 2>&1; then
+      echo 'Image already pulled by another runner'
+    else
+      echo 'Pulling image...'
+      timeout 900 docker pull '${IMAGE}'
+    fi
+  "
+
+  echo "Pull step completed"
+fi
+
 remove_docker_container() {
   docker rm -f "${container_name}" || true
 }
@@ -346,12 +357,9 @@ export HF_TOKEN ZE_AFFINITY_MASK
 
 {
   flock 9
-  if docker image inspect "${IMAGE}" >/dev/null 2>&1; then
+  if ! docker image inspect "${IMAGE}" >/dev/null 2>&1; then
-    echo "Image already exists locally, skipping pull"
+    echo 'Image missing before container creation, pulling again...'
-  else
-    echo "Image not found locally, pulling image..."
     timeout 900 docker pull "${IMAGE}"
-    echo "Pull step completed"
   fi
 
   docker create \
@@ -364,12 +372,10 @@ export HF_TOKEN ZE_AFFINITY_MASK
     --entrypoint='' \
     -e HF_TOKEN \
     -e ZE_AFFINITY_MASK \
-    -e BUILDKITE_PARALLEL_JOB \
-    -e BUILDKITE_PARALLEL_JOB_COUNT \
     -e CMDS \
     --name "${container_name}" \
     "${IMAGE}" \
-    bash -c 'set -e; source /opt/intel/oneapi/setvars.sh --force; source /opt/intel/oneapi/ccl/2021.15/env/vars.sh --force; echo "ZE_AFFINITY_MASK is ${ZE_AFFINITY_MASK:-}"; eval "$CMDS"' \
+    bash -c 'set -e; echo "ZE_AFFINITY_MASK is ${ZE_AFFINITY_MASK:-}"; eval "$CMDS"' \
     >/dev/null
 } 9>/tmp/docker-pull.lock
 

.buildkite/scripts/install-kv-connectors.sh

@@ -4,11 +4,6 @@
 
 set -euo pipefail
 
-if python3 -c "import torch; raise SystemExit(0 if torch.version.hip is not None else 1)"; then
-    uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
-    exit 0
-fi
-
 REQUIREMENTS_FILE="${KV_CONNECTORS_REQUIREMENTS:-/vllm-workspace/requirements/kv_connectors.txt}"
 
 uv pip install --system -r "${REQUIREMENTS_FILE}"

.buildkite/scripts/run-rust-frontend-cargo-ci.sh

@@ -90,16 +90,6 @@ install_cargo_sort() {
   cargo binstall --no-confirm cargo-sort
 }
 
-install_cargo_deny() {
-  if command -v cargo-deny >/dev/null 2>&1; then
-    return
-  fi
-
-  log_section "Installing cargo-deny"
-  install_cargo_binstall
-  cargo binstall --no-confirm cargo-deny
-}
-
 install_cargo_nextest() {
   if command -v cargo-nextest >/dev/null 2>&1; then
     return
@@ -120,39 +110,8 @@ install_uv() {
     | env UV_INSTALL_DIR="$CARGO_HOME/bin" sh
 }
 
-setup_pyo3_python() {
-  local python_version="${PYO3_PYTHON_VERSION:-3.12}"
-
-  log_section "Installing Python ${python_version} for PyO3 tests"
-  uv python install "$python_version"
-  PYO3_PYTHON="$(uv python find \
-    --managed-python \
-    --no-project \
-    --resolve-links \
-    "$python_version")"
-  export PYO3_PYTHON
-
-  local python_libdir
-  python_libdir="$("$PYO3_PYTHON" - <<'PY'
-import pathlib
-import sysconfig
-
-libdir = pathlib.Path(sysconfig.get_config_var("LIBDIR"))
-ldlibrary = sysconfig.get_config_var("LDLIBRARY")
-assert sysconfig.get_config_var("Py_ENABLE_SHARED") == 1
-assert ldlibrary
-assert (libdir / ldlibrary).exists(), libdir / ldlibrary
-print(libdir)
-PY
-)"
-
-  export LD_LIBRARY_PATH="${python_libdir}:${LD_LIBRARY_PATH:-}"
-  export LIBRARY_PATH="${python_libdir}:${LIBRARY_PATH:-}"
-}
-
 run_style_clippy() {
   install_cargo_sort
-  install_cargo_deny
 
   log_section "Checking Rust formatting"
   cargo fmt --manifest-path rust/Cargo.toml --all -- --check
@@ -160,13 +119,6 @@ run_style_clippy() {
   log_section "Checking Cargo.toml ordering"
   cargo sort --workspace --check rust
 
-  log_section "Checking Rust dependency bans"
-  cargo deny \
-    --manifest-path rust/Cargo.toml \
-    check \
-    --config rust/deny.toml \
-    bans
-
   log_section "Running clippy"
   cargo clippy \
     --manifest-path rust/Cargo.toml \
@@ -180,7 +132,6 @@ run_style_clippy() {
 
 run_tests() {
   install_uv
-  setup_pyo3_python
   install_cargo_nextest
 
   log_section "Running cargo nextest"

.buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_prefetch_offload.sh

@@ -33,14 +33,6 @@ if [[ -n "${ATTENTION_BACKEND:-}" ]]; then
   EXTRA_ARGS+=(--attention-backend "${ATTENTION_BACKEND}")
 fi
 
-# ROCm: run eager to avoid intermittent HIP-graph decode corruption.
-# See https://github.com/ROCm/clr/issues/279
-# TODO(aarushjain29): Revert after TheRock 7.14
-if command -v rocm-smi &> /dev/null || command -v amd-smi &> /dev/null || [[ -d /opt/rocm ]] || [[ -n "${ROCM_PATH:-}" ]]; then
-  echo "ROCm platform detected: adding --enforce-eager to avoid HIP-graph decode corruption"
-  EXTRA_ARGS+=(--enforce-eager)
-fi
-
 cleanup() {
   if [[ -n "${SERVER_PID:-}" ]] && kill -0 "${SERVER_PID}" 2>/dev/null; then
     kill "${SERVER_PID}" 2>/dev/null || true

.buildkite/test_areas/attention.yaml

@@ -15,7 +15,7 @@ steps:
     - pytest -v -s v1/attention
   mirror:
     amd:
-      device: mi325_1
+      device: mi300_1
       timeout_in_minutes: 70
       depends_on:
       - image-build-amd

.buildkite/test_areas/basic_correctness.yaml

@@ -16,9 +16,3 @@ steps:
   - pytest -v -s basic_correctness/test_mem.py
   - pytest -v -s basic_correctness/test_basic_correctness.py
   - pytest -v -s basic_correctness/test_cpu_offload.py
-  mirror:
-    amd:
-      device: mi325_1
-      timeout_in_minutes: 50
-      depends_on:
-      - image-build-amd

.buildkite/test_areas/benchmarks.yaml

@@ -11,11 +11,6 @@ steps:
   - tests/benchmarks/
   commands:
   - pytest -v -s benchmarks/
-  mirror:
-    amd:
-      device: mi300_1
-      depends_on:
-      - image-build-amd
 
 - label: Attention Benchmarks Smoke Test (B200)
   key: attention-benchmarks-smoke-test-b200
@@ -28,4 +23,4 @@ steps:
   - benchmarks/attention_benchmarks/
   - vllm/v1/attention/
   commands:
-  - python3 benchmarks/attention_benchmarks/benchmark.py --backends flash flashinfer --batch-specs "8q1s1k"
+  - python3 benchmarks/attention_benchmarks/benchmark.py --backends flash flashinfer --batch-specs "8q1s1k" --repeats 1 --warmup-iters 1

.buildkite/test_areas/cuda.yaml

@@ -2,8 +2,8 @@ group: CUDA
 depends_on: 
   - image-build
 steps:
-- label: Platform Tests
+- label: Platform Tests (CUDA)
-  key: platform-tests
+  key: platform-tests-cuda
   timeout_in_minutes: 15
   device: h200_18gb
   source_file_dependencies:

.buildkite/test_areas/disaggregated.yaml

@@ -13,20 +13,6 @@ steps:
   commands:
     - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
     - bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
-  mirror:
-    amd:
-      device: mi300_4
-      timeout_in_minutes: 110
-      depends_on:
-        - image-build-amd
-      source_file_dependencies:
-        - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
-        - tests/v1/kv_connector/nixl_integration/
-        - vllm/platforms/rocm.py
-      commands:
-        - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
-        - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
-
 - label: Distributed FlashInfer NixlConnector PD accuracy (4 GPUs)
   key: distributed-flashinfer-nixlconnector-pd-accuracy-4-gpus
   timeout_in_minutes: 30
@@ -50,19 +36,6 @@ steps:
   commands:
     - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
     - DP_EP=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
-  mirror:
-    amd:
-      device: mi300_4
-      timeout_in_minutes: 50
-      depends_on:
-        - image-build-amd
-      source_file_dependencies:
-        - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
-        - tests/v1/kv_connector/nixl_integration/
-        - vllm/platforms/rocm.py
-      commands:
-        - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
-        - DP_EP=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
 
 - label: CrossLayer KV layout Distributed NixlConnector PD accuracy tests (4 GPUs)
   key: crosslayer-kv-layout-distributed-nixlconnector-pd-accuracy-tests-4-gpus
@@ -75,19 +48,6 @@ steps:
   commands:
     - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
     - CROSS_LAYERS_BLOCKS=True bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
-  mirror:
-    amd:
-      device: mi300_4
-      timeout_in_minutes: 110
-      depends_on:
-        - image-build-amd
-      source_file_dependencies:
-        - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
-        - tests/v1/kv_connector/nixl_integration/
-        - vllm/platforms/rocm.py
-      commands:
-        - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
-        - CROSS_LAYERS_BLOCKS=True ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
 
 - label: Hybrid SSM NixlConnector PD accuracy tests (4 GPUs)
   key: hybrid-ssm-nixlconnector-pd-accuracy-tests-4-gpus
@@ -100,33 +60,6 @@ steps:
   commands:
     - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
     - HYBRID_SSM=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
-  mirror:
-    amd:
-      device: mi300_4
-      timeout_in_minutes: 60
-      depends_on:
-        - image-build-amd
-      source_file_dependencies:
-        - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
-        - tests/v1/kv_connector/nixl_integration/
-        - vllm/platforms/rocm.py
-      commands:
-        - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
-        - HYBRID_SSM=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
-
-- label: Hybrid SSM NixlConnector PD prefix cache test (2 GPUs)
-  key: hybrid-ssm-nixlconnector-pd-prefix-cache-2-gpus
-  timeout_in_minutes: 25
-  working_dir: "/vllm-workspace/tests"
-  num_devices: 2
-  source_file_dependencies:
-    - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
-    - vllm/v1/core/sched/
-    - vllm/v1/core/kv_cache_coordinator.py
-    - tests/v1/kv_connector/nixl_integration/
-  commands:
-    - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
-    - bash v1/kv_connector/nixl_integration/run_mamba_prefix_cache_test.sh
 
 - label: MultiConnector (Nixl+Offloading) PD accuracy (2 GPUs)
   key: multiconnector-nixl-offloading-pd-accuracy-2-gpus
@@ -156,20 +89,6 @@ steps:
   commands:
     - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
     - bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh
-  mirror:
-    amd:
-      device: mi300_2
-      timeout_in_minutes: 60
-      depends_on:
-        - image-build-amd
-      source_file_dependencies:
-        - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
-        - vllm/v1/worker/kv_connector_model_runner_mixin.py
-        - tests/v1/kv_connector/nixl_integration/
-        - vllm/platforms/rocm.py
-      commands:
-        - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
-        - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh
 
 - label: MultiConnector (Nixl+Offloading) PD edge cases (2 GPUs)
   key: multiconnector-nixl-offloading-pd-edge-cases-2-gpus

.buildkite/test_areas/distributed.yaml

@@ -37,21 +37,6 @@ steps:
   - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
   - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py
   - DP_SIZE=2 pytest -v -s entrypoints/openai/test_multi_api_servers.py
-  mirror:
-    amd:
-      device: mi300_2
-      depends_on:
-      - image-build-amd
-      source_file_dependencies:
-      - vllm/distributed/
-      - vllm/engine/
-      - vllm/executor/
-      - vllm/worker/worker_base.py
-      - vllm/v1/engine/
-      - vllm/v1/worker/
-      - tests/v1/distributed
-      - tests/entrypoints/openai/test_multi_api_servers.py
-      - vllm/platforms/rocm.py
 
 - label: Distributed Compile + RPC Tests (2 GPUs)
   key: distributed-compile-rpc-tests-2-gpus
@@ -174,8 +159,8 @@ steps:
   # test multi-node TP with multiproc executor (simulated on single node)
   - pytest -v -s distributed/test_multiproc_executor.py::test_multiproc_executor_multi_node
 
-- label: Distributed Tests (8xH100)
+- label: Distributed Tests (8 GPUs)(H100)
-  key: distributed-tests-8xh100
+  key: distributed-tests-8-gpus-h100
   timeout_in_minutes: 10
   device: h100
   num_devices: 8
@@ -195,8 +180,8 @@ steps:
   # test with torchrun tp=2 and dp=4 with ep
   - torchrun --nproc-per-node=8 ../examples/features/torchrun/torchrun_dp_example_offline.py --tp-size=2 --pp-size=1 --dp-size=4 --enable-ep
 
-- label: Distributed Tests (4xA100)
+- label: Distributed Tests (4 GPUs)(A100)
-  key: distributed-tests-4xa100
+  key: distributed-tests-4-gpus-a100
   device: a100
   optional: true
   num_devices: 4
@@ -210,8 +195,8 @@ steps:
   - TARGET_TEST_SUITE=A100 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
   - pytest -v -s -x lora/test_mixtral.py
 
-- label: Distributed Tests (2xH100-2xMI300)
+- label: Distributed Tests (2 GPUs)(H100)
-  key: distributed-tests-2xh100-2xmi300
+  key: distributed-tests-2-gpus-h100
   timeout_in_minutes: 15
   device: h100
   optional: true
@@ -225,8 +210,8 @@ steps:
     - VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer.py
     - pytest -v -s tests/distributed/test_packed_tensor.py
 
-- label: Distributed Tests (2xB200)
+- label: Distributed Tests (2 GPUs)(B200)
-  key: distributed-tests-2xb200
+  key: distributed-tests-2-gpus-b200
   device: b200-k8s
   optional: true
   working_dir: "/vllm-workspace/"

.buildkite/test_areas/e2e_integration.yaml

@@ -2,8 +2,8 @@ group: E2E Integration
 depends_on: 
   - image-build
 steps:
-- label: DeepSeek V2-Lite Sync EPLB Accuracy (4xH100)
+- label: DeepSeek V2-Lite Sync EPLB Accuracy
-  key: deepseek-v2-lite-sync-eplb-accuracy-4xh100
+  key: deepseek-v2-lite-sync-eplb-accuracy
   timeout_in_minutes: 60
   device: h100
   optional: true
@@ -12,8 +12,8 @@ steps:
   commands:
   - bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh 0.25 200 8010
 
-- label: Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy (4xH100)
+- label: Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy
-  key: qwen3-30b-a3b-fp8-block-sync-eplb-accuracy-4xh100
+  key: qwen3-30b-a3b-fp8-block-sync-eplb-accuracy
   timeout_in_minutes: 60
   device: h100
   optional: true
@@ -22,8 +22,8 @@ steps:
   commands:
   - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020
 
-- label: Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy (2xB200)
+- label: Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy (B200)
-  key: qwen3-30b-a3b-fp8-block-sync-eplb-accuracy-2xb200
+  key: qwen3-30b-a3b-fp8-block-sync-eplb-accuracy-b200
   timeout_in_minutes: 60
   device: b200-k8s
   optional: true

.buildkite/test_areas/engine.yaml

@@ -28,7 +28,7 @@ steps:
   - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py test_jit_monitor.py
   mirror:
     amd:
-      device: mi325_1
+      device: mi300_1
       timeout_in_minutes: 60
       depends_on:
       - image-build-amd
@@ -44,7 +44,7 @@ steps:
     - pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py
   mirror:
     amd:
-      device: mi325_1
+      device: mi300_1
       timeout_in_minutes: 40
       depends_on:
       - image-build-amd
@@ -74,16 +74,6 @@ steps:
     - tests/v1/e2e/general/
   commands:
     - pytest -v -s v1/e2e/general --ignore v1/e2e/general/test_async_scheduling.py
-  mirror:
-    amd:
-      device: mi250_1
-      timeout_in_minutes: 35
-      depends_on:
-      - image-build-amd
-      source_file_dependencies:
-      - vllm/v1/
-      - tests/v1/e2e/general/
-      - vllm/platforms/rocm.py
 
 - label: V1 e2e (2 GPUs)
   key: v1-e2e-2-gpus
@@ -112,11 +102,6 @@ steps:
   commands:
     # Only run tests that need exactly 2 GPUs
     - pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "tensor_parallelism"
-  mirror:
-    amd:
-      device: mi300_2
-      depends_on:
-      - image-build-amd
 
 - label: V1 e2e (4 GPUs)
   key: v1-e2e-4-gpus

.buildkite/test_areas/entrypoints.yaml

@@ -8,11 +8,10 @@ steps:
   working_dir: "/vllm-workspace/tests"
   source_file_dependencies:
   - vllm/entrypoints
-  - tests/entrypoints/unit_tests
+  - tests/entrypoints/
-  - tests/entrypoints/weight_transfer
   commands:
-  - pytest -v -s entrypoints/unit_tests
+  - pytest -v -s entrypoints/openai/tool_parsers
-  - pytest -v -s entrypoints/weight_transfer
+  - pytest -v -s entrypoints/ --ignore=entrypoints/llm --ignore=entrypoints/openai --ignore=entrypoints/serve --ignore=entrypoints/test_chat_utils.py  --ignore=entrypoints/pooling --ignore=entrypoints/speech_to_text --ignore=tests/entrypoints/generate
 
 - label: Entrypoints Integration (LLM)
   key: entrypoints-integration-llm
@@ -29,32 +28,10 @@ steps:
   mirror:
     amd:
       device: mi325_1
-      # TODO(akaratza): Test after Torch >= 2.12 bump
-      soft_fail: true
       depends_on:
       - image-build-amd
 
-- label: Entrypoints Integration (API Server)
+- label: Entrypoints Integration (API Server openai - Part 1)
-  key: entrypoints-integration-api-server
-  device: h200_35gb
-  timeout_in_minutes: 130
-  working_dir: "/vllm-workspace/tests"
-  source_file_dependencies:
-  - vllm/
-  - tests/entrypoints/serve
-  - tests/entrypoints/scale_out
-  commands:
-  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-  - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc
-  - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc
-  - pytest -v -s entrypoints/scale_out
-  mirror:
-    amd:
-      device: mi325_1
-      depends_on:
-      - image-build-amd
-
-- label: Entrypoints Integration (API Server OpenAI - Part 1)
   key: entrypoints-integration-api-server-openai-part-1
   timeout_in_minutes: 50
   working_dir: "/vllm-workspace/tests"
@@ -64,15 +41,15 @@ steps:
   - tests/entrypoints/test_chat_utils
   commands:
   - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-  - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/correctness
+  - pytest -v -s entrypoints/openai/chat_completion --ignore=entrypoints/openai/chat_completion/test_oot_registration.py
   mirror:
     amd:
-      device: mi325_1
+      device: mi300_1
       timeout_in_minutes: 80
       depends_on:
       - image-build-amd
 
-- label: Entrypoints Integration (API Server OpenAI - Part 2)
+- label: Entrypoints Integration (API Server openai - Part 2)
   key: entrypoints-integration-api-server-openai-part-2
   timeout_in_minutes: 50
   working_dir: "/vllm-workspace/tests"
@@ -80,32 +57,32 @@ steps:
   - vllm/
   - tests/entrypoints/openai
   - tests/entrypoints/test_chat_utils
+  - tests/entrypoints/generate
+  - tests/tool_use
   commands:
-  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-  - pytest -v -s entrypoints/openai/chat_completion
   - pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py
+  - pytest -v -s entrypoints/test_chat_utils.py
+  - pytest -v -s entrypoints/generate
+  - pytest -v -s tool_use
   mirror:
     amd:
-      device: mi325_1
+      device: mi300_1
-      timeout_in_minutes: 80
+      timeout_in_minutes: 60
       depends_on:
       - image-build-amd
 
-- label: Entrypoints Integration (API Server Generate)
+- label: Entrypoints Integration (API Server openai - Part 3)
-  key: entrypoints-integration-api-server-generate
+  key: entrypoints-integration-api-server-openai-part-3
   timeout_in_minutes: 50
+  device: h200_18gb
   working_dir: "/vllm-workspace/tests"
   source_file_dependencies:
   - vllm/
-  - tests/tool_use
+  - tests/entrypoints/openai
-  - tests/entrypoints/tool_parsers
+  - tests/entrypoints/test_chat_utils
-  - tests/entrypoints/anthropic
-  - tests/entrypoints/generate
   commands:
-  - pytest -v -s tool_use
+  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-  - pytest -v -s entrypoints/tool_parsers
+  - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/correctness/ --ignore=entrypoints/openai/tool_parsers/ --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/test_multi_api_servers.py
-  - pytest -v -s entrypoints/generate
-  - pytest -v -s entrypoints/anthropic
   mirror:
     amd:
       device: mi325_1
@@ -113,15 +90,23 @@ steps:
       depends_on:
       - image-build-amd
 
-- label: Entrypoints Integration (Responses API)
+- label: Entrypoints Integration (API Server 2)
-  key: entrypoints-integration-responses-api
+  device: h200_35gb
-  timeout_in_minutes: 50
+  key: entrypoints-integration-api-server-2
+  timeout_in_minutes: 130
   working_dir: "/vllm-workspace/tests"
   source_file_dependencies:
   - vllm/
-  - tests/entrypoints/openai/responses
+  - tests/entrypoints/serve
   commands:
-  - pytest -v -s entrypoints/openai/responses
+  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+  - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc
+  - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc
+  mirror:
+    amd:
+      device: mi300_1
+      depends_on:
+      - image-build-amd
 
 - label: Entrypoints Integration (Speech to Text)
   device: h200_35gb
@@ -135,18 +120,6 @@ steps:
   - export VLLM_WORKER_MULTIPROC_METHOD=spawn
   - pytest -v -s entrypoints/speech_to_text
 
-- label: Entrypoints Integration (Multimodal)
-  device: h200_35gb
-  key: entrypoints-integration-multimodal
-  timeout_in_minutes: 50
-  working_dir: "/vllm-workspace/tests"
-  source_file_dependencies:
-  - vllm/
-  - tests/entrypoints/multimodal
-  commands:
-  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-  - pytest -v -s entrypoints/multimodal
-
 - label: Entrypoints Integration (Pooling)
   key: entrypoints-integration-pooling
   timeout_in_minutes: 50
@@ -158,6 +131,16 @@ steps:
   - export VLLM_WORKER_MULTIPROC_METHOD=spawn
   - pytest -v -s entrypoints/pooling
 
+- label: Entrypoints Integration (Responses API)
+  key: entrypoints-integration-responses-api
+  timeout_in_minutes: 50
+  working_dir: "/vllm-workspace/tests"
+  source_file_dependencies:
+  - vllm/
+  - tests/entrypoints/openai/responses
+  commands:
+  - pytest -v -s entrypoints/openai/responses
+
 - label: OpenAI API Correctness
   key: openai-api-correctness
   timeout_in_minutes: 30
@@ -169,12 +152,13 @@ steps:
   - pytest -s entrypoints/openai/correctness/
   mirror:
     amd:
-      device: mi325_1
+      device: mi300_1
       depends_on:
       - image-build-amd
       source_file_dependencies:
       - csrc/
       - vllm/entrypoints/openai/
+      - vllm/model_executor/models/whisper.py
       - vllm/model_executor/layers/
       - vllm/v1/attention/backends/
       - vllm/v1/attention/selector.py

.buildkite/test_areas/expert_parallelism.yaml

@@ -14,16 +14,6 @@ steps:
   commands:
   - pytest -v -s distributed/test_eplb_algo.py
   - pytest -v -s distributed/test_eplb_utils.py
-  mirror:
-    amd:
-      device: mi300_1
-      depends_on:
-      - image-build-amd
-      source_file_dependencies:
-      - vllm/distributed/eplb
-      - tests/distributed/test_eplb_algo.py
-      - tests/distributed/test_eplb_utils.py
-      - vllm/platforms/rocm.py
 
 - label: EPLB Execution # 17min
   key: eplb-execution

.buildkite/test_areas/kernels.yaml

@@ -21,9 +21,8 @@ steps:
   - csrc/
   - tests/kernels/core
   - tests/kernels/test_concat_mla_q.py
-  - tests/kernels/test_fused_qk_norm_rope_gate.py
   commands:
-    - pytest -v -s kernels/core --ignore=kernels/core/test_minimax_reduce_rms.py  kernels/test_concat_mla_q.py kernels/test_fused_qk_norm_rope_gate.py
+    - pytest -v -s kernels/core --ignore=kernels/core/test_minimax_reduce_rms.py  kernels/test_concat_mla_q.py
 
 - label: Kernels MiniMax Reduce RMS Test (2 GPUs)
   key: kernels-minimax-reduce-rms-test-2-gpus
@@ -47,10 +46,8 @@ steps:
   - csrc/fused_deepseek_v4_qnorm_rope_kv_insert_kernel.cu
   - vllm/models/deepseek_v4/common/ops/
   - tests/kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py
-  - tests/kernels/test_top_k_per_row.py # it runs on Blackwell too - some kernels have arch-specific optimizations
   commands:
     - pytest -v -s kernels/test_fused_deepseek_v4_*.py
-    - pytest -v -s kernels/test_top_k_per_row.py
 
 - label: Deepseek V4 Kernel Test (B200)
   key: deepseek-v4-kernel-test-b200
@@ -76,33 +73,6 @@ steps:
   commands:
     - pytest -v -s kernels/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
   parallelism: 2
-  mirror:
-    amd:
-      device: mi325_1
-      timeout_in_minutes: 55
-      depends_on:
-      - image-build-amd
-      source_file_dependencies:
-      - csrc/attention/
-      - vllm/v1/attention
-      - vllm/model_executor/layers/attention
-      - tests/kernels/attention
-      - vllm/_aiter_ops.py
-      - vllm/envs.py
-      - vllm/platforms/rocm.py
-
-- label: Kernels Attention DiffKV Test (H100)
-  key: kernels-attention-diffkv-test-h100
-  timeout_in_minutes: 20
-  device: h100
-  num_devices: 1
-  source_file_dependencies:
-  - vllm/v1/attention/ops/triton_unified_attention_diffkv.py
-  - vllm/v1/attention/backends/triton_attn_diffkv.py
-  - vllm/v1/attention/backends/flash_attn_diffkv.py
-  - tests/kernels/attention/test_triton_unified_attention_diffkv.py
-  commands:
-    - pytest -v -s kernels/attention/test_triton_unified_attention_diffkv.py
 
 - label: Kernels Quantization Test %N
   key: kernels-quantization-test
@@ -116,11 +86,10 @@ steps:
   parallelism: 2
   mirror:
     amd:
-      device: mi325_1
+      device: mi300_1
       source_file_dependencies:
       - csrc/quantization/
       - vllm/model_executor/layers/quantization
-      - vllm/config/
       - tests/kernels/quantization
       - tests/kernels/quantization/test_rocm_skinny_gemms.py
       - vllm/_aiter_ops.py
@@ -144,22 +113,6 @@ steps:
     - pytest -v -s kernels/moe --ignore=kernels/moe/test_modular_oai_triton_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
     - pytest -v -s kernels/moe/test_modular_oai_triton_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
   parallelism: 5
-  mirror:
-    amd:
-      device: mi325_1
-      timeout_in_minutes: 50
-      source_file_dependencies:
-      - csrc/quantization/cutlass_w8a8/moe/
-      - csrc/moe/
-      - tests/kernels/moe
-      - vllm/model_executor/layers/fused_moe/
-      - vllm/distributed/device_communicators/
-      - vllm/envs.py
-      - vllm/config
-      - vllm/_aiter_ops.py
-      - vllm/platforms/rocm.py
-      depends_on:
-      - image-build-amd
 
 - label: Kernels Mamba Test
   key: kernels-mamba-test
@@ -270,12 +223,12 @@ steps:
   - vllm/utils/import_utils.py
   - tests/kernels/helion/
   commands:
-    - pip install helion==1.1.0
+    - pip install helion==1.0.0
     - pytest -v -s kernels/helion/
 
  
-- label: Kernels FP8 MoE Test (1xH100)
+- label: Kernels FP8 MoE Test (1 H100)
-  key: kernels-fp8-moe-test-1xh100
+  key: kernels-fp8-moe-test-1-h100
   timeout_in_minutes: 90
   device: h100
   num_devices: 1
@@ -291,8 +244,8 @@ steps:
     - pytest -v -s kernels/moe/test_triton_moe_no_act_mul.py
     - pytest -v -s kernels/moe/test_triton_moe_ptpc_fp8.py
 
-- label: Kernels FP8 MoE Test (2xH100)
+- label: Kernels FP8 MoE Test (2 H100s)
-  key: kernels-fp8-moe-test-2xh100
+  key: kernels-fp8-moe-test-2-h100s
   timeout_in_minutes: 90
   device: h100
   num_devices: 2
@@ -346,4 +299,3 @@ steps:
   - vllm/config
   commands:
     - pytest -v -s kernels/moe/test_moe_layer.py
-    - pytest -v -s kernels/moe/test_deepep_v2_moe.py

.buildkite/test_areas/lm_eval.yaml

@@ -14,7 +14,7 @@ steps:
   - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small.txt
   mirror:
     amd:
-      device: mi325_1
+      device: mi300_1
       timeout_in_minutes: 55
       depends_on:
       - image-build-amd
@@ -28,8 +28,7 @@ steps:
       - vllm/_aiter_ops.py
       - vllm/platforms/rocm.py
 
-# - label: LM Eval Large Models (4xA100)
+# - label: LM Eval Large Models (4 GPUs)(A100)
-#   key: lm-eval-large-models-4xa100
 #   device: a100
 #   optional: true
 #   num_devices: 4
@@ -41,8 +40,8 @@ steps:
 #   - export VLLM_WORKER_MULTIPROC_METHOD=spawn
 #   - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large.txt --tp-size=4
 
-- label: LM Eval Large Models (4xH100)
+- label: LM Eval Large Models (4 GPUs)(H100)
-  key: lm-eval-large-models-4xh100
+  key: lm-eval-large-models-4-gpus-h100
   device: h100
   optional: true
   num_devices: 4
@@ -54,8 +53,8 @@ steps:
     - export VLLM_USE_DEEP_GEMM=0  # We found Triton is faster than DeepGEMM for H100
     - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-hopper.txt --tp-size=4
 
-- label: LM Eval Small Models (2xB200)
+- label: LM Eval Small Models (B200)
-  key: lm-eval-small-models-2xb200
+  key: lm-eval-small-models-b200
   timeout_in_minutes: 120
   device: b200-k8s
   optional: true
@@ -65,20 +64,8 @@ steps:
   commands:
   - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell.txt
 
-- label: LM Eval Small Models (2xL4)
+- label: LM Eval Large Models (B200, EP)
-  key: lm-eval-small-models-tp
+  key: lm-eval-large-models-b200-ep
-  timeout_in_minutes: 10
-  num_devices: 2
-  optional: true
-  source_file_dependencies:
-  - csrc/
-  - vllm/model_executor/layers/quantization
-  autorun_on_main: true
-  commands:
-  - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small-tp.txt
-
-- label: LM Eval Large Models EP (2xB200)
-  key: lm-eval-large-models-ep-2xb200
   timeout_in_minutes: 120
   device: b200-k8s
   optional: true
@@ -89,8 +76,8 @@ steps:
   commands:
   - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell-ep.txt
 
-- label: LM Eval Qwen3.5 Models (2xB200)
+- label: LM Eval Qwen3.5 Models (B200)
-  key: lm-eval-qwen3-5-models-2xb200
+  key: lm-eval-qwen3-5-models-b200
   timeout_in_minutes: 120
   device: b200-k8s
   optional: true
@@ -106,24 +93,14 @@ steps:
   commands:
   - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-qwen35-blackwell.txt
 
-- label: LM Eval Large Models (8xH200)
+- label: LM Eval Large Models (H200)
-  key: lm-eval-large-models-8xh200
+  key: lm-eval-large-models-h200
   timeout_in_minutes: 60
   device: h200
   optional: true
   num_devices: 8
   commands:
     - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200.txt
-  mirror:
-    amd:
-      device: mi300_8
-      timeout_in_minutes: 180
-      depends_on:
-      - image-build-amd
-      commands:
-      - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-      - export PYTORCH_ROCM_ARCH=gfx942 # Limit Quark compilation to save time
-      - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx.txt
 
 - label: MoE Refactor Integration Test (H100 - TEMPORARY)
   key: moe-refactor-integration-test-h100-temporary
@@ -149,49 +126,6 @@ steps:
   commands:
     - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor-dp-ep/config-b200.txt
 
-- label: LM Eval Humming (A100 - TEMPORARY)
-  key: lm-eval-humming-a100
-  timeout_in_minutes: 30
-  device: a100
-  optional: true
-  num_devices: 1
-  source_file_dependencies:
-  - vllm/model_executor/layers/quantization/humming.py
-  - vllm/model_executor/layers/quantization/utils/humming_utils.py
-  - vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
-  - vllm/model_executor/layers/fused_moe/oracle/mxfp4.py
-  commands:
-  - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config.txt
-
-- label: LM Eval Humming (H100 - TEMPORARY)
-  key: lm-eval-humming-h100
-  timeout_in_minutes: 30
-  device: h100
-  optional: true
-  num_devices: 1
-  source_file_dependencies:
-  - vllm/model_executor/layers/quantization/humming.py
-  - vllm/model_executor/layers/quantization/utils/humming_utils.py
-  - vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
-  - vllm/model_executor/layers/fused_moe/oracle/mxfp4.py
-  commands:
-  - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config.txt
-  - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-fp8.txt
-
-- label: LM Eval Humming (B200 - TEMPORARY)
-  key: lm-eval-humming-b200
-  timeout_in_minutes: 30
-  device: b200-k8s
-  optional: true
-  num_devices: 1
-  source_file_dependencies:
-  - vllm/model_executor/layers/quantization/humming.py
-  - vllm/model_executor/layers/quantization/utils/humming_utils.py
-  - vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
-  - vllm/model_executor/layers/fused_moe/oracle/mxfp4.py
-  commands:
-  - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config.txt
-  - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-fp8.txt
 
 - label: LM Eval TurboQuant KV Cache
   key: lm-eval-turboquant-kv-cache
@@ -205,8 +139,8 @@ steps:
   commands:
   - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant.txt
 
-- label: GPQA Eval (GPT-OSS) (2xH100)
+- label: GPQA Eval (GPT-OSS) (H100)
-  key: gpqa-eval-gpt-oss-2xh100
+  key: gpqa-eval-gpt-oss-h100
   timeout_in_minutes: 120
   device: h100
   optional: true
@@ -219,8 +153,8 @@ steps:
     - uv pip install --system 'gpt-oss[eval]==0.0.5'
     - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-h100.txt
 
-- label: GPQA Eval (GPT-OSS) (2xB200)
+- label: GPQA Eval (GPT-OSS) (B200)
-  key: gpqa-eval-gpt-oss-2xb200
+  key: gpqa-eval-gpt-oss-b200
   timeout_in_minutes: 120
   device: b200-k8s
   optional: true
@@ -233,22 +167,6 @@ steps:
     - uv pip install --system 'gpt-oss[eval]==0.0.5'
     - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-b200.txt
 
-- label: GPQA Eval (GPT-OSS) (DGX Spark)
-  key: gpqa-eval-gpt-oss-spark
-  timeout_in_minutes: 120
-  device: dgx-spark
-  optional: true
-  num_devices: 1
-  depends_on:
-  - arm64-image-build
-  source_file_dependencies:
-  - csrc/
-  - vllm/model_executor/layers/quantization
-  - tests/evals/gpt_oss/
-  commands:
-    - uv pip install --system 'gpt-oss[eval]==0.0.5'
-    - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-spark.txt
-
 - label: MRCR Eval Small Models
   device: h200_35gb
   timeout_in_minutes: 30

.buildkite/test_areas/lora.yaml

@@ -12,17 +12,6 @@ steps:
   commands:
     - pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --ignore=lora/test_chatglm3_tp.py --ignore=lora/test_llama_tp.py --ignore=lora/test_qwen3_with_multi_loras.py --ignore=lora/test_olmoe_tp.py --ignore=lora/test_deepseekv2_tp.py --ignore=lora/test_gptoss_tp.py --ignore=lora/test_qwen3moe_tp.py --ignore=lora/test_qwen35_densemodel_lora.py 
   parallelism: 4
-  mirror:
-    amd:
-      device: mi325_1
-      working_dir: "/vllm-workspace/tests"
-      timeout_in_minutes: 60
-      source_file_dependencies:
-      - vllm/lora
-      - tests/lora
-      - vllm/platforms/rocm.py
-      depends_on:
-      - image-build-amd
 
 
 - label: LoRA TP (Distributed)

.buildkite/test_areas/misc.yaml

@@ -21,12 +21,6 @@ steps:
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
     # TODO: create another `optional` test group for slow tests
     - pytest -v -s -m 'not slow_test' v1/spec_decode
-  mirror:
-    amd:
-      device: mi300_1
-      timeout_in_minutes: 65
-      depends_on:
-      - image-build-amd
 
 - label: V1 Sample + Logits
   key: v1-sample-logits
@@ -105,12 +99,6 @@ steps:
     # Integration test for streaming correctness (requires special branch).
     - pip install -U git+https://github.com/robertgshaw2-redhat/lm-evaluation-harness.git@streaming-api
     - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
-  mirror:
-    amd:
-      device: mi325_1
-      timeout_in_minutes: 60
-      depends_on:
-      - image-build-amd
 
 - label: V1 Others (CPU)
   key: v1-others-cpu
@@ -150,26 +138,11 @@ steps:
     - vllm/v1/spec_decode/extract_hidden_states.py
     - vllm/model_executor/models/extract_hidden_states.py
     - vllm/transformers_utils/configs/extract_hidden_states.py
-    - vllm/distributed/kv_transfer/kv_connector/v1/example_hidden_states_connector.py
     - tests/v1/kv_connector/extract_hidden_states_integration
   commands:
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
     - pytest -v -s v1/kv_connector/extract_hidden_states_integration
 
-- label: Extract Hidden States Integration (2 GPUs)
-  key: extract-hidden-states-integration-2-gpus
-  timeout_in_minutes: 20
-  num_devices: 2
-  source_file_dependencies:
-    - vllm/v1/spec_decode/extract_hidden_states.py
-    - vllm/model_executor/models/extract_hidden_states.py
-    - vllm/transformers_utils/configs/extract_hidden_states.py
-    - vllm/distributed/kv_transfer/kv_connector/v1/example_hidden_states_connector.py
-    - tests/v1/kv_connector/extract_hidden_states_integration
-  commands:
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -v -s -m 'distributed' v1/kv_connector/extract_hidden_states_integration
-
 - label: Regression
   key: regression
   timeout_in_minutes: 20
@@ -224,16 +197,6 @@ steps:
     - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048
     # https://github.com/vllm-project/vllm/pull/26682 uses slightly more memory in PyTorch 2.9+ causing this test to OOM in 1xL4 GPU
     - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536
-  mirror:
-    amd:
-      device: mi325_1
-      source_file_dependencies:
-      - vllm/entrypoints
-      - vllm/multimodal
-      - examples/
-      - vllm/platforms/rocm.py
-      depends_on:
-      - image-build-amd
 
 - label: Metrics, Tracing (2 GPUs)
   key: metrics-tracing-2-gpus
@@ -260,12 +223,6 @@ steps:
       'opentelemetry-exporter-otlp>=1.26.0' \
       'opentelemetry-semantic-conventions-ai>=0.4.1'"
   - pytest -v -s v1/tracing
-  mirror:
-    amd:
-      device: mi325_2
-      depends_on:
-      - image-build-amd
-      optional: true
 
 - label: Python-only Installation
   key: python-only-installation
@@ -278,16 +235,6 @@ steps:
   - setup.py
   commands:
   - bash standalone_tests/python_only_compile.sh
-  mirror:
-    amd:
-      device: mi325_1
-      timeout_in_minutes: 20
-      depends_on:
-      - image-build-amd
-      source_file_dependencies:
-      - tests/standalone_tests/python_only_compile.sh
-      - setup.py
-      - vllm/platforms/rocm.py
 
 - label: Async Engine, Inputs, Utils, Worker
   device: h200_35gb
@@ -346,7 +293,6 @@ steps:
   - vllm/transformers_utils/
   - vllm/utils/
   - vllm/v1/
-  - tests/test_envs.py
   - tests/test_inputs.py
   - tests/test_outputs.py
   - tests/test_pooling_params.py
@@ -354,25 +300,24 @@ steps:
   - tests/multimodal
   - tests/renderers
   - tests/standalone_tests/lazy_imports.py
+  - tests/tokenizers_
   - tests/reasoning
   - tests/tool_parsers
-  - tests/tokenizers_
   - tests/parser
   - tests/transformers_utils
   - tests/config
   device: cpu-small
   commands:
   - python3 standalone_tests/lazy_imports.py
-  - pytest -v -s test_envs.py
   - pytest -v -s test_inputs.py
   - pytest -v -s test_outputs.py
   - pytest -v -s test_pooling_params.py
   - pytest -v -s test_ray_env.py
   - pytest -v -s -m 'cpu_test' multimodal
   - pytest -v -s renderers
-  - pytest -v -s reasoning
-  - pytest -v -s tool_parsers
   - pytest -v -s tokenizers_
+  - pytest -v -s reasoning --ignore=reasoning/test_seedoss_reasoning_parser.py --ignore=reasoning/test_glm4_moe_reasoning_parser.py
+  - pytest -v -s tool_parsers
   - pytest -v -s parser
   - pytest -v -s transformers_utils
   - pytest -v -s config

.buildkite/test_areas/model_executor.yaml

@@ -23,16 +23,3 @@ steps:
     # calls that the signal method cannot interrupt.
     - pytest -v -s model_executor -m '(not slow_test)' --timeout=900 --timeout-method=thread
     - pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py --timeout=900 --timeout-method=thread
-  mirror:
-    amd:
-      device: mi300_1
-      depends_on:
-      - image-build-amd
-      source_file_dependencies:
-      - vllm/engine/arg_utils.py
-      - vllm/config/model.py
-      - vllm/model_executor
-      - tests/model_executor
-      - tests/entrypoints/openai/completion/test_tensorizer_entrypoint.py
-      - vllm/_aiter_ops.py
-      - vllm/platforms/rocm.py

.buildkite/test_areas/models_basic.yaml

@@ -45,11 +45,6 @@ steps:
   - tests/models/test_registry.py
   commands:
     - pytest -v -s models/test_terratorch.py models/test_transformers.py models/test_registry.py
-  mirror:
-    amd:
-      device: mi325_1
-      depends_on:
-      - image-build-amd
 
 - label: Basic Models Test (Other CPU) # 5min
   key: basic-models-test-other-cpu

.buildkite/test_areas/models_language.yaml

@@ -15,10 +15,6 @@ steps:
     - pytest -v -s models/language -m 'core_model and (not slow_test)'
   mirror:
     torch_nightly: {}
-    amd:
-      device: mi300_1
-      depends_on:
-      - image-build-amd
 
 - label: Language Models Tests (Extra Standard) %N
   key: language-models-tests-extra-standard
@@ -36,21 +32,6 @@ steps:
   parallelism: 2
   mirror:
     torch_nightly: {}
-    amd:
-      device: mi300_1
-      depends_on:
-      - image-build-amd
-      source_file_dependencies:
-      - vllm/model_executor/models/
-      - vllm/model_executor/model_loader/
-      - vllm/model_executor/layers/
-      - vllm/v1/attention/backends/
-      - vllm/v1/attention/selector.py
-      - tests/models/language/pooling/test_embedding.py
-      - tests/models/language/generation/test_common.py
-      - tests/models/language/pooling/test_classification.py
-      - vllm/_aiter_ops.py
-      - vllm/platforms/rocm.py
 
 - label: Language Models Tests (Hybrid) %N
   key: language-models-tests-hybrid

.buildkite/test_areas/models_multimodal.yaml

@@ -15,7 +15,7 @@ steps:
     - pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model
   mirror:
     amd:
-      device: mi325_1
+      device: mi300_1
       depends_on:
       - image-build-amd
 
@@ -30,9 +30,10 @@ steps:
     - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
     - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen3 or gemma"
     - pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model
+    - pytest -v -s models/multimodal/generation/test_vit_cudagraph.py -m core_model
   mirror:
     amd:
-      device: mi325_1
+      device: mi300_1
       depends_on:
       - image-build-amd
 
@@ -49,7 +50,7 @@ steps:
     - pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model
   mirror:
     amd:
-      device: mi325_1
+      device: mi300_1
       depends_on:
       - image-build-amd
 
@@ -62,15 +63,9 @@ steps:
   - tests/models/multimodal
   commands:
     - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
-    - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py  --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/generation/test_vit_cudagraph.py --ignore models/multimodal/processing
+    - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py  --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/processing
-    - pytest -v -s models/multimodal/generation/test_vit_cudagraph.py -m core_model
     - pytest models/multimodal/generation/test_memory_leak.py -m core_model
     - cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model  # Otherwise, mp_method="spawn" doesn't work
-  mirror:
-    amd:
-      device: mi325_1
-      depends_on:
-      - image-build-amd
 
 - label: Multi-Modal Processor (CPU)
   key: multi-modal-processor-cpu
@@ -109,17 +104,6 @@ steps:
   - vllm/v1/core/
   commands:
   - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-mm-small.txt --tp-size=1
-  mirror:
-    amd:
-      device: mi300_1
-      depends_on:
-      - image-build-amd
-      source_file_dependencies:
-      - vllm/multimodal/
-      - vllm/inputs/
-      - vllm/v1/core/
-      - vllm/platforms/rocm.py
-      - vllm/model_executor/model_loader/
 
 - label: Multi-Modal Models (Extended Generation 1)
   key: multi-modal-models-extended-generation-1
@@ -171,7 +155,7 @@ steps:
     - pytest -v -s models/multimodal/pooling -m 'not core_model'
   mirror:
     amd:
-      device: mi325_1
+      device: mi300_1
       timeout_in_minutes: 60
       depends_on:
       - image-build-amd

.buildkite/test_areas/plugins.yaml

@@ -27,10 +27,6 @@ steps:
   - pip install -e ./plugins/bge_m3_sparse_plugin
   - pytest -v -s plugins_tests/test_bge_m3_sparse_io_processor_plugins.py
   - pip uninstall bge_m3_sparse_plugin -y
-  # test colbert_query io_processor plugin
-  - pip install -e ./plugins/colbert_query_plugin
-  - pytest -v -s plugins_tests/test_colbert_query_io_processor_plugins.py
-  - pip uninstall colbert_query_plugin -y
   # end io_processor plugins test
   # begin stat_logger plugins test
   - pip install -e ./plugins/vllm_add_dummy_stat_logger
@@ -41,20 +37,6 @@ steps:
   - pytest -v -s plugins_tests/test_scheduler_plugins.py
   - pip install -e ./plugins/vllm_add_dummy_model
   - pytest -v -s distributed/test_distributed_oot.py
-  - pytest -v -s plugins_tests/test_oot_registration_online.py # it needs a clean process
+  - pytest -v -s entrypoints/openai/chat_completion/test_oot_registration.py # it needs a clean process
-  - pytest -v -s plugins_tests/test_oot_registration_offline.py # it needs a clean process
+  - pytest -v -s models/test_oot_registration.py # it needs a clean process
-  - pytest -v -s plugins_tests/lora_resolvers # unit tests for in-tree lora resolver plugins
+  - pytest -v -s plugins/lora_resolvers # unit tests for in-tree lora resolver plugins
-
-
-- label: GGUF Plugin
-  key: gguf-plugin
-  device: h200_18gb
-  timeout_in_minutes: 30
-  soft_fail: true
-  optional: true
-  source_file_dependencies:
-  - vllm/model_executor/layers/quantization
-  - tests/plugins_tests/test_gguf_plugin.py
-  commands:
-    - pip install "vllm-gguf-plugin >= 0.0.2"
-    - pytest -v -s plugins_tests/gguf

.buildkite/test_areas/pytorch.yaml

@@ -107,12 +107,6 @@ steps:
     - tests/compile/passes
   commands:
   - pytest -s -v compile/passes --ignore compile/passes/distributed
-  mirror:
-    amd:
-      device: mi300_1
-      timeout_in_minutes: 180
-      depends_on:
-      - image-build-amd
 
 - label: PyTorch Fullgraph Smoke Test
   key: pytorch-fullgraph-smoke-test
@@ -195,11 +189,3 @@ steps:
   - requirements/test/nightly-torch.txt
   commands:
   - bash standalone_tests/pytorch_nightly_dependency.sh
-  mirror:
-    amd:
-      device: mi300_1
-      depends_on:
-      - image-build-amd
-      source_file_dependencies:
-      - requirements/test/nightly-torch.txt
-      - vllm/platforms/rocm.py

.buildkite/test_areas/quantization.yaml

@@ -21,18 +21,6 @@ steps:
   - uv pip install --system conch-triton-kernels
   - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py
 
-- label: Quantized Fusions
-  key: quantized-fusions
-  timeout_in_minutes: 30
-  source_file_dependencies:
-  - tests/fusion
-  - vllm/model_executor/layers/fusion
-  - vllm/model_executor/kernels/linear
-  - vllm/model_executor/layers/quantization/compressed_tensors
-  - vllm/model_executor/layers/quantization/modelopt.py
-  commands:
-    - pytest -v -s fusion/
-
 - label: Quantized MoE Test (B200)
   key: quantized-moe-test-b200
   timeout_in_minutes: 60

.buildkite/test_areas/rust_frontend.yaml

@@ -26,7 +26,7 @@ steps:
   - export VLLM_USE_RUST_FRONTEND=1
   - export VLLM_WORKER_MULTIPROC_METHOD=spawn
   - pytest -v -s benchmarks/test_serve_cli.py -k "not insecure and not (test_bench_serve and not test_bench_serve_chat)"
-  - pytest -v -s entrypoints/openai/chat_completion/test_chat_completion.py -k "not test_invalid_json_schema and not test_invalid_regex"
+  - pytest -v -s entrypoints/openai/chat_completion/test_chat_completion.py
   # - pytest -v -s entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py -k "not invalid"
 
   # - pytest -v -s entrypoints/openai/completion/test_prompt_validation.py -k "not prompt_embeds"
@@ -46,7 +46,7 @@ steps:
   - vllm/v1/engine/
   - tests/utils.py
   # - tests/entrypoints/serve/dev/rpc/test_collective_rpc.py
-  - tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py
+  - tests/entrypoints/serve/disagg/test_serving_tokens.py
   - tests/entrypoints/serve/instrumentator/test_basic.py
   - tests/entrypoints/serve/instrumentator/test_metrics.py
   # - tests/entrypoints/serve/dev/test_sleep.py
@@ -55,7 +55,7 @@ steps:
   - export VLLM_WORKER_MULTIPROC_METHOD=spawn
   # - pytest -v -s entrypoints/serve/dev/rpc/test_collective_rpc.py
   - pytest -v -s entrypoints/serve/instrumentator/test_basic.py -k "not show_version and not server_load"
-  - pytest -v -s entrypoints/scale_out/token_in_token_out/test_serving_tokens.py -k "not stream and not lora and not test_generate_logprobs and not stop_string_workflow"
+  - pytest -v -s entrypoints/serve/disagg/test_serving_tokens.py -k "not stream and not lora and not test_generate_logprobs and not stop_string_workflow"
   - pytest -v -s entrypoints/serve/instrumentator/test_metrics.py -k "text and not show and not run_batch and not test_metrics_counts and not test_metrics_exist"
   # - pytest -v -s entrypoints/serve/dev/test_sleep.py
 
@@ -99,13 +99,9 @@ steps:
   - vllm/v1/engine/
   - vllm/v1/worker/
   - tests/utils.py
-  - tests/v1/distributed/test_external_lb_dp.py
-  - tests/v1/distributed/test_hybrid_lb_dp.py
   - tests/v1/distributed/test_internal_lb_dp.py
   commands:
   - export VLLM_USE_RUST_FRONTEND=1
   - export VLLM_WORKER_MULTIPROC_METHOD=spawn
   - export NCCL_CUMEM_HOST_ENABLE=0
   - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_internal_lb_dp.py -k "not 4 and not server_info"
-  - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py -k "not 4 and not server_info"
-  - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_hybrid_lb_dp.py -k "not 4 and not server_info"

.buildkite/test_areas/spec_decode.yaml

@@ -12,20 +12,6 @@ steps:
     - tests/v1/e2e/spec_decode/
   commands:
     - pytest -v -s v1/e2e/spec_decode -k "eagle_correctness"
-  mirror:
-    amd:
-      device: mi325_1
-      timeout_in_minutes: 45
-      depends_on:
-      - image-build-amd
-      source_file_dependencies:
-      - vllm/v1/spec_decode/
-      - vllm/v1/worker/gpu/spec_decode/
-      - vllm/model_executor/model_loader/
-      - vllm/v1/sample/
-      - vllm/model_executor/layers/
-      - tests/v1/e2e/spec_decode/
-      - vllm/platforms/rocm.py
 
 - label: Spec Decode Eagle Nightly B200
   key: spec-decode-eagle-nightly-b200
@@ -53,7 +39,7 @@ steps:
     - pytest -v -s v1/e2e/spec_decode -k "speculators or mtp_correctness"
   mirror:
     amd:
-      device: mi325_1
+      device: mi300_1
       timeout_in_minutes: 65
       depends_on:
       - image-build-amd
@@ -92,10 +78,8 @@ steps:
     - pytest -v -s v1/e2e/spec_decode -k "ngram or suffix"
   mirror:
     amd:
-      device: mi325_1
+      device: mi300_1
       timeout_in_minutes: 65
-      # TODO(akaratza): Test after Torch >= 2.12 bump
-      soft_fail: true
       depends_on:
       - image-build-amd
       source_file_dependencies:
@@ -119,7 +103,7 @@ steps:
     - pytest -v -s v1/e2e/spec_decode -k "draft_model or no_sync or batch_inference"
   mirror:
     amd:
-      device: mi325_1
+      device: mi300_1
       timeout_in_minutes: 50
       depends_on:
       - image-build-amd

.buildkite/test_areas/weight_loading.yaml

@@ -13,13 +13,6 @@ steps:
   - tests/weight_loading
   commands:
     - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models.txt
-  mirror:
-    amd:
-      device: mi300_2
-      depends_on:
-      - image-build-amd
-      commands:
-      - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-amd.txt
 
 # - label: Weight Loading Multiple GPU - Large Models # optional
 #   working_dir: "/vllm-workspace/tests"

.claude/skills/ci-fails-buildkite/SKILL.md

@@ -1,35 +0,0 @@
----
-name: ci-fails-buildkite
-description: Fetch and diagnose vLLM Buildkite CI failure logs. Use when investigating failing CI jobs on a PR or build, when the user pastes a buildkite.com URL, or asks to fetch/diagnose CI logs.
----
-
-# Diagnosing vLLM Buildkite CI Failures
-
-Buildkite logs are public; no login needed.
-
-`.buildkite/scripts/ci-fetch-log.sh` saves each log as `ci-<build>-<job-name>.log`, stripped of timestamps and ANSI codes. Existing files are kept; set `CI_FETCH_LOG_FORCE=1` to refetch.
-
-## Fetching logs
-
-```bash
-# All failed jobs in a PR's latest build (current branch's PR if omitted):
-.buildkite/scripts/ci-fetch-log.sh --pr <PR>
-
-# All failed jobs in a build (--soft also includes soft-failed jobs;
-# --all fetches every finished job):
-.buildkite/scripts/ci-fetch-log.sh "https://buildkite.com/vllm/ci/builds/<N>"
-
-# One job — `gh pr checks` URLs (#<job_uuid>) and web UI URLs (?sid=) both
-# work; pass "-" as a second argument to stream to stdout:
-.buildkite/scripts/ci-fetch-log.sh "https://buildkite.com/vllm/ci/builds/<N>#<job_uuid>"
-```
-
-To clean an already-downloaded log with `.buildkite/scripts/ci-clean-log.sh`:
-
-```bash
-./ci-clean-log.sh ci.log
-```
-
-## Reference
-
-See [docs/contributing/ci/failures.md](../../../docs/contributing/ci/failures.md) for the full guide: filing CI failure issues, investigating/bisecting, reproducing flaky tests, and daily triage.

.github/CODEOWNERS

@@ -2,16 +2,17 @@
 # for more info about CODEOWNERS file
 
 # This lists cover the "core" components of vLLM that require careful review
-/vllm/compilation @zou3519 @youkaichao @ProExpertProg @BoyuanFeng
+/vllm/compilation @zou3519 @youkaichao @ProExpertProg @BoyuanFeng @vadiklyutiy
-/vllm/distributed/kv_transfer @NickLucche @ApostaC @orozery @xuechendi @ivanium
+/vllm/distributed/kv_transfer @NickLucche @ApostaC @orozery @xuechendi
 /vllm/lora @jeejeelee
 /vllm/model_executor/layers/attention @LucasWilkinson @MatthewBonanni
 /vllm/model_executor/layers/fused_moe @mgoin @pavanimajety @zyongye
 /vllm/model_executor/layers/quantization @mgoin @robertgshaw2-redhat @tlrmchlsmth @yewentao256 @pavanimajety @zyongye
 /vllm/model_executor/layers/mamba @tdoublep @tomeras91
-/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py @tdoublep @ZJY0516 @vadiklyutiy
+/vllm/model_executor/layers/mamba/gdn_linear_attn.py @tdoublep @ZJY0516 @vadiklyutiy
+/vllm/model_executor/layers/rotary_embedding.py @vadiklyutiy
 /vllm/model_executor/model_loader @22quinn
-/vllm/model_executor/layers/batch_invariant.py @yewentao256
+/vllm/model_executor/layers/batch_invariant.py @yewentao256 
 /vllm/ir @ProExpertProg
 /vllm/kernels/ @ProExpertProg @tjtanaa
 /vllm/kernels/helion @ProExpertProg @zou3519
@@ -22,13 +23,8 @@
 
 # Any change to the VllmConfig changes can have a large user-facing impact,
 # so spam a lot of people
-/vllm/config @WoosukKwon @youkaichao @robertgshaw2-redhat @mgoin @tlrmchlsmth @houseroad @yewentao256 @ProExpertProg
+/vllm/config @WoosukKwon @youkaichao @robertgshaw2-redhat @mgoin @tlrmchlsmth @houseroad @hmellor @yewentao256 @ProExpertProg
-/vllm/config/cache.py @heheda12345 @ivanium
+/vllm/config/cache.py @heheda12345
-
-# Config utils
-/vllm/config/utils.py @hmellor
-/vllm/engine/arg_utils.py @hmellor
-/vllm/utils/argparse_utils.py
 
 # Entrypoints
 /vllm/entrypoints/anthropic @mgoin @DarkLight1337
@@ -67,20 +63,19 @@
 /vllm/v1/attention/backends/flashinfer.py @mgoin @pavanimajety @vadiklyutiy
 /vllm/v1/attention/backends/triton_attn.py @tdoublep
 /vllm/v1/attention/backends/gdn_attn.py @ZJY0516 @vadiklyutiy
-/vllm/v1/core @WoosukKwon @robertgshaw2-redhat @njhill @ywang96 @alexm-redhat @heheda12345 @ApostaC @orozery @ivanium
+/vllm/v1/core @WoosukKwon @robertgshaw2-redhat @njhill @ywang96 @alexm-redhat @heheda12345 @ApostaC @orozery
 /vllm/v1/sample @22quinn @houseroad @njhill
 /vllm/v1/spec_decode @benchislett @luccafong @MatthewBonanni
 /vllm/v1/structured_output @mgoin @russellb @aarnphm @benchislett
-/vllm/v1/kv_cache_interface.py @heheda12345 @ivanium
+/vllm/v1/kv_cache_interface.py @heheda12345
 /vllm/v1/kv_offload @ApostaC @orozery
-/vllm/v1/simple_kv_offload @ivanium
 /vllm/v1/engine @njhill
 /vllm/v1/executor @njhill
 /vllm/v1/worker @njhill
-/vllm/v1/worker/kv_connector_model_runner_mixin.py @orozery @NickLucche @ivanium
+/vllm/v1/worker/kv_connector_model_runner_mixin.py @orozery @NickLucche
 
 # Model runner V2
-/vllm/v1/worker/gpu @WoosukKwon @njhill @yewentao256
+/vllm/v1/worker/gpu @WoosukKwon @njhill
 /vllm/v1/worker/gpu/kv_connector.py @orozery
 
 # CI & building
@@ -104,14 +99,13 @@
 /tests/test_inputs.py @DarkLight1337 @ywang96
 /tests/entrypoints/llm/test_struct_output_generate.py @mgoin @russellb @aarnphm
 /tests/v1/structured_output @mgoin @russellb @aarnphm
-/tests/v1/core @WoosukKwon @robertgshaw2-redhat @njhill @ywang96 @alexm-redhat @heheda12345 @ApostaC @orozery @ivanium
+/tests/v1/core @WoosukKwon @robertgshaw2-redhat @njhill @ywang96 @alexm-redhat @heheda12345 @ApostaC @orozery
 /tests/weight_loading @mgoin @youkaichao @yewentao256
 /tests/lora @jeejeelee
 /tests/models/language/generation/test_hybrid.py @tdoublep @tomeras91
 /tests/v1/kv_connector/nixl_integration @NickLucche
-/tests/v1/kv_connector @ApostaC @orozery @ivanium
+/tests/v1/kv_connector @ApostaC @orozery
 /tests/v1/kv_offload @ApostaC @orozery
-/tests/v1/simple_kv_offload @ivanium
 /tests/v1/determinism @yewentao256
 /tests/reasoning @aarnphm @chaunceyjiang @sfeng33 @bbrowning
 /tests/tool_parsers @aarnphm @chaunceyjiang @sfeng33 @bbrowning
@@ -121,6 +115,16 @@
 /vllm/model_executor/models/transformers @hmellor
 /tests/models/test_transformers.py @hmellor
 
+# Observability
+/vllm/config/observability.py @markmc
+/vllm/v1/metrics @markmc
+/tests/v1/metrics @markmc
+/vllm/tracing.py @markmc
+/tests/v1/tracing/test_tracing.py @markmc
+/vllm/config/kv_events.py @markmc
+/vllm/distributed/kv_events.py @markmc
+/tests/distributed/test_events.py @markmc
+
 # Docs
 /docs/mkdocs @hmellor
 /docs/**/*.yml @hmellor

.github/actionlint.yaml

@@ -1,7 +0,0 @@
-# Custom self-hosted runner labels (e.g. the autoscaling vllm-runners pool) so
-# actionlint doesn't flag them as unknown in `runs-on`.
-self-hosted-runner:
-  labels:
-    - vllm-runners
-    # Not yet in actionlint's known-label set.
-    - macos-26

.github/dependabot.yml

@@ -21,6 +21,7 @@ updates:
       - dependency-name: "torchvision"
       - dependency-name: "xformers"
       - dependency-name: "lm-format-enforcer"
+      - dependency-name: "gguf"
       - dependency-name: "compressed-tensors"
       - dependency-name: "ray[cgraph]" # Ray Compiled Graph
       - dependency-name: "lm-eval"

.github/mergify.yml

@@ -144,12 +144,12 @@ pull_request_rules:
     - label != stale
     - or:
       - files~=^examples/.*mistral.*\.py
-      - files~=^tests/.*(?:mistral|voxtral|mixtral|pixtral).*\.py
+      - files~=^tests/.*mistral.*\.py
-      - files~=^vllm/model_executor/models/.*(?:mistral|voxtral|mixtral|pixtral).*\.py
+      - files~=^vllm/model_executor/models/.*mistral.*\.py
       - files~=^vllm/reasoning/.*mistral.*\.py
       - files~=^vllm/tool_parsers/.*mistral.*\.py
-      - files~=^vllm/transformers_utils/.*(?:mistral|voxtral|pixtral).*\.py
+      - files~=^vllm/transformers_utils/.*mistral.*\.py
-      - title~=(?i)(?:mistral|ministral|voxtral|mixtral|pixtral)
+      - title~=(?i)Mistral
   actions:
     label:
       add:
@@ -388,13 +388,9 @@ pull_request_rules:
     - or:
       - files~=^tests/tool_use/
       - files~=^tests/tool_parsers/
-      - files~=^tests/parser/
-      - files~=^tests/reasoning/
       - files~=^tests/entrypoints/openai/.*tool.*
       - files~=^tests/entrypoints/anthropic/.*tool.*
       - files~=^vllm/tool_parsers/
-      - files~=^vllm/parser/
-      - files~=^vllm/reasoning/
       - files=docs/features/tool_calling.md
       - files~=^examples/tool_calling/
   actions:

.github/workflows/issue_autolabel.yml

@@ -327,7 +327,7 @@ jobs:
                 message: 'CC {users} for ROCm-related issue',
               },
               mistral: {
-                users: ['patrickvonplaten', 'juliendenize', 'andylolu2', 'NickLucche'],
+                users: ['patrickvonplaten', 'juliendenize', 'andylolu2'],
                 message: 'CC {users} for Mistral-related issue',
               },
               // Add more label -> user mappings here

.github/workflows/macos-smoke-test.yml

@@ -11,25 +11,13 @@ permissions:
 
 jobs:
   macos-m1-smoke-test:
-    # macos-26 (the supported target) is still a preview runner, so gate on GA
+    runs-on: macos-latest
-    # macos-15 and keep macos-26 non-blocking.
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - os: macos-15
-            required: true
-          - os: macos-26
-            required: false
-    name: macos-m1-smoke-test (${{ matrix.os }})
-    runs-on: ${{ matrix.os }}
-    continue-on-error: ${{ !matrix.required }}
     timeout-minutes: 30
 
     steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+      - uses: actions/checkout@v6.0.1
 
-      - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+      - uses: astral-sh/setup-uv@v7
         with:
           enable-cache: true
           cache-dependency-glob: |
@@ -84,11 +72,14 @@ jobs:
           # Test health endpoint
           curl -f http://localhost:8000/health
 
-          # Long prompt: hits the split-KV path that short prompts skip (#46769).
+          # Test completion
-          PAYLOAD=$(python -c "import json; print(json.dumps({'model': 'Qwen/Qwen3-0.6B', 'prompt': 'The quick brown fox jumps over the lazy dog. ' * 24, 'max_tokens': 16}))")
+          curl -f http://localhost:8000/v1/completions \
-          curl -f --max-time 120 http://localhost:8000/v1/completions \
             -H "Content-Type: application/json" \
-            -d "$PAYLOAD"
+            -d '{
+              "model": "Qwen/Qwen3-0.6B",
+              "prompt": "Hello",
+              "max_tokens": 5
+            }'
 
           # Cleanup
           kill "$SERVER_PID"

.github/workflows/pre-commit.yml

@@ -46,16 +46,12 @@ jobs:
   pre-commit:
     needs: pre-run-check
     if: always() && (needs.pre-run-check.result == 'success' || needs.pre-run-check.result == 'skipped')
-    runs-on: [self-hosted, linux, x64, vllm-runners]
+    runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+    - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
-    - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+    - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
       with:
         python-version: "3.12"
-    # Provide shellcheck on PATH so tools/pre_commit/shellcheck.sh skips its
-    # wget + tar -xJ self-download, which the self-hosted runner image lacks
-    # (no wget/xz). Pinned to shellcheck 0.10.0 to match the script's "stable".
-    - run: python -m pip install shellcheck-py==0.10.0.1
     - run: echo "::add-matcher::.github/workflows/matchers/actionlint.json"
     - run: echo "::add-matcher::.github/workflows/matchers/markdownlint.json"
     - run: echo "::add-matcher::.github/workflows/matchers/mypy.json"

.github/workflows/scripts/build.sh

@@ -9,7 +9,7 @@ PATH=${cuda_home}/bin:$PATH
 LD_LIBRARY_PATH=${cuda_home}/lib64:$LD_LIBRARY_PATH
 
 # Install requirements
-if [ "$(echo "$2" | cut -d. -f1)" = "12" ]; then
+if [ "$(echo $2 | cut -d. -f1)" = "12" ]; then
     sed -i 's/^nvidia-cutlass-dsl\[cu13\]>=/nvidia-cutlass-dsl>=/' requirements/cuda.txt
 fi
 $python_executable -m pip install -r requirements/build/cuda.txt -r requirements/cuda.txt
@@ -17,10 +17,7 @@ $python_executable -m pip install -r requirements/build/cuda.txt -r requirements
 # Limit the number of parallel jobs to avoid OOM
 export MAX_JOBS=1
 # Make sure release wheels are built for the following architectures
-# Do not add +PTX here: vLLM filters torch's top-level PTX flag when it
+export TORCH_CUDA_ARCH_LIST="7.5 8.0 8.6 8.9 9.0 10.0 12.0+PTX"
-# converts global gencode flags into per-kernel arch lists. If a specific
-# kernel needs PTX, add +PTX to that kernel's CMake arch list instead.
-export TORCH_CUDA_ARCH_LIST="7.5 8.0 8.6 8.9 9.0 10.0 12.0"
 
 bash tools/check_repo.sh
 

.gitignore

@@ -15,9 +15,6 @@ vllm/third_party/flashmla/flash_mla_interface.py
 # DeepGEMM vendored package built from source
 vllm/third_party/deep_gemm/
 
-# fmha_sm100 vendored package built from source
-vllm/third_party/fmha_sm100/
-
 # triton jit
 .triton
 
@@ -199,9 +196,7 @@ cython_debug/
 .vscode/
 
 # Claude
-.claude/*
+.claude/
-!.claude/skills/
-!.claude/skills/**
 
 # Codex
 .codex/
@@ -238,7 +233,7 @@ actionlint
 shellcheck*/
 
 # Ignore moe/marlin_moe gen code
-csrc/libtorch_stable/moe/marlin_moe_wna16/kernel_*
+csrc/moe/marlin_moe_wna16/kernel_*
 
 # Ignore ep_kernels_workspace folder
 ep_kernels_workspace/

.pre-commit-config.yaml

@@ -21,7 +21,7 @@ repos:
   rev: v21.1.2
   hooks:
   - id: clang-format
-    exclude: 'csrc/libtorch_stable/moe/topk_softmax_kernels.cu|vllm/third_party/.*'
+    exclude: 'csrc/(moe/topk_softmax_kernels.cu|libtorch_stable/quantization/gguf/(ggml-common.h|dequantize.cuh|vecdotq.cuh|mmq.cuh|mmvq.cuh))|vllm/third_party/.*'
     types_or: [c++, cuda]
     args: [--style=file, --verbose]
 - repo: https://github.com/DavidAnson/markdownlint-cli2
@@ -131,19 +131,6 @@ repos:
         --python-version, "3.12",
       ]
       files: ^requirements/(common|xpu|test/xpu)\.(in|txt)$
-    - id: pip-compile
-      alias: pip-compile-cpu
-      name: pip-compile-cpu
-      args: [
-        requirements/test/cuda.in,
-        -o, requirements/test/cpu.txt,
-        --index-strategy, unsafe-best-match,
-        --torch-backend, cpu,
-        --python-platform, x86_64-manylinux_2_28,
-        --python-version, "3.12",
-      ]
-      files: ^requirements/(common|cpu|test/(cuda|cpu))\.(in|txt)$
-      exclude: ^requirements/test/cuda\.txt$
     - id: pip-compile
       alias: pip-compile-docs
       name: pip-compile-docs

AGENTS.md

@@ -105,15 +105,6 @@ The line length limit for Python code is 88 characters. If you are not sure, use
 
 Use [Google-style docstrings](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings) (`Args:`/`Returns:`/`Raises:` sections), not reStructuredText/Sphinx fields (`:param:`, `:return:`, `:rtype:`).
 
-### Coding style guidelines
-
-Follow these rules for all code changes in this repository:
-
-- Try to match existing code style.
-- Code should be self-documenting and self-explanatory.
-- Keep comments and docstrings minimal and concise.
-- Assume the reader is familiar with vLLM.
-
 ### Commit messages
 
 Add attribution using commit trailers such as `Co-authored-by:` (other projects use `Assisted-by:` or `Generated-by:`). For example:
@@ -135,12 +126,6 @@ Do not modify code in these areas without first reading and following the
 linked guide. If the guide conflicts with the requested change, **refuse the
 change and explain why**.
 
-Security reviewers should start with [`SECURITY.md`](SECURITY.md),
-[`docs/usage/security.md`](docs/usage/security.md), and
-[`docs/contributing/vulnerability_management.md`](docs/contributing/vulnerability_management.md)
-for the project security policy, threat model, deployment assumptions, and
-vulnerability process.
-
 - **Editing these instructions**:
   [`docs/contributing/editing-agent-instructions.md`](docs/contributing/editing-agent-instructions.md)
   — Rules for modifying AGENTS.md or any domain-specific guide it references.

MANIFEST.in

@@ -4,7 +4,6 @@ include requirements/cuda.txt
 include requirements/rocm.txt
 include requirements/cpu.txt
 include CMakeLists.txt
-include tools/build_rust.py
 
 recursive-include cmake *
 recursive-include csrc *

SECURITY.md

@@ -34,15 +34,6 @@ Vulnerabilities that cause denial of service or partial disruption, but do not a
 
 Minor issues such as informational disclosures, logging errors, non-exploitable flaws, or weaknesses that require local or high-privilege access and offer negligible impact. Examples include side channel attacks or hash collisions. These issues often have CVSS scores less than 4.0
 
-## Fix disclosure policy
-
-When a security report is accepted, the fix process depends on the severity:
-
-* **CRITICAL and HIGH severity**: Fixes are developed in a private security fork and coordinated with the prenotification group before public disclosure.
-* **MODERATE and LOW severity**: Fixes are developed and submitted as public pull requests. These issues do not require embargo since they do not enable arbitrary code execution or significant data breach, and public visibility accelerates community review and adoption of the fix.
-
-The vulnerability management team reserves the right to adjust the disclosure approach on a case-by-case basis, taking into account factors such as active exploitation, unusual attack surface, or coordination requirements with downstream vendors.
-
 ## Prenotification policy
 
 For certain security issues of CRITICAL, HIGH, or MODERATE severity level, we may prenotify certain organizations or vendors that ship vLLM. The purpose of this prenotification is to allow for a coordinated release of fixes for severe issues.

benchmarks/attention_benchmarks/README.md

@@ -108,6 +108,7 @@ python benchmark.py \
     --backends flash triton flashinfer \
     --batch-specs "q2k" "8q1s1k" "2q2k_32q1s1k" \
     --num-layers 10 \
+    --repeats 5 \
     --output-csv results.csv
 ```
 
@@ -163,17 +164,14 @@ python benchmark.py \
 # Model configuration
 --num-layers N                      # Number of layers
 --head-dim N                        # Head dimension
---v-head-dim N                      # Value head dimension (defaults to --head-dim)
 --num-q-heads N                     # Query heads
 --num-kv-heads N                    # KV heads
 --block-size N                      # Block size
---kv-lora-rank N                    # MLA KV LoRA rank
---qk-nope-head-dim N                # MLA non-RoPE QK head dim
---qk-rope-head-dim N                # MLA RoPE QK head dim
 
 # Benchmark settings
 --device DEVICE                     # Device (default: cuda:0)
---warmup-ms N                       # Warmup window in ms for triton do_bench
+--repeats N                         # Repetitions
+--warmup-iters N                    # Warmup iterations
 --profile-memory                    # Profile memory usage
 
 # Parameter sweeps
@@ -213,6 +211,8 @@ config = BenchmarkConfig(
     num_kv_heads=1,
     block_size=128,
     device="cuda:0",
+    repeats=5,
+    warmup_iters=3,
 )
 
 # CUTLASS MLA with specific num_kv_splits
@@ -253,10 +253,14 @@ formatter.save_json(results, "output.json")
 
 ## Tips
 
-**1. Save results** - Always use `--output-csv` or `--output-json`
+**1. Warmup matters** - Use `--warmup-iters 10` for stable results
 
-**2. Test incrementally** - Start with `--num-layers 1`
+**2. Multiple repeats** - Use `--repeats 20` for low variance
 
-**3. Extended grammar** - Leverage spec decode, chunked prefill patterns
+**3. Save results** - Always use `--output-csv` or `--output-json`
 
-**4. Parameter sweeps** - Use `--sweep-param` and `--sweep-values` to find optimal values
+**4. Test incrementally** - Start with `--num-layers 1 --repeats 1`
+
+**5. Extended grammar** - Leverage spec decode, chunked prefill patterns
+
+**6. Parameter sweeps** - Use `--sweep-param` and `--sweep-values` to find optimal values

benchmarks/attention_benchmarks/benchmark.py

@@ -26,9 +26,6 @@ Examples:
 """
 
 import argparse
-import os
-import shutil
-import subprocess
 import sys
 from dataclasses import replace
 from pathlib import Path
@@ -53,16 +50,6 @@ from common import (
 from vllm.v1.worker.workspace import init_workspace_manager
 
 
-def _str2bool(v) -> bool:
-    if isinstance(v, bool):
-        return v
-    if v.lower() in ("true", "1", "yes", "t"):
-        return True
-    if v.lower() in ("false", "0", "no", "f"):
-        return False
-    raise argparse.ArgumentTypeError(f"expected a boolean, got {v!r}")
-
-
 def run_standard_attention_benchmark(config: BenchmarkConfig) -> BenchmarkResult:
     """Run standard attention benchmark (Flash/Triton/FlashInfer)."""
     from runner import run_attention_benchmark
@@ -96,15 +83,13 @@ def run_benchmark(config: BenchmarkConfig, **kwargs) -> BenchmarkResult:
         else:
             return run_standard_attention_benchmark(config)
     except Exception as e:
-        error_msg = str(e) or repr(e)
         return BenchmarkResult(
             config=config,
             mean_time=float("inf"),
-            median_time=float("inf"),
             std_time=0,
             min_time=float("inf"),
             max_time=float("inf"),
-            error=error_msg,
+            error=str(e),
         )
 
 
@@ -130,12 +115,9 @@ def run_model_parameter_sweep(
     """
     all_results = []
 
-    sweep_desc = (
+    console.print(
-        f"{sweep.param_name} = {sweep.values}"
+        f"[yellow]Model sweep mode: testing {sweep.param_name} = {sweep.values}[/]"
-        if sweep.param_name
-        else f"{len(sweep.values)} configurations"
     )
-    console.print(f"[yellow]Model sweep mode: testing {sweep_desc}[/]")
 
     total = len(backends) * len(batch_specs) * len(sweep.values)
 
@@ -143,9 +125,9 @@ def run_model_parameter_sweep(
         for backend in backends:
             for spec in batch_specs:
                 for value in sweep.values:
-                    # Create config with modified model parameter(s)
+                    # Create config with modified model parameter
                     config_args = base_config_args.copy()
-                    sweep.apply(config_args, value)
+                    config_args[sweep.param_name] = value
 
                     # Create config with original backend for running
                     clean_config = BenchmarkConfig(
@@ -162,21 +144,13 @@ def run_model_parameter_sweep(
                     all_results.append(result)
 
                     if not result.success:
-                        err_label = (
-                            f"{sweep.param_name}={value}"
-                            if sweep.param_name
-                            else f"{value}"
-                        )
                         console.print(
-                            f"[red]Error {backend} {spec} {err_label}"
+                            f"[red]Error {backend} {spec} {sweep.param_name}="
-                            f": {result.error}[/]"
+                            f"{value}: {result.error}[/]"
                         )
 
                     pbar.update(1)
 
-    if base_config_args.get("ncu_profile"):
-        return all_results
-
     # Display sweep results - create separate table for each parameter value
     console.print("\n[bold green]Model Parameter Sweep Results:[/]")
     formatter = ResultsFormatter(console)
@@ -210,10 +184,7 @@ def run_model_parameter_sweep(
     )
 
     for param_value in sorted_param_values:
-        label = (
+        console.print(f"\n[bold cyan]{sweep.param_name} = {param_value}[/]")
-            f"{sweep.param_name} = {param_value}" if sweep.param_name else param_value
-        )
-        console.print(f"\n[bold cyan]{label}[/]")
         param_results = by_param_value[param_value]
 
         # Create modified results with original backend names
@@ -229,9 +200,8 @@ def run_model_parameter_sweep(
         formatter.print_table(modified_results, backends, compare_to_fastest=True)
 
     # Show optimal backend for each (param_value, batch_spec) combination
-    sweep_name = sweep.param_name or "config"
     console.print(
-        f"\n[bold cyan]Optimal backend for each ({sweep_name}, batch_spec):[/]"
+        f"\n[bold cyan]Optimal backend for each ({sweep.param_name}, batch_spec):[/]"
     )
 
     # Group by (param_value, batch_spec)
@@ -266,10 +236,7 @@ def run_model_parameter_sweep(
     for param_value, spec in sorted_keys:
         # Print header when param value changes
         if param_value != current_param_value:
-            header = (
+            console.print(f"\n  [bold]{sweep.param_name}={param_value}:[/]")
-                f"{sweep.param_name}={param_value}" if sweep.param_name else param_value
-            )
-            console.print(f"\n  [bold]{header}:[/]")
             current_param_value = param_value
 
         results = by_param_and_spec[(param_value, spec)]
@@ -355,9 +322,6 @@ def run_parameter_sweep(
 
                     pbar.update(1)
 
-    if base_config_args.get("ncu_profile"):
-        return all_results
-
     # Display sweep results
     console.print("\n[bold green]Sweep Results:[/]")
     backend_labels = [sweep.get_label(b, v) for b in backends for v in sweep_values]
@@ -495,20 +459,6 @@ def main():
         help="Prefill backends to compare (fa2, fa3, fa4). "
         "Uses the first decode backend for impl construction.",
     )
-    parser.add_argument(
-        "--fp8-output-scale",
-        type=float,
-        help="Static per-tensor scale enabling the MLA prefill FP8-output "
-        "comparison on FA4 (fused write vs standalone post-quant).",
-    )
-    parser.add_argument(
-        "--fuse-quant-op",
-        nargs="+",
-        type=_str2bool,
-        help="FP8-output write path(s) to run: false = bf16 attention + "
-        "standalone static-FP8 quant, true = FA4 writes FP8 directly. "
-        "Default: both.",
-    )
 
     # Batch specifications
     parser.add_argument(
@@ -524,35 +474,11 @@ def main():
     parser.add_argument("--num-q-heads", type=int, default=32, help="Query heads")
     parser.add_argument("--num-kv-heads", type=int, default=8, help="KV heads")
     parser.add_argument("--block-size", type=int, default=16, help="Block size")
-    parser.add_argument(
-        "--v-head-dim",
-        type=int,
-        default=None,
-        help="Value head dimension (defaults to --head-dim if unset)",
-    )
-
-    # MLA-specific model dimensions
-    parser.add_argument(
-        "--kv-lora-rank", type=int, default=None, help="MLA KV LoRA rank"
-    )
-    parser.add_argument(
-        "--qk-nope-head-dim", type=int, default=None, help="MLA non-RoPE QK head dim"
-    )
-    parser.add_argument(
-        "--qk-rope-head-dim", type=int, default=None, help="MLA RoPE QK head dim"
-    )
 
     # Benchmark settings
     parser.add_argument("--device", default="cuda:0", help="Device")
-    parser.add_argument(
+    parser.add_argument("--repeats", type=int, default=1, help="Repetitions")
-        "--warmup-ms",
+    parser.add_argument("--warmup-iters", type=int, default=3, help="Warmup iterations")
-        type=int,
-        default=None,
-        help=(
-            "Warmup window in ms for triton's do_bench (default: triton's own). "
-            "Has no effect with CUDA graphs; pass --no-cuda-graphs to use it."
-        ),
-    )
     parser.add_argument("--profile-memory", action="store_true", help="Profile memory")
     parser.add_argument(
         "--kv-cache-dtype",
@@ -565,33 +491,10 @@ def main():
         action=argparse.BooleanOptionalAction,
         default=True,
         help=(
-            "Use triton do_bench_cudagraph (True) or do_bench (False) "
+            "Launch kernels with CUDA graphs to eliminate CPU overhead"
-            "for timing. CUDA graphs eliminate CPU launch overhead "
+            "in measurements (default: True)"
-            "(default: True)"
         ),
     )
-    parser.add_argument(
-        "--num-splits",
-        type=int,
-        default=None,
-        help="FlashAttention split-K factor (0=auto heuristic, 1=disabled, >1=force N)",
-    )
-    parser.add_argument(
-        "--ncu-profile",
-        action="store_true",
-        default=False,
-        help=(
-            "Enable Nsight Compute profiling mode. Automatically wraps the "
-            "script with ncu, capturing a profile with source correlation. "
-            "Use --ncu-output to set the output file name."
-        ),
-    )
-    parser.add_argument(
-        "--ncu-output",
-        type=str,
-        default="profile",
-        help="Output file name for ncu profile (default: 'profile').",
-    )
 
     # Parameter sweep (use YAML config for advanced sweeps)
     parser.add_argument(
@@ -642,12 +545,6 @@ def main():
         # Prefill backends (e.g., ["fa3", "fa4"])
         args.prefill_backends = yaml_config.get("prefill_backends", None)
 
-        # FP8 output benchmark knobs; CLI wins.
-        if args.fp8_output_scale is None:
-            args.fp8_output_scale = yaml_config.get("fp8_output_scale", None)
-        if args.fuse_quant_op is None:
-            args.fuse_quant_op = yaml_config.get("fuse_quant_op", None)
-
         # Check for special modes
         args.mode = yaml_config.get("mode", None)
 
@@ -679,28 +576,23 @@ def main():
             model = yaml_config["model"]
             args.num_layers = model.get("num_layers", args.num_layers)
             args.head_dim = model.get("head_dim", args.head_dim)
-            args.v_head_dim = model.get("v_head_dim", args.v_head_dim)
             args.num_q_heads = model.get("num_q_heads", args.num_q_heads)
             args.num_kv_heads = model.get("num_kv_heads", args.num_kv_heads)
             args.block_size = model.get("block_size", args.block_size)
-            # MLA-specific dimensions
-            args.kv_lora_rank = model.get("kv_lora_rank", args.kv_lora_rank)
-            args.qk_nope_head_dim = model.get("qk_nope_head_dim", args.qk_nope_head_dim)
-            args.qk_rope_head_dim = model.get("qk_rope_head_dim", args.qk_rope_head_dim)
 
         # Benchmark settings (top-level keys)
         if "device" in yaml_config:
             args.device = yaml_config["device"]
-        if "warmup_ms" in yaml_config:
+        if "repeats" in yaml_config:
-            args.warmup_ms = yaml_config["warmup_ms"]
+            args.repeats = yaml_config["repeats"]
+        if "warmup_iters" in yaml_config:
+            args.warmup_iters = yaml_config["warmup_iters"]
         if "profile_memory" in yaml_config:
             args.profile_memory = yaml_config["profile_memory"]
         if "kv_cache_dtype" in yaml_config:
             args.kv_cache_dtype = yaml_config["kv_cache_dtype"]
         if "cuda_graphs" in yaml_config:
             args.cuda_graphs = yaml_config["cuda_graphs"]
-        if "ncu_profile" in yaml_config:
-            args.ncu_profile = yaml_config["ncu_profile"]
 
         # Parameter sweep configuration
         if "parameter_sweep" in yaml_config:
@@ -720,7 +612,7 @@ def main():
         if "model_parameter_sweep" in yaml_config:
             sweep_config = yaml_config["model_parameter_sweep"]
             args.model_parameter_sweep = ModelParameterSweep(
-                param_name=sweep_config.get("param_name"),
+                param_name=sweep_config["param_name"],
                 values=sweep_config["values"],
                 label_format=sweep_config.get(
                     "label_format", "{backend}_{param_name}_{value}"
@@ -739,32 +631,6 @@ def main():
 
         console.print()
 
-    # Re-exec under ncu if --ncu-profile and not already inside ncu. This runs
-    # after YAML processing so ncu_profile set via config file is honored.
-    if args.ncu_profile and "_NCU_INNER" not in os.environ:
-        ncu = shutil.which("ncu")
-        if ncu is None:
-            print("Error: 'ncu' not found in PATH", file=sys.stderr)
-            sys.exit(1)
-        cmd = [
-            ncu,
-            "--profile-from-start",
-            "off",
-            "--set",
-            "full",
-            "--import-source",
-            "yes",
-            "-o",
-            args.ncu_output,
-            sys.executable,
-            *sys.argv,
-        ]
-        env = os.environ.copy()
-        env["CUTE_DSL_LINEINFO"] = "1"
-        env["_NCU_INNER"] = "1"
-        print(f"Launching: {' '.join(cmd)}")
-        sys.exit(subprocess.call(cmd, env=env))
-
     # Handle CLI-based parameter sweep (if not from YAML)
     if (
         (not hasattr(args, "parameter_sweep") or args.parameter_sweep is None)
@@ -789,18 +655,6 @@ def main():
     console.print(f"Batch specs: {', '.join(args.batch_specs)}")
     console.print(f"KV cache dtype: {args.kv_cache_dtype}")
     console.print(f"CUDA graphs: {args.cuda_graphs}")
-    if args.warmup_ms is not None and args.cuda_graphs:
-        console.print(
-            "[yellow]Warning: --warmup-ms is ignored with CUDA graphs "
-            "(do_bench_cudagraph warms up internally). Pass --no-cuda-graphs "
-            "to use it.[/]"
-        )
-    if args.num_splits == 0 and args.cuda_graphs:
-        console.print(
-            "[yellow]Warning: --num-splits 0 (FA3 heuristic) is not CUDA-graph "
-            "compatible and may fail or fall back. Pass --no-cuda-graphs or use "
-            "--num-splits >=1.[/]"
-        )
     console.print()
 
     init_workspace_manager(args.device)
@@ -808,68 +662,8 @@ def main():
     # Run benchmarks
     all_results = []
 
-    # Under ncu profiling the kernels run only to be captured by the profiler;
-    # timings are placeholder zeros, so the result tables and saved metrics are
-    # skipped. The Nsight Compute report (--ncu-output) holds the real data.
-    if args.ncu_profile:
-        console.print(
-            "[dim]ncu profiling enabled: result tables and saved metrics are "
-            "skipped (timings are placeholder zeros).[/]"
-        )
-
-    # FA4 fused FP8 output vs standalone post-quant, on the same fa4 kernel:
-    # the delta is the post-quant kernel the fused path removes.
-    fp8_output_scale = getattr(args, "fp8_output_scale", None)
-    if fp8_output_scale is not None:
-        decode_backend = backends[0]
-        fuse_variants = args.fuse_quant_op or [False, True]
-        label_of = {False: "post_quant", True: "fused"}
-        console.print(
-            f"[yellow]FP8 output comparison @ scale={fp8_output_scale} "
-            f"(prefill=fa4, decode impl={decode_backend})[/]"
-        )
-        fp8_results = []
-        total = len(fuse_variants) * len(args.batch_specs)
-        with tqdm(total=total, desc="FP8 output benchmarking") as pbar:
-            for spec in args.batch_specs:
-                for fuse in fuse_variants:
-                    config = BenchmarkConfig(
-                        backend=decode_backend,
-                        batch_spec=spec,
-                        num_layers=args.num_layers,
-                        head_dim=args.head_dim,
-                        num_q_heads=args.num_q_heads,
-                        num_kv_heads=args.num_kv_heads,
-                        block_size=args.block_size,
-                        device=args.device,
-                        repeats=args.repeats,
-                        warmup_iters=args.warmup_iters,
-                        profile_memory=args.profile_memory,
-                        kv_cache_dtype=args.kv_cache_dtype,
-                        use_cuda_graphs=args.cuda_graphs,
-                        prefill_backend="fa4",
-                    )
-                    result = run_benchmark(
-                        config, output_scale=fp8_output_scale, fuse_quant_op=fuse
-                    )
-                    label = label_of[fuse]
-                    labeled_config = replace(result.config, backend=label)
-                    result = replace(result, config=labeled_config)
-                    fp8_results.append(result)
-
-                    if not result.success:
-                        console.print(f"[red]Error {label} {spec}: {result.error}[/]")
-
-                    pbar.update(1)
-
-        console.print("\n[bold green]FP8 Output Results:[/]")
-        formatter = ResultsFormatter(console)
-        labels = [label_of[f] for f in fuse_variants]
-        formatter.print_table(fp8_results, labels, compare_to_fastest=True)
-        all_results = fp8_results
-
     # Handle special mode: decode_vs_prefill comparison
-    elif hasattr(args, "mode") and args.mode == "decode_vs_prefill":
+    if hasattr(args, "mode") and args.mode == "decode_vs_prefill":
         console.print("[yellow]Mode: Decode vs Prefill pipeline comparison[/]")
         console.print(
             "[dim]For each query length, testing both decode and prefill pipelines[/]"
@@ -914,11 +708,11 @@ def main():
                         num_kv_heads=args.num_kv_heads,
                         block_size=args.block_size,
                         device=args.device,
+                        repeats=args.repeats,
+                        warmup_iters=args.warmup_iters,
                         profile_memory=args.profile_memory,
                         kv_cache_dtype=args.kv_cache_dtype,
                         use_cuda_graphs=args.cuda_graphs,
-                        ncu_profile=args.ncu_profile,
-                        warmup_ms=args.warmup_ms,
                     )
 
                     # Add decode pipeline config
@@ -955,7 +749,6 @@ def main():
                         result = BenchmarkResult(
                             config=config,
                             mean_time=timing["mean"],
-                            median_time=timing.get("median", timing["mean"]),
                             std_time=timing["std"],
                             min_time=timing["min"],
                             max_time=timing["max"],
@@ -977,7 +770,6 @@ def main():
                         result = BenchmarkResult(
                             config=config,
                             mean_time=float("inf"),
-                            median_time=float("inf"),
                             std_time=0,
                             min_time=float("inf"),
                             max_time=float("inf"),
@@ -987,9 +779,6 @@ def main():
 
                 pbar.update(1)
 
-        if args.ncu_profile:
-            return
-
         # Display decode vs prefill results
         console.print("\n[bold green]Decode vs Prefill Results:[/]")
 
@@ -1069,20 +858,15 @@ def main():
         base_config_args = {
             "num_layers": args.num_layers,
             "head_dim": args.head_dim,
-            "v_head_dim": args.v_head_dim,
             "num_q_heads": args.num_q_heads,
             "num_kv_heads": args.num_kv_heads,
             "block_size": args.block_size,
             "device": args.device,
+            "repeats": args.repeats,
+            "warmup_iters": args.warmup_iters,
             "profile_memory": args.profile_memory,
             "kv_cache_dtype": args.kv_cache_dtype,
             "use_cuda_graphs": args.cuda_graphs,
-            "ncu_profile": args.ncu_profile,
-            "warmup_ms": args.warmup_ms,
-            "num_splits": args.num_splits,
-            "kv_lora_rank": args.kv_lora_rank,
-            "qk_nope_head_dim": args.qk_nope_head_dim,
-            "qk_rope_head_dim": args.qk_rope_head_dim,
         }
         all_results = run_model_parameter_sweep(
             backends,
@@ -1098,17 +882,15 @@ def main():
         base_config_args = {
             "num_layers": args.num_layers,
             "head_dim": args.head_dim,
-            "v_head_dim": args.v_head_dim,
             "num_q_heads": args.num_q_heads,
             "num_kv_heads": args.num_kv_heads,
             "block_size": args.block_size,
             "device": args.device,
+            "repeats": args.repeats,
+            "warmup_iters": args.warmup_iters,
             "profile_memory": args.profile_memory,
             "kv_cache_dtype": args.kv_cache_dtype,
             "use_cuda_graphs": args.cuda_graphs,
-            "ncu_profile": args.ncu_profile,
-            "warmup_ms": args.warmup_ms,
-            "num_splits": args.num_splits,
         }
         all_results = run_parameter_sweep(
             backends, args.batch_specs, base_config_args, args.parameter_sweep, console
@@ -1132,17 +914,15 @@ def main():
                             batch_spec=spec,
                             num_layers=args.num_layers,
                             head_dim=args.head_dim,
-                            v_head_dim=getattr(args, "v_head_dim", None),
                             num_q_heads=args.num_q_heads,
                             num_kv_heads=args.num_kv_heads,
                             block_size=args.block_size,
                             device=args.device,
+                            repeats=args.repeats,
+                            warmup_iters=args.warmup_iters,
                             profile_memory=args.profile_memory,
                             kv_cache_dtype=args.kv_cache_dtype,
                             use_cuda_graphs=args.cuda_graphs,
-                            ncu_profile=args.ncu_profile,
-                            warmup_ms=args.warmup_ms,
-                            num_splits=args.num_splits,
                         )
 
                         result = run_benchmark(config)
@@ -1155,10 +935,9 @@ def main():
 
                         pbar.update(1)
 
-            if not args.ncu_profile:
+            console.print("\n[bold green]Results:[/]")
-                console.print("\n[bold green]Results:[/]")
+            formatter = ResultsFormatter(console)
-                formatter = ResultsFormatter(console)
+            formatter.print_table(decode_results, backends)
-                formatter.print_table(decode_results, backends)
 
         # Run prefill backend comparison
         if prefill_backends:
@@ -1183,8 +962,9 @@ def main():
                             num_kv_heads=args.num_kv_heads,
                             block_size=args.block_size,
                             device=args.device,
+                            repeats=args.repeats,
+                            warmup_iters=args.warmup_iters,
                             profile_memory=args.profile_memory,
-                            warmup_ms=args.warmup_ms,
                             prefill_backend=pb,
                         )
 
@@ -1200,17 +980,16 @@ def main():
 
                         pbar.update(1)
 
-            if not args.ncu_profile:
+            console.print("\n[bold green]Prefill Backend Results:[/]")
-                console.print("\n[bold green]Prefill Backend Results:[/]")
+            formatter = ResultsFormatter(console)
-                formatter = ResultsFormatter(console)
+            formatter.print_table(
-                formatter.print_table(
+                prefill_results, prefill_backends, compare_to_fastest=True
-                    prefill_results, prefill_backends, compare_to_fastest=True
+            )
-                )
 
         all_results = decode_results + prefill_results
 
-    # Save results (skip ncu profiling runs: timings are placeholder zeros)
+    # Save results
-    if all_results and not args.ncu_profile:
+    if all_results:
         formatter = ResultsFormatter(console)
         if args.output_csv:
             formatter.save_csv(all_results, args.output_csv)

benchmarks/attention_benchmarks/common.py

@@ -15,8 +15,6 @@ from batch_spec import get_batch_type, parse_batch_spec
 from rich.console import Console
 from rich.table import Table
 
-from vllm.triton_utils import triton
-
 
 def batch_spec_sort_key(spec: str) -> tuple[int, int, int]:
     """
@@ -36,30 +34,6 @@ def batch_spec_sort_key(spec: str) -> tuple[int, int, int]:
         return (0, 0, 0)
 
 
-def run_do_bench(
-    benchmark_fn,
-    use_cuda_graphs: bool,
-    warmup_ms: int | None = None,
-) -> list[float]:
-    kwargs: dict[str, Any] = {"return_mode": "all"}
-    if use_cuda_graphs:
-        result = triton.testing.do_bench_cudagraph(benchmark_fn, **kwargs)
-    else:
-        if warmup_ms is not None:
-            kwargs["warmup"] = warmup_ms
-        result = triton.testing.do_bench(benchmark_fn, **kwargs)
-    return result
-
-
-def run_ncu_profile(benchmark_fn) -> None:
-    benchmark_fn()
-    torch.accelerator.synchronize()
-    torch.cuda.cudart().cudaProfilerStart()
-    benchmark_fn()
-    torch.accelerator.synchronize()
-    torch.cuda.cudart().cudaProfilerStop()
-
-
 # Mock classes for vLLM attention infrastructure
 
 
@@ -208,37 +182,18 @@ class ParameterSweep:
 
 @dataclass
 class ModelParameterSweep:
-    """Configuration for sweeping model configuration parameter(s).
+    """Configuration for sweeping a model configuration parameter."""
 
-    Supports two modes:
+    param_name: str  # Name of the model config parameter to sweep (e.g., "num_q_heads")
-    - Single param: param_name="head_dim", values=[128, 256, 512]
+    values: list[Any]  # List of values to test
-    - Multi param: values=[{head_dim: 192, v_head_dim: 128}, {head_dim: 256}]
+    label_format: str = "{backend}_{param_name}_{value}"  # Result label template
-      When values are dicts, each dict's keys are applied as config overrides.
-    """
-
-    param_name: str | None = None
-    values: list[Any] | None = None
-    label_format: str = "{backend}_{param_name}_{value}"
 
     def get_label(self, backend: str, value: Any) -> str:
         """Generate a label for a specific parameter value."""
-        if isinstance(value, dict):
-            return self.label_format.format(
-                backend=backend, param_name=self.param_name, value=value, **value
-            )
         return self.label_format.format(
             backend=backend, param_name=self.param_name, value=value
         )
 
-    def apply(self, config_args: dict, value: Any) -> None:
-        """Apply a sweep value to config args."""
-        if isinstance(value, dict):
-            config_args.update(value)
-        elif self.param_name is not None:
-            config_args[self.param_name] = value
-        else:
-            raise ValueError("param_name must be set if sweep values are not dicts")
-
 
 @dataclass
 class BenchmarkConfig:
@@ -253,10 +208,10 @@ class BenchmarkConfig:
     block_size: int
     device: str
     dtype: torch.dtype = torch.float16
+    repeats: int = 1
+    warmup_iters: int = 3
     profile_memory: bool = False
     use_cuda_graphs: bool = False
-    ncu_profile: bool = False
-    warmup_ms: int | None = None
 
     # "auto" or "fp8"
     kv_cache_dtype: str = "auto"
@@ -271,7 +226,6 @@ class BenchmarkConfig:
     # Backend-specific tuning
     num_kv_splits: int | None = None  # CUTLASS MLA
     reorder_batch_threshold: int | None = None  # FlashAttn MLA, FlashMLA
-    num_splits: int | None = None  # FlashAttention split-K (0=auto, 1=disabled)
 
 
 @dataclass
@@ -280,7 +234,6 @@ class BenchmarkResult:
 
     config: BenchmarkConfig
     mean_time: float  # seconds
-    median_time: float  # seconds
     std_time: float  # seconds
     min_time: float  # seconds
     max_time: float  # seconds
@@ -299,7 +252,6 @@ class BenchmarkResult:
         return {
             "config": asdict(self.config),
             "mean_time": self.mean_time,
-            "median_time": self.median_time,
             "std_time": self.std_time,
             "min_time": self.min_time,
             "max_time": self.max_time,

benchmarks/attention_benchmarks/configs/mla_decode.yaml

@@ -56,6 +56,8 @@ backends:
   - TOKENSPEED_MLA  # Blackwell + R1 dims + FP8 KV (use --kv-cache-dtype fp8)
 
 device: "cuda:0"
+repeats: 100
+warmup_iters: 10
 profile_memory: true
 
 # Backend-specific tuning

benchmarks/attention_benchmarks/configs/mla_fa4_fp8_output.yaml

@@ -1,44 +0,0 @@
-# MLA prefill FP8-output microbenchmark (FA4).
-# Compares the fused FP8 write against bf16 attention + a standalone static-FP8
-# quant; the delta is the post-quant kernel the fused path removes.
-# DeepSeek-Coder-V2-Lite dims; FA4 needs SM100/110.
-#
-# Usage:
-#   python benchmark.py --config configs/mla_fa4_fp8_output.yaml
-
-description: "MLA prefill FA4 fused-FP8 output vs post-quant"
-
-model:
-  name: "deepseek-v2-lite"
-  num_layers: 27
-  num_q_heads: 16
-  num_kv_heads: 1
-  head_dim: 576
-  kv_lora_rank: 512
-  qk_nope_head_dim: 128
-  qk_rope_head_dim: 64
-  v_head_dim: 128
-  block_size: 128
-
-# Pure prefill (q_len == kv_len) so every token goes through forward_mha.
-batch_specs:
-  - "q512"
-  - "q1k"
-  - "q2k"
-  - "q4k"
-  - "q8k"
-  - "2q4k"
-  - "4q4k"
-  - "8q4k"
-
-# Only used to construct the MLA impl; the pure-prefill specs skip decode.
-decode_backends:
-  - CUTLASS_MLA
-
-# Sweep the two FP8 write paths (prefill backend is fixed to fa4).
-fp8_output_scale: 0.1
-fuse_quant_op: [false, true]
-
-device: "cuda:0"
-repeats: 50
-warmup_iters: 10

benchmarks/attention_benchmarks/configs/mla_mixed_batch.yaml

@@ -51,6 +51,8 @@ backends:
   - FLASHMLA         # Hopper only
 
 device: "cuda:0"
+repeats: 5
+warmup_iters: 3
 profile_memory: true
 
 # Analyze chunked prefill workspace size impact

benchmarks/attention_benchmarks/configs/mla_prefill.yaml

@@ -124,3 +124,5 @@ prefill_backends:
   - tokenspeed
 
 device: "cuda:0"
+repeats: 20
+warmup_iters: 5

benchmarks/attention_benchmarks/configs/mla_sparse_decode.yaml

@@ -53,4 +53,6 @@ backends:
   - FLASHINFER_MLA_SPARSE
 
 device: "cuda:0"
+repeats: 100
+warmup_iters: 10
 profile_memory: true

benchmarks/attention_benchmarks/configs/mla_sparse_prefill.yaml

@@ -57,4 +57,6 @@ backends:
   - FLASHINFER_MLA_SPARSE
 
 device: "cuda:0"
+repeats: 10
+warmup_iters: 3
 profile_memory: true

benchmarks/attention_benchmarks/configs/reorder_threshold.yaml

@@ -63,6 +63,8 @@ model:
 
 # Benchmark settings
 device: "cuda:0"
+repeats: 15          # More repeats for spec decode variance
+warmup_iters: 5
 profile_memory: false
 
 # Output

benchmarks/attention_benchmarks/configs/speculative_decode.yaml

@@ -49,6 +49,8 @@ backends:
 
 # Benchmark settings
 device: "cuda:0"
+repeats: 10  # More repeats for statistical significance
+warmup_iters: 5
 profile_memory: false
 
 # Test these threshold values for optimization

benchmarks/attention_benchmarks/configs/standard_attention.yaml

@@ -43,4 +43,6 @@ backends:
   - FLASHINFER
 
 device: "cuda:0"
+repeats: 5
+warmup_iters: 3
 profile_memory: false

benchmarks/attention_benchmarks/configs/standard_decode.yaml

@@ -1,142 +0,0 @@
-# Standard attention decode benchmark configuration
-# Sweeps num_q_heads and num_kv_heads to isolate effects of:
-#   1. GQA ratio (fixed num_q_heads=32, vary num_kv_heads)
-#   2. Absolute head count (fixed 4:1 ratio, vary scale)
-
-model:
-  num_layers: 32
-  num_q_heads: 32 # Base value, overridden by sweep
-  num_kv_heads: 8 # Base value, overridden by sweep
-  head_dim: 128
-  block_size: 16
-
-# Head count sweep: each entry overrides num_q_heads, num_kv_heads, and
-# head_dim where it differs from the base (128). Head counts are per-GPU
-# (i.e. after TP sharding).
-#
-# Group A — vary GQA ratio (fixed q=32, head_dim=128):
-#   32:32 (MHA), 32:8 (GQA 4:1), 32:4 (GQA 8:1), 32:1 (MQA)
-#
-# Groups B-E — real model configs at various TP degrees:
-#   Model              head_dim  Full    TP2      TP4      TP8
-#   Llama 3 8B           128    32:8   16:4      8:2      4:1
-#   Llama 3 70B          128    64:8   32:4     16:2      8:1
-#   GPT-OSS 120B          64    64:8   32:4     16:2      8:1
-#   Llama 3 405B         128   128:8   64:4     32:2     16:1
-model_parameter_sweep:
-  values:
-    # --- head_dim=128 (Llama 3 family) ---
-    - { num_q_heads: 32, num_kv_heads: 32, head_dim: 128 } # MHA 1:1
-    - { num_q_heads: 32, num_kv_heads: 1, head_dim: 128 } # MQA 32:1
-    - { num_q_heads: 4, num_kv_heads: 1, head_dim: 128 } # Llama 3 8B TP8
-    - { num_q_heads: 8, num_kv_heads: 2, head_dim: 128 } # Llama 3 8B TP4
-    - { num_q_heads: 16, num_kv_heads: 4, head_dim: 128 } # Llama 3 8B TP2
-    - { num_q_heads: 32, num_kv_heads: 8, head_dim: 128 } # Llama 3 8B TP1 / GQA 4:1
-    - { num_q_heads: 8, num_kv_heads: 1, head_dim: 128 } # Llama 3 70B TP8
-    - { num_q_heads: 16, num_kv_heads: 2, head_dim: 128 } # Llama 3 70B TP4
-    - { num_q_heads: 32, num_kv_heads: 4, head_dim: 128 } # Llama 3 70B TP2 / GQA 8:1
-    - { num_q_heads: 64, num_kv_heads: 8, head_dim: 128 } # Llama 3 70B TP1
-    - { num_q_heads: 16, num_kv_heads: 1, head_dim: 128 } # Llama 3 405B TP8
-    - { num_q_heads: 32, num_kv_heads: 2, head_dim: 128 } # Llama 3 405B TP4
-    - { num_q_heads: 64, num_kv_heads: 4, head_dim: 128 } # Llama 3 405B TP2
-    - { num_q_heads: 128, num_kv_heads: 8, head_dim: 128 } # Llama 3 405B TP1
-    # --- head_dim=64 (GPT-OSS 120B) ---
-    - { num_q_heads: 8, num_kv_heads: 1, head_dim: 64 } # GPT-OSS 120B TP8
-    - { num_q_heads: 16, num_kv_heads: 2, head_dim: 64 } # GPT-OSS 120B TP4
-    - { num_q_heads: 32, num_kv_heads: 4, head_dim: 64 } # GPT-OSS 120B TP2
-    - { num_q_heads: 64, num_kv_heads: 8, head_dim: 64 } # GPT-OSS 120B TP1
-  label_format: "{backend}_q{num_q_heads}kv{num_kv_heads}d{head_dim}"
-
-batch_specs:
-  # ---- batch_size x seq_len grid (decode: q_len=1) ----
-  # Small grid for quick iteration. Uncomment for full sweep.
-
-  # Batch size 1
-  - "q1s1k"
-  - "q1s512"
-  - "q1s2k"
-  - "q1s4k"
-  - "q1s8k"
-  - "q1s16k"
-  - "q1s32k"
-
-  # Batch size 2
-  - "2q1s512"
-  - "2q1s1k"
-  - "2q1s2k"
-  - "2q1s4k"
-  - "2q1s8k"
-  - "2q1s16k"
-  - "2q1s32k"
-
-  # Batch size 4
-  - "4q1s512"
-  - "4q1s1k"
-  - "4q1s2k"
-  - "4q1s4k"
-  - "4q1s8k"
-  - "4q1s16k"
-  - "4q1s32k"
-
-  # Batch size 8
-  - "8q1s1k"
-  - "8q1s512"
-  - "8q1s2k"
-  - "8q1s4k"
-  - "8q1s8k"
-  - "8q1s16k"
-  - "8q1s32k"
-
-  # Batch size 16
-  - "16q1s512"
-  - "16q1s1k"
-  - "16q1s2k"
-  - "16q1s4k"
-  - "16q1s8k"
-  - "16q1s16k"
-  - "16q1s32k"
-
-  # Batch size 32
-  - "32q1s512"
-  - "32q1s1k"
-  - "32q1s2k"
-  - "32q1s4k"
-  - "32q1s8k"
-  - "32q1s16k"
-  - "32q1s32k"
-
-  # Batch size 64
-  - "64q1s1k"
-  - "64q1s512"
-  - "64q1s2k"
-  - "64q1s4k"
-  - "64q1s8k"
-  - "64q1s16k"
-  - "64q1s32k"
-
-  # Batch size 128
-  - "128q1s512"
-  - "128q1s1k"
-  - "128q1s2k"
-  - "128q1s4k"
-  - "128q1s8k"
-  - "128q1s16k"
-  - "128q1s32k"
-
-  # Batch size 256
-  - "256q1s1k"
-  - "256q1s512"
-  - "256q1s2k"
-  - "256q1s4k"
-  - "256q1s8k"
-  - "256q1s16k"
-  - "256q1s32k"
-
-# Available backends: FLASH_ATTN, TRITON_ATTN, FLASHINFER
-backends:
-  - FLASH_ATTN
-  - TRITON_ATTN
-  - FLASHINFER
-
-device: "cuda:0"
-profile_memory: false

benchmarks/attention_benchmarks/configs/standard_prefill.yaml

@@ -1,108 +0,0 @@
-# Standard attention prefill benchmark configuration
-# Sweeps num_q_heads and num_kv_heads to isolate effects of:
-#   1. GQA ratio (fixed num_q_heads=32, vary num_kv_heads)
-#   2. Absolute head count (fixed 4:1 ratio, vary scale)
-
-model:
-  num_layers: 32
-  num_q_heads: 32 # Base value, overridden by sweep
-  num_kv_heads: 8 # Base value, overridden by sweep
-  head_dim: 128
-  block_size: 16
-
-# Head count sweep: each entry overrides num_q_heads, num_kv_heads, and
-# head_dim where it differs from the base (128). Head counts are per-GPU
-# (i.e. after TP sharding).
-#
-# Group A — vary GQA ratio (fixed q=32, head_dim=128):
-#   32:32 (MHA), 32:8 (GQA 4:1), 32:4 (GQA 8:1), 32:1 (MQA)
-#
-# Groups B-E — real model configs at various TP degrees:
-#   Model              head_dim  Full    TP2      TP4      TP8
-#   Llama 3 8B           128    32:8   16:4      8:2      4:1
-#   Llama 3 70B          128    64:8   32:4     16:2      8:1
-#   GPT-OSS 120B          64    64:8   32:4     16:2      8:1
-#   Llama 3 405B         128   128:8   64:4     32:2     16:1
-model_parameter_sweep:
-  values:
-    # --- head_dim=128 (Llama 3 family) ---
-    - { num_q_heads: 32, num_kv_heads: 32, head_dim: 128 } # MHA 1:1
-    - { num_q_heads: 32, num_kv_heads: 1, head_dim: 128 } # MQA 32:1
-    - { num_q_heads: 4, num_kv_heads: 1, head_dim: 128 } # Llama 3 8B TP8
-    - { num_q_heads: 8, num_kv_heads: 2, head_dim: 128 } # Llama 3 8B TP4
-    - { num_q_heads: 16, num_kv_heads: 4, head_dim: 128 } # Llama 3 8B TP2
-    - { num_q_heads: 32, num_kv_heads: 8, head_dim: 128 } # Llama 3 8B TP1 / GQA 4:1
-    - { num_q_heads: 8, num_kv_heads: 1, head_dim: 128 } # Llama 3 70B TP8
-    - { num_q_heads: 16, num_kv_heads: 2, head_dim: 128 } # Llama 3 70B TP4
-    - { num_q_heads: 32, num_kv_heads: 4, head_dim: 128 } # Llama 3 70B TP2 / GQA 8:1
-    - { num_q_heads: 64, num_kv_heads: 8, head_dim: 128 } # Llama 3 70B TP1
-    - { num_q_heads: 16, num_kv_heads: 1, head_dim: 128 } # Llama 3 405B TP8
-    - { num_q_heads: 32, num_kv_heads: 2, head_dim: 128 } # Llama 3 405B TP4
-    - { num_q_heads: 64, num_kv_heads: 4, head_dim: 128 } # Llama 3 405B TP2
-    - { num_q_heads: 128, num_kv_heads: 8, head_dim: 128 } # Llama 3 405B TP1
-    # --- head_dim=64 (GPT-OSS 120B) ---
-    - { num_q_heads: 8, num_kv_heads: 1, head_dim: 64 } # GPT-OSS 120B TP8
-    - { num_q_heads: 16, num_kv_heads: 2, head_dim: 64 } # GPT-OSS 120B TP4
-    - { num_q_heads: 32, num_kv_heads: 4, head_dim: 64 } # GPT-OSS 120B TP2
-    - { num_q_heads: 64, num_kv_heads: 8, head_dim: 64 } # GPT-OSS 120B TP1
-  label_format: "{backend}_q{num_q_heads}kv{num_kv_heads}d{head_dim}"
-
-batch_specs:
-  # ---- batch_size x prefill_len grid (prefill: q_len == seq_len) ----
-  # Total tokens = batch_size * prefill_len, and prefill compute scales with
-  # prefill_len^2, so the largest cells are expensive. Trim batch sizes or
-  # lengths for quick iteration.
-
-  # Batch size 1
-  - "q512"
-  - "q1k"
-  - "q2k"
-  - "q4k"
-  - "q8k"
-  - "q16k"
-  - "q32k"
-
-  # Batch size 2
-  - "2q512"
-  - "2q1k"
-  - "2q2k"
-  - "2q4k"
-  - "2q8k"
-  - "2q16k"
-  - "2q32k"
-
-  # Batch size 4
-  - "4q512"
-  - "4q1k"
-  - "4q2k"
-  - "4q4k"
-  - "4q8k"
-  - "4q16k"
-  - "4q32k"
-
-  # Batch size 8
-  - "8q512"
-  - "8q1k"
-  - "8q2k"
-  - "8q4k"
-  - "8q8k"
-  - "8q16k"
-  - "8q32k"
-
-  # Batch size 16
-  - "16q512"
-  - "16q1k"
-  - "16q2k"
-  - "16q4k"
-  - "16q8k"
-  - "16q16k"
-  - "16q32k"
-
-# Available backends: FLASH_ATTN, TRITON_ATTN, FLASHINFER
-backends:
-  - FLASH_ATTN
-  - TRITON_ATTN
-  - FLASHINFER
-
-device: "cuda:0"
-profile_memory: false

benchmarks/attention_benchmarks/mla_runner.py

@@ -8,8 +8,6 @@ This module provides helpers for running MLA backends without
 needing full VllmConfig integration.
 """
 
-import statistics
-
 import numpy as np
 import torch
 from batch_spec import parse_batch_spec
@@ -19,8 +17,6 @@ from common import (
     MockIndexer,
     MockKVBProj,
     MockLayer,
-    run_do_bench,
-    run_ncu_profile,
     setup_mla_dims,
 )
 
@@ -708,8 +704,6 @@ def _run_single_benchmark(
     device: torch.device,
     indexer=None,
     kv_cache_dtype: str | None = None,
-    output_scale: float | None = None,
-    fuse_quant_op: bool = False,
 ) -> BenchmarkResult:
     """
     Run a single benchmark iteration.
@@ -723,11 +717,6 @@ def _run_single_benchmark(
         mla_dims: MLA dimension configuration
         device: Target device
         indexer: Optional MockIndexer for sparse backends
-        output_scale: Static per-tensor FP8 scale for prefill output. None
-            keeps the plain bf16 output (no quantization).
-        fuse_quant_op: With output_scale set, True lets the prefill kernel write
-            FP8 directly; False runs bf16 attention then a standalone static-FP8
-            quant. The delta isolates the saved post-quant kernel.
 
     Returns:
         BenchmarkResult with timing statistics
@@ -831,86 +820,63 @@ def _run_single_benchmark(
             num_prefill, mla_dims, query_fmt, device, torch.bfloat16
         )
 
-    # Prefill FP8 output: fused (kernel writes e4m3) vs separate post-quant.
+    # Build forward function
-    prefill_fp8_output = None
-    prefill_output_scale = None
-    prefill_quant_op = None
-    if has_prefill and output_scale is not None:
-        from vllm.platforms import current_platform
-
-        prefill_output_scale = torch.tensor(
-            [output_scale], device=device, dtype=torch.float32
-        )
-        if fuse_quant_op:
-            prefill_fp8_output = torch.empty_like(
-                prefill_inputs["output"], dtype=current_platform.fp8_dtype()
-            )
-        else:
-            from vllm.model_executor.layers.quantization.input_quant_fp8 import (
-                QuantFP8,
-            )
-            from vllm.model_executor.layers.quantization.utils.quant_utils import (
-                GroupShape,
-            )
-
-            prefill_quant_op = QuantFP8(static=True, group_shape=GroupShape.PER_TENSOR)
-
-    fused_output = output_scale is not None and fuse_quant_op
-
-    # Build forward function (runs a single decode/prefill pass)
     def forward_fn():
         results = []
         if has_decode:
             results.append(impl.forward_mqa(decode_inputs, kv_cache, metadata, layer))
         if has_prefill:
-            out = impl.forward_mha(
+            results.append(
-                prefill_inputs["q"],
+                impl.forward_mha(
-                prefill_inputs["k_c_normed"],
+                    prefill_inputs["q"],
-                prefill_inputs["k_pe"],
+                    prefill_inputs["k_c_normed"],
-                kv_cache,
+                    prefill_inputs["k_pe"],
-                metadata,
+                    kv_cache,
-                prefill_inputs["k_scale"],
+                    metadata,
-                prefill_fp8_output if fused_output else prefill_inputs["output"],
+                    prefill_inputs["k_scale"],
-                prefill_output_scale if fused_output else None,
+                    prefill_inputs["output"],
-            )
-            if fused_output:
-                out = prefill_fp8_output
-            elif prefill_quant_op is not None:
-                out, _ = prefill_quant_op(
-                    prefill_inputs["output"], prefill_output_scale
                 )
-            results.append(out)
+            )
         return results[0] if len(results) == 1 else tuple(results)
 
-    def benchmark_fn():
+    # Warmup
-        for _ in range(config.num_layers):
+    for _ in range(config.warmup_iters):
+        forward_fn()
+    torch.accelerator.synchronize()
+
+    # Optionally capture a CUDA graph after warmup.
+    # Graph replay eliminates CPU launch overhead so timings reflect pure
+    # kernel time.
+    if config.use_cuda_graphs:
+        graph = torch.cuda.CUDAGraph()
+        with torch.cuda.graph(graph):
             forward_fn()
+        benchmark_fn = graph.replay
+    else:
+        benchmark_fn = forward_fn
 
-    if config.ncu_profile:
+    # Benchmark
-        run_ncu_profile(benchmark_fn)
+    times = []
-        return BenchmarkResult(
+    for _ in range(config.repeats):
-            config=config,
+        start = torch.cuda.Event(enable_timing=True)
-            mean_time=0.0,
+        end = torch.cuda.Event(enable_timing=True)
-            median_time=0.0,
-            std_time=0.0,
-            min_time=0.0,
-            max_time=0.0,
-            throughput_tokens_per_sec=0.0,
-        )
 
-    all_ms = run_do_bench(benchmark_fn, config.use_cuda_graphs, config.warmup_ms)
+        start.record()
+        for _ in range(config.num_layers):
+            benchmark_fn()
+        end.record()
 
-    # Convert ms to seconds per layer
+        torch.accelerator.synchronize()
-    times = [t / 1000.0 / config.num_layers for t in all_ms]
+        elapsed_ms = start.elapsed_time(end)
-    mean_time = statistics.mean(times)
+        times.append(elapsed_ms / 1000.0 / config.num_layers)
 
+    mean_time = float(np.mean(times))
     return BenchmarkResult(
         config=config,
         mean_time=mean_time,
-        median_time=statistics.median(times),
+        std_time=float(np.std(times)),
-        std_time=statistics.stdev(times) if len(times) > 1 else 0.0,
+        min_time=float(np.min(times)),
-        min_time=min(times),
+        max_time=float(np.max(times)),
-        max_time=max(times),
         throughput_tokens_per_sec=total_q / mean_time if mean_time > 0 else 0,
     )
 
@@ -920,8 +886,6 @@ def _run_mla_benchmark_batched(
     configs_with_params: list[tuple],  # [(config, threshold, num_splits), ...]
     index_topk: int = 2048,
     prefill_backend: str | None = None,
-    output_scale: float | None = None,
-    fuse_quant_op: bool = False,
 ) -> list[BenchmarkResult]:
     """
     Unified batched MLA benchmark runner for all backends.
@@ -1061,8 +1025,6 @@ def _run_mla_benchmark_batched(
                     device,
                     indexer=indexer,
                     kv_cache_dtype=kv_cache_dtype,
-                    output_scale=output_scale,
-                    fuse_quant_op=fuse_quant_op,
                 )
                 results.append(result)
 
@@ -1090,8 +1052,6 @@ def run_mla_benchmark(
     num_kv_splits: int | None = None,
     index_topk: int = 2048,
     prefill_backend: str | None = None,
-    output_scale: float | None = None,
-    fuse_quant_op: bool = False,
 ) -> BenchmarkResult | list[BenchmarkResult]:
     """
     Unified MLA benchmark runner for all backends.
@@ -1111,9 +1071,6 @@ def run_mla_benchmark(
         index_topk: Topk value for sparse MLA backends (default 2048)
         prefill_backend: Prefill backend name (e.g., "fa3", "fa4").
             When set, forces the specified FlashAttention version for prefill.
-        output_scale: Static per-tensor FP8 scale for prefill output (None = bf16).
-        fuse_quant_op: With output_scale set, fuse the FP8 write into the prefill
-            kernel vs a standalone post-quant kernel. See _run_single_benchmark.
 
     Returns:
         BenchmarkResult (single mode) or list of BenchmarkResult (batched mode)
@@ -1138,12 +1095,7 @@ def run_mla_benchmark(
 
     # Use unified batched execution
     results = _run_mla_benchmark_batched(
-        backend,
+        backend, configs_with_params, index_topk, prefill_backend=prefill_backend
-        configs_with_params,
-        index_topk,
-        prefill_backend=prefill_backend,
-        output_scale=output_scale,
-        fuse_quant_op=fuse_quant_op,
     )
 
     # Return single result or list based on input

benchmarks/attention_benchmarks/runner.py

@@ -9,20 +9,13 @@ This module provides helpers for running standard attention backends
 """
 
 import logging
-import statistics
 import types
 from contextlib import contextmanager
 
+import numpy as np
 import torch
 from batch_spec import parse_batch_spec, reorder_for_flashinfer
-from common import (
+from common import BenchmarkConfig, BenchmarkResult, MockLayer, get_attention_scale
-    BenchmarkConfig,
-    BenchmarkResult,
-    MockLayer,
-    get_attention_scale,
-    run_do_bench,
-    run_ncu_profile,
-)
 
 from vllm.config import (
     CacheConfig,
@@ -215,13 +208,6 @@ def _create_backend_impl(
 
     scale = get_attention_scale(config.head_dim)
 
-    # Set v_head_dim for diff-headdim backends. Always reset (defaulting to
-    # head_dim) so a prior run's value doesn't leak into this one via the
-    # backend's class-level state.
-    if hasattr(backend_class, "set_head_size_v"):
-        v_dim = config.v_head_dim if config.v_head_dim is not None else config.head_dim
-        backend_class.set_head_size_v(v_dim)
-
     impl = backend_class.get_impl_cls()(
         num_heads=config.num_q_heads,
         head_size=config.head_dim,
@@ -314,7 +300,6 @@ def _create_input_tensors(
         from vllm.platforms import current_platform
 
         q_dtype = current_platform.fp8_dtype()
-    v_dim = config.v_head_dim if config.v_head_dim is not None else config.head_dim
     q_list = [
         torch.randn(
             total_q, config.num_q_heads, config.head_dim, device=device, dtype=dtype
@@ -328,7 +313,9 @@ def _create_input_tensors(
         for _ in range(config.num_layers)
     ]
     v_list = [
-        torch.randn(total_q, config.num_kv_heads, v_dim, device=device, dtype=dtype)
+        torch.randn(
+            total_q, config.num_kv_heads, config.head_dim, device=device, dtype=dtype
+        )
         for _ in range(config.num_layers)
     ]
     return q_list, k_list, v_list
@@ -402,17 +389,14 @@ def _run_single_benchmark(
     device: torch.device,
     dtype: torch.dtype,
 ) -> tuple:
-    """Run single benchmark using triton's do_bench_cudagraph/do_bench.
+    """Run single benchmark iteration with warmup and timing loop."""
-
-    Returns:
-        (timing_stats, mem_stats) where timing_stats is a dict with
-        mean/std/min/max in seconds per layer.
-    """
     total_q = q_list[0].shape[0]
-    v_dim = config.v_head_dim if config.v_head_dim is not None else config.head_dim
+    out = torch.empty(
-    out = torch.empty(total_q, config.num_q_heads, v_dim, device=device, dtype=dtype)
+        total_q, config.num_q_heads, config.head_dim, device=device, dtype=dtype
+    )
 
-    def benchmark_fn():
+    # Warmup
+    for _ in range(config.warmup_iters):
         for i in range(config.num_layers):
             impl.forward(
                 layer,
@@ -423,22 +407,52 @@ def _run_single_benchmark(
                 attn_metadata,
                 output=out,
             )
+    torch.accelerator.synchronize()
 
-    if config.ncu_profile:
+    # Optionally capture a CUDA graph after warmup.
-        run_ncu_profile(benchmark_fn)
+    # Graph replay eliminates CPU launch overhead so timings reflect pure
-        timing_stats = dict.fromkeys(("mean", "median", "std", "min", "max"), 0.0)
+    # kernel time.
+    if config.use_cuda_graphs:
+        graph = torch.cuda.CUDAGraph()
+        with torch.cuda.graph(graph):
+            for i in range(config.num_layers):
+                impl.forward(
+                    layer,
+                    q_list[i],
+                    k_list[i],
+                    v_list[i],
+                    cache_list[i],
+                    attn_metadata,
+                    output=out,
+                )
+        benchmark_fn = graph.replay
     else:
-        all_ms = run_do_bench(benchmark_fn, config.use_cuda_graphs, config.warmup_ms)
 
-        # Convert ms to seconds per layer
+        def benchmark_fn():
-        times = [t / 1000.0 / config.num_layers for t in all_ms]
+            for i in range(config.num_layers):
-        timing_stats = {
+                impl.forward(
-            "mean": statistics.mean(times),
+                    layer,
-            "std": statistics.stdev(times) if len(times) > 1 else 0.0,
+                    q_list[i],
-            "min": min(times),
+                    k_list[i],
-            "max": max(times),
+                    v_list[i],
-            "median": statistics.median(times),
+                    cache_list[i],
-        }
+                    attn_metadata,
+                    output=out,
+                )
+
+    # Benchmark
+    times = []
+    for _ in range(config.repeats):
+        start = torch.cuda.Event(enable_timing=True)
+        end = torch.cuda.Event(enable_timing=True)
+
+        start.record()
+        benchmark_fn()
+        end.record()
+
+        torch.accelerator.synchronize()
+        elapsed_ms = start.elapsed_time(end)
+        times.append(elapsed_ms / 1000.0 / config.num_layers)  # seconds per layer
 
     mem_stats = {}
     if config.profile_memory:
@@ -447,7 +461,7 @@ def _run_single_benchmark(
             "reserved_mb": torch.accelerator.memory_reserved(device) / 1024**2,
         }
 
-    return timing_stats, mem_stats
+    return times, mem_stats
 
 
 # ============================================================================
@@ -527,12 +541,6 @@ def run_attention_benchmark(config: BenchmarkConfig) -> BenchmarkResult:
                 common_attn_metadata=common_metadata,
             )
 
-            # Override num_splits for split-K testing (FlashAttention only)
-            if config.num_splits is not None and hasattr(
-                attn_metadata, "max_num_splits"
-            ):
-                attn_metadata.max_num_splits = config.num_splits
-
             # Only quantize queries when the impl supports it
             quantize_query = config.kv_cache_dtype.startswith("fp8") and getattr(
                 impl, "supports_quant_query_input", False
@@ -545,7 +553,7 @@ def run_attention_benchmark(config: BenchmarkConfig) -> BenchmarkResult:
                 config, max_num_blocks, backend_class, device, dtype
             )
 
-            timing_stats, mem_stats = _run_single_benchmark(
+            times, mem_stats = _run_single_benchmark(
                 config,
                 impl,
                 layer,
@@ -558,16 +566,15 @@ def run_attention_benchmark(config: BenchmarkConfig) -> BenchmarkResult:
                 dtype,
             )
 
-    mean_time = timing_stats["mean"]
+    mean_time = np.mean(times)
     throughput = total_q / mean_time if mean_time > 0 else 0
 
     return BenchmarkResult(
         config=config,
         mean_time=mean_time,
-        median_time=timing_stats["median"],
+        std_time=np.std(times),
-        std_time=timing_stats["std"],
+        min_time=np.min(times),
-        min_time=timing_stats["min"],
+        max_time=np.max(times),
-        max_time=timing_stats["max"],
         throughput_tokens_per_sec=throughput,
         memory_allocated_mb=mem_stats.get("allocated_mb"),
         memory_reserved_mb=mem_stats.get("reserved_mb"),

benchmarks/benchmark_hidden_state_extraction.py

@@ -92,6 +92,7 @@ def run_baseline(
     llm = LLM(
         model=model,
         enable_prefix_caching=False,
+        enable_chunked_prefill=False,
         **extra_args,
     )
     sampling_params = SamplingParams(max_tokens=1)
@@ -193,6 +194,7 @@ async def _run_extraction_async(
     engine_args = AsyncEngineArgs(
         model=model,
         enable_prefix_caching=False,
+        enable_chunked_prefill=False,
         max_num_batched_tokens=40960,
         max_model_len=40960,
         speculative_config={

benchmarks/benchmark_pin_memory.py

@@ -1,358 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""Benchmark and regression-test pinned (page-locked) CPU memory for vLLM.
-
-Verifies that enabling pinned memory does not regress throughput or latency
-compared to unpinned memory.  Each condition runs in an isolated ``spawn``
-subprocess so both start from a cold CUDA context, giving an unbiased
-comparison.
-
-Usage
------
-Run all tests with the default model::
-
-    python benchmarks/benchmark_pin_memory.py -v
-
-Override the model and optional max-model-len::
-
-    python benchmarks/benchmark_pin_memory.py --model unsloth/Qwen3-1.7B -v
-    python benchmarks/benchmark_pin_memory.py --model unsloth/Qwen3-1.7B \
-        --max-model-len 8192 -v
-
-Run only throughput or latency tests::
-
-    python benchmarks/benchmark_pin_memory.py -v -k test_throughput
-    python benchmarks/benchmark_pin_memory.py -v -k test_latency
-
-Run only the v1 or v2 runner variant::
-
-    python benchmarks/benchmark_pin_memory.py -v -k v1
-    python benchmarks/benchmark_pin_memory.py -v -k v2
-
-Note: on WSL2, v1 runner tests are skipped because pin memory is not available
-for the v1 runner without cpu_offload_gb.  Run on other platforms to exercise v1.
-"""
-
-import argparse
-import json
-import multiprocessing
-import sys
-import tempfile
-
-import pytest
-
-# Allow up to 2% degradation.  Both benchmark runs start from an identical
-# cold CUDA context (separate spawn subprocesses), so the measured difference
-# reflects the genuine pin_memory overhead rather than cold/warm ordering bias.
-_THROUGHPUT_TOLERANCE = 0.98
-_THROUGHPUT_NUM_REQUESTS = 200
-_THROUGHPUT_INPUT_LEN = 128
-_THROUGHPUT_OUTPUT_LEN = 512
-_THROUGHPUT_MAX_NUM_SEQS = 128
-
-# Latency benchmark constants — match latency.py defaults.
-_LATENCY_TOLERANCE = 1.02  # Allow up to 2% latency regression.
-_LATENCY_BATCH_SIZE = 64
-_LATENCY_INPUT_LEN = 32
-_LATENCY_OUTPUT_LEN = 128
-_LATENCY_WARMUP_ITERS = 5
-_LATENCY_BENCH_ITERS = 15
-
-_DEFAULT_MODEL = "unsloth/Qwen3-1.7B"
-_DEFAULT_MAX_MODEL_LEN = 16384
-
-
-def _benchmark_args() -> argparse.Namespace:
-    parser = argparse.ArgumentParser(add_help=False)
-    parser.add_argument("--model", default=_DEFAULT_MODEL)
-    parser.add_argument("--max-model-len", type=int, default=_DEFAULT_MAX_MODEL_LEN)
-    args, _ = parser.parse_known_args()
-    return args
-
-
-@pytest.fixture
-def model() -> str:
-    return _benchmark_args().model
-
-
-@pytest.fixture
-def max_model_len() -> int:
-    return _benchmark_args().max_model_len
-
-
-def _skip_if_pin_memory_not_available(engine_args_kwargs: dict) -> None:
-    """Skip the current pytest test if pin_memory is unavailable for this config."""
-    import vllm.utils.platform_utils as pu
-    from vllm.config import set_current_vllm_config
-    from vllm.engine.arg_utils import EngineArgs
-
-    vllm_config = EngineArgs(**engine_args_kwargs).create_engine_config()
-    with set_current_vllm_config(vllm_config):
-        pu.is_pin_memory_available.cache_clear()
-        if not pu.is_pin_memory_available():
-            import os
-
-            runner = "v2" if os.environ.get("VLLM_USE_V2_MODEL_RUNNER") == "1" else "v1"
-            model = engine_args_kwargs.get("model", "unknown")
-            print(
-                f"\033[33mSKIP: pin_memory not available for "
-                f"{runner} runner, model={model}\033[0m"
-            )
-            pytest.skip("pin_memory not available for this configuration")
-
-
-def _throughput_worker(
-    pin: bool,
-    engine_args_kwargs: dict,
-    q: "multiprocessing.Queue[float]",
-    v2_mode: bool = False,
-) -> None:
-    """Run throughput benchmark in a fresh spawn subprocess.
-
-    Delegates to vllm/benchmarks/throughput.py main() using the random dataset,
-    so the methodology matches the official benchmark.  Results are written to a
-    temp JSON file and forwarded through the queue as tokens/s.
-
-    v2_mode: when True, monkeypatches is_uva_available() to always return True
-    so the v2 model runner's UVA buffers remain functional even when pin=False.
-    This isolates the non-UVA pin_memory paths in v2.
-    """
-    import vllm.utils.platform_utils as pu
-    from vllm.platforms import current_platform
-
-    pu.is_pin_memory_available.cache_clear()
-    pu.is_uva_available.cache_clear()
-    type(current_platform).is_pin_memory_available = classmethod(lambda cls: pin)
-    if v2_mode:
-        pu.is_uva_available = lambda: True
-
-    from vllm.benchmarks.throughput import add_cli_args
-    from vllm.benchmarks.throughput import main as throughput_main
-
-    parser = argparse.ArgumentParser()
-    add_cli_args(parser)
-    args = parser.parse_args([])
-
-    for key, val in engine_args_kwargs.items():
-        setattr(args, key, val)
-    args.max_num_seqs = _THROUGHPUT_MAX_NUM_SEQS
-    args.dataset_name = "random"
-    args.input_len = _THROUGHPUT_INPUT_LEN
-    args.output_len = _THROUGHPUT_OUTPUT_LEN
-    # Nullify defaults that conflict with explicit input/output_len.
-    args.random_input_len = None
-    args.random_output_len = None
-    args.random_prefix_len = None
-    args.num_prompts = _THROUGHPUT_NUM_REQUESTS
-    args.seed = 0
-    args.disable_detokenize = True
-
-    with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
-        tmp_path = f.name
-    args.output_json = tmp_path
-
-    throughput_main(args)
-
-    with open(tmp_path) as f:
-        results = json.load(f)
-    q.put(results["tokens_per_second"])
-
-
-def _run_throughput_benchmark(
-    pin: bool,
-    engine_args_kwargs: dict,
-    v2_mode: bool = False,
-) -> float:
-    ctx = multiprocessing.get_context("spawn")
-    q = ctx.Queue()
-    p = ctx.Process(
-        target=_throughput_worker,
-        args=(pin, engine_args_kwargs, q, v2_mode),
-    )
-    p.start()
-    p.join()
-    if p.exitcode != 0:
-        raise RuntimeError(
-            f"Throughput benchmark subprocess (pin={pin}) exited with code {p.exitcode}"
-        )
-    return q.get()
-
-
-def _latency_worker(
-    pin: bool,
-    engine_args_kwargs: dict,
-    q: "multiprocessing.Queue[dict]",
-    v2_mode: bool = False,
-) -> None:
-    """Run latency benchmark in a fresh spawn subprocess.
-
-    Follows latency.py methodology: fixed batch of dummy token IDs, warmup
-    iterations to reach steady state, then timed iterations reduced to avg
-    and percentiles.  Results are written to a temp JSON file by latency_main
-    and forwarded through the queue.
-    """
-    import vllm.utils.platform_utils as pu
-    from vllm.platforms import current_platform
-
-    pu.is_pin_memory_available.cache_clear()
-    pu.is_uva_available.cache_clear()
-    type(current_platform).is_pin_memory_available = classmethod(lambda cls: pin)
-    if v2_mode:
-        pu.is_uva_available = lambda: True
-
-    from vllm.benchmarks.latency import add_cli_args
-    from vllm.benchmarks.latency import main as latency_main
-
-    parser = argparse.ArgumentParser()
-    add_cli_args(parser)
-    args = parser.parse_args([])
-
-    for key, val in engine_args_kwargs.items():
-        setattr(args, key, val)
-    args.input_len = _LATENCY_INPUT_LEN
-    args.output_len = _LATENCY_OUTPUT_LEN
-    args.batch_size = _LATENCY_BATCH_SIZE
-    args.num_iters_warmup = _LATENCY_WARMUP_ITERS
-    args.num_iters = _LATENCY_BENCH_ITERS
-    args.profile = False
-    args.disable_detokenize = True
-
-    with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
-        tmp_path = f.name
-    args.output_json = tmp_path
-
-    latency_main(args)
-
-    with open(tmp_path) as f:
-        results = json.load(f)
-    q.put(results)
-
-
-def _run_latency_benchmark(
-    pin: bool,
-    engine_args_kwargs: dict,
-    v2_mode: bool = False,
-) -> dict:
-    ctx = multiprocessing.get_context("spawn")
-    q = ctx.Queue()
-    p = ctx.Process(
-        target=_latency_worker,
-        args=(pin, engine_args_kwargs, q, v2_mode),
-    )
-    p.start()
-    p.join()
-    if p.exitcode != 0:
-        raise RuntimeError(
-            f"Latency benchmark subprocess (pin={pin}) exited with code {p.exitcode}"
-        )
-    return q.get()
-
-
-@pytest.mark.parametrize(
-    "test_v2_runner",
-    [
-        pytest.param(False, id="v1"),
-        pytest.param(True, id="v2"),
-    ],
-)
-class TestPinnedMemory:
-    """Verify pinned memory yields >= throughput vs unpinned via real vLLM inference."""
-
-    def test_throughput(self, monkeypatch, test_v2_runner, model, max_model_len):
-        """Benchmark throughput with pin_memory forced on then off.
-
-        Delegates to vllm/benchmarks/throughput.py main() with the random
-        dataset.  Each condition runs in an isolated spawn subprocess so both
-        start from a cold CUDA context, giving an unbiased comparison.
-        """
-        monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
-        monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1" if test_v2_runner else "0")
-
-        engine_args_kwargs = dict(
-            model=model,
-            gpu_memory_utilization=0.88,
-            max_model_len=max_model_len,
-            enable_prefix_caching=False,
-        )
-
-        _skip_if_pin_memory_not_available(engine_args_kwargs)
-
-        unpinned_tps = _run_throughput_benchmark(
-            False, engine_args_kwargs, v2_mode=test_v2_runner
-        )
-        pinned_tps = _run_throughput_benchmark(
-            True, engine_args_kwargs, v2_mode=test_v2_runner
-        )
-
-        pct_diff = (pinned_tps - unpinned_tps) / unpinned_tps * 100
-        runner = "v2" if test_v2_runner else "v1"
-        print(
-            f"\n=== Throughput results ({runner} runner, {model}) ==="
-            f"\npin_memory=True:  {pinned_tps:.1f} tok/s"
-            f"\npin_memory=False: {unpinned_tps:.1f} tok/s"
-            f"\nDifference: {pct_diff:+.1f}% (pinned vs unpinned)"
-        )
-
-        assert pinned_tps >= unpinned_tps * _THROUGHPUT_TOLERANCE, (
-            f"Pinned throughput ({pinned_tps:.1f} tok/s) fell more than "
-            f"{(1.0 - _THROUGHPUT_TOLERANCE) * 100:.1f}% below "
-            f"unpinned ({unpinned_tps:.1f} tok/s)."
-        )
-
-    def test_latency(self, monkeypatch, test_v2_runner, model, max_model_len):
-        """Benchmark per-batch latency with pin_memory forced on then off.
-
-        Follows vllm/benchmarks/latency.py: fixed dummy-token batch, warmup
-        iterations to reach steady state, then timed iterations reduced to avg
-        and percentiles.  Subprocesses run serially so each gets a cold CUDA
-        context without GPU memory pressure from the other run.
-        """
-        monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
-        monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1" if test_v2_runner else "0")
-
-        engine_args_kwargs = dict(
-            model=model,
-            gpu_memory_utilization=0.88,
-            max_model_len=max_model_len,
-            enable_prefix_caching=False,
-        )
-
-        _skip_if_pin_memory_not_available(engine_args_kwargs)
-
-        unpinned = _run_latency_benchmark(
-            False, engine_args_kwargs, v2_mode=test_v2_runner
-        )
-        pinned = _run_latency_benchmark(
-            True, engine_args_kwargs, v2_mode=test_v2_runner
-        )
-
-        pct_diff = (
-            (pinned["avg_latency"] - unpinned["avg_latency"])
-            / unpinned["avg_latency"]
-            * 100
-        )
-        runner = "v2" if test_v2_runner else "v1"
-        print(
-            f"\n=== Latency results ({runner} runner, {model}) ==="
-            f"\npin_memory=True:  avg={pinned['avg_latency']:.3f}s"
-            f"  p50={pinned['percentiles']['50']:.3f}s"
-            f"  p99={pinned['percentiles']['99']:.3f}s"
-            f"\npin_memory=False: avg={unpinned['avg_latency']:.3f}s"
-            f"  p50={unpinned['percentiles']['50']:.3f}s"
-            f"  p99={unpinned['percentiles']['99']:.3f}s"
-            f"\nDifference: {pct_diff:+.1f}% (pinned vs unpinned)"
-        )
-
-        assert pinned["avg_latency"] <= unpinned["avg_latency"] * _LATENCY_TOLERANCE, (
-            f"Pinned avg latency ({pinned['avg_latency']:.3f}s) exceeded "
-            f"unpinned ({unpinned['avg_latency']:.3f}s) by more than "
-            f"{(_LATENCY_TOLERANCE - 1.0) * 100:.1f}%."
-        )
-
-
-if __name__ == "__main__":
-    _parser = argparse.ArgumentParser(add_help=False)
-    _parser.add_argument("--model", default=_DEFAULT_MODEL)
-    _parser.add_argument("--max-model-len", type=int, default=_DEFAULT_MAX_MODEL_LEN)
-    _, _remaining = _parser.parse_known_args()
-    sys.exit(pytest.main([__file__] + _remaining))

benchmarks/disagg_benchmarks/disagg_overhead_benchmark.sh

@@ -0,0 +1,143 @@
+#!/bin/bash
+
+# benchmark the overhead of disaggregated prefill.
+# methodology:
+# - send all request to prefill vLLM instance. It will buffer KV cache.
+# - then send all request to decode instance.
+# - The TTFT of decode instance is the overhead.
+
+set -ex
+
+kill_gpu_processes() {
+  # kill all processes on GPU.
+  pgrep pt_main_thread | xargs -r kill -9
+  pgrep python3 | xargs -r kill -9
+  # vLLM now names the process with VLLM prefix after https://github.com/vllm-project/vllm/pull/21445
+  pgrep VLLM | xargs -r kill -9
+  sleep 10
+
+  # remove vllm config file
+  rm -rf ~/.config/vllm
+
+  # Print the GPU memory usage
+  # so that we know if all GPU processes are killed.
+  gpu_memory_usage=$(nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits -i 0)
+  # The memory usage should be 0 MB.
+  echo "GPU 0 Memory Usage: $gpu_memory_usage MB"
+}
+
+wait_for_server() {
+  # wait for vllm server to start
+  # return 1 if vllm server crashes
+  local port=$1
+  timeout 1200 bash -c "
+    until curl -s localhost:${port}/v1/completions > /dev/null; do
+      sleep 1
+    done" && return 0 || return 1
+}
+
+
+benchmark() {
+
+  export VLLM_LOGGING_LEVEL=DEBUG
+  export VLLM_HOST_IP=$(hostname -I | awk '{print $1}')
+
+  # compare chunked prefill with disaggregated prefill
+
+  results_folder="./results"
+  model="meta-llama/Meta-Llama-3.1-8B-Instruct"
+  dataset_name="sonnet"
+  dataset_path="../sonnet_4x.txt"
+  num_prompts=10
+  qps=$1
+  prefix_len=50
+  input_len=2048
+  output_len=$2
+
+
+  CUDA_VISIBLE_DEVICES=0 vllm serve $model \
+    --port 8100 \
+    --max-model-len 10000 \
+    --gpu-memory-utilization 0.6 \
+    --kv-transfer-config \
+    '{"kv_connector":"P2pNcclConnector","kv_role":"kv_producer","kv_rank":0,"kv_parallel_size":2,"kv_buffer_size":5e9}' &
+
+
+  CUDA_VISIBLE_DEVICES=1 vllm serve $model \
+    --port 8200 \
+    --max-model-len 10000 \
+    --gpu-memory-utilization 0.6 \
+    --kv-transfer-config \
+    '{"kv_connector":"P2pNcclConnector","kv_role":"kv_consumer","kv_rank":1,"kv_parallel_size":2,"kv_buffer_size":5e9}' &
+
+  wait_for_server 8100
+  wait_for_server 8200
+
+  # let the prefill instance finish prefill
+  vllm bench serve \
+    --backend vllm \
+    --model $model \
+    --dataset-name $dataset_name \
+    --dataset-path $dataset_path \
+    --sonnet-input-len $input_len \
+    --sonnet-output-len "$output_len" \
+    --sonnet-prefix-len $prefix_len \
+    --num-prompts $num_prompts \
+    --port 8100 \
+    --save-result \
+    --result-dir $results_folder \
+    --result-filename disagg_prefill_tp1.json \
+    --request-rate "inf"
+
+
+  # send the request to decode.
+  # The TTFT of this command will be the overhead of disagg prefill impl.
+  vllm bench serve \
+    --backend vllm \
+    --model $model \
+    --dataset-name $dataset_name \
+    --dataset-path $dataset_path \
+    --sonnet-input-len $input_len \
+    --sonnet-output-len "$output_len" \
+    --sonnet-prefix-len $prefix_len \
+    --num-prompts $num_prompts \
+    --port 8200 \
+    --save-result \
+    --result-dir $results_folder \
+    --result-filename disagg_prefill_tp1_overhead.json \
+    --request-rate "$qps"
+  kill_gpu_processes
+
+}
+
+
+main() {
+
+  (which wget && which curl) || (apt-get update && apt-get install -y wget curl)
+  (which jq) || (apt-get -y install jq)
+  (which socat) || (apt-get -y install socat)
+
+  pip install quart httpx datasets
+
+  cd "$(dirname "$0")"
+
+  cd ..
+  # create sonnet-4x.txt
+  echo "" > sonnet_4x.txt
+  for _ in {1..4}
+  do
+    cat sonnet.txt >> sonnet_4x.txt
+  done
+  cd disagg_benchmarks
+
+  rm -rf results
+  mkdir results
+
+  default_qps=1
+  default_output_len=1
+  benchmark $default_qps $default_output_len
+
+}
+
+
+main "$@"

benchmarks/disagg_benchmarks/disagg_performance_benchmark.sh

@@ -0,0 +1,157 @@
+#!/bin/bash
+
+# Requirement: 2x GPUs.
+
+
+# Model: meta-llama/Meta-Llama-3.1-8B-Instruct
+# Query: 1024 input tokens, 6 output tokens, QPS 2/4/6/8, 100 requests
+# Resource: 2x GPU
+# Approaches:
+# 2. Chunked prefill: 2 vllm instance with tp=4, equivalent to 1 tp=4 instance with QPS 4
+# 3. Disaggregated prefill: 1 prefilling instance and 1 decoding instance
+# Prefilling instance: max_output_token=1
+# Decoding instance: force the input tokens be the same across requests to bypass prefilling
+
+set -ex
+
+kill_gpu_processes() {
+  # kill all processes on GPU.
+  pgrep pt_main_thread | xargs -r kill -9
+  pgrep python3 | xargs -r kill -9
+  # vLLM now names the process with VLLM prefix after https://github.com/vllm-project/vllm/pull/21445
+  pgrep VLLM | xargs -r kill -9
+  for port in 8000 8100 8200; do lsof -t -i:$port | xargs -r kill -9; done
+  sleep 1
+}
+
+wait_for_server() {
+  # wait for vllm server to start
+  # return 1 if vllm server crashes
+  local port=$1
+  timeout 1200 bash -c "
+    until curl -s localhost:${port}/v1/completions > /dev/null; do
+      sleep 1
+    done" && return 0 || return 1
+}
+
+
+launch_chunked_prefill() {
+  model="meta-llama/Meta-Llama-3.1-8B-Instruct"
+  # disagg prefill
+  CUDA_VISIBLE_DEVICES=0 vllm serve $model \
+    --port 8100 \
+    --max-model-len 10000 \
+    --enable-chunked-prefill \
+    --gpu-memory-utilization 0.6 &
+  CUDA_VISIBLE_DEVICES=1 vllm serve $model \
+    --port 8200 \
+    --max-model-len 10000 \
+    --enable-chunked-prefill \
+    --gpu-memory-utilization 0.6 &
+  wait_for_server 8100
+  wait_for_server 8200
+  python3 round_robin_proxy.py &
+  sleep 1
+}
+
+
+launch_disagg_prefill() {
+  model="meta-llama/Meta-Llama-3.1-8B-Instruct"
+  # disagg prefill
+  CUDA_VISIBLE_DEVICES=0 vllm serve $model \
+    --port 8100 \
+    --max-model-len 10000 \
+    --gpu-memory-utilization 0.6 \
+    --kv-transfer-config \
+    '{"kv_connector":"P2pNcclConnector","kv_role":"kv_producer","kv_rank":0,"kv_parallel_size":2,"kv_buffer_size":5e9}' &
+
+  CUDA_VISIBLE_DEVICES=1 vllm serve $model \
+    --port 8200 \
+    --max-model-len 10000 \
+    --gpu-memory-utilization 0.6 \
+    --kv-transfer-config \
+    '{"kv_connector":"P2pNcclConnector","kv_role":"kv_consumer","kv_rank":1,"kv_parallel_size":2,"kv_buffer_size":5e9}' &
+
+  wait_for_server 8100
+  wait_for_server 8200
+  python3 disagg_prefill_proxy_server.py &
+  sleep 1
+}
+
+
+benchmark() {
+  results_folder="./results"
+  model="meta-llama/Meta-Llama-3.1-8B-Instruct"
+  dataset_name="sonnet"
+  dataset_path="../sonnet_4x.txt"
+  num_prompts=100
+  qps=$1
+  prefix_len=50
+  input_len=1024
+  output_len=$2
+  tag=$3
+
+  vllm bench serve \
+    --backend vllm \
+    --model $model \
+    --dataset-name $dataset_name \
+    --dataset-path $dataset_path \
+    --sonnet-input-len $input_len \
+    --sonnet-output-len "$output_len" \
+    --sonnet-prefix-len $prefix_len \
+    --num-prompts $num_prompts \
+    --port 8000 \
+    --save-result \
+    --result-dir $results_folder \
+    --result-filename "$tag"-qps-"$qps".json \
+    --request-rate "$qps"
+
+  sleep 2
+}
+
+
+main() {
+
+  (which wget && which curl) || (apt-get update && apt-get install -y wget curl)
+  (which jq) || (apt-get -y install jq)
+  (which socat) || (apt-get -y install socat)
+  (which lsof) || (apt-get -y install lsof)
+
+  pip install quart httpx matplotlib aiohttp datasets
+
+  cd "$(dirname "$0")"
+
+  cd ..
+  # create sonnet-4x.txt so that we can sample 2048 tokens for input
+  echo "" > sonnet_4x.txt
+  for _ in {1..4}
+  do
+    cat sonnet.txt >> sonnet_4x.txt
+  done
+  cd disagg_benchmarks
+
+  rm -rf results
+  mkdir results
+
+  default_output_len=6
+
+  export VLLM_HOST_IP=$(hostname -I | awk '{print $1}')
+
+  launch_chunked_prefill
+  for qps in 2 4 6 8; do
+  benchmark $qps $default_output_len chunked_prefill
+  done
+  kill_gpu_processes
+
+  launch_disagg_prefill
+  for qps in 2 4 6 8; do
+  benchmark $qps $default_output_len disagg_prefill
+  done
+  kill_gpu_processes
+
+  python3 visualize_benchmark_results.py
+
+}
+
+
+main "$@"

benchmarks/disagg_benchmarks/disagg_prefill_proxy_server.py

@@ -0,0 +1,260 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import argparse
+import asyncio
+import logging
+import os
+import time
+import uuid
+from urllib.parse import urlparse
+
+import aiohttp
+from quart import Quart, Response, make_response, request
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+def parse_args():
+    """parse command line arguments"""
+    parser = argparse.ArgumentParser(description="vLLM P/D disaggregation proxy server")
+
+    # Add args
+    parser.add_argument(
+        "--timeout",
+        type=float,
+        default=6 * 60 * 60,
+        help="Timeout for backend service requests in seconds (default: 21600)",
+    )
+    parser.add_argument(
+        "--port",
+        type=int,
+        default=8000,
+        help="Port to run the server on (default: 8000)",
+    )
+    parser.add_argument(
+        "--prefill-url",
+        type=str,
+        default="http://localhost:8100",
+        help="Prefill service base URL (protocol + host[:port])",
+    )
+    parser.add_argument(
+        "--decode-url",
+        type=str,
+        default="http://localhost:8200",
+        help="Decode service base URL (protocol + host[:port])",
+    )
+    parser.add_argument(
+        "--kv-host",
+        type=str,
+        default="localhost",
+        help="Hostname or IP used by KV transfer (default: localhost)",
+    )
+    parser.add_argument(
+        "--prefill-kv-port",
+        type=int,
+        default=14579,
+        help="Prefill KV port (default: 14579)",
+    )
+    parser.add_argument(
+        "--decode-kv-port",
+        type=int,
+        default=14580,
+        help="Decode KV port (default: 14580)",
+    )
+
+    return parser.parse_args()
+
+
+def main():
+    """parse command line arguments"""
+    args = parse_args()
+
+    # Initialize configuration using command line parameters
+    AIOHTTP_TIMEOUT = aiohttp.ClientTimeout(total=args.timeout)
+    PREFILL_SERVICE_URL = args.prefill_url
+    DECODE_SERVICE_URL = args.decode_url
+    PORT = args.port
+
+    PREFILL_KV_ADDR = f"{args.kv_host}:{args.prefill_kv_port}"
+    DECODE_KV_ADDR = f"{args.kv_host}:{args.decode_kv_port}"
+
+    logger.info(
+        "Proxy resolved KV addresses -> prefill: %s, decode: %s",
+        PREFILL_KV_ADDR,
+        DECODE_KV_ADDR,
+    )
+
+    app = Quart(__name__)
+
+    # Attach the configuration object to the application instance so helper
+    # coroutines can read the resolved backend URLs and timeouts without using
+    # globals.
+    app.config.update(
+        {
+            "AIOHTTP_TIMEOUT": AIOHTTP_TIMEOUT,
+            "PREFILL_SERVICE_URL": PREFILL_SERVICE_URL,
+            "DECODE_SERVICE_URL": DECODE_SERVICE_URL,
+            "PREFILL_KV_ADDR": PREFILL_KV_ADDR,
+            "DECODE_KV_ADDR": DECODE_KV_ADDR,
+        }
+    )
+
+    def _normalize_base_url(url: str) -> str:
+        """Remove any trailing slash so path joins behave predictably."""
+        return url.rstrip("/")
+
+    def _get_host_port(url: str) -> str:
+        """Return the hostname:port portion for logging and KV headers."""
+        parsed = urlparse(url)
+        host = parsed.hostname or "localhost"
+        port = parsed.port
+        if port is None:
+            port = 80 if parsed.scheme == "http" else 443
+        return f"{host}:{port}"
+
+    PREFILL_BASE = _normalize_base_url(PREFILL_SERVICE_URL)
+    DECODE_BASE = _normalize_base_url(DECODE_SERVICE_URL)
+    KV_TARGET = _get_host_port(DECODE_SERVICE_URL)
+
+    def _build_headers(request_id: str) -> dict[str, str]:
+        """Construct the headers expected by vLLM's P2P disagg connector."""
+        headers: dict[str, str] = {"X-Request-Id": request_id, "X-KV-Target": KV_TARGET}
+        api_key = os.environ.get("OPENAI_API_KEY")
+        if api_key:
+            headers["Authorization"] = f"Bearer {api_key}"
+        return headers
+
+    async def _run_prefill(
+        request_path: str,
+        payload: dict,
+        headers: dict[str, str],
+        request_id: str,
+    ):
+        url = f"{PREFILL_BASE}{request_path}"
+        start_ts = time.perf_counter()
+        logger.info("[prefill] start request_id=%s url=%s", request_id, url)
+        try:
+            async with (
+                aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session,
+                session.post(url=url, json=payload, headers=headers) as resp,
+            ):
+                if resp.status != 200:
+                    error_text = await resp.text()
+                    raise RuntimeError(
+                        f"Prefill backend error {resp.status}: {error_text}"
+                    )
+                await resp.read()
+                logger.info(
+                    "[prefill] done request_id=%s status=%s elapsed=%.2fs",
+                    request_id,
+                    resp.status,
+                    time.perf_counter() - start_ts,
+                )
+        except asyncio.TimeoutError as exc:
+            raise RuntimeError(f"Prefill service timeout at {url}") from exc
+        except aiohttp.ClientError as exc:
+            raise RuntimeError(f"Prefill service unavailable at {url}") from exc
+
+    async def _stream_decode(
+        request_path: str,
+        payload: dict,
+        headers: dict[str, str],
+        request_id: str,
+    ):
+        url = f"{DECODE_BASE}{request_path}"
+        # Stream tokens from the decode service once the prefill stage has
+        # materialized KV caches on the target workers.
+        logger.info("[decode] start request_id=%s url=%s", request_id, url)
+        try:
+            async with (
+                aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session,
+                session.post(url=url, json=payload, headers=headers) as resp,
+            ):
+                if resp.status != 200:
+                    error_text = await resp.text()
+                    logger.error(
+                        "Decode backend error %s - %s", resp.status, error_text
+                    )
+                    err_msg = (
+                        '{"error": "Decode backend error ' + str(resp.status) + '"}'
+                    )
+                    yield err_msg.encode()
+                    return
+                logger.info(
+                    "[decode] streaming response request_id=%s status=%s",
+                    request_id,
+                    resp.status,
+                )
+                async for chunk_bytes in resp.content.iter_chunked(1024):
+                    yield chunk_bytes
+                logger.info("[decode] finished streaming request_id=%s", request_id)
+        except asyncio.TimeoutError:
+            logger.error("Decode service timeout at %s", url)
+            yield b'{"error": "Decode service timeout"}'
+        except aiohttp.ClientError as exc:
+            logger.error("Decode service error at %s: %s", url, exc)
+            yield b'{"error": "Decode service unavailable"}'
+
+    async def process_request():
+        """Process a single request through prefill and decode stages"""
+        try:
+            original_request_data = await request.get_json()
+
+            # Create prefill request (max_tokens=1)
+            prefill_request = original_request_data.copy()
+            prefill_request["max_tokens"] = 1
+            if "max_completion_tokens" in prefill_request:
+                prefill_request["max_completion_tokens"] = 1
+
+            # Execute prefill stage
+            # The request id encodes both KV socket addresses so the backend can
+            # shuttle tensors directly via NCCL once the prefill response
+            # completes.
+            request_id = (
+                f"___prefill_addr_{PREFILL_KV_ADDR}___decode_addr_"
+                f"{DECODE_KV_ADDR}_{uuid.uuid4().hex}"
+            )
+
+            headers = _build_headers(request_id)
+            await _run_prefill(request.path, prefill_request, headers, request_id)
+
+            # Execute decode stage and stream response
+            # Pass the unmodified user request so the decode phase can continue
+            # sampling with the already-populated KV cache.
+            generator = _stream_decode(
+                request.path, original_request_data, headers, request_id
+            )
+            response = await make_response(generator)
+            response.timeout = None  # Disable timeout for streaming response
+            return response
+
+        except Exception:
+            logger.exception("Error processing request")
+            return Response(
+                response=b'{"error": "Internal server error"}',
+                status=500,
+                content_type="application/json",
+            )
+
+    @app.route("/v1/completions", methods=["POST"])
+    async def handle_request():
+        """Handle incoming API requests with concurrency and rate limiting"""
+        try:
+            return await process_request()
+        except asyncio.CancelledError:
+            logger.warning("Request cancelled")
+            return Response(
+                response=b'{"error": "Request cancelled"}',
+                status=503,
+                content_type="application/json",
+            )
+
+    # Start the Quart server with host can be set to 0.0.0.0
+    app.run(port=PORT)
+
+
+if __name__ == "__main__":
+    main()

benchmarks/disagg_benchmarks/round_robin_proxy.py

@@ -0,0 +1,63 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import asyncio
+import itertools
+
+import aiohttp
+from aiohttp import web
+
+
+class RoundRobinProxy:
+    def __init__(self, target_ports):
+        self.target_ports = target_ports
+        self.port_cycle = itertools.cycle(self.target_ports)
+
+    async def handle_request(self, request):
+        target_port = next(self.port_cycle)
+        target_url = f"http://localhost:{target_port}{request.path_qs}"
+
+        async with aiohttp.ClientSession() as session:
+            try:
+                # Forward the request
+                async with session.request(
+                    method=request.method,
+                    url=target_url,
+                    headers=request.headers,
+                    data=request.content,
+                ) as response:
+                    # Start sending the response
+                    resp = web.StreamResponse(
+                        status=response.status, headers=response.headers
+                    )
+                    await resp.prepare(request)
+
+                    # Stream the response content
+                    async for chunk in response.content.iter_any():
+                        await resp.write(chunk)
+
+                    await resp.write_eof()
+                    return resp
+
+            except Exception as e:
+                return web.Response(text=f"Error: {str(e)}", status=500)
+
+
+async def main():
+    proxy = RoundRobinProxy([8100, 8200])
+    app = web.Application()
+    app.router.add_route("*", "/{path:.*}", proxy.handle_request)
+
+    runner = web.AppRunner(app)
+    await runner.setup()
+    site = web.TCPSite(runner, "localhost", 8000)
+    await site.start()
+
+    print("Proxy server started on http://localhost:8000")
+
+    # Keep the server running
+    await asyncio.Event().wait()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

benchmarks/disagg_benchmarks/visualize_benchmark_results.py

@@ -0,0 +1,47 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import json
+
+import matplotlib.pyplot as plt
+import pandas as pd
+
+if __name__ == "__main__":
+    data = []
+    for name in ["disagg_prefill", "chunked_prefill"]:
+        for qps in [2, 4, 6, 8]:
+            with open(f"results/{name}-qps-{qps}.json") as f:
+                x = json.load(f)
+                x["name"] = name
+                x["qps"] = qps
+                data.append(x)
+
+    df = pd.DataFrame.from_dict(data)
+    dis_df = df[df["name"] == "disagg_prefill"]
+    chu_df = df[df["name"] == "chunked_prefill"]
+
+    plt.style.use("bmh")
+    plt.rcParams["font.size"] = 20
+
+    for key in [
+        "mean_ttft_ms",
+        "median_ttft_ms",
+        "p99_ttft_ms",
+        "mean_itl_ms",
+        "median_itl_ms",
+        "p99_itl_ms",
+    ]:
+        fig, ax = plt.subplots(figsize=(11, 7))
+        plt.plot(
+            dis_df["qps"], dis_df[key], label="disagg_prefill", marker="o", linewidth=4
+        )
+        plt.plot(
+            chu_df["qps"], chu_df[key], label="chunked_prefill", marker="o", linewidth=4
+        )
+        ax.legend()
+
+        ax.set_xlabel("QPS")
+        ax.set_ylabel(key)
+        ax.set_ylim(bottom=0)
+        fig.savefig(f"results/{key}.png")
+        plt.close(fig)

benchmarks/kernels/benchmark_flydsl_moe_w4a16.py

@@ -1,277 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-# Copyright (c) 2025 FlyDSL Project Contributors
-
-import json
-import os
-
-import torch
-from aiter.test_common import run_perftest
-
-from vllm.model_executor.layers.fused_moe import fused_experts
-from vllm.model_executor.layers.fused_moe.activation import MoEActivation
-from vllm.model_executor.layers.fused_moe.config import (
-    int4_w4a16_moe_quant_config,
-)
-from vllm.model_executor.layers.fused_moe.fused_flydsl_moe import fused_flydsl_moe
-from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import (  # noqa: E501
-    compressed_tensors_moe_w4a16_flydsl,
-)
-from vllm.platforms import current_platform
-
-RoutingBuffers = tuple[
-    torch.Tensor,  # sorted_token_ids
-    torch.Tensor,  # sorted_weights
-    torch.Tensor,  # sorted_expert_ids
-    torch.Tensor,  # num_valid_ids (shape [1], i32)
-    int,  # sorted_size
-    int,  # blocks
-]
-
-MODEL_PARAMS_TO_TUNE = [
-    # (num_experts, inter_dim, hidden_size, topk)
-    (384, 256, 7168, 8),  # Kimi K2.5 TP=8
-    (384, 512, 7168, 8),  # Kimi K2.5 TP=4
-]
-
-NUM_TOKENS_TO_TUNE = [
-    1,
-    2,
-    4,
-    8,
-    16,
-    24,
-    32,
-    48,
-    64,
-    128,
-    256,
-    512,
-    1024,
-    2048,
-    4096,
-    8192,
-]
-
-TILE_M_SEARCH_SPACE = [16, 32, 64, 128, 256]
-TILE_N_SEARCH_SPACE = [16, 32, 64, 128, 256]
-TILE_K_SEARCH_SPACE = [16, 32, 64, 128, 256, 512]
-TILE_N2_SEARCH_SPACE = [16, 32, 64, 128, 256]
-TILE_K2_SEARCH_SPACE = [16, 32, 64, 128, 256, 512]
-
-TILE_CONFIGS = []
-for tile_m in TILE_M_SEARCH_SPACE:
-    for tile_n in TILE_N_SEARCH_SPACE:
-        for tile_k in TILE_K_SEARCH_SPACE:
-            for tile_n2 in TILE_N2_SEARCH_SPACE:
-                for tile_k2 in TILE_K2_SEARCH_SPACE:
-                    TILE_CONFIGS.append(
-                        {
-                            "tile_m": tile_m,
-                            "tile_n": tile_n,
-                            "tile_k": tile_k,
-                            "tile_n2": tile_n2,
-                            "tile_k2": tile_k2,
-                        }
-                    )
-
-
-def tune_flydsl_moe_w4a16(
-    device: str = "cuda", num_iters: int = 100, num_warmup: int = 10
-):
-    packed_factor = 8
-    w13_num_shards = 2
-    params_dtype = torch.bfloat16
-    group_size = 32
-    scale_factor = 0.01
-
-    for model_params in MODEL_PARAMS_TO_TUNE:
-        num_experts = model_params[0]
-        inter_dim = model_params[1]
-        hidden_size = model_params[2]
-        topk = model_params[3]
-        print(
-            f"\nTuning: num_experts={num_experts}, inter_dim={inter_dim}, "
-            f"hidden_size={hidden_size}, topk={topk}...\n"
-        )
-
-        w2_scales_size = inter_dim
-        num_groups_w2 = w2_scales_size // group_size
-        num_groups_w13 = hidden_size // group_size
-
-        w13_weight = torch.randint(
-            0,
-            255,
-            (num_experts, hidden_size // packed_factor, w13_num_shards * inter_dim),
-            dtype=torch.int32,
-            device=device,
-        )
-
-        w2_weight = torch.randint(
-            0,
-            255,
-            (num_experts, inter_dim // packed_factor, hidden_size),
-            dtype=torch.int32,
-            device=device,
-        )
-        w13_scale = scale_factor * torch.randn(
-            num_experts,
-            num_groups_w13,
-            w13_num_shards * inter_dim,
-            dtype=params_dtype,
-            device=device,
-        )
-        w2_scale = scale_factor * torch.randn(
-            num_experts, num_groups_w2, hidden_size, dtype=params_dtype, device=device
-        )
-
-        w13 = w13_weight
-        w13 = compressed_tensors_moe_w4a16_flydsl._gptq_int32_to_flydsl_packed(w13)
-        w13 = w13.view(-1).contiguous()
-
-        w2 = w2_weight
-        w2 = compressed_tensors_moe_w4a16_flydsl._gptq_int32_to_flydsl_packed(w2)
-        w2 = w2.view(-1).contiguous()
-
-        w13_scale_flydsl = w13_scale
-        w2_scale_flydsl = w2_scale
-
-        if group_size > 0 and w13_scale.dim() == 3 and w13_scale.shape[1] > 1:
-            E, G, N = w13_scale.shape
-            w13_scale_flydsl = (
-                w13_scale_flydsl.view(E, G // 2, 2, N)
-                .permute(0, 1, 3, 2)
-                .contiguous()
-                .view(-1)
-                .contiguous()
-            )
-        elif w13_scale.dim() == 3 and w13_scale.shape[1] == 1:
-            w13_scale_flydsl = w13_scale_flydsl.squeeze(1)
-
-        if group_size > 0 and w2_scale.dim() == 3 and w2_scale.shape[1] > 1:
-            E, G, N = w2_scale.shape
-            w2_scale_flydsl = (
-                w2_scale_flydsl.view(E, G // 2, 2, N)
-                .permute(0, 1, 3, 2)
-                .contiguous()
-                .view(-1)
-                .contiguous()
-            )
-        elif w2_scale.dim() == 3 and w2_scale.shape[1] == 1:
-            w2_scale_flydsl = w2_scale_flydsl.squeeze(1)
-
-        w13_scale_flydsl = w13_scale_flydsl.contiguous()
-        w2_scale_flydsl = w2_scale_flydsl.contiguous()
-
-        w13.is_shuffled = True
-        w2.is_shuffled = True
-
-        w13_weight_scale = w13_scale.transpose(1, 2).contiguous()
-        w2_weight_scale = w2_scale.transpose(1, 2).contiguous()
-        w13_weight_packed = w13_weight.transpose(1, 2).contiguous().view(torch.uint8)
-        w2_weight_packed = w2_weight.transpose(1, 2).contiguous().view(torch.uint8)
-
-        moe_quant_config = int4_w4a16_moe_quant_config(
-            w1_scale=w13_weight_scale,
-            w2_scale=w2_weight_scale,
-            w1_zp=None,
-            w2_zp=None,
-            block_shape=[0, group_size],
-        )
-
-        tuned_config = {}
-
-        for num_tokens in NUM_TOKENS_TO_TUNE:
-            score = torch.rand(
-                (num_tokens, num_experts), device=device, dtype=torch.float32
-            )
-            topk_vals, topk_ids = torch.topk(score, k=topk, dim=1)
-            topk_weights = torch.softmax(topk_vals, dim=1).to(torch.float32)
-            x = torch.randn(
-                (num_tokens, hidden_size), dtype=torch.bfloat16, device=device
-            )
-            us_best = float("inf")
-            for tile_config in TILE_CONFIGS:
-                try:
-                    tile_m = tile_config["tile_m"]
-                    tile_n = tile_config["tile_n"]
-                    tile_k = tile_config["tile_k"]
-                    tile_n2 = tile_config["tile_n2"]
-                    tile_k2 = tile_config["tile_k2"]
-
-                    model_dim = x.shape[1]
-                    assert model_dim % 64 == 0
-                    assert model_dim % tile_k == 0
-                    assert inter_dim % tile_n == 0
-                    assert model_dim % tile_n2 == 0
-                    assert inter_dim % tile_k2 == 0
-                    assert ((tile_m * tile_k2) % 256) == 0
-                    bytes_per_thread_x = (tile_m * tile_k2) // 256
-                    assert (bytes_per_thread_x % 4) == 0
-
-                    out, _us = run_perftest(
-                        fused_flydsl_moe,
-                        x,
-                        w13,
-                        w2,
-                        num_experts,
-                        inter_dim,
-                        topk_weights,
-                        topk_ids,
-                        num_iters=num_iters,
-                        num_warmup=num_warmup,
-                        w1_scale=w13_scale_flydsl,
-                        w2_scale=w2_scale_flydsl,
-                        topk=topk_weights.shape[-1],
-                        group_size=group_size,
-                        doweight_stage1=False,
-                        scale_is_bf16=True,
-                        config=tile_config,
-                    )
-                    torch.accelerator.synchronize()
-                except Exception:
-                    torch.accelerator.synchronize()
-                    continue
-                else:
-                    us = _us.item()
-                    if us < us_best:
-                        out_ref = fused_experts(
-                            x,
-                            w13_weight_packed,
-                            w2_weight_packed,
-                            topk_weights=topk_weights,
-                            topk_ids=topk_ids,
-                            activation=MoEActivation.SILU,
-                            apply_router_weight_on_input=False,
-                            global_num_experts=num_experts,
-                            expert_map=None,
-                            quant_config=moe_quant_config,
-                        )
-                        try:
-                            assert torch.allclose(out, out_ref, atol=0.5, rtol=0.1)
-                        except Exception:
-                            continue
-                        else:
-                            print(
-                                f"For [num_tokens={num_tokens}, num_experts={num_experts}, "  # noqa: E501
-                                f"inter_dim={inter_dim}] found new best "  # noqa: E501
-                                f"config={tile_config}, us={us:0.3f}"
-                            )
-                            us_best = us
-                            tuned_config[str(num_tokens)] = tile_config
-            device_name = current_platform.get_device_name().replace(" ", "_")
-            tuned_config_file_name = (
-                f"E={num_experts},N={inter_dim},device_name={device_name},"
-                f"dtype=int4_w4a16,backend=flydsl.json"
-            )
-            tuner_dir_path = os.path.dirname(os.path.realpath(__file__))
-            store_path = os.path.join(tuner_dir_path, tuned_config_file_name)
-            with open(store_path, "w") as f:
-                json.dump(tuned_config, f, indent=4)
-            print(
-                f"\nTuned config for num_tokens={num_tokens} was stored at {store_path}\n"  # noqa: E501
-            )
-
-
-if __name__ == "__main__":
-    tune_flydsl_moe_w4a16(device="cuda")

benchmarks/kernels/benchmark_fused_collective.py

@@ -80,17 +80,13 @@ _FI_MAX_SIZES = {
     2: 64 * MiB,  # 64MB
     4: 64 * MiB,  # 64MB
     8: 64 * MiB,  # 64MB
-    16: 64 * MiB,  # 64MB (multi-node)
 }
 
 # Global workspace tensors for FlashInfer (keyed by backend name)
 _FI_WORKSPACES: dict = {}
 
-# Backends to benchmark. trtllm is single-node only and can hang cross-node, so
+# Backends to benchmark
-# multi-node sweeps can restrict to mnnvl via FI_BACKENDS=mnnvl.
+FLASHINFER_BACKENDS = ["trtllm", "mnnvl"]
-FLASHINFER_BACKENDS = [
-    b for b in os.environ.get("FI_BACKENDS", "trtllm,mnnvl").split(",") if b
-]
 
 
 def setup_flashinfer_workspace(
@@ -999,10 +995,7 @@ def main():
     rank = int(os.environ["RANK"])
     world_size = int(os.environ["WORLD_SIZE"])
 
-    # Use LOCAL_RANK for the device so multi-node runs (global rank >= GPUs per
+    device = torch.device(f"cuda:{rank}")
-    # node) map to a valid local GPU; falls back to global rank single-node.
-    local_rank = int(os.environ.get("LOCAL_RANK", rank))
-    device = torch.device(f"cuda:{local_rank}")
     torch.accelerator.set_device_index(device)
     torch.set_default_device(device)
 

benchmarks/kernels/benchmark_moe.py

@@ -250,7 +250,7 @@ def benchmark_config(
                     num_experts=num_experts,
                     experts_per_token=topk,
                     hidden_dim=hidden_size,
-                    intermediate_size=shard_intermediate_size,
+                    intermediate_size_per_partition=shard_intermediate_size,
                     num_local_experts=num_experts,
                     num_logical_experts=num_experts,
                     activation=MoEActivation.SILU,
@@ -391,19 +391,16 @@ def get_configs_compute_bound(use_fp16, block_quant_shape) -> list[dict[str, int
         config = dict(zip(keys, config_values))
         configs.append(config)
 
-    # Drop configs incompatible with fp8 block quantization. A tile must align
+    # Remove configs that are not compatible with fp8 block quantization
-    # to the quant-block scale grid, i.e. tile and block must divide one
+    # BLOCK_SIZE_K must be a multiple of block_k
-    # another. The kernel indexes scales per element (offs_bn // group_n,
+    # BLOCK_SIZE_N must be a multiple of block_n
-    # k_start // group_k), so a tile narrower than the block (e.g. N=64 with
-    # block_n=128) is valid -- and often faster at small batch. An exact
-    # multiple was required before, which dropped those smaller tiles entirely.
     if block_quant_shape is not None and not use_fp16:
         block_n, block_k = block_quant_shape[0], block_quant_shape[1]
         for config in configs[:]:
-            bn, bk = config["BLOCK_SIZE_N"], config["BLOCK_SIZE_K"]
+            if (
-            n_aligned = bn % block_n == 0 or block_n % bn == 0
+                config["BLOCK_SIZE_K"] % block_k != 0
-            k_aligned = bk % block_k == 0 or block_k % bk == 0
+                or config["BLOCK_SIZE_N"] % block_n != 0
-            if not (n_aligned and k_aligned):
+            ):
                 configs.remove(config)
     return configs
 
@@ -795,12 +792,6 @@ def get_model_params(config):
         topk = text_config.num_experts_per_tok
         intermediate_size = text_config.moe_intermediate_size
         hidden_size = text_config.hidden_size
-    elif architecture == "DiffusionGemmaForBlockDiffusion":
-        text_config = config.get_text_config()
-        E = text_config.num_experts
-        topk = text_config.top_k_experts
-        intermediate_size = text_config.moe_intermediate_size
-        hidden_size = text_config.hidden_size
     elif architecture == "HunYuanMoEV1ForCausalLM":
         E = config.num_experts
         topk = config.moe_topk[0]

benchmarks/kernels/cpu/benchmark_cpu_fused_moe.py

@@ -7,7 +7,6 @@ import time
 import numpy as np
 import torch
 
-from vllm.platforms import CpuArchEnum, current_platform
 from vllm.utils.argparse_utils import FlexibleArgumentParser
 from vllm.utils.torch_utils import set_random_seed
 
@@ -15,15 +14,17 @@ from vllm.utils.torch_utils import set_random_seed
 try:
     from vllm._custom_ops import cpu_fused_moe, cpu_prepack_moe_weight
 except (ImportError, AttributeError) as e:
+    print("ERROR: CPU fused MoE operations are not available on this platform.")
+    print("This benchmark requires x86 CPU with proper vLLM CPU extensions compiled.")
+    print(
+        "The cpu_fused_moe kernel is typically available on Linux x86_64 "
+        "with AVX2/AVX512."
+    )
     print(f"Import error: {e}")
     sys.exit(1)
 
 # ISA selection following test_cpu_fused_moe.py pattern
-ISA_CHOICES = ["vec"]
+ISA_CHOICES = ["amx", "vec"] if torch.cpu._is_amx_tile_supported() else ["vec"]
-if torch.cpu._is_amx_tile_supported():
-    ISA_CHOICES.append("amx")
-if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
-    ISA_CHOICES.append("neon")
 
 
 @torch.inference_mode()
@@ -144,7 +145,7 @@ if __name__ == "__main__":
         "--isa",
         type=str,
         choices=ISA_CHOICES,
-        default="vec",
+        default=ISA_CHOICES[0],
         help=f"ISA to use (available: {ISA_CHOICES})",
     )
     parser.add_argument("--seed", type=int, default=0)

benchmarks/kv_cache_watermark.sh

@@ -1,248 +0,0 @@
-#!/bin/bash
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-#
-# Reproducible demonstration of the KV cache watermark (`--watermark`) for
-# reducing preemption thrashing.
-#
-# The watermark is the fraction of total KV cache blocks the scheduler keeps
-# free when admitting a waiting/preempted request into the running queue.
-#
-# Why this workload triggers thrashing:
-#   Requests are admitted based on the KV cache they need *at admission time*.
-#   With `--scheduler-reserve-full-isl` (default) the input length is reserved up
-#   front, but the *output* length is unknown and unreserved. A decode-heavy
-#   workload (output >> input) at high concurrency therefore over-admits while
-#   requests are short, then runs out of KV cache as they all grow during decode
-#   -> the scheduler preempts (recompute) recently-admitted requests, re-prefills
-#   them later, and repeats. The watermark keeps a block of KV cache free so
-#   running requests can grow into it instead of triggering this churn.
-#
-# This script launches `vllm serve` under a deliberately KV-constrained config
-# and a decode-heavy workload, sweeping the watermark across several values, and
-# reports the preemption count (scraped from /metrics), throughput, and latency
-# percentiles for each. It then plots the results.
-#
-# Default workload: concurrency 200, input ~300 tokens, output ~4000 tokens
-# (+/- 20% variance), sized to run each config for ~5 minutes.
-#
-# Usage:
-#   benchmarks/kv_cache_watermark.sh
-#   MODEL=Qwen/Qwen2.5-14B-Instruct TP=2 benchmarks/kv_cache_watermark.sh
-#
-# Run inside the vLLM virtualenv (so `vllm` and `python` resolve to it).
-set -euo pipefail
-
-# ---- Config (override via environment) -------------------------------------
-MODEL=${MODEL:-Qwen/Qwen2.5-7B-Instruct}
-TP=${TP:-1}
-PORT=${PORT:-8000}
-URL="http://127.0.0.1:${PORT}"
-# Constrain the KV cache to a *near-critical* size: large enough that the engine
-# can run stably, but small enough that greedy over-admission tips it into
-# preemption thrashing. (Independent of GPU size, so the demo is reproducible.)
-# At the default workload this fits ~1.5x the mean concurrent KV demand.
-KV_CACHE_MEMORY_GB=${KV_CACHE_MEMORY_GB:-16}
-MAX_MODEL_LEN=${MAX_MODEL_LEN:-8192}
-MAX_NUM_SEQS=${MAX_NUM_SEQS:-256}
-# Optional weight loader (e.g. fastsafetensors on the GCP cluster).
-LOAD_FORMAT=${LOAD_FORMAT:-auto}
-# Decode-heavy workload: moderate input, long output, with length variance. The
-# long output means preempted requests have generated a lot before eviction, so
-# resuming them re-prefills a long sequence (high recomputation cost).
-INPUT_LEN=${INPUT_LEN:-1000}
-OUTPUT_LEN=${OUTPUT_LEN:-5000}
-RANGE_RATIO=${RANGE_RATIO:-0.2}
-CONCURRENCY=${CONCURRENCY:-128}
-# Enough prompts to keep each config saturated for ~5+ minutes.
-NUM_PROMPTS=${NUM_PROMPTS:-450}
-OUTDIR=${OUTDIR:-./watermark_bench_results}
-# Watermark fractions compared. "label value" per line; value=0 disables it.
-CONFIGS=${CONFIGS:-"off    0
-w0.02  0.02
-w0.05  0.05
-w0.10  0.10
-w0.15  0.15"}
-
-KV_CACHE_MEMORY_BYTES=$((KV_CACHE_MEMORY_GB * 1024 * 1024 * 1024))
-mkdir -p "$OUTDIR"
-
-SERVER_PID=""
-cleanup() { [[ -n "$SERVER_PID" ]] && kill "$SERVER_PID" 2>/dev/null || true; }
-trap cleanup EXIT
-
-scrape_preemptions() {
-  # Sum the vllm:num_preemptions_total counter across engines.
-  python - "${URL}/metrics" <<'PY'
-import sys, urllib.request
-total = 0.0
-try:
-    body = urllib.request.urlopen(sys.argv[1], timeout=10).read().decode("utf-8", "replace")
-    for line in body.splitlines():
-        if line.startswith("vllm:num_preemptions_total"):
-            total += float(line.rsplit(" ", 1)[-1])
-except Exception as e:  # noqa: BLE001
-    print(f"scrape error: {e}", file=sys.stderr)
-print(int(total))
-PY
-}
-
-wait_for_server() {
-  for _ in $(seq 1 300); do
-    if curl -s "${URL}/health" >/dev/null 2>&1; then return 0; fi
-    if ! kill -0 "$SERVER_PID" 2>/dev/null; then
-      echo "ERROR: server process exited during startup" >&2; return 1
-    fi
-    sleep 5
-  done
-  echo "ERROR: server did not become ready" >&2; return 1
-}
-
-run_one() {
-  local label=$1 watermark=$2
-  echo
-  echo "==================== watermark: ${label} (${watermark}) ===================="
-  vllm serve "$MODEL" \
-    --tensor-parallel-size "$TP" \
-    --load-format "$LOAD_FORMAT" \
-    --kv-cache-memory-bytes "$KV_CACHE_MEMORY_BYTES" \
-    --max-model-len "$MAX_MODEL_LEN" \
-    --max-num-seqs "$MAX_NUM_SEQS" \
-    --no-enable-prefix-caching \
-    --watermark "$watermark" \
-    --port "$PORT" >"${OUTDIR}/serve_${label}.log" 2>&1 &
-  SERVER_PID=$!
-  wait_for_server
-  sleep 5
-
-  local pre post
-  pre=$(scrape_preemptions)
-  vllm bench serve \
-    --backend vllm \
-    --base-url "$URL" \
-    --model "$MODEL" \
-    --dataset-name random \
-    --random-input-len "$INPUT_LEN" \
-    --random-output-len "$OUTPUT_LEN" \
-    --random-range-ratio "$RANGE_RATIO" \
-    --ignore-eos \
-    --num-prompts "$NUM_PROMPTS" \
-    --max-concurrency "$CONCURRENCY" \
-    --percentile-metrics "ttft,tpot,itl,e2el" \
-    --metric-percentiles "50,90,99" \
-    --save-result \
-    --result-dir "$OUTDIR" \
-    --result-filename "bench_${label}.json"
-  post=$(scrape_preemptions)
-  echo "${label} ${watermark} $((post - pre))" >>"${OUTDIR}/preemptions.txt"
-
-  kill "$SERVER_PID" 2>/dev/null || true
-  for _ in $(seq 1 60); do curl -s "${URL}/health" >/dev/null 2>&1 || break; sleep 2; done
-  SERVER_PID=""
-  sleep 10
-}
-
-: >"${OUTDIR}/preemptions.txt"
-while read -r label watermark; do
-  [[ -z "${label:-}" ]] && continue
-  run_one "$label" "$watermark"
-done <<<"$CONFIGS"
-
-echo
-echo "==================== summary ===================="
-python - "$OUTDIR" <<'PY'
-import json, os, sys
-outdir = sys.argv[1]
-pre = {}
-order = []
-for line in open(os.path.join(outdir, "preemptions.txt")):
-    label, watermark, n = line.split()
-    pre[label] = (float(watermark), int(n))
-    order.append(label)
-
-def g(d, *names):
-    for n in names:
-        if d.get(n) is not None:
-            return d[n]
-    return float("nan")
-
-cols = ["watermark", "frac", "preempt", "out_tok/s", "req/s",
-        "TTFT_p50", "TTFT_p99", "ITL_p99", "E2EL_p50"]
-print("  ".join(f"{c:>10}" for c in cols))
-rows = []
-for label in order:
-    watermark, n = pre[label]
-    d = json.load(open(os.path.join(outdir, f"bench_{label}.json")))
-    rows.append(dict(
-        label=label, watermark=watermark, preempt=n,
-        out_tok_s=g(d, "output_throughput"),
-        req_s=g(d, "request_throughput"),
-        ttft_p50=g(d, "p50_ttft_ms", "median_ttft_ms"),
-        ttft_p99=g(d, "p99_ttft_ms"),
-        itl_p99=g(d, "p99_itl_ms"),
-        e2el_p50=g(d, "p50_e2el_ms", "median_e2el_ms"),
-    ))
-    print("  ".join(f"{str(v):>10}" for v in [
-        label, watermark, n,
-        f"{rows[-1]['out_tok_s']:.0f}",
-        f"{rows[-1]['req_s']:.3f}",
-        f"{rows[-1]['ttft_p50']/1000:.2f}",
-        f"{rows[-1]['ttft_p99']/1000:.2f}",
-        f"{rows[-1]['itl_p99']:.2f}",
-        f"{rows[-1]['e2el_p50']/1000:.1f}",
-    ]))
-print("\n(TTFT/E2EL in seconds; ITL in ms. Lower preempt is better.)")
-
-# ---- Plot -------------------------------------------------------------------
-try:
-    import matplotlib
-    matplotlib.use("Agg")
-    import matplotlib.pyplot as plt
-except Exception as e:  # noqa: BLE001
-    print(f"\n(skip plot: matplotlib unavailable: {e})")
-    sys.exit(0)
-
-x = [r["watermark"] for r in rows]
-xt = [f"{r['watermark']:g}\n({r['label']})" for r in rows]
-idx = list(range(len(rows)))
-
-fig, axes = plt.subplots(2, 2, figsize=(12, 8))
-fig.suptitle(
-    f"KV cache watermark sweep — {os.path.basename(os.path.abspath(outdir))}",
-    fontsize=12,
-)
-
-ax = axes[0][0]
-ax.bar(idx, [r["preempt"] for r in rows], color="tab:red")
-ax.set_title("Preemptions (lower is better)")
-ax.set_ylabel("preemptions")
-ax.set_xticks(idx); ax.set_xticklabels(xt)
-
-ax = axes[0][1]
-ax.plot(idx, [r["out_tok_s"] for r in rows], "o-", color="tab:green")
-ax.set_title("Output throughput (higher is better)")
-ax.set_ylabel("tokens/s")
-ax.set_xticks(idx); ax.set_xticklabels(xt)
-
-ax = axes[1][0]
-ax.plot(idx, [r["itl_p99"] for r in rows], "o-", color="tab:blue")
-ax.set_title("Inter-token latency p99 (lower is better)")
-ax.set_ylabel("ITL p99 (ms)")
-ax.set_xlabel("watermark fraction")
-ax.set_xticks(idx); ax.set_xticklabels(xt)
-
-ax = axes[1][1]
-ax.plot(idx, [r["ttft_p50"] / 1000 for r in rows], "o-", label="TTFT p50")
-ax.plot(idx, [r["ttft_p99"] / 1000 for r in rows], "o-", label="TTFT p99")
-ax.plot(idx, [r["e2el_p50"] / 1000 for r in rows], "o-", label="E2EL p50")
-ax.set_title("Latency (lower is better)")
-ax.set_ylabel("seconds")
-ax.set_xlabel("watermark fraction")
-ax.set_xticks(idx); ax.set_xticklabels(xt)
-ax.legend()
-
-fig.tight_layout(rect=(0, 0, 1, 0.95))
-out_png = os.path.join(outdir, "watermark_results.png")
-fig.savefig(out_png, dpi=120)
-print(f"\nWrote plot: {out_png}")
-PY

benchmarks/multi_turn/benchmark_serving_multi_turn.py

@@ -65,32 +65,6 @@ class RequestArgs(NamedTuple):
     limit_min_tokens: int  # Use negative value for no limit
     limit_max_tokens: int  # Use negative value for no limit
     timeout_sec: int
-    send_conversation_id: bool
-    headers: dict[str, str]
-
-
-def parse_custom_header(header: str) -> tuple[str, str]:
-    separators = (":", "=")
-    for separator in separators:
-        if separator in header:
-            key, value = header.split(separator, 1)
-            key = key.strip()
-            value = value.strip()
-            if key:
-                return key, value
-            break
-    raise argparse.ArgumentTypeError(
-        "Headers must be provided as 'Header-Name: value' or 'Header-Name=value'"
-    )
-
-
-def build_request_headers(
-    api_key: str | None, custom_headers: list[tuple[str, str]] | None
-) -> dict[str, str]:
-    headers = dict(custom_headers or [])
-    if api_key:
-        headers["Authorization"] = f"Bearer {api_key}"
-    return headers
 
 
 class BenchmarkArgs(NamedTuple):
@@ -244,11 +218,12 @@ async def send_request(
     max_tokens: int | None = None,
     timeout_sec: int = 120,
     conversation_id: str | None = None,
-    headers: dict[str, str] | None = None,
 ) -> ServerResponse:
     payload = {
         "model": model,
         "messages": messages,
+        "seed": 0,
+        "temperature": 0.0,
     }
 
     if conversation_id is not None:
@@ -258,17 +233,13 @@ async def send_request(
         payload["stream"] = True
         payload["stream_options"] = {"include_usage": False}
 
-    # if min_tokens is not None:
+    if min_tokens is not None:
-    #     payload["min_tokens"] = min_tokens
+        payload["min_tokens"] = min_tokens
 
     if max_tokens is not None:
         payload["max_tokens"] = max_tokens
 
-    request_headers = {"Content-Type": "application/json"}
+    headers = {"Content-Type": "application/json"}
-    if conversation_id is not None:
-        request_headers["X-Session-ID"] = str(conversation_id)
-    if headers is not None:
-        request_headers.update(headers)
 
     # Calculate the timeout for the request
     if max_tokens is not None:
@@ -294,7 +265,7 @@ async def send_request(
     most_recent_timestamp: int = start_time
 
     async with session.post(
-        url=chat_url, json=payload, headers=request_headers, timeout=timeout
+        url=chat_url, json=payload, headers=headers, timeout=timeout
     ) as response:
         http_status = HTTPStatus(response.status)
         if http_status == HTTPStatus.OK:
@@ -346,8 +317,6 @@ async def send_request(
             latency = time.perf_counter_ns() - start_time
 
     if ttft is None:
-        if stream:
-            valid_response = False
         # The response was a single chunk
         ttft = latency
 
@@ -454,8 +423,7 @@ async def send_turn(
         min_tokens,
         max_tokens,
         req_args.timeout_sec,
-        conversation_id=conv_id if req_args.send_conversation_id else None,
+        conversation_id=conv_id,
-        headers=req_args.headers,
     )
 
     if response.valid is False:
@@ -904,7 +872,6 @@ def get_client_config(
     # Arguments for API requests
     chat_url = f"{args.url}/v1/chat/completions"
     model_name = args.served_model_name if args.served_model_name else args.model
-    headers = build_request_headers(args.api_key, args.header)
 
     req_args = RequestArgs(
         chat_url=chat_url,
@@ -913,8 +880,6 @@ def get_client_config(
         limit_min_tokens=args.limit_min_tokens,
         limit_max_tokens=args.limit_max_tokens,
         timeout_sec=args.request_timeout_sec,
-        send_conversation_id=args.send_conversation_id,
-        headers=headers,
     )
 
     return client_args, req_args
@@ -1280,19 +1245,19 @@ def process_statistics(
         )
 
 
-async def get_server_info(url: str, headers: dict[str, str] | None = None) -> None:
+async def get_server_info(url: str) -> None:
     logger.info(f"{Color.BLUE}Collecting information from server: {url}{Color.RESET}")
     async with aiohttp.ClientSession() as session:
         # Get server version (not mandatory, "version" endpoint may not exist)
         url_version = f"{url}/version"
-        async with session.get(url_version, headers=headers) as response:
+        async with session.get(url_version) as response:
             if HTTPStatus(response.status) == HTTPStatus.OK:
                 text = await response.text()
                 logger.info(f"{Color.BLUE}Server version: {text}{Color.RESET}")
 
         # Get available models
         url_models = f"{url}/v1/models"
-        async with session.get(url_models, headers=headers) as response:
+        async with session.get(url_models) as response:
             if HTTPStatus(response.status) == HTTPStatus.OK:
                 text = await response.text()
                 logger.info(f"{Color.BLUE}Models:{Color.RESET}")
@@ -1358,22 +1323,6 @@ async def main() -> None:
         help="Base URL for the LLM API server",
     )
 
-    parser.add_argument(
-        "--api-key",
-        type=str,
-        default=None,
-        help="API key to send as an Authorization bearer token",
-    )
-    parser.add_argument(
-        "--header",
-        action="append",
-        type=parse_custom_header,
-        default=None,
-        metavar="KEY=VALUE",
-        help="Custom request header. Can be specified multiple times. "
-        "Accepts 'Header-Name: value' or 'Header-Name=value'.",
-    )
-
     parser.add_argument(
         "-p",
         "--num-clients",
@@ -1488,22 +1437,6 @@ async def main() -> None:
         help="Disable stream/streaming mode (set 'stream' to False in the API request)",
     )
 
-    parser.add_argument(
-        "--send-conversation-id",
-        default=False,
-        action="store_true",
-        help=(
-            "Inject a `conversation_id` field into each Chat Completions "
-            "payload. This is a non-standard OpenAI extension consumed by "
-            "vLLM's disaggregated multi-turn proxy "
-            "(examples/disaggregated/disaggregated_serving/"
-            "disagg_proxy_multiturn.py) to key cross-turn KV cache reuse. "
-            "Leave disabled (default) when targeting strict "
-            "OpenAI-compatible endpoints; enable when benchmarking the "
-            "disaggregated proxy."
-        ),
-    )
-
     parser.add_argument(
         "-e",
         "--excel-output",
@@ -1592,8 +1525,7 @@ async def main() -> None:
         args.model, trust_remote_code=args.trust_remote_code
     )
 
-    headers = build_request_headers(args.api_key, args.header)
+    await get_server_info(args.url)
-    await get_server_info(args.url, headers=headers)
 
     # Load the input file (either conversations of configuration file)
     logger.info(f"Reading input file: {args.input_file}")

build_rust.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Build vLLM Rust artifacts and install them into the vllm package.
+# Build the vllm-rs Rust frontend binary and install it into the vllm package.
 # Usage: ./build_rust.sh [--debug]
 #
 # By default builds in release mode. Pass --debug for faster compile times
@@ -8,6 +8,8 @@
 set -euo pipefail
 
 REPO_ROOT="$(cd "$(dirname "$0")" && pwd)"
+RUST_DIR="$REPO_ROOT/rust"
+TARGET_PATH="${VLLM_RS_TARGET_PATH:-$REPO_ROOT/vllm/vllm-rs}"
 
 # Read the required toolchain from rust-toolchain.toml.
 TOOLCHAIN=$(grep '^channel' "$REPO_ROOT/rust-toolchain.toml" | sed 's/.*= *"\(.*\)"/\1/')
@@ -25,9 +27,18 @@ if ! rustup run "$TOOLCHAIN" rustc --version &>/dev/null; then
 fi
 
 if [[ "${1:-}" == "--debug" ]]; then
-    PROFILE_ARG="--debug"
+    PROFILE_ARGS=()
+    PROFILE_DIR="debug"
 else
-    PROFILE_ARG="--release"
+    PROFILE_ARGS=(--release)
+    PROFILE_DIR="release"
 fi
 
-python3 "$REPO_ROOT/tools/build_rust.py" "$PROFILE_ARG"
+cargo +"$TOOLCHAIN" build "${PROFILE_ARGS[@]}" \
+    --manifest-path "$RUST_DIR/Cargo.toml" \
+    --bin vllm-rs \
+    --features native-tls-vendored
+
+mkdir -p "$(dirname "$TARGET_PATH")"
+cp "$RUST_DIR/target/$PROFILE_DIR/vllm-rs" "$TARGET_PATH"
+echo "Installed vllm-rs to $TARGET_PATH"

cmake/cpu_extension.cmake

@@ -24,10 +24,7 @@ set (ENABLE_NUMA TRUE)
 # Check the compile flags
 #
 if(MACOSX_FOUND)
-    # Apple clang needs -Xpreprocessor to enable OpenMP. No runtime link is
-    # needed: _C is a dynamic_lookup bundle and resolves libomp from torch.
     list(APPEND CXX_COMPILE_FLAGS
-        "-Xpreprocessor" "-fopenmp"
         "-DVLLM_CPU_EXTENSION")
 else()
     list(APPEND CXX_COMPILE_FLAGS
@@ -169,13 +166,8 @@ elseif (S390_FOUND)
         "-mtune=native")
 elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "riscv64")
     message(STATUS "RISC-V detected")
-    if(DEFINED VLLM_RVV_VLEN AND VLLM_RVV_VLEN LESS 0)
-        message(FATAL_ERROR
-            "VLLM_RVV_VLEN must be zero or a positive integer; got '${VLLM_RVV_VLEN}'")
-    endif()
     # VLLM_RVV_VLEN selects the target VLEN. Auto-detected from /proc/cpuinfo
-    # by default; set -DVLLM_RVV_VLEN=0 to force scalar RISC-V build.
+    # by default; override with -DVLLM_RVV_VLEN=128 or -DVLLM_RVV_VLEN=256.
-    # Override with -DVLLM_RVV_VLEN=128 or -DVLLM_RVV_VLEN=256 for RVV.
     if(NOT DEFINED VLLM_RVV_VLEN)
         # Auto-detect: find the largest zvl<N>b in /proc/cpuinfo isa line.
         if(EXISTS /proc/cpuinfo)
@@ -197,7 +189,8 @@ elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "riscv64")
                 "RISC-V RVV is available but VLEN could not be auto-detected. "
                 "Please specify VLEN explicitly:\n"
                 "  -DVLLM_RVV_VLEN=128   (for VLEN=128 hardware)\n"
-                "  -DVLLM_RVV_VLEN=256   (for VLEN=256 hardware, e.g. Spacemit X100)")
+                "  -DVLLM_RVV_VLEN=256   (for VLEN=256 hardware, e.g. Spacemit X100)\n"
+                "  -DVLLM_RVV_VLEN=0     (force scalar, no RVV)")
         endif()
     endif()
     if(VLLM_RVV_VLEN AND VLLM_RVV_VLEN GREATER 0)
@@ -226,7 +219,7 @@ endif()
 
 
 # Build oneDNN for GEMM kernels
-if (ENABLE_X86_ISA OR (ASIMD_FOUND AND NOT APPLE_SILICON_FOUND) OR POWER9_FOUND OR POWER10_FOUND OR POWER11_FOUND OR RVV_FP16_FOUND OR RVV_BF16_FOUND)
+if (ENABLE_X86_ISA OR (ASIMD_FOUND AND NOT APPLE_SILICON_FOUND) OR POWER9_FOUND OR POWER10_FOUND OR POWER11_FOUND)
     # Fetch and build Arm Compute Library (ACL) as oneDNN's backend for AArch64
     # TODO [fadara01]: remove this once ACL can be fetched and built automatically as a dependency of oneDNN
     set(ONEDNN_AARCH64_USE_ACL OFF CACHE BOOL "")
@@ -427,7 +420,6 @@ if (ASIMD_FOUND AND NOT APPLE_SILICON_FOUND)
     set(VLLM_EXT_SRC
         "csrc/cpu/shm.cpp"
         "csrc/cpu/activation_lut_bf16.cpp"
-        "csrc/cpu/cpu_fused_moe.cpp"
         ${VLLM_EXT_SRC})
 endif()
 
@@ -443,12 +435,6 @@ if(USE_ONEDNN)
         ${VLLM_EXT_SRC})
 endif()
 
-if (CMAKE_SYSTEM_PROCESSOR MATCHES "riscv64")
-    set(VLLM_EXT_SRC
-        "csrc/cpu/sgl-kernels/gemm_int4.cpp"
-        ${VLLM_EXT_SRC})
-endif()
-
 if (ENABLE_X86_ISA)
     set(VLLM_EXT_SRC_SGL
         "csrc/cpu/sgl-kernels/conv.cpp"