Apply pre-commit formatting and use torch.accelerator API

- Apply ruff, clang-format formatting fixes - Replace torch.cuda.set_device/device_count/synchronize with torch.accelerator equivalents per project convention Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Signed-off-by: Alexander Matveev <alexm-redhat@dgx-b200-02.mgmt.accl-001.lab.rdu2.dc.redhat.com> Signed-off-by: <>
Address review: add push_ar to benchmark and backend logging
2026-06-15 18:21:59 -04:00 · 2026-06-15 18:09:24 -04:00 · 2026-06-15 18:09:24 -04:00 · 2026-06-15 18:09:24 -04:00 · 2026-06-15 18:09:24 -04:00 · 2026-06-15 18:09:24 -04:00
1497 changed files with 37950 additions and 122158 deletions
@@ -2,16 +2,17 @@ name: vllm_intel_ci
 job_dirs:
  - ".buildkite/intel_jobs"
 run_all_patterns:
-  - ".buildkite/ci_config_intel.yaml"
  - "docker/Dockerfile"
-  - "docker/Dockerfile.xpu"
  - "CMakeLists.txt"
  - "requirements/common.txt"
  - "requirements/xpu.txt"
+  - "requirements/build/cuda.txt"
+  - "requirements/test/cuda.txt"
  - "setup.py"
  - "csrc/"
  - "cmake/"
 run_all_exclude_patterns:
+  - "docker/Dockerfile."
  - "csrc/cpu/"
  - "csrc/rocm/"
  - "cmake/hipify.py"
@@ -6,7 +6,6 @@ steps:
  # differ ci_base is rebuilt and pushed automatically.
  - label: "AMD: :docker: ensure ci_base"
    key: ensure-ci-base-amd
-    soft_fail: false
    depends_on: []
    device: amd_cpu
    no_plugin: true
@@ -27,7 +26,6 @@ steps:

  - label: "AMD: :docker: build test image and artifacts"
    key: image-build-amd
-    soft_fail: false
    depends_on:
      - ensure-ci-base-amd
    device: amd_cpu
@@ -53,7 +53,7 @@ steps:
  - tests/models/language/pooling/
  commands:
    - |
-      bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 50m "
+      bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 40m "
      pytest -x -v -s tests/models/language/generation -m cpu_model
      pytest -x -v -s tests/models/language/pooling -m cpu_model"

@@ -68,15 +68,13 @@ steps:
  - vllm/v1/sample/ops/topk_topp_triton.py
  - vllm/v1/sample/ops/topk_topp_sampler.py
  - tests/v1/sample/test_topk_topp_sampler.py
-  - tests/v1/e2e/test_cpu_linear_attn_chunked_prefix.py
  commands:
    - |
      bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 45m "
      uv pip install git+https://github.com/triton-lang/triton-cpu.git@270e696d
      VLLM_USE_V2_MODEL_RUNNER=1 pytest -x -v -s tests/models/language/generation/test_granite.py -m cpu_model
      # TODO: move to CPU-Kernel Tests once triton-cpu has a pre-built wheel
-      pytest -x -v -s tests/v1/sample/test_topk_topp_sampler.py::TestTritonTopkTopp
-      pytest -x -v -s tests/v1/e2e/test_cpu_linear_attn_chunked_prefix.py"
+      pytest -x -v -s tests/v1/sample/test_topk_topp_sampler.py::TestTritonTopkTopp"

 - label: CPU-Quantization Model Tests
  depends_on: []
@@ -91,13 +89,11 @@ steps:
  - vllm/model_executor/layers/fused_moe/experts/cpu_moe.py
  - tests/quantization/test_compressed_tensors.py
  - tests/quantization/test_cpu_wna16.py
-  - tests/quantization/test_cpu_w8a8.py
  commands:
    - |
      bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 45m "
      pytest -x -v -s tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_logprobs
-      pytest -x -v -s tests/quantization/test_cpu_wna16.py
-      pytest -x -v -s tests/quantization/test_cpu_w8a8.py"
+      pytest -x -v -s tests/quantization/test_cpu_wna16.py"
      
 - label: CPU-Distributed Tests (PP+TP)
  depends_on: []
@@ -140,7 +136,7 @@ steps:
    - |
      bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 45m "
      pytest -x -v -s tests/models/multimodal/generation --ignore=tests/models/multimodal/generation/test_pixtral.py -m cpu_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
-  parallelism: 4
+  parallelism: 3

 - label: "Arm CPU Test"
  depends_on: []
@@ -1,80 +0,0 @@
-group: Intel
-steps:
-  - label: ":docker: Build XPU image"
-    soft_fail: true
-    optional: true
-    depends_on: []
-    key: image-build-xpu
-    commands:
-      - bash -lc '.buildkite/image_build/image_build_xpu.sh "public.ecr.aws/q9t5s3a7" "vllm-ci-test-repo" "$BUILDKITE_COMMIT"'
-    env:
-      DOCKER_BUILDKIT: "1"
-    retry:
-      automatic:
-        - exit_status: -1  # Agent was lost
-          limit: 2
-        - exit_status: -10  # Agent was lost
-          limit: 2
-  - label: "XPU example Test"
-    depends_on:
-      - image-build-xpu
-    timeout_in_minutes: 30
-    optional: true
-    device: intel_gpu
-    agent_tags:
-      label: production
-      gpu: 2+
-      mem: 24+
-    no_plugin: true
-    env:
-      REGISTRY: "public.ecr.aws/q9t5s3a7"
-      REPO: "vllm-ci-test-repo"
-    source_file_dependencies:
-      - .buildkite/hardware_tests/intel_xpu_ci/test-intel.yaml
-      - .buildkite/scripts/hardware_ci/run-intel-ci-test.sh
-    commands:
-      - >-
-        bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-        'bash .buildkite/scripts/hardware_ci/run-intel-ci-test.sh example'
-  - label: "XPU V1 test"
-    depends_on:
-      - image-build-xpu
-    timeout_in_minutes: 30
-    optional: true
-    device: intel_gpu
-    agent_tags:
-      label: production
-      gpu: 1+
-      mem: 16+
-    no_plugin: true
-    env:
-      REGISTRY: "public.ecr.aws/q9t5s3a7"
-      REPO: "vllm-ci-test-repo"
-    source_file_dependencies:
-      - .buildkite/hardware_tests/intel_xpu_ci/test-intel.yaml
-      - .buildkite/scripts/hardware_ci/run-intel-ci-test.sh
-    commands:
-      - >-
-        bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-        'bash .buildkite/scripts/hardware_ci/run-intel-ci-test.sh v1'
-  - label: "XPU server test"
-    depends_on:
-      - image-build-xpu
-    timeout_in_minutes: 30
-    optional: true
-    device: intel_gpu
-    agent_tags:
-      label: production
-      gpu: 1+
-      mem: 16+
-    no_plugin: true
-    env:
-      REGISTRY: "public.ecr.aws/q9t5s3a7"
-      REPO: "vllm-ci-test-repo"
-    source_file_dependencies:
-      - .buildkite/hardware_tests/intel_xpu_ci/test-intel.yaml
-      - .buildkite/scripts/hardware_ci/run-intel-ci-test.sh
-    commands:
-      - >-
-        bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-        'bash .buildkite/scripts/hardware_ci/run-intel-ci-test.sh server'
@@ -21,13 +21,12 @@ else
  exit 0
 fi

-# build for arm64 GPU targets: Grace/GH200 (sm_90) and DGX Spark/GB10
-# (sm_121, family-covered by 12.0 under CUDA 13)
+# build (Grace/GH200 is the arm64 GPU target; sm_90)
 docker build --file docker/Dockerfile \
  --platform linux/arm64 \
  --build-arg max_jobs=16 \
  --build-arg nvcc_threads=4 \
-  --build-arg torch_cuda_arch_list="9.0 12.0" \
+  --build-arg torch_cuda_arch_list="9.0" \
  --build-arg USE_SCCACHE=1 \
  --build-arg buildkite_commit="$BUILDKITE_COMMIT" \
  --tag "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64 \
@@ -5,10 +5,6 @@ steps:
 - label: XPU Sleep Mode
  timeout_in_minutes: 30
  device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
@@ -23,5 +19,4 @@ steps:
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'cd tests &&
      export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
-      pytest -v -s basic_correctness/test_cpu_offload.py &&
      pytest -v -s basic_correctness/test_mem.py::test_end_to_end'
@@ -5,10 +5,6 @@ steps:
 - label: Engine (1 GPU)
  timeout_in_minutes: 30
  device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
@@ -1,15 +1,11 @@
 group: Expert Parallelism
-depends_on:
+depends_on: 
  - image-build-xpu
 steps:
 - label: EPLB Algorithm
  key: eplb-algorithm
  timeout_in_minutes: 45
  device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
@@ -5,10 +5,6 @@ steps:
 - label: vLLM IR Tests
  timeout_in_minutes: 30
  device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
@@ -5,10 +5,6 @@ steps:
 - label: LoRA Runtime + Utils
  timeout_in_minutes: 45
  device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 24+
  no_plugin: true
  working_dir: "."
  env:
@@ -38,10 +34,6 @@ steps:
 - label: LoRA Fused/MoE Kernels
  timeout_in_minutes: 45
  device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
@@ -62,10 +54,6 @@ steps:
 - label: LoRA Punica Kernels
  timeout_in_minutes: 45
  device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
@@ -86,10 +74,6 @@ steps:
 - label: LoRA Punica FP8/XPU Ops
  timeout_in_minutes: 45
  device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
@@ -110,10 +94,6 @@ steps:
 - label: LoRA Models
  timeout_in_minutes: 45
  device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 2+
-    mem: 24+
  no_plugin: true
  working_dir: "."
  env:
@@ -128,19 +108,15 @@ steps:
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'cd tests &&
      export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
+      (pytest -v -s lora/test_mixtral.py --deselect="tests/lora/test_mixtral.py::test_mixtral_lora[4]" || true) &&
      pytest -v -s lora/test_quant_model.py --deselect="tests/lora/test_quant_model.py::test_quant_model_lora[model0]" --deselect="tests/lora/test_quant_model.py::test_quant_model_lora[model1]" --deselect="tests/lora/test_quant_model.py::test_quant_model_tp_equality[model0]" &&
      pytest -v -s lora/test_transformers_model.py &&
      pytest -v -s lora/test_chatglm3_tp.py &&
-      pytest -v -s lora/test_llama_tp.py::test_llama_lora &&
      pytest -s -v lora/test_minicpmv_tp.py'

 - label: LoRA Multimodal
  timeout_in_minutes: 45
  device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
@@ -5,10 +5,6 @@ steps:
 - label: V1 Core + KV + Metrics
  timeout_in_minutes: 30
  device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
@@ -35,10 +31,6 @@ steps:
 - label: V1 Sample + Logits
  timeout_in_minutes: 30
  device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
@@ -65,24 +57,17 @@ steps:
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-      'pip install lm_eval[api]>=0.4.12 &&
-      export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
+      'export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
      cd tests &&
      pytest -v -s v1/logits_processors --ignore=v1/logits_processors/test_custom_online.py --ignore=v1/logits_processors/test_custom_offline.py &&
      pytest -v -s v1/test_oracle.py &&
      pytest -v -s v1/test_request.py &&
      pytest -v -s v1/test_outputs.py &&
-      pytest -v -s v1/sample/test_topk_topp_sampler.py &&
-      pytest -v -s v1/sample/test_logprobs.py &&
-      pytest -v -s v1/sample/test_logprobs_e2e.py'
+      pytest -v -s v1/sample/test_topk_topp_sampler.py'

 - label: XPU CPU Offload
  timeout_in_minutes: 60
  device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
@@ -107,10 +92,6 @@ steps:
  key: regression
  timeout_in_minutes: 30
  device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
@@ -142,10 +123,6 @@ steps:
  timeout_in_minutes: 30
  num_devices: 2
  device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 2+
-    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
@@ -177,10 +154,6 @@ steps:
  key: async-engine-inputs-utils-worker
  timeout_in_minutes: 30
  device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 24+
  no_plugin: true
  working_dir: "."
  env:
@@ -1,62 +0,0 @@
-group: Model Runner V2 Intel
-depends_on:
-  - image-build-xpu
-steps:
- label: Model Runner V2 Core Tests (Intel)
-  timeout_in_minutes: 45
-  device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 2+
-    mem: 16+
-  no_plugin: true
-  working_dir: "."
-  env:
-    REGISTRY: "public.ecr.aws/q9t5s3a7"
-    REPO: "vllm-ci-test-repo"
-    VLLM_TEST_DEVICE: "xpu"
-  source_file_dependencies:
-    - vllm/v1/worker/gpu/
-    - vllm/v1/worker/gpu_worker.py
-    - vllm/v1/core/sched/
-    - vllm/v1/attention/
-    - tests/v1/engine/test_llm_engine.py
-    - tests/v1/e2e/
-  commands:
-    - >-
-      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-      'export VLLM_USE_V2_MODEL_RUNNER=1 &&
-      cd tests &&
-      pytest -v -s v1/engine/test_llm_engine.py -k "not test_engine_metrics" &&
-      ENFORCE_EAGER=1 pytest -v -s v1/e2e/general/test_async_scheduling.py -k "not ngram" &&
-      pytest -v -s v1/e2e/general/test_min_tokens.py'
-
- label: Model Runner V2 Examples (Intel)
-  timeout_in_minutes: 45
-  device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 24+
-  no_plugin: true
-  working_dir: "."
-  env:
-    REGISTRY: "public.ecr.aws/q9t5s3a7"
-    REPO: "vllm-ci-test-repo"
-    VLLM_TEST_DEVICE: "xpu"
-  source_file_dependencies:
-    - vllm/v1/worker/gpu/
-    - vllm/v1/core/sched/
-    - vllm/v1/worker/gpu_worker.py
-    - examples/basic/offline_inference/
-    - examples/generate/multimodal/
-    - examples/features/
-  commands:
-    - >-
-      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-      'export VLLM_USE_V2_MODEL_RUNNER=1 &&
-      cd examples &&
-      python3 basic/offline_inference/chat.py &&
-      python3 basic/offline_inference/generate.py --model facebook/opt-125m &&
-      python3 generate/multimodal/vision_language_offline.py --seed 0 &&
-      python3 features/automatic_prefix_caching/prefix_caching_offline.py'
@@ -1,27 +0,0 @@
-group: Models - Distributed
-depends_on: 
-  - image-build-xpu
-steps:
- label: Distributed Model Tests (2 GPUs)
-  key: distributed-model-tests-2-gpus
-  timeout_in_minutes: 50
-  device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 2+
-    mem: 24+
-  no_plugin: true
-  working_dir: "."
-  env:
-    REGISTRY: "public.ecr.aws/q9t5s3a7"
-    REPO: "vllm-ci-test-repo"
-    VLLM_TEST_DEVICE: "xpu"
-  source_file_dependencies:
-  - vllm/model_executor/model_loader/sharded_state_loader.py
-  - vllm/model_executor/models/
-  - tests/model_executor/model_loader/test_sharded_state_loader.py
-  commands:
-    - >-
-      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-      'cd tests &&
-      pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m "not slow_test"'
@@ -1,15 +1,11 @@
 group: Models - Multimodal
-depends_on:
+depends_on: 
  - image-build-xpu
 steps:
 - label: "Multi-Modal Models (Standard) 1: qwen2"
  key: multi-modal-models-standard-1-qwen2
  timeout_in_minutes: 45
  device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
@@ -31,10 +27,6 @@ steps:
  key: multi-modal-models-standard-2-qwen3-gemma
  timeout_in_minutes: 45
  device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
@@ -55,10 +47,6 @@ steps:
  key: multi-modal-models-standard-3-llava-qwen2-vl
  timeout_in_minutes: 45
  device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 24+
  no_plugin: true
  working_dir: "."
  env:
@@ -80,10 +68,6 @@ steps:
  key: multi-modal-models-standard-4-other-whisper
  timeout_in_minutes: 45
  device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
@@ -104,10 +88,6 @@ steps:
  key: multi-modal-processor
  timeout_in_minutes: 45
  device: intel_gpu
-  agent_tags:
-    label: production
-    gpu: 1+
-    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
@@ -125,5 +105,7 @@ steps:
      pip install open-clip-torch --no-deps &&
      cd tests &&
      pytest -v -s models/multimodal/processing/test_tensor_schema.py
+      --deselect "tests/models/multimodal/processing/test_tensor_schema.py::test_model_tensor_schema[mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4]"
+      --deselect "tests/models/multimodal/processing/test_tensor_schema.py::test_model_tensor_schema[Qwen/Qwen2.5-Omni-7B-AWQ]"
      --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB'
  parallelism: 4
@@ -19,10 +19,6 @@ steps:
      - image-build-xpu
    timeout_in_minutes: 30
    device: intel_gpu
-    agent_tags:
-      label: production
-      gpu: 2+
-      mem: 24+
    no_plugin: true
    env:
      REGISTRY: "public.ecr.aws/q9t5s3a7"
@@ -53,10 +49,6 @@ steps:
      - image-build-xpu
    timeout_in_minutes: 30
    device: intel_gpu
-    agent_tags:
-      label: production
-      gpu: 1+
-      mem: 16+
    no_plugin: true
    env:
      REGISTRY: "public.ecr.aws/q9t5s3a7"
@@ -68,24 +60,19 @@ steps:
      - >-
        bash .buildkite/scripts/hardware_ci/run-intel-test.sh
        'cd tests &&
-        bash v1/kv_connector/nixl_integration/run_xpu_disagg_accuracy_test.sh &&
        pytest -v -s v1/core --ignore=v1/core/test_reset_prefix_cache_e2e.py --ignore=v1/core/test_scheduler_e2e.py &&
        pytest -v -s v1/engine --ignore=v1/engine/test_output_processor.py &&
        pytest -v -s v1/sample --ignore=v1/sample/test_logprobs.py --ignore=v1/sample/test_logprobs_e2e.py -k "not test_topk_only and not test_topp_only and not test_topk_and_topp" &&
        pytest -v -s v1/worker --ignore=v1/worker/test_gpu_model_runner.py --ignore=v1/worker/test_worker_memory_snapshot.py &&
        pytest -v -s v1/structured_output &&
        pytest -v -s v1/test_serial_utils.py &&
-        pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py --ignore=v1/spec_decode/test_speculators_correctness.py &&
+        pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py &&
        pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_multi_connector.py --ignore=v1/kv_connector/unit/test_example_connector.py --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py --ignore=v1/kv_connector/unit/test_offloading_connector.py'
  - label: "XPU server test"
    depends_on:
      - image-build-xpu
    timeout_in_minutes: 30
    device: intel_gpu
-    agent_tags:
-      label: production
-      gpu: 1+
-      mem: 16+
    no_plugin: true
    env:
      REGISTRY: "public.ecr.aws/q9t5s3a7"
@@ -100,24 +87,3 @@ steps:
        cd tests &&
        pytest -v -s entrypoints/multimodal/openai/chat_completion/test_audio_in_video.py &&
        pytest -v -s benchmarks/test_serve_cli.py'
-  - label: "XPU quantization test"
-    depends_on:
-      - image-build-xpu
-    timeout_in_minutes: 30
-    device: intel_gpu
-    agent_tags:
-      label: production
-      gpu: 1+
-      mem: 16+
-    no_plugin: true
-    env:
-      REGISTRY: "public.ecr.aws/q9t5s3a7"
-      REPO: "vllm-ci-test-repo"
-    source_file_dependencies:
-      - vllm/
-      - .buildkite/intel_jobs/test-intel.yaml 
-    commands:
-      - >-
-        bash .buildkite/scripts/hardware_ci/run-intel-test.sh
-        'cd tests &&
-        pytest -v -s quantization/test_auto_round.py'
@@ -15,10 +15,9 @@ set -euo pipefail

 DEFAULT_REPO_SLUG="vllm-project/vllm"
 DEFAULT_CI_HCL_SOURCE="docker/ci-rocm.hcl"
-DEFAULT_CI_BASE_CONTENT_FILES="requirements/common.txt requirements/rocm.txt requirements/test/rocm.txt docker/Dockerfile.rocm_base docker/ci-rocm.hcl docker/docker-bake-rocm.hcl tools/install_torchcodec_rocm.sh tests/vllm_test_utils .buildkite/scripts/ci-bake-rocm.sh"
+DEFAULT_CI_BASE_CONTENT_FILES="requirements/common.txt requirements/rocm.txt requirements/test/rocm.txt docker/Dockerfile.rocm_base tools/install_torchcodec_rocm.sh tests/vllm_test_utils"
 DEFAULT_CI_BASE_DOCKERFILE="docker/Dockerfile.rocm"
 DEFAULT_CI_BASE_DOCKERFILE_STAGES="base build_rixl build_rocshmem build_deepep mori_base ci_base"
-DEFAULT_CI_BASE_METADATA_VERSION="1"
 IMAGE_EXISTED_BEFORE_BUILD=0

 TARGET=""
@@ -526,22 +525,6 @@ get_remote_image_label_with_retry() {
    return 0
 }

-remote_ci_base_metadata_is_current() {
-    local image_ref="$1"
-    local metadata_version=""
-
-    metadata_version=$(get_remote_image_label "${image_ref}" "vllm.ci_base.metadata_version")
-    [[ "${metadata_version}" == "${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}" ]]
-}
-
-remote_ci_base_metadata_is_current_with_retry() {
-    local image_ref="$1"
-    local metadata_version=""
-
-    metadata_version=$(get_remote_image_label_with_retry "${image_ref}" "vllm.ci_base.metadata_version")
-    [[ "${metadata_version}" == "${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}" ]]
-}
-
 remote_image_exists() {
    local image_ref="$1"
    docker manifest inspect "${image_ref}" >/dev/null 2>&1
@@ -598,7 +581,6 @@ init_config() {
    CI_BASE_CONTENT_FILES="${CI_BASE_CONTENT_FILES:-${DEFAULT_CI_BASE_CONTENT_FILES}}"
    CI_BASE_DOCKERFILE="${CI_BASE_DOCKERFILE:-${DEFAULT_CI_BASE_DOCKERFILE}}"
    CI_BASE_DOCKERFILE_STAGES="${CI_BASE_DOCKERFILE_STAGES:-${DEFAULT_CI_BASE_DOCKERFILE_STAGES}}"
-    CI_BASE_METADATA_VERSION="${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}"
    CI_BASE_IMAGE_TAG="${CI_BASE_IMAGE_TAG:-rocm/vllm-dev:ci_base}"
    export PYTORCH_ROCM_ARCH

@@ -653,10 +635,6 @@ load_ci_hcl() {
    echo "Copied ${CI_HCL_SOURCE} to ${CI_HCL_PATH}"
 }

-init_bake_files() {
-    BAKE_FILES=(-f "${VLLM_BAKE_FILE}" -f "${CI_HCL_PATH}")
-}
-
 compute_ci_base_hash_if_needed() {
    if [[ -z "${CI_BASE_CONTENT_FILES:-}" ]]; then
        return 0
@@ -698,14 +676,12 @@ configure_ci_base_image_refs() {
    fi

    content_tag=$(ci_base_tag_with_suffix "${stable_tag}" "${CI_BASE_CONTENT_HASH}")
-    CI_BASE_IMAGE_TAG_CONTENT_REF="${content_tag}"
    if [[ -n "${BUILDKITE_COMMIT:-}" ]]; then
        commit_tag=$(ci_base_tag_with_suffix "${stable_tag}" "${BUILDKITE_COMMIT}")
+        CI_BASE_IMAGE_TAG_COMMIT="${commit_tag}"
+        export CI_BASE_IMAGE_TAG_COMMIT
    fi
-    CI_BASE_IMAGE_TAG_COMMIT_REF="${commit_tag}"

-    # *_REF is the logical tag recorded in metadata. *_EXTRA is only passed to
-    # bake when that tag is not already the primary tag, avoiding duplicates.
    if should_push_stable_ci_base_tag; then
        primary_tag="${content_tag}"
        CI_BASE_IMAGE_TAG_STABLE="${stable_tag}"
@@ -715,33 +691,19 @@ configure_ci_base_image_refs() {
    fi
    CI_BASE_IMAGE_TAG="${primary_tag}"
    if [[ "${primary_tag}" == "${content_tag}" ]]; then
-        CI_BASE_IMAGE_TAG_CONTENT_EXTRA=""
+        CI_BASE_IMAGE_TAG_CONTENT=""
    else
-        CI_BASE_IMAGE_TAG_CONTENT_EXTRA="${content_tag}"
+        CI_BASE_IMAGE_TAG_CONTENT="${content_tag}"
    fi
-    if [[ -n "${commit_tag}" && "${commit_tag}" != "${primary_tag}" ]]; then
-        CI_BASE_IMAGE_TAG_COMMIT_EXTRA="${commit_tag}"
-    else
-        CI_BASE_IMAGE_TAG_COMMIT_EXTRA=""
-    fi
-    export CI_BASE_IMAGE_TAG
-    export CI_BASE_IMAGE_TAG_COMMIT_EXTRA
-    export CI_BASE_IMAGE_TAG_CONTENT_EXTRA
-    export CI_BASE_IMAGE_TAG_CONTENT_REF
-    export CI_BASE_IMAGE_TAG_COMMIT_REF
-    export CI_BASE_IMAGE_TAG_STABLE
+    export CI_BASE_IMAGE_TAG CI_BASE_IMAGE_TAG_CONTENT CI_BASE_IMAGE_TAG_STABLE

    if is_ci_base_target; then
        IMAGE_TAG="${primary_tag}"
        export IMAGE_TAG

        echo "ci_base primary image tag: ${CI_BASE_IMAGE_TAG}"
-        if [[ -n "${commit_tag}" ]]; then
-            if [[ "${commit_tag}" == "${primary_tag}" ]]; then
-                echo "ci_base commit image tag: ${commit_tag} (primary)"
-            else
-                echo "ci_base commit image tag: ${commit_tag}"
-            fi
+        if [[ -n "${CI_BASE_IMAGE_TAG_COMMIT:-}" ]]; then
+            echo "ci_base commit image tag: ${CI_BASE_IMAGE_TAG_COMMIT}"
        fi
        echo "ci_base content image tag: ${content_tag}"
        if [[ -n "${CI_BASE_IMAGE_TAG_STABLE}" ]]; then
@@ -766,8 +728,8 @@ ci_base_candidate_refs() {
    printf '%s\n' \
        "${IMAGE_TAG:-}" \
        "${CI_BASE_IMAGE_TAG:-}" \
-        "${CI_BASE_IMAGE_TAG_COMMIT_EXTRA:-}" \
-        "${CI_BASE_IMAGE_TAG_CONTENT_EXTRA:-}" \
+        "${CI_BASE_IMAGE_TAG_COMMIT:-}" \
+        "${CI_BASE_IMAGE_TAG_CONTENT:-}" \
        "${CI_BASE_IMAGE_TAG_STABLE:-}" \
        | awk 'NF && !seen[$0]++'
 }
@@ -781,10 +743,6 @@ find_matching_ci_base_ref() {
        remote_image_exists "${candidate}" || continue
        candidate_hash=$(get_remote_image_label "${candidate}" "vllm.ci_base.content_hash")
        if [[ "${candidate_hash}" == "${CI_BASE_CONTENT_HASH}" ]]; then
-            if ! remote_ci_base_metadata_is_current "${candidate}"; then
-                echo "Found matching ci_base content hash but stale metadata: ${candidate}" >&2
-                continue
-            fi
            printf '%s\n' "${candidate}"
            return 0
        fi
@@ -859,10 +817,6 @@ maybe_skip_existing_image() {
        if [[ -n "${remote_hash}" ]]; then
            echo "Remote ci_base content hash: ${remote_hash:0:16}..."
            if [[ "${remote_hash}" == "${CI_BASE_CONTENT_HASH}" ]]; then
-                if ! remote_ci_base_metadata_is_current "${IMAGE_TAG}"; then
-                    echo "Content hashes match but ci_base metadata is stale; rebuilding to refresh metadata"
-                    return 0
-                fi
                if ! refresh_ci_base_tags_from_ref "${IMAGE_TAG}"; then
                    echo "ci_base tag refresh failed; rebuilding to push expected tags"
                    return 0
@@ -1044,104 +998,12 @@ prepare_git_cache_metadata() {
    fi
 }

-ci_base_metadata_pairs() {
-    local dockerfile="${CI_BASE_DOCKERFILE:-${DEFAULT_CI_BASE_DOCKERFILE}}"
-    local stages="${CI_BASE_DOCKERFILE_STAGES:-${DEFAULT_CI_BASE_DOCKERFILE_STAGES}}"
-    local content_files="${CI_BASE_CONTENT_FILES:-${DEFAULT_CI_BASE_CONTENT_FILES}}"
-    local content_files_hash=""
-    local base_image=""
-    local base_image_digest=""
-    local git_branch=""
-    local -a content_paths=()
-    local -a content_args=()
-
-    read -r -a content_paths <<< "${content_files}"
-    if [[ ${#content_paths[@]} -gt 0 ]]; then
-        content_files_hash=$(compute_content_hash "${content_paths[@]}")
-    fi
-    mapfile -t content_args < <(
-        get_content_arg_names "${dockerfile}" "${stages}" "${CI_BASE_CONTENT_ARGS:-}"
-    )
-
-    base_image=$(resolve_dockerfile_arg_value "${dockerfile}" "BASE_IMAGE")
-    if [[ -n "${base_image}" ]]; then
-        base_image_digest=$(resolve_image_digest "${base_image}")
-    fi
-    git_branch="${BUILDKITE_BRANCH:-${VLLM_BRANCH:-}}"
-
-    metadata_pair "vllm.ci_base.metadata_version" "${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}"
-    metadata_pair "vllm.ci_base.content_hash" "${CI_BASE_CONTENT_HASH:-}"
-    metadata_pair "vllm.ci_base.content_files_hash" "${content_files_hash}"
-    metadata_pair "vllm.ci_base.content_files" "${content_files}"
-    metadata_pair "vllm.ci_base.content_args" "$(join_words "${content_args[@]}")"
-    metadata_pair "vllm.ci_base.dockerfile" "${dockerfile}"
-    metadata_pair "vllm.ci_base.dockerfile_stages" "${stages}"
-    metadata_pair "vllm.ci_base.image.primary" "${CI_BASE_IMAGE_TAG:-}"
-    metadata_pair "vllm.ci_base.image.content" "${CI_BASE_IMAGE_TAG_CONTENT_REF:-${CI_BASE_IMAGE_TAG_CONTENT_EXTRA:-}}"
-    metadata_pair "vllm.ci_base.image.commit" "${CI_BASE_IMAGE_TAG_COMMIT_REF:-${CI_BASE_IMAGE_TAG_COMMIT_EXTRA:-}}"
-    metadata_pair "vllm.ci_base.image.stable" "${CI_BASE_IMAGE_TAG_STABLE:-}"
-    metadata_pair "vllm.ci_base.git_commit" "${BUILDKITE_COMMIT:-}"
-    metadata_pair "vllm.ci_base.git_branch" "${git_branch}"
-    metadata_pair "vllm.ci_base.vllm_branch" "${VLLM_BRANCH:-}"
-    metadata_pair "vllm.ci_base.stable_branch" "${CI_BASE_STABLE_BRANCH:-main}"
-
-    metadata_pair "vllm.rocm.base_image" "${base_image}"
-    metadata_pair "vllm.rocm.base_image_digest" "${base_image_digest}"
-    metadata_pair "vllm.rocm.pytorch_rocm_arch" "${PYTORCH_ROCM_ARCH:-}"
-    metadata_pair "vllm.rocm.nic_backend" "$(resolve_dockerfile_arg_value "${dockerfile}" "NIC_BACKEND")"
-    metadata_pair "vllm.rocm.ainic_version" "$(resolve_dockerfile_arg_value "${dockerfile}" "AINIC_VERSION")"
-    metadata_pair "vllm.rocm.ubuntu_codename" "$(resolve_dockerfile_arg_value "${dockerfile}" "UBUNTU_CODENAME")"
-    metadata_pair "vllm.rocm.rixl_repo" "$(resolve_dockerfile_arg_value "${dockerfile}" "RIXL_REPO")"
-    metadata_pair "vllm.rocm.rixl_commit" "${RIXL_BRANCH:-$(resolve_dockerfile_arg_value "${dockerfile}" "RIXL_BRANCH")}"
-    metadata_pair "vllm.rocm.ucx_repo" "$(resolve_dockerfile_arg_value "${dockerfile}" "UCX_REPO")"
-    metadata_pair "vllm.rocm.ucx_commit" "${UCX_BRANCH:-$(resolve_dockerfile_arg_value "${dockerfile}" "UCX_BRANCH")}"
-    metadata_pair "vllm.rocm.rocshmem_repo" "$(resolve_dockerfile_arg_value "${dockerfile}" "ROCSHMEM_REPO")"
-    metadata_pair "vllm.rocm.rocshmem_commit" "${ROCSHMEM_BRANCH:-$(resolve_dockerfile_arg_value "${dockerfile}" "ROCSHMEM_BRANCH")}"
-    metadata_pair "vllm.rocm.deepep_repo" "$(resolve_dockerfile_arg_value "${dockerfile}" "DEEPEP_REPO")"
-    metadata_pair "vllm.rocm.deepep_commit" "${DEEPEP_BRANCH:-$(resolve_dockerfile_arg_value "${dockerfile}" "DEEPEP_BRANCH")}"
-    metadata_pair "vllm.rocm.deepep_nic" "$(resolve_dockerfile_arg_value "${dockerfile}" "DEEPEP_NIC")"
-    metadata_pair "vllm.rocm.deepep_rocm_arch" "$(resolve_dockerfile_arg_value "${dockerfile}" "DEEPEP_ROCM_ARCH")"
-    metadata_pair "vllm.rocm.rixl_cache_key" "${RIXL_CACHE_KEY:-}"
-    metadata_pair "vllm.rocm.rocshmem_cache_key" "${ROCSHMEM_CACHE_KEY:-}"
-    metadata_pair "vllm.rocm.deepep_cache_key" "${DEEPEP_CACHE_KEY:-}"
-
-    metadata_pair "vllm.buildkite.build_number" "${BUILDKITE_BUILD_NUMBER:-}"
-    metadata_pair "vllm.buildkite.build_id" "${BUILDKITE_BUILD_ID:-}"
-}
-
-write_ci_base_metadata_annotations() {
-    local metadata="$1"
-    local key=""
-    local value=""
-    local annotation=""
-
-    [[ -n "${metadata}" ]] || return 0
-    while IFS=$'\t' read -r key value; do
-        [[ -n "${key}" && -n "${value}" ]] || continue
-        annotation="manifest:${key}=${value}"
-        printf '    "%s",\n' "$(hcl_escape_string "${annotation}")"
-    done <<< "${metadata}"
-}
-
-write_ci_base_metadata_labels() {
-    local metadata="$1"
-    local key=""
-    local value=""
-
-    [[ -n "${metadata}" ]] || return 0
-    while IFS=$'\t' read -r key value; do
-        [[ -n "${key}" && -n "${value}" ]] || continue
-        printf '    "%s" = "%s"\n' \
-            "$(hcl_escape_string "${key}")" \
-            "$(hcl_escape_string "${value}")"
-    done <<< "${metadata}"
-}
-
 write_ci_base_label_override() {
    local target_name=""
-    local metadata=""
    local -a ci_base_targets=()

+    BAKE_FILES=(-f "${VLLM_BAKE_FILE}" -f "${CI_HCL_PATH}")
+
    if [[ -z "${CI_BASE_CONTENT_HASH:-}" ]]; then
        return 0
    fi
@@ -1157,23 +1019,16 @@ write_ci_base_label_override() {
        return 0
    fi

-    metadata=$(ci_base_metadata_pairs)
-
    : > "${CI_BASE_LABEL_OVERRIDE_PATH}"
    for target_name in "${ci_base_targets[@]}"; do
        cat >> "${CI_BASE_LABEL_OVERRIDE_PATH}" <<EOF
 target "${target_name}" {
  annotations = [
    "manifest:org.opencontainers.image.revision=",
-EOF
-        write_ci_base_metadata_annotations "${metadata}" >> "${CI_BASE_LABEL_OVERRIDE_PATH}"
-        cat >> "${CI_BASE_LABEL_OVERRIDE_PATH}" <<EOF
  ]
  labels = {
    "org.opencontainers.image.revision" = ""
-EOF
-        write_ci_base_metadata_labels "${metadata}" >> "${CI_BASE_LABEL_OVERRIDE_PATH}"
-        cat >> "${CI_BASE_LABEL_OVERRIDE_PATH}" <<EOF
+    "vllm.ci_base.content_hash" = "${CI_BASE_CONTENT_HASH}"
  }
 }

@@ -1181,7 +1036,7 @@ EOF
    done

    BAKE_FILES+=(-f "${CI_BASE_LABEL_OVERRIDE_PATH}")
-    echo "Appended ci_base metadata label override for targets: ${ci_base_targets[*]}"
+    echo "Appended ci_base content-hash label override for targets: ${ci_base_targets[*]}"
 }

 uses_rocm_csrc_cache() {
@@ -1264,18 +1119,6 @@ hcl_escape_string() {
    printf '%s' "${value}"
 }

-join_words() {
-    local IFS=" "
-    printf '%s' "$*"
-}
-
-metadata_pair() {
-    local key="$1"
-    local value="${2:-}"
-
-    printf '%s\t%s\n' "${key}" "${value}"
-}
-
 write_hcl_string_list() {
    local indent="$1"
    shift
@@ -1698,13 +1541,7 @@ confirm_remote_image_push() {

        remote_hash=$(get_remote_image_label_with_retry "${image_ref}" "vllm.ci_base.content_hash")
        if [[ -n "${remote_hash}" && "${remote_hash}" == "${CI_BASE_CONTENT_HASH}" ]]; then
-            if remote_ci_base_metadata_is_current_with_retry "${image_ref}"; then
-                return 0
-            fi
-
-            echo "Remote image exists with the expected ci_base content hash but stale metadata."
-            echo "  expected metadata version: ${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}"
-            return 1
+            return 0
        fi

        echo "Remote image exists but does not have the expected ci_base content hash."
@@ -1887,16 +1724,15 @@ main() {
    print_header
    validate_inputs
    load_ci_hcl
-    init_bake_files
    compute_ci_base_hash_if_needed
    configure_ci_base_image_refs
    maybe_skip_existing_image
    setup_builder
    prepare_git_cache_metadata
+    write_ci_base_label_override
    extract_dependency_pins
    write_rocm_build_arg_override
    compute_dependency_cache_keys
-    write_ci_base_label_override
    compute_rocm_csrc_content_hash_if_needed
    write_rocm_cache_override
    resolve_ci_base_dependency_targets
@@ -367,20 +367,6 @@ remove_docker_container() {
 }
 trap remove_docker_container EXIT

-# python_only_compile.sh runs `python setup.py develop` and needs the full repo tree
-# under /vllm-workspace (Dockerfile.rocm test stage: mkdir src && mv vllm).
-# The ROCm wheel artifact tarball only ships a thin tree (tests, etc.), so
-# artifact images cannot satisfy that test — use the full rocm/vllm-ci image.
-_cmd_probe="${VLLM_TEST_COMMANDS:-}"
-if [[ -z "${_cmd_probe}" ]]; then
-  _cmd_probe="$*"
-fi
-if [[ "${VLLM_CI_USE_ARTIFACTS:-0}" == "1" && "${_cmd_probe}" == *python_only_compile.sh* ]]; then
-  echo "INFO: disabling VLLM_CI_USE_ARTIFACTS for python_only_compile (requires full /vllm-workspace tree)"
-  export VLLM_CI_USE_ARTIFACTS=0
-fi
-unset -v _cmd_probe
-
 if ! prepare_artifact_image; then
  echo "Using full ROCm CI image: ${image_name}"
  docker pull "${image_name}" || exit 1
@@ -440,24 +426,6 @@ fi

 echo "Final commands: $commands"

-# The ROCm test image often ships /vllm-workspace without .git (artifact tarball unpack).
-# tests/standalone_tests/python_only_compile.sh uses merge-base(HEAD, origin/main) for
-# wheels.vllm.ai; compute on the agent (full git checkout) and pass into the container.
-vllm_standalone_merge_base=""
-checkout="${BUILDKITE_BUILD_CHECKOUT_PATH:-}"
-if [[ -z "${checkout}" || ! -d "${checkout}" ]]; then
-  checkout="."
-fi
-if git -C "${checkout}" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
-  vllm_standalone_merge_base="$(
-    git -C "${checkout}" merge-base HEAD origin/main 2>/dev/null || true
-  )"
-fi
-if [[ -z "${vllm_standalone_merge_base}" ]]; then
-  vllm_standalone_merge_base="${BUILDKITE_COMMIT:-}"
-fi
-echo "INFO: passing VLLM_STANDALONE_MERGE_BASE into container: ${vllm_standalone_merge_base}"
-
 MYPYTHONPATH="/vllm-workspace"

 container_job_id="${BUILDKITE_JOB_ID:-${BUILDKITE_PARALLEL_JOB:-0}}"
@@ -557,7 +525,6 @@ else
    -e "VLLM_CACHE_ROOT=${CONTAINER_CACHE_ROOT}/vllm" \
    -e "XDG_CACHE_HOME=${CONTAINER_CACHE_ROOT}/xdg" \
    -e "PYTORCH_ROCM_ARCH=" \
-    -e "VLLM_STANDALONE_MERGE_BASE=${vllm_standalone_merge_base}" \
    --name "${container_name}" \
    "${image_name}" \
    /bin/bash -c "${CONTAINER_PREFLIGHT} && ${commands}"
@@ -8,7 +8,7 @@ set -ex
 CORE_RANGE=${CORE_RANGE:-0-31}
 OMP_CORE_RANGE=${OMP_CORE_RANGE:-0-31}

-export CMAKE_BUILD_PARALLEL_LEVEL=32
+export CMAKE_BUILD_PARALLEL_LEVEL=16

 # Setup cleanup
 remove_docker_container() {
@@ -37,7 +37,7 @@ function cpu_tests() {
    pytest -x -v -s tests/kernels/test_onednn.py
    pytest -x -v -s tests/kernels/attention/test_cpu_attn.py
    pytest -x -v -s tests/kernels/core/test_cpu_activation.py
-    pytest -x -v -s tests/kernels/moe/test_cpu_fused_moe.py
+    pytest -x -v -s tests/kernels/moe/test_moe.py -k test_cpu_fused_moe_basic
    pytest -x -v -s tests/kernels/mamba/cpu/test_cpu_gdn_ops.py"

  # skip tests requiring model downloads if HF_TOKEN is not set
@@ -7,49 +7,10 @@ set -euox pipefail
 # allow to bind to different cores
 CORE_RANGE=${CORE_RANGE:-48-95}
 NUMA_NODE=${NUMA_NODE:-1}
-AGENT_SLOT=${AGENT_SLOT:-}
-IMAGE_NAME="cpu-test-${NUMA_NODE}${AGENT_SLOT:+-${AGENT_SLOT}}"
+IMAGE_NAME="cpu-test-$NUMA_NODE"
 TIMEOUT_VAL=$1
 TEST_COMMAND=$2

-# Disk hygiene knobs. Reclaim space only once the Docker root filesystem crosses
-# DISK_USAGE_THRESHOLD percent, and cap the shared BuildKit cache at
-# BUILDKIT_CACHE_MAX so subsequent builds keep reusing the hottest layers.
-DISK_USAGE_THRESHOLD=${DISK_USAGE_THRESHOLD:-70}
-BUILDKIT_CACHE_MAX=${BUILDKIT_CACHE_MAX:-80GB}
-
-# Reclaim disk only when the host is under pressure. We trim (not purge) the
-# shared BuildKit cache so cross-job/cross-agent reuse stays intact, and only
-# touch dangling images; other agents' uniquely tagged images are left alone.
-prune_if_disk_pressure() {
-    local docker_root disk_usage
-    docker_root=$(docker info -f '{{.DockerRootDir}}' 2>/dev/null || true)
-    if [ -z "$docker_root" ]; then
-        return 0
-    fi
-    disk_usage=$(df "$docker_root" 2>/dev/null | tail -1 | awk '{print $5}' | tr -d '%')
-    if [ "${disk_usage:-0}" -gt "$DISK_USAGE_THRESHOLD" ]; then
-        echo "--- :broom: Disk usage ${disk_usage}% exceeds ${DISK_USAGE_THRESHOLD}%, reclaiming space"
-        docker image prune -f || true
-        docker builder prune -f --keep-storage="$BUILDKIT_CACHE_MAX" || true
-    else
-        echo "Disk usage ${disk_usage:-unknown}% within ${DISK_USAGE_THRESHOLD}% threshold; skipping prune"
-    fi
-}
-
-# Always drop this agent's image once the job ends (the default builder never
-# uses it as a cache source, so removing it costs no rebuild speed), then
-# reclaim space if needed. Guard every docker call with `|| true` so the trap
-# never overrides the test's exit code.
-cleanup() {
-    docker image rm -f "$IMAGE_NAME" || true
-    prune_if_disk_pressure
-}
-trap cleanup EXIT
-
-# Free space up front so a nearly-full host doesn't fail the build.
-prune_if_disk_pressure
-
 # building the docker image
 echo "--- :docker: Building Docker image"
 docker build --progress plain --tag "$IMAGE_NAME" --target vllm-test -f docker/Dockerfile.cpu .
@@ -1,51 +0,0 @@
-#!/bin/bash
-
-set -euo pipefail
-
-test_suite="${1:-}"
-
-if [[ -z "${test_suite}" ]]; then
-  echo "Usage: $0 <example|v1|server>" >&2
-  exit 1
-fi
-
-case "${test_suite}" in
-  example)
-    pip install tblib==3.1.0
-
-    python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager
-    python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 -O3 -cc.cudagraph_mode=NONE
-    python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager -tp 2 --distributed-executor-backend mp
-    python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --attention-backend=TRITON_ATTN
-    python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --quantization fp8
-    python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --kv-cache-dtype fp8
-    python3 examples/basic/offline_inference/generate.py --model nvidia/Llama-3.1-8B-Instruct-FP8 --block-size 64 --enforce-eager --quantization modelopt --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --max-model-len 4096
-    python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --block-size 64 --enforce-eager --max-model-len 8192
-    python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2
-    python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 --enable-expert-parallel
-    python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --max-model-len 8192
-    ;;
-  v1)
-    cd tests
-
-    pytest -v -s v1/core --ignore=v1/core/test_reset_prefix_cache_e2e.py --ignore=v1/core/test_scheduler_e2e.py
-    pytest -v -s v1/engine --ignore=v1/engine/test_output_processor.py
-    pytest -v -s v1/sample --ignore=v1/sample/test_logprobs.py --ignore=v1/sample/test_logprobs_e2e.py -k "not test_topk_only and not test_topp_only and not test_topk_and_topp"
-    pytest -v -s v1/worker --ignore=v1/worker/test_gpu_model_runner.py --ignore=v1/worker/test_worker_memory_snapshot.py
-    pytest -v -s v1/structured_output
-    pytest -v -s v1/test_serial_utils.py
-    pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py --ignore=v1/spec_decode/test_speculators_correctness.py
-    pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_multi_connector.py --ignore=v1/kv_connector/unit/test_example_connector.py --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py --ignore=v1/kv_connector/unit/test_offloading_connector.py
-    ;;
-  server)
-    pip install av
-    cd tests
-
-    pytest -v -s entrypoints/multimodal/openai/chat_completion/test_audio_in_video.py
-    pytest -v -s benchmarks/test_serve_cli.py
-    ;;
-  *)
-    echo "Unknown Intel test suite: ${test_suite}" >&2
-    exit 1
-    ;;
-esac
@@ -243,10 +243,8 @@ container_name="xpu_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head

 # ---- Command source selection ----
 commands=""
-commands_source=""
 if [[ -n "${VLLM_TEST_COMMANDS:-}" ]]; then
  commands="${VLLM_TEST_COMMANDS}"
-  commands_source="env"
  echo "Commands sourced from VLLM_TEST_COMMANDS (quoting preserved)"
 elif [[ $# -gt 0 ]]; then
  all_yaml=true
@@ -305,12 +303,8 @@ if [[ -z "$commands" ]]; then
 fi

 echo "Raw commands: $commands"
-if [[ "$commands_source" != "env" ]]; then
-  commands=$(re_quote_pytest_markers "$commands")
-  echo "After re-quoting: $commands"
-else
-  echo "Skipping re-quoting for VLLM_TEST_COMMANDS input"
-fi
+commands=$(re_quote_pytest_markers "$commands")
+echo "After re-quoting: $commands"
 commands=$(apply_intel_test_overrides "$commands")
 echo "Final commands: $commands"

@@ -369,7 +363,7 @@ export HF_TOKEN ZE_AFFINITY_MASK
    -e CMDS \
    --name "${container_name}" \
    "${IMAGE}" \
-    bash -c 'set -e; source /opt/intel/oneapi/setvars.sh --force; source /opt/intel/oneapi/ccl/2021.15/env/vars.sh --force; echo "ZE_AFFINITY_MASK is ${ZE_AFFINITY_MASK:-}"; eval "$CMDS"' \
+    bash -c 'set -e; echo "ZE_AFFINITY_MASK is ${ZE_AFFINITY_MASK:-}"; eval "$CMDS"' \
    >/dev/null
 } 9>/tmp/docker-pull.lock

@@ -4,11 +4,6 @@

 set -euo pipefail

-if python3 -c "import torch; raise SystemExit(0 if torch.version.hip is not None else 1)"; then
-    uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
-    exit 0
-fi
-
 REQUIREMENTS_FILE="${KV_CONNECTORS_REQUIREMENTS:-/vllm-workspace/requirements/kv_connectors.txt}"

 uv pip install --system -r "${REQUIREMENTS_FILE}"
@@ -90,16 +90,6 @@ install_cargo_sort() {
  cargo binstall --no-confirm cargo-sort
 }

-install_cargo_deny() {
-  if command -v cargo-deny >/dev/null 2>&1; then
-    return
-  fi
-
-  log_section "Installing cargo-deny"
-  install_cargo_binstall
-  cargo binstall --no-confirm cargo-deny
-}
-
 install_cargo_nextest() {
  if command -v cargo-nextest >/dev/null 2>&1; then
    return
@@ -152,7 +142,6 @@ PY

 run_style_clippy() {
  install_cargo_sort
-  install_cargo_deny

  log_section "Checking Rust formatting"
  cargo fmt --manifest-path rust/Cargo.toml --all -- --check
@@ -160,13 +149,6 @@ run_style_clippy() {
  log_section "Checking Cargo.toml ordering"
  cargo sort --workspace --check rust

-  log_section "Checking Rust dependency bans"
-  cargo deny \
-    --manifest-path rust/Cargo.toml \
-    check \
-    --config rust/deny.toml \
-    bans
-
  log_section "Running clippy"
  cargo clippy \
    --manifest-path rust/Cargo.toml \
@@ -33,14 +33,6 @@ if [[ -n "${ATTENTION_BACKEND:-}" ]]; then
  EXTRA_ARGS+=(--attention-backend "${ATTENTION_BACKEND}")
 fi

-# ROCm: run eager to avoid intermittent HIP-graph decode corruption.
-# See https://github.com/ROCm/clr/issues/279
-# TODO(aarushjain29): Revert after TheRock 7.14
-if command -v rocm-smi &> /dev/null || command -v amd-smi &> /dev/null || [[ -d /opt/rocm ]] || [[ -n "${ROCM_PATH:-}" ]]; then
-  echo "ROCm platform detected: adding --enforce-eager to avoid HIP-graph decode corruption"
-  EXTRA_ARGS+=(--enforce-eager)
-fi
-
 cleanup() {
  if [[ -n "${SERVER_PID:-}" ]] && kill -0 "${SERVER_PID}" 2>/dev/null; then
    kill "${SERVER_PID}" 2>/dev/null || true
@@ -16,9 +16,3 @@ steps:
  - pytest -v -s basic_correctness/test_mem.py
  - pytest -v -s basic_correctness/test_basic_correctness.py
  - pytest -v -s basic_correctness/test_cpu_offload.py
-  mirror:
-    amd:
-      device: mi325_1
-      timeout_in_minutes: 50
-      depends_on:
-      - image-build-amd
@@ -11,11 +11,6 @@ steps:
  - tests/benchmarks/
  commands:
  - pytest -v -s benchmarks/
-  mirror:
-    amd:
-      device: mi300_1
-      depends_on:
-      - image-build-amd

 - label: Attention Benchmarks Smoke Test (B200)
  key: attention-benchmarks-smoke-test-b200
@@ -28,4 +23,4 @@ steps:
  - benchmarks/attention_benchmarks/
  - vllm/v1/attention/
  commands:
-  - python3 benchmarks/attention_benchmarks/benchmark.py --backends flash flashinfer --batch-specs "8q1s1k"
+  - python3 benchmarks/attention_benchmarks/benchmark.py --backends flash flashinfer --batch-specs "8q1s1k" --repeats 1 --warmup-iters 1
@@ -2,8 +2,8 @@ group: CUDA
 depends_on: 
  - image-build
 steps:
- label: Platform Tests
-  key: platform-tests
+- label: Platform Tests (CUDA)
+  key: platform-tests-cuda
  timeout_in_minutes: 15
  device: h200_18gb
  source_file_dependencies:
@@ -13,20 +13,6 @@ steps:
  commands:
    - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
    - bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
-  mirror:
-    amd:
-      device: mi300_4
-      timeout_in_minutes: 110
-      depends_on:
-        - image-build-amd
-      source_file_dependencies:
-        - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
-        - tests/v1/kv_connector/nixl_integration/
-        - vllm/platforms/rocm.py
-      commands:
-        - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
-        - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
-
 - label: Distributed FlashInfer NixlConnector PD accuracy (4 GPUs)
  key: distributed-flashinfer-nixlconnector-pd-accuracy-4-gpus
  timeout_in_minutes: 30
@@ -50,19 +36,6 @@ steps:
  commands:
    - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
    - DP_EP=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
-  mirror:
-    amd:
-      device: mi300_4
-      timeout_in_minutes: 50
-      depends_on:
-        - image-build-amd
-      source_file_dependencies:
-        - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
-        - tests/v1/kv_connector/nixl_integration/
-        - vllm/platforms/rocm.py
-      commands:
-        - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
-        - DP_EP=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh

 - label: CrossLayer KV layout Distributed NixlConnector PD accuracy tests (4 GPUs)
  key: crosslayer-kv-layout-distributed-nixlconnector-pd-accuracy-tests-4-gpus
@@ -75,19 +48,6 @@ steps:
  commands:
    - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
    - CROSS_LAYERS_BLOCKS=True bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
-  mirror:
-    amd:
-      device: mi300_4
-      timeout_in_minutes: 110
-      depends_on:
-        - image-build-amd
-      source_file_dependencies:
-        - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
-        - tests/v1/kv_connector/nixl_integration/
-        - vllm/platforms/rocm.py
-      commands:
-        - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
-        - CROSS_LAYERS_BLOCKS=True ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh

 - label: Hybrid SSM NixlConnector PD accuracy tests (4 GPUs)
  key: hybrid-ssm-nixlconnector-pd-accuracy-tests-4-gpus
@@ -100,19 +60,6 @@ steps:
  commands:
    - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
    - HYBRID_SSM=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
-  mirror:
-    amd:
-      device: mi300_4
-      timeout_in_minutes: 60
-      depends_on:
-        - image-build-amd
-      source_file_dependencies:
-        - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
-        - tests/v1/kv_connector/nixl_integration/
-        - vllm/platforms/rocm.py
-      commands:
-        - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
-        - HYBRID_SSM=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh

 - label: Hybrid SSM NixlConnector PD prefix cache test (2 GPUs)
  key: hybrid-ssm-nixlconnector-pd-prefix-cache-2-gpus
@@ -156,20 +103,6 @@ steps:
  commands:
    - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
    - bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh
-  mirror:
-    amd:
-      device: mi300_2
-      timeout_in_minutes: 60
-      depends_on:
-        - image-build-amd
-      source_file_dependencies:
-        - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
-        - vllm/v1/worker/kv_connector_model_runner_mixin.py
-        - tests/v1/kv_connector/nixl_integration/
-        - vllm/platforms/rocm.py
-      commands:
-        - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
-        - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh

 - label: MultiConnector (Nixl+Offloading) PD edge cases (2 GPUs)
  key: multiconnector-nixl-offloading-pd-edge-cases-2-gpus
@@ -37,21 +37,6 @@ steps:
  - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
  - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py
  - DP_SIZE=2 pytest -v -s entrypoints/openai/test_multi_api_servers.py
-  mirror:
-    amd:
-      device: mi300_2
-      depends_on:
-      - image-build-amd
-      source_file_dependencies:
-      - vllm/distributed/
-      - vllm/engine/
-      - vllm/executor/
-      - vllm/worker/worker_base.py
-      - vllm/v1/engine/
-      - vllm/v1/worker/
-      - tests/v1/distributed
-      - tests/entrypoints/openai/test_multi_api_servers.py
-      - vllm/platforms/rocm.py

 - label: Distributed Compile + RPC Tests (2 GPUs)
  key: distributed-compile-rpc-tests-2-gpus
@@ -174,8 +159,8 @@ steps:
  # test multi-node TP with multiproc executor (simulated on single node)
  - pytest -v -s distributed/test_multiproc_executor.py::test_multiproc_executor_multi_node

- label: Distributed Tests (8xH100)
-  key: distributed-tests-8xh100
+- label: Distributed Tests (8 GPUs)(H100)
+  key: distributed-tests-8-gpus-h100
  timeout_in_minutes: 10
  device: h100
  num_devices: 8
@@ -195,8 +180,8 @@ steps:
  # test with torchrun tp=2 and dp=4 with ep
  - torchrun --nproc-per-node=8 ../examples/features/torchrun/torchrun_dp_example_offline.py --tp-size=2 --pp-size=1 --dp-size=4 --enable-ep

- label: Distributed Tests (4xA100)
-  key: distributed-tests-4xa100
+- label: Distributed Tests (4 GPUs)(A100)
+  key: distributed-tests-4-gpus-a100
  device: a100
  optional: true
  num_devices: 4
@@ -210,8 +195,8 @@ steps:
  - TARGET_TEST_SUITE=A100 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
  - pytest -v -s -x lora/test_mixtral.py

- label: Distributed Tests (2xH100-2xMI300)
-  key: distributed-tests-2xh100-2xmi300
+- label: Distributed Tests (2 GPUs)(H100)
+  key: distributed-tests-2-gpus-h100
  timeout_in_minutes: 15
  device: h100
  optional: true
@@ -225,8 +210,8 @@ steps:
    - VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer.py
    - pytest -v -s tests/distributed/test_packed_tensor.py

- label: Distributed Tests (2xB200)
-  key: distributed-tests-2xb200
+- label: Distributed Tests (2 GPUs)(B200)
+  key: distributed-tests-2-gpus-b200
  device: b200-k8s
  optional: true
  working_dir: "/vllm-workspace/"
@@ -2,8 +2,8 @@ group: E2E Integration
 depends_on: 
  - image-build
 steps:
- label: DeepSeek V2-Lite Sync EPLB Accuracy (4xH100)
-  key: deepseek-v2-lite-sync-eplb-accuracy-4xh100
+- label: DeepSeek V2-Lite Sync EPLB Accuracy
+  key: deepseek-v2-lite-sync-eplb-accuracy
  timeout_in_minutes: 60
  device: h100
  optional: true
@@ -12,8 +12,8 @@ steps:
  commands:
  - bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh 0.25 200 8010

- label: Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy (4xH100)
-  key: qwen3-30b-a3b-fp8-block-sync-eplb-accuracy-4xh100
+- label: Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy
+  key: qwen3-30b-a3b-fp8-block-sync-eplb-accuracy
  timeout_in_minutes: 60
  device: h100
  optional: true
@@ -22,8 +22,8 @@ steps:
  commands:
  - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020

- label: Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy (2xB200)
-  key: qwen3-30b-a3b-fp8-block-sync-eplb-accuracy-2xb200
+- label: Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy (B200)
+  key: qwen3-30b-a3b-fp8-block-sync-eplb-accuracy-b200
  timeout_in_minutes: 60
  device: b200-k8s
  optional: true
@@ -74,16 +74,6 @@ steps:
    - tests/v1/e2e/general/
  commands:
    - pytest -v -s v1/e2e/general --ignore v1/e2e/general/test_async_scheduling.py
-  mirror:
-    amd:
-      device: mi250_1
-      timeout_in_minutes: 35
-      depends_on:
-      - image-build-amd
-      source_file_dependencies:
-      - vllm/v1/
-      - tests/v1/e2e/general/
-      - vllm/platforms/rocm.py

 - label: V1 e2e (2 GPUs)
  key: v1-e2e-2-gpus
@@ -112,11 +102,6 @@ steps:
  commands:
    # Only run tests that need exactly 2 GPUs
    - pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "tensor_parallelism"
-  mirror:
-    amd:
-      device: mi300_2
-      depends_on:
-      - image-build-amd

 - label: V1 e2e (4 GPUs)
  key: v1-e2e-4-gpus
@@ -29,8 +29,6 @@ steps:
  mirror:
    amd:
      device: mi325_1
-      # TODO(akaratza): Test after Torch >= 2.12 bump
-      soft_fail: true
      depends_on:
      - image-build-amd

@@ -42,12 +40,10 @@ steps:
  source_file_dependencies:
  - vllm/
  - tests/entrypoints/serve
-  - tests/entrypoints/scale_out
  commands:
  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
  - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc
  - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc
-  - pytest -v -s entrypoints/scale_out
  mirror:
    amd:
      device: mi325_1
@@ -14,16 +14,6 @@ steps:
  commands:
  - pytest -v -s distributed/test_eplb_algo.py
  - pytest -v -s distributed/test_eplb_utils.py
-  mirror:
-    amd:
-      device: mi300_1
-      depends_on:
-      - image-build-amd
-      source_file_dependencies:
-      - vllm/distributed/eplb
-      - tests/distributed/test_eplb_algo.py
-      - tests/distributed/test_eplb_utils.py
-      - vllm/platforms/rocm.py

 - label: EPLB Execution # 17min
  key: eplb-execution
@@ -47,10 +47,8 @@ steps:
  - csrc/fused_deepseek_v4_qnorm_rope_kv_insert_kernel.cu
  - vllm/models/deepseek_v4/common/ops/
  - tests/kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py
-  - tests/kernels/test_top_k_per_row.py # it runs on Blackwell too - some kernels have arch-specific optimizations
  commands:
    - pytest -v -s kernels/test_fused_deepseek_v4_*.py
-    - pytest -v -s kernels/test_top_k_per_row.py

 - label: Deepseek V4 Kernel Test (B200)
  key: deepseek-v4-kernel-test-b200
@@ -76,20 +74,6 @@ steps:
  commands:
    - pytest -v -s kernels/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
  parallelism: 2
-  mirror:
-    amd:
-      device: mi325_1
-      timeout_in_minutes: 55
-      depends_on:
-      - image-build-amd
-      source_file_dependencies:
-      - csrc/attention/
-      - vllm/v1/attention
-      - vllm/model_executor/layers/attention
-      - tests/kernels/attention
-      - vllm/_aiter_ops.py
-      - vllm/envs.py
-      - vllm/platforms/rocm.py

 - label: Kernels Attention DiffKV Test (H100)
  key: kernels-attention-diffkv-test-h100
@@ -120,7 +104,6 @@ steps:
      source_file_dependencies:
      - csrc/quantization/
      - vllm/model_executor/layers/quantization
-      - vllm/config/
      - tests/kernels/quantization
      - tests/kernels/quantization/test_rocm_skinny_gemms.py
      - vllm/_aiter_ops.py
@@ -144,22 +127,6 @@ steps:
    - pytest -v -s kernels/moe --ignore=kernels/moe/test_modular_oai_triton_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
    - pytest -v -s kernels/moe/test_modular_oai_triton_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
  parallelism: 5
-  mirror:
-    amd:
-      device: mi325_1
-      timeout_in_minutes: 50
-      source_file_dependencies:
-      - csrc/quantization/cutlass_w8a8/moe/
-      - csrc/moe/
-      - tests/kernels/moe
-      - vllm/model_executor/layers/fused_moe/
-      - vllm/distributed/device_communicators/
-      - vllm/envs.py
-      - vllm/config
-      - vllm/_aiter_ops.py
-      - vllm/platforms/rocm.py
-      depends_on:
-      - image-build-amd

 - label: Kernels Mamba Test
  key: kernels-mamba-test
@@ -274,8 +241,8 @@ steps:
    - pytest -v -s kernels/helion/

 
- label: Kernels FP8 MoE Test (1xH100)
-  key: kernels-fp8-moe-test-1xh100
+- label: Kernels FP8 MoE Test (1 H100)
+  key: kernels-fp8-moe-test-1-h100
  timeout_in_minutes: 90
  device: h100
  num_devices: 1
@@ -291,8 +258,8 @@ steps:
    - pytest -v -s kernels/moe/test_triton_moe_no_act_mul.py
    - pytest -v -s kernels/moe/test_triton_moe_ptpc_fp8.py

- label: Kernels FP8 MoE Test (2xH100)
-  key: kernels-fp8-moe-test-2xh100
+- label: Kernels FP8 MoE Test (2 H100s)
+  key: kernels-fp8-moe-test-2-h100s
  timeout_in_minutes: 90
  device: h100
  num_devices: 2
@@ -28,8 +28,7 @@ steps:
      - vllm/_aiter_ops.py
      - vllm/platforms/rocm.py

-# - label: LM Eval Large Models (4xA100)
-#   key: lm-eval-large-models-4xa100
+# - label: LM Eval Large Models (4 GPUs)(A100)
 #   device: a100
 #   optional: true
 #   num_devices: 4
@@ -41,8 +40,8 @@ steps:
 #   - export VLLM_WORKER_MULTIPROC_METHOD=spawn
 #   - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large.txt --tp-size=4

- label: LM Eval Large Models (4xH100)
-  key: lm-eval-large-models-4xh100
+- label: LM Eval Large Models (4 GPUs)(H100)
+  key: lm-eval-large-models-4-gpus-h100
  device: h100
  optional: true
  num_devices: 4
@@ -54,8 +53,8 @@ steps:
    - export VLLM_USE_DEEP_GEMM=0  # We found Triton is faster than DeepGEMM for H100
    - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-hopper.txt --tp-size=4

- label: LM Eval Small Models (2xB200)
-  key: lm-eval-small-models-2xb200
+- label: LM Eval Small Models (B200)
+  key: lm-eval-small-models-b200
  timeout_in_minutes: 120
  device: b200-k8s
  optional: true
@@ -65,20 +64,8 @@ steps:
  commands:
  - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell.txt

- label: LM Eval Small Models (2xL4)
-  key: lm-eval-small-models-tp
-  timeout_in_minutes: 10
-  num_devices: 2
-  optional: true
-  source_file_dependencies:
-  - csrc/
-  - vllm/model_executor/layers/quantization
-  autorun_on_main: true
-  commands:
-  - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small-tp.txt
-
- label: LM Eval Large Models EP (2xB200)
-  key: lm-eval-large-models-ep-2xb200
+- label: LM Eval Large Models (B200, EP)
+  key: lm-eval-large-models-b200-ep
  timeout_in_minutes: 120
  device: b200-k8s
  optional: true
@@ -89,8 +76,8 @@ steps:
  commands:
  - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell-ep.txt

- label: LM Eval Qwen3.5 Models (2xB200)
-  key: lm-eval-qwen3-5-models-2xb200
+- label: LM Eval Qwen3.5 Models (B200)
+  key: lm-eval-qwen3-5-models-b200
  timeout_in_minutes: 120
  device: b200-k8s
  optional: true
@@ -106,24 +93,14 @@ steps:
  commands:
  - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-qwen35-blackwell.txt

- label: LM Eval Large Models (8xH200)
-  key: lm-eval-large-models-8xh200
+- label: LM Eval Large Models (H200)
+  key: lm-eval-large-models-h200
  timeout_in_minutes: 60
  device: h200
  optional: true
  num_devices: 8
  commands:
    - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200.txt
-  mirror:
-    amd:
-      device: mi300_8
-      timeout_in_minutes: 180
-      depends_on:
-      - image-build-amd
-      commands:
-      - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-      - export PYTORCH_ROCM_ARCH=gfx942 # Limit Quark compilation to save time
-      - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx.txt

 - label: MoE Refactor Integration Test (H100 - TEMPORARY)
  key: moe-refactor-integration-test-h100-temporary
@@ -149,49 +126,6 @@ steps:
  commands:
    - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor-dp-ep/config-b200.txt

- label: LM Eval Humming (A100 - TEMPORARY)
-  key: lm-eval-humming-a100
-  timeout_in_minutes: 30
-  device: a100
-  optional: true
-  num_devices: 1
-  source_file_dependencies:
-  - vllm/model_executor/layers/quantization/humming.py
-  - vllm/model_executor/layers/quantization/utils/humming_utils.py
-  - vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
-  - vllm/model_executor/layers/fused_moe/oracle/mxfp4.py
-  commands:
-  - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config.txt
-
- label: LM Eval Humming (H100 - TEMPORARY)
-  key: lm-eval-humming-h100
-  timeout_in_minutes: 30
-  device: h100
-  optional: true
-  num_devices: 1
-  source_file_dependencies:
-  - vllm/model_executor/layers/quantization/humming.py
-  - vllm/model_executor/layers/quantization/utils/humming_utils.py
-  - vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
-  - vllm/model_executor/layers/fused_moe/oracle/mxfp4.py
-  commands:
-  - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config.txt
-  - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-fp8.txt
-
- label: LM Eval Humming (B200 - TEMPORARY)
-  key: lm-eval-humming-b200
-  timeout_in_minutes: 30
-  device: b200-k8s
-  optional: true
-  num_devices: 1
-  source_file_dependencies:
-  - vllm/model_executor/layers/quantization/humming.py
-  - vllm/model_executor/layers/quantization/utils/humming_utils.py
-  - vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
-  - vllm/model_executor/layers/fused_moe/oracle/mxfp4.py
-  commands:
-  - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config.txt
-  - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-fp8.txt

 - label: LM Eval TurboQuant KV Cache
  key: lm-eval-turboquant-kv-cache
@@ -205,8 +139,8 @@ steps:
  commands:
  - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant.txt

- label: GPQA Eval (GPT-OSS) (2xH100)
-  key: gpqa-eval-gpt-oss-2xh100
+- label: GPQA Eval (GPT-OSS) (H100)
+  key: gpqa-eval-gpt-oss-h100
  timeout_in_minutes: 120
  device: h100
  optional: true
@@ -219,8 +153,8 @@ steps:
    - uv pip install --system 'gpt-oss[eval]==0.0.5'
    - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-h100.txt

- label: GPQA Eval (GPT-OSS) (2xB200)
-  key: gpqa-eval-gpt-oss-2xb200
+- label: GPQA Eval (GPT-OSS) (B200)
+  key: gpqa-eval-gpt-oss-b200
  timeout_in_minutes: 120
  device: b200-k8s
  optional: true
@@ -233,22 +167,6 @@ steps:
    - uv pip install --system 'gpt-oss[eval]==0.0.5'
    - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-b200.txt

- label: GPQA Eval (GPT-OSS) (DGX Spark)
-  key: gpqa-eval-gpt-oss-spark
-  timeout_in_minutes: 120
-  device: dgx-spark
-  optional: true
-  num_devices: 1
-  depends_on:
-  - arm64-image-build
-  source_file_dependencies:
-  - csrc/
-  - vllm/model_executor/layers/quantization
-  - tests/evals/gpt_oss/
-  commands:
-    - uv pip install --system 'gpt-oss[eval]==0.0.5'
-    - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-spark.txt
-
 - label: MRCR Eval Small Models
  device: h200_35gb
  timeout_in_minutes: 30
@@ -12,17 +12,6 @@ steps:
  commands:
    - pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --ignore=lora/test_chatglm3_tp.py --ignore=lora/test_llama_tp.py --ignore=lora/test_qwen3_with_multi_loras.py --ignore=lora/test_olmoe_tp.py --ignore=lora/test_deepseekv2_tp.py --ignore=lora/test_gptoss_tp.py --ignore=lora/test_qwen3moe_tp.py --ignore=lora/test_qwen35_densemodel_lora.py 
  parallelism: 4
-  mirror:
-    amd:
-      device: mi325_1
-      working_dir: "/vllm-workspace/tests"
-      timeout_in_minutes: 60
-      source_file_dependencies:
-      - vllm/lora
-      - tests/lora
-      - vllm/platforms/rocm.py
-      depends_on:
-      - image-build-amd


 - label: LoRA TP (Distributed)
@@ -21,12 +21,6 @@ steps:
    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
    # TODO: create another `optional` test group for slow tests
    - pytest -v -s -m 'not slow_test' v1/spec_decode
-  mirror:
-    amd:
-      device: mi300_1
-      timeout_in_minutes: 65
-      depends_on:
-      - image-build-amd

 - label: V1 Sample + Logits
  key: v1-sample-logits
@@ -105,12 +99,6 @@ steps:
    # Integration test for streaming correctness (requires special branch).
    - pip install -U git+https://github.com/robertgshaw2-redhat/lm-evaluation-harness.git@streaming-api
    - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
-  mirror:
-    amd:
-      device: mi325_1
-      timeout_in_minutes: 60
-      depends_on:
-      - image-build-amd

 - label: V1 Others (CPU)
  key: v1-others-cpu
@@ -224,16 +212,6 @@ steps:
    - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048
    # https://github.com/vllm-project/vllm/pull/26682 uses slightly more memory in PyTorch 2.9+ causing this test to OOM in 1xL4 GPU
    - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536
-  mirror:
-    amd:
-      device: mi325_1
-      source_file_dependencies:
-      - vllm/entrypoints
-      - vllm/multimodal
-      - examples/
-      - vllm/platforms/rocm.py
-      depends_on:
-      - image-build-amd

 - label: Metrics, Tracing (2 GPUs)
  key: metrics-tracing-2-gpus
@@ -260,12 +238,6 @@ steps:
      'opentelemetry-exporter-otlp>=1.26.0' \
      'opentelemetry-semantic-conventions-ai>=0.4.1'"
  - pytest -v -s v1/tracing
-  mirror:
-    amd:
-      device: mi325_2
-      depends_on:
-      - image-build-amd
-      optional: true

 - label: Python-only Installation
  key: python-only-installation
@@ -278,16 +250,6 @@ steps:
  - setup.py
  commands:
  - bash standalone_tests/python_only_compile.sh
-  mirror:
-    amd:
-      device: mi325_1
-      timeout_in_minutes: 20
-      depends_on:
-      - image-build-amd
-      source_file_dependencies:
-      - tests/standalone_tests/python_only_compile.sh
-      - setup.py
-      - vllm/platforms/rocm.py

 - label: Async Engine, Inputs, Utils, Worker
  device: h200_35gb
@@ -370,7 +332,7 @@ steps:
  - pytest -v -s test_ray_env.py
  - pytest -v -s -m 'cpu_test' multimodal
  - pytest -v -s renderers
-  - pytest -v -s reasoning
+  - pytest -v -s reasoning --ignore=reasoning/test_seedoss_reasoning_parser.py --ignore=reasoning/test_glm4_moe_reasoning_parser.py
  - pytest -v -s tool_parsers
  - pytest -v -s tokenizers_
  - pytest -v -s parser
@@ -23,16 +23,3 @@ steps:
    # calls that the signal method cannot interrupt.
    - pytest -v -s model_executor -m '(not slow_test)' --timeout=900 --timeout-method=thread
    - pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py --timeout=900 --timeout-method=thread
-  mirror:
-    amd:
-      device: mi300_1
-      depends_on:
-      - image-build-amd
-      source_file_dependencies:
-      - vllm/engine/arg_utils.py
-      - vllm/config/model.py
-      - vllm/model_executor
-      - tests/model_executor
-      - tests/entrypoints/openai/completion/test_tensorizer_entrypoint.py
-      - vllm/_aiter_ops.py
-      - vllm/platforms/rocm.py
@@ -45,11 +45,6 @@ steps:
  - tests/models/test_registry.py
  commands:
    - pytest -v -s models/test_terratorch.py models/test_transformers.py models/test_registry.py
-  mirror:
-    amd:
-      device: mi325_1
-      depends_on:
-      - image-build-amd

 - label: Basic Models Test (Other CPU) # 5min
  key: basic-models-test-other-cpu
@@ -15,10 +15,6 @@ steps:
    - pytest -v -s models/language -m 'core_model and (not slow_test)'
  mirror:
    torch_nightly: {}
-    amd:
-      device: mi300_1
-      depends_on:
-      - image-build-amd

 - label: Language Models Tests (Extra Standard) %N
  key: language-models-tests-extra-standard
@@ -36,21 +32,6 @@ steps:
  parallelism: 2
  mirror:
    torch_nightly: {}
-    amd:
-      device: mi300_1
-      depends_on:
-      - image-build-amd
-      source_file_dependencies:
-      - vllm/model_executor/models/
-      - vllm/model_executor/model_loader/
-      - vllm/model_executor/layers/
-      - vllm/v1/attention/backends/
-      - vllm/v1/attention/selector.py
-      - tests/models/language/pooling/test_embedding.py
-      - tests/models/language/generation/test_common.py
-      - tests/models/language/pooling/test_classification.py
-      - vllm/_aiter_ops.py
-      - vllm/platforms/rocm.py

 - label: Language Models Tests (Hybrid) %N
  key: language-models-tests-hybrid
@@ -30,6 +30,7 @@ steps:
    - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
    - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen3 or gemma"
    - pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model
+    - pytest -v -s models/multimodal/generation/test_vit_cudagraph.py -m core_model
  mirror:
    amd:
      device: mi325_1
@@ -62,15 +63,9 @@ steps:
  - tests/models/multimodal
  commands:
    - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
-    - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py  --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/generation/test_vit_cudagraph.py --ignore models/multimodal/processing
-    - pytest -v -s models/multimodal/generation/test_vit_cudagraph.py -m core_model
+    - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py  --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/processing
    - pytest models/multimodal/generation/test_memory_leak.py -m core_model
    - cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model  # Otherwise, mp_method="spawn" doesn't work
-  mirror:
-    amd:
-      device: mi325_1
-      depends_on:
-      - image-build-amd

 - label: Multi-Modal Processor (CPU)
  key: multi-modal-processor-cpu
@@ -109,17 +104,6 @@ steps:
  - vllm/v1/core/
  commands:
  - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-mm-small.txt --tp-size=1
-  mirror:
-    amd:
-      device: mi300_1
-      depends_on:
-      - image-build-amd
-      source_file_dependencies:
-      - vllm/multimodal/
-      - vllm/inputs/
-      - vllm/v1/core/
-      - vllm/platforms/rocm.py
-      - vllm/model_executor/model_loader/

 - label: Multi-Modal Models (Extended Generation 1)
  key: multi-modal-models-extended-generation-1
@@ -27,10 +27,6 @@ steps:
  - pip install -e ./plugins/bge_m3_sparse_plugin
  - pytest -v -s plugins_tests/test_bge_m3_sparse_io_processor_plugins.py
  - pip uninstall bge_m3_sparse_plugin -y
-  # test colbert_query io_processor plugin
-  - pip install -e ./plugins/colbert_query_plugin
-  - pytest -v -s plugins_tests/test_colbert_query_io_processor_plugins.py
-  - pip uninstall colbert_query_plugin -y
  # end io_processor plugins test
  # begin stat_logger plugins test
  - pip install -e ./plugins/vllm_add_dummy_stat_logger
@@ -107,12 +107,6 @@ steps:
    - tests/compile/passes
  commands:
  - pytest -s -v compile/passes --ignore compile/passes/distributed
-  mirror:
-    amd:
-      device: mi300_1
-      timeout_in_minutes: 180
-      depends_on:
-      - image-build-amd

 - label: PyTorch Fullgraph Smoke Test
  key: pytorch-fullgraph-smoke-test
@@ -195,11 +189,3 @@ steps:
  - requirements/test/nightly-torch.txt
  commands:
  - bash standalone_tests/pytorch_nightly_dependency.sh
-  mirror:
-    amd:
-      device: mi300_1
-      depends_on:
-      - image-build-amd
-      source_file_dependencies:
-      - requirements/test/nightly-torch.txt
-      - vllm/platforms/rocm.py
@@ -26,7 +26,7 @@ steps:
  - export VLLM_USE_RUST_FRONTEND=1
  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
  - pytest -v -s benchmarks/test_serve_cli.py -k "not insecure and not (test_bench_serve and not test_bench_serve_chat)"
-  - pytest -v -s entrypoints/openai/chat_completion/test_chat_completion.py -k "not test_invalid_json_schema and not test_invalid_regex"
+  - pytest -v -s entrypoints/openai/chat_completion/test_chat_completion.py
  # - pytest -v -s entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py -k "not invalid"

  # - pytest -v -s entrypoints/openai/completion/test_prompt_validation.py -k "not prompt_embeds"
@@ -46,7 +46,7 @@ steps:
  - vllm/v1/engine/
  - tests/utils.py
  # - tests/entrypoints/serve/dev/rpc/test_collective_rpc.py
-  - tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py
+  - tests/entrypoints/serve/disagg/test_serving_tokens.py
  - tests/entrypoints/serve/instrumentator/test_basic.py
  - tests/entrypoints/serve/instrumentator/test_metrics.py
  # - tests/entrypoints/serve/dev/test_sleep.py
@@ -55,7 +55,7 @@ steps:
  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
  # - pytest -v -s entrypoints/serve/dev/rpc/test_collective_rpc.py
  - pytest -v -s entrypoints/serve/instrumentator/test_basic.py -k "not show_version and not server_load"
-  - pytest -v -s entrypoints/scale_out/token_in_token_out/test_serving_tokens.py -k "not stream and not lora and not test_generate_logprobs and not stop_string_workflow"
+  - pytest -v -s entrypoints/serve/disagg/test_serving_tokens.py -k "not stream and not lora and not test_generate_logprobs and not stop_string_workflow"
  - pytest -v -s entrypoints/serve/instrumentator/test_metrics.py -k "text and not show and not run_batch and not test_metrics_counts and not test_metrics_exist"
  # - pytest -v -s entrypoints/serve/dev/test_sleep.py

@@ -99,13 +99,9 @@ steps:
  - vllm/v1/engine/
  - vllm/v1/worker/
  - tests/utils.py
-  - tests/v1/distributed/test_external_lb_dp.py
-  - tests/v1/distributed/test_hybrid_lb_dp.py
  - tests/v1/distributed/test_internal_lb_dp.py
  commands:
  - export VLLM_USE_RUST_FRONTEND=1
  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
  - export NCCL_CUMEM_HOST_ENABLE=0
  - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_internal_lb_dp.py -k "not 4 and not server_info"
-  - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py -k "not 4 and not server_info"
-  - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_hybrid_lb_dp.py -k "not 4 and not server_info"
@@ -12,20 +12,6 @@ steps:
    - tests/v1/e2e/spec_decode/
  commands:
    - pytest -v -s v1/e2e/spec_decode -k "eagle_correctness"
-  mirror:
-    amd:
-      device: mi325_1
-      timeout_in_minutes: 45
-      depends_on:
-      - image-build-amd
-      source_file_dependencies:
-      - vllm/v1/spec_decode/
-      - vllm/v1/worker/gpu/spec_decode/
-      - vllm/model_executor/model_loader/
-      - vllm/v1/sample/
-      - vllm/model_executor/layers/
-      - tests/v1/e2e/spec_decode/
-      - vllm/platforms/rocm.py

 - label: Spec Decode Eagle Nightly B200
  key: spec-decode-eagle-nightly-b200
@@ -94,8 +80,6 @@ steps:
    amd:
      device: mi325_1
      timeout_in_minutes: 65
-      # TODO(akaratza): Test after Torch >= 2.12 bump
-      soft_fail: true
      depends_on:
      - image-build-amd
      source_file_dependencies:
@@ -13,13 +13,6 @@ steps:
  - tests/weight_loading
  commands:
    - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models.txt
-  mirror:
-    amd:
-      device: mi300_2
-      depends_on:
-      - image-build-amd
-      commands:
-      - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-amd.txt

 # - label: Weight Loading Multiple GPU - Large Models # optional
 #   working_dir: "/vllm-workspace/tests"
@@ -1,35 +0,0 @@
---
-name: ci-fails-buildkite
-description: Fetch and diagnose vLLM Buildkite CI failure logs. Use when investigating failing CI jobs on a PR or build, when the user pastes a buildkite.com URL, or asks to fetch/diagnose CI logs.
---
-
-# Diagnosing vLLM Buildkite CI Failures
-
-Buildkite logs are public; no login needed.
-
-`.buildkite/scripts/ci-fetch-log.sh` saves each log as `ci-<build>-<job-name>.log`, stripped of timestamps and ANSI codes. Existing files are kept; set `CI_FETCH_LOG_FORCE=1` to refetch.
-
-## Fetching logs
-
-```bash
-# All failed jobs in a PR's latest build (current branch's PR if omitted):
-.buildkite/scripts/ci-fetch-log.sh --pr <PR>
-
-# All failed jobs in a build (--soft also includes soft-failed jobs;
-# --all fetches every finished job):
-.buildkite/scripts/ci-fetch-log.sh "https://buildkite.com/vllm/ci/builds/<N>"
-
-# One job — `gh pr checks` URLs (#<job_uuid>) and web UI URLs (?sid=) both
-# work; pass "-" as a second argument to stream to stdout:
-.buildkite/scripts/ci-fetch-log.sh "https://buildkite.com/vllm/ci/builds/<N>#<job_uuid>"
-```
-
-To clean an already-downloaded log with `.buildkite/scripts/ci-clean-log.sh`:
-
-```bash
-./ci-clean-log.sh ci.log
-```
-
-## Reference
-
-See [docs/contributing/ci/failures.md](../../../docs/contributing/ci/failures.md) for the full guide: filing CI failure issues, investigating/bisecting, reproducing flaky tests, and daily triage.
@@ -2,16 +2,17 @@
 # for more info about CODEOWNERS file

 # This lists cover the "core" components of vLLM that require careful review
-/vllm/compilation @zou3519 @youkaichao @ProExpertProg @BoyuanFeng
-/vllm/distributed/kv_transfer @NickLucche @ApostaC @orozery @xuechendi @ivanium
+/vllm/compilation @zou3519 @youkaichao @ProExpertProg @BoyuanFeng @vadiklyutiy
+/vllm/distributed/kv_transfer @NickLucche @ApostaC @orozery @xuechendi
 /vllm/lora @jeejeelee
 /vllm/model_executor/layers/attention @LucasWilkinson @MatthewBonanni
 /vllm/model_executor/layers/fused_moe @mgoin @pavanimajety @zyongye
 /vllm/model_executor/layers/quantization @mgoin @robertgshaw2-redhat @tlrmchlsmth @yewentao256 @pavanimajety @zyongye
 /vllm/model_executor/layers/mamba @tdoublep @tomeras91
-/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py @tdoublep @ZJY0516 @vadiklyutiy
+/vllm/model_executor/layers/mamba/gdn_linear_attn.py @tdoublep @ZJY0516 @vadiklyutiy
+/vllm/model_executor/layers/rotary_embedding.py @vadiklyutiy
 /vllm/model_executor/model_loader @22quinn
-/vllm/model_executor/layers/batch_invariant.py @yewentao256
+/vllm/model_executor/layers/batch_invariant.py @yewentao256 
 /vllm/ir @ProExpertProg
 /vllm/kernels/ @ProExpertProg @tjtanaa
 /vllm/kernels/helion @ProExpertProg @zou3519
@@ -23,7 +24,7 @@
 # Any change to the VllmConfig changes can have a large user-facing impact,
 # so spam a lot of people
 /vllm/config @WoosukKwon @youkaichao @robertgshaw2-redhat @mgoin @tlrmchlsmth @houseroad @yewentao256 @ProExpertProg
-/vllm/config/cache.py @heheda12345 @ivanium
+/vllm/config/cache.py @heheda12345

 # Config utils
 /vllm/config/utils.py @hmellor
@@ -67,17 +68,16 @@
 /vllm/v1/attention/backends/flashinfer.py @mgoin @pavanimajety @vadiklyutiy
 /vllm/v1/attention/backends/triton_attn.py @tdoublep
 /vllm/v1/attention/backends/gdn_attn.py @ZJY0516 @vadiklyutiy
-/vllm/v1/core @WoosukKwon @robertgshaw2-redhat @njhill @ywang96 @alexm-redhat @heheda12345 @ApostaC @orozery @ivanium
+/vllm/v1/core @WoosukKwon @robertgshaw2-redhat @njhill @ywang96 @alexm-redhat @heheda12345 @ApostaC @orozery
 /vllm/v1/sample @22quinn @houseroad @njhill
 /vllm/v1/spec_decode @benchislett @luccafong @MatthewBonanni
 /vllm/v1/structured_output @mgoin @russellb @aarnphm @benchislett
-/vllm/v1/kv_cache_interface.py @heheda12345 @ivanium
+/vllm/v1/kv_cache_interface.py @heheda12345
 /vllm/v1/kv_offload @ApostaC @orozery
-/vllm/v1/simple_kv_offload @ivanium
 /vllm/v1/engine @njhill
 /vllm/v1/executor @njhill
 /vllm/v1/worker @njhill
-/vllm/v1/worker/kv_connector_model_runner_mixin.py @orozery @NickLucche @ivanium
+/vllm/v1/worker/kv_connector_model_runner_mixin.py @orozery @NickLucche

 # Model runner V2
 /vllm/v1/worker/gpu @WoosukKwon @njhill @yewentao256
@@ -104,14 +104,13 @@
 /tests/test_inputs.py @DarkLight1337 @ywang96
 /tests/entrypoints/llm/test_struct_output_generate.py @mgoin @russellb @aarnphm
 /tests/v1/structured_output @mgoin @russellb @aarnphm
-/tests/v1/core @WoosukKwon @robertgshaw2-redhat @njhill @ywang96 @alexm-redhat @heheda12345 @ApostaC @orozery @ivanium
+/tests/v1/core @WoosukKwon @robertgshaw2-redhat @njhill @ywang96 @alexm-redhat @heheda12345 @ApostaC @orozery
 /tests/weight_loading @mgoin @youkaichao @yewentao256
 /tests/lora @jeejeelee
 /tests/models/language/generation/test_hybrid.py @tdoublep @tomeras91
 /tests/v1/kv_connector/nixl_integration @NickLucche
-/tests/v1/kv_connector @ApostaC @orozery @ivanium
+/tests/v1/kv_connector @ApostaC @orozery
 /tests/v1/kv_offload @ApostaC @orozery
-/tests/v1/simple_kv_offload @ivanium
 /tests/v1/determinism @yewentao256
 /tests/reasoning @aarnphm @chaunceyjiang @sfeng33 @bbrowning
 /tests/tool_parsers @aarnphm @chaunceyjiang @sfeng33 @bbrowning
@@ -121,6 +120,16 @@
 /vllm/model_executor/models/transformers @hmellor
 /tests/models/test_transformers.py @hmellor

+# Observability
+/vllm/config/observability.py @markmc
+/vllm/v1/metrics @markmc
+/tests/v1/metrics @markmc
+/vllm/tracing.py @markmc
+/tests/v1/tracing/test_tracing.py @markmc
+/vllm/config/kv_events.py @markmc
+/vllm/distributed/kv_events.py @markmc
+/tests/distributed/test_events.py @markmc
+
 # Docs
 /docs/mkdocs @hmellor
 /docs/**/*.yml @hmellor
@@ -1,7 +0,0 @@
-# Custom self-hosted runner labels (e.g. the autoscaling vllm-runners pool) so
-# actionlint doesn't flag them as unknown in `runs-on`.
-self-hosted-runner:
-  labels:
-    - vllm-runners
-    # Not yet in actionlint's known-label set.
-    - macos-26
@@ -388,13 +388,9 @@ pull_request_rules:
    - or:
      - files~=^tests/tool_use/
      - files~=^tests/tool_parsers/
-      - files~=^tests/parser/
-      - files~=^tests/reasoning/
      - files~=^tests/entrypoints/openai/.*tool.*
      - files~=^tests/entrypoints/anthropic/.*tool.*
      - files~=^vllm/tool_parsers/
-      - files~=^vllm/parser/
-      - files~=^vllm/reasoning/
      - files=docs/features/tool_calling.md
      - files~=^examples/tool_calling/
  actions:
@@ -327,7 +327,7 @@ jobs:
                message: 'CC {users} for ROCm-related issue',
              },
              mistral: {
-                users: ['patrickvonplaten', 'juliendenize', 'andylolu2', 'NickLucche'],
+                users: ['patrickvonplaten', 'juliendenize', 'andylolu2'],
                message: 'CC {users} for Mistral-related issue',
              },
              // Add more label -> user mappings here
@@ -11,25 +11,13 @@ permissions:

 jobs:
  macos-m1-smoke-test:
-    # macos-26 (the supported target) is still a preview runner, so gate on GA
-    # macos-15 and keep macos-26 non-blocking.
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - os: macos-15
-            required: true
-          - os: macos-26
-            required: false
-    name: macos-m1-smoke-test (${{ matrix.os }})
-    runs-on: ${{ matrix.os }}
-    continue-on-error: ${{ !matrix.required }}
+    runs-on: macos-latest
    timeout-minutes: 30

    steps:
-      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+      - uses: actions/checkout@v6.0.1

-      - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+      - uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true
          cache-dependency-glob: |
@@ -84,11 +72,14 @@ jobs:
          # Test health endpoint
          curl -f http://localhost:8000/health

-          # Long prompt: hits the split-KV path that short prompts skip (#46769).
-          PAYLOAD=$(python -c "import json; print(json.dumps({'model': 'Qwen/Qwen3-0.6B', 'prompt': 'The quick brown fox jumps over the lazy dog. ' * 24, 'max_tokens': 16}))")
-          curl -f --max-time 120 http://localhost:8000/v1/completions \
+          # Test completion
+          curl -f http://localhost:8000/v1/completions \
            -H "Content-Type: application/json" \
-            -d "$PAYLOAD"
+            -d '{
+              "model": "Qwen/Qwen3-0.6B",
+              "prompt": "Hello",
+              "max_tokens": 5
+            }'

          # Cleanup
          kill "$SERVER_PID"
@@ -46,16 +46,12 @@ jobs:
  pre-commit:
    needs: pre-run-check
    if: always() && (needs.pre-run-check.result == 'success' || needs.pre-run-check.result == 'skipped')
-    runs-on: [self-hosted, linux, x64, vllm-runners]
+    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
-    - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+    - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+    - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
      with:
        python-version: "3.12"
-    # Provide shellcheck on PATH so tools/pre_commit/shellcheck.sh skips its
-    # wget + tar -xJ self-download, which the self-hosted runner image lacks
-    # (no wget/xz). Pinned to shellcheck 0.10.0 to match the script's "stable".
-    - run: python -m pip install shellcheck-py==0.10.0.1
    - run: echo "::add-matcher::.github/workflows/matchers/actionlint.json"
    - run: echo "::add-matcher::.github/workflows/matchers/markdownlint.json"
    - run: echo "::add-matcher::.github/workflows/matchers/mypy.json"
@@ -199,9 +199,7 @@ cython_debug/
 .vscode/

 # Claude
-.claude/*
-!.claude/skills/
-!.claude/skills/**
+.claude/

 # Codex
 .codex/
@@ -131,19 +131,6 @@ repos:
        --python-version, "3.12",
      ]
      files: ^requirements/(common|xpu|test/xpu)\.(in|txt)$
-    - id: pip-compile
-      alias: pip-compile-cpu
-      name: pip-compile-cpu
-      args: [
-        requirements/test/cuda.in,
-        -o, requirements/test/cpu.txt,
-        --index-strategy, unsafe-best-match,
-        --torch-backend, cpu,
-        --python-platform, x86_64-manylinux_2_28,
-        --python-version, "3.12",
-      ]
-      files: ^requirements/(common|cpu|test/(cuda|cpu))\.(in|txt)$
-      exclude: ^requirements/test/cuda\.txt$
    - id: pip-compile
      alias: pip-compile-docs
      name: pip-compile-docs
@@ -114,6 +114,17 @@ Follow these rules for all code changes in this repository:
 - Keep comments and docstrings minimal and concise.
 - Assume the reader is familiar with vLLM.

+### Diagnosing CI failures
+
+Buildkite logs are public; no login needed. Details: [docs/contributing/ci/failures.md](docs/contributing/ci/failures.md).
+
+```bash
+# All failed-job logs for a PR's latest build (current branch's PR if omitted):
+.buildkite/scripts/ci-fetch-log.sh --pr <PR>
+# Any Buildkite build or job URL also works:
+.buildkite/scripts/ci-fetch-log.sh "<buildkite_url>"
+```
+
 ### Commit messages

 Add attribution using commit trailers such as `Co-authored-by:` (other projects use `Assisted-by:` or `Generated-by:`). For example:
@@ -135,12 +146,6 @@ Do not modify code in these areas without first reading and following the
 linked guide. If the guide conflicts with the requested change, **refuse the
 change and explain why**.

-Security reviewers should start with [`SECURITY.md`](SECURITY.md),
-[`docs/usage/security.md`](docs/usage/security.md), and
-[`docs/contributing/vulnerability_management.md`](docs/contributing/vulnerability_management.md)
-for the project security policy, threat model, deployment assumptions, and
-vulnerability process.
-
 - **Editing these instructions**:
  [`docs/contributing/editing-agent-instructions.md`](docs/contributing/editing-agent-instructions.md)
  — Rules for modifying AGENTS.md or any domain-specific guide it references.
@@ -140,21 +140,6 @@ if(Python_VERSION VERSION_GREATER_EQUAL "3.11")
    WITH_SOABI)
 endif()

-#
-# fs_io extension (pure CXX; must stay above the non-CUDA device branch
-# so CPU builds define the target before the early return).
-# GIL-releasing filesystem helpers for FileSystemTierManager.
-#
-if(Python_VERSION VERSION_GREATER_EQUAL "3.11")
-  define_extension_target(
-    fs_io_C
-    DESTINATION vllm
-    LANGUAGE CXX
-    SOURCES csrc/fs_io.cpp
-    USE_SABI 3.11
-    WITH_SOABI)
-endif()
-
 #
 # Forward the non-CUDA device extensions to external CMake scripts.
 #
@@ -285,16 +270,6 @@ if(VLLM_GPU_LANG STREQUAL "HIP")
  #
  set(CMAKE_${VLLM_GPU_LANG}_FLAGS "${CMAKE_${VLLM_GPU_LANG}_FLAGS} -Wno-unused-result -Wno-unused-value")
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-result -Wno-unused-value")
-
-  # When using LTO then *.cpp files must be compiled with same compiler as used linker
-  # So if HIP uses clang linker we also must use it
-  # Otherwise symbols will be missing from .so
-  if (CMAKE_CXX_FLAGS MATCHES "\-flto")
-    if(NOT CMAKE_CXX_COMPILER_ID STREQUAL CMAKE_HIP_COMPILER_ID)
-      message(FATAL_ERROR "LTO is enabled for ROCm build, but the C++ compiler (${CMAKE_CXX_COMPILER_ID}) and HIP compiler (${CMAKE_HIP_COMPILER_ID}) are different which is not supported. "
-                          "Please ensure they are same by setting CXX=${CMAKE_HIP_COMPILER} environment variable. Or alternatively disable LTO.")
-    endif()
-   endif()
 endif()

 #
@@ -344,35 +319,112 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
 endif()

 #
-# Legacy _C extension (ROCm only — CUDA ops migrated to _C_stable_libtorch)
+# _C extension
 #

-if(VLLM_GPU_LANG STREQUAL "HIP")
-  set(VLLM_EXT_SRC
-    "csrc/torch_bindings.cpp"
+set(VLLM_EXT_SRC
+  "csrc/quantization/activation_kernels.cu"
+  "csrc/push_all_reduce.cu"
+  "csrc/torch_bindings.cpp")
+
+if(VLLM_GPU_LANG STREQUAL "CUDA")
+  SET(CUTLASS_ENABLE_HEADERS_ONLY ON CACHE BOOL "Enable only the header library")
+
+  # Set CUTLASS_REVISION. Used for FetchContent. Also fixes some bogus messages when building.
+  set(CUTLASS_REVISION "v4.4.2")
+
+  # Use the specified CUTLASS source directory for compilation if VLLM_CUTLASS_SRC_DIR is provided
+  if (DEFINED ENV{VLLM_CUTLASS_SRC_DIR})
+    set(VLLM_CUTLASS_SRC_DIR $ENV{VLLM_CUTLASS_SRC_DIR})
+  endif()
+
+  if(VLLM_CUTLASS_SRC_DIR)
+    if(NOT IS_ABSOLUTE VLLM_CUTLASS_SRC_DIR)
+      get_filename_component(VLLM_CUTLASS_SRC_DIR "${VLLM_CUTLASS_SRC_DIR}" ABSOLUTE)
+    endif()
+    message(STATUS "The VLLM_CUTLASS_SRC_DIR is set, using ${VLLM_CUTLASS_SRC_DIR} for compilation")
+    FetchContent_Declare(cutlass SOURCE_DIR ${VLLM_CUTLASS_SRC_DIR})
+  else()
+    FetchContent_Declare(
+        cutlass
+        GIT_REPOSITORY https://github.com/nvidia/cutlass.git
+        # Please keep this in sync with CUTLASS_REVISION line above.
+        GIT_TAG ${CUTLASS_REVISION}
+        GIT_PROGRESS TRUE
+
+        # Speed up CUTLASS download by retrieving only the specified GIT_TAG instead of the history.
+        # Important: If GIT_SHALLOW is enabled then GIT_TAG works only with branch names and tags.
+        # So if the GIT_TAG above is updated to a commit hash, GIT_SHALLOW must be set to FALSE
+        GIT_SHALLOW TRUE
+    )
+  endif()
+  FetchContent_MakeAvailable(cutlass)
+
+  set_gencode_flags_for_srcs(
+    SRCS "${VLLM_EXT_SRC}"
+    CUDA_ARCHS "${CUDA_ARCHS}")
+
+  # Expert-specialization MXFP8 blockscaled grouped kernels (SM100+).
+  if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0)
+    cuda_archs_loose_intersection(ES_MXFP8_GROUPED_MM_ARCHS "10.0f;11.0f" "${CUDA_ARCHS}")
+  else()
+    cuda_archs_loose_intersection(ES_MXFP8_GROUPED_MM_ARCHS "10.0a;10.1a;10.3a" "${CUDA_ARCHS}")
+  endif()
+  if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8 AND ES_MXFP8_GROUPED_MM_ARCHS)
+    set(ES_MXFP8_GROUPED_MM_SRCS
+      "csrc/libtorch_stable/moe/mxfp8_moe/cutlass_mxfp8_grouped_mm.cu"
+      "csrc/libtorch_stable/moe/mxfp8_moe/mxfp8_experts_quant.cu")
+    set_gencode_flags_for_srcs(
+      SRCS "${ES_MXFP8_GROUPED_MM_SRCS}"
+      CUDA_ARCHS "${ES_MXFP8_GROUPED_MM_ARCHS}")
+    list(APPEND VLLM_STABLE_EXT_SRC "${ES_MXFP8_GROUPED_MM_SRCS}")
+    list(APPEND VLLM_GPU_FLAGS "-DENABLE_ES_MXFP8_GROUPED_MM_SM100=1")
+    message(STATUS "Building ES MXFP8 grouped kernels for archs: ${ES_MXFP8_GROUPED_MM_ARCHS}")
+  else()
+    if (NOT ${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8
+        AND ES_MXFP8_GROUPED_MM_ARCHS)
+      message(STATUS "Not building ES MXFP8 grouped kernels as CUDA Compiler version is "
+                     "not >= 12.8.")
+    else()
+      message(STATUS "Not building ES MXFP8 grouped kernels as no compatible archs found "
+                     "in CUDA target architectures.")
+    endif()
+  endif()
+
+
+
+# if CUDA endif
+endif()
+
+if (VLLM_GPU_LANG STREQUAL "HIP")
+  # Add QuickReduce kernels (ROCm-only; not part of stable ABI migration).
+  # TODO: Remove the cuda_view when ROCm upgrade to torch 2.11.
+  list(APPEND VLLM_EXT_SRC
    "csrc/custom_quickreduce.cu"
    "csrc/cuda_view.cu"
-    "csrc/libtorch_stable/cuda_utils_kernels.cu")
+    "csrc/libtorch_stable/cuda_utils_kernels.cu"
+  )
+# if ROCM endif
+endif()

-  message(STATUS "Enabling C extension.")
-  define_extension_target(
-    _C
-    DESTINATION vllm
-    LANGUAGE ${VLLM_GPU_LANG}
-    SOURCES ${VLLM_EXT_SRC}
-    COMPILE_FLAGS ${VLLM_GPU_FLAGS}
-    ARCHITECTURES ${VLLM_GPU_ARCHES}
-    INCLUDE_DIRECTORIES ${CUTLASS_INCLUDE_DIR}
-    INCLUDE_DIRECTORIES ${CUTLASS_TOOLS_UTIL_INCLUDE_DIR}
-    USE_SABI 3
-    WITH_SOABI)
+message(STATUS "Enabling C extension.")
+define_extension_target(
+  _C
+  DESTINATION vllm
+  LANGUAGE ${VLLM_GPU_LANG}
+  SOURCES ${VLLM_EXT_SRC}
+  COMPILE_FLAGS ${VLLM_GPU_FLAGS}
+  ARCHITECTURES ${VLLM_GPU_ARCHES}
+  INCLUDE_DIRECTORIES ${CUTLASS_INCLUDE_DIR}
+  INCLUDE_DIRECTORIES ${CUTLASS_TOOLS_UTIL_INCLUDE_DIR}
+  USE_SABI 3
+  WITH_SOABI)

-  # If CUTLASS is compiled on NVCC >= 12.5, it by default uses
-  # cudaGetDriverEntryPointByVersion as a wrapper to avoid directly calling the
-  # driver API. This causes problems when linking with earlier versions of CUDA.
-  # Setting this variable sidesteps the issue by calling the driver directly.
-  target_compile_definitions(_C PRIVATE CUTLASS_ENABLE_DIRECT_CUDA_DRIVER_CALL=1)
-endif() # _C HIP endif
+# If CUTLASS is compiled on NVCC >= 12.5, it by default uses
+# cudaGetDriverEntryPointByVersion as a wrapper to avoid directly calling the
+# driver API. This causes problems when linking with earlier versions of CUDA.
+# Setting this variable sidesteps the issue by calling the driver directly.
+target_compile_definitions(_C PRIVATE CUTLASS_ENABLE_DIRECT_CUDA_DRIVER_CALL=1)

 if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
  #
@@ -381,7 +433,6 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
  set(VLLM_STABLE_EXT_SRC
    "csrc/libtorch_stable/torch_bindings.cpp"
    "csrc/libtorch_stable/activation_kernels.cu"
-    "csrc/libtorch_stable/quantization/activation_kernels.cu"
    "csrc/libtorch_stable/quantization/w8a8/int8/scaled_quant.cu"
    "csrc/libtorch_stable/quantization/w8a8/fp8/common.cu"
    "csrc/libtorch_stable/quantization/w8a8/fp8/per_token_group_quant.cu"
@@ -407,57 +458,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
    "csrc/libtorch_stable/custom_all_reduce.cu"
    "csrc/libtorch_stable/fused_deepseek_v4_qnorm_rope_kv_insert_kernel.cu")

-  if(VLLM_GPU_LANG STREQUAL "CUDA" AND
-     DEFINED CMAKE_CUDA_COMPILER_VERSION AND
-     CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 12.0)
-
-    if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0)
-      cuda_archs_loose_intersection(COOPERATIVE_TOPK_ARCHS
-        "9.0a;10.0f;10.1f;10.3f;11.0f;12.0f;12.1f" "${CUDA_ARCHS}")
-    else()
-      cuda_archs_loose_intersection(COOPERATIVE_TOPK_ARCHS
-        "9.0a;10.0a;10.1a;10.3a;12.0a;12.1a" "${CUDA_ARCHS}")
-    endif()
-
-    if(COOPERATIVE_TOPK_ARCHS)
-      list(APPEND VLLM_GPU_FLAGS "-DVLLM_ENABLE_COOPERATIVE_TOPK=1")
-
-    endif()
-  endif()
-
  if(VLLM_GPU_LANG STREQUAL "CUDA")
-    SET(CUTLASS_ENABLE_HEADERS_ONLY ON CACHE BOOL "Enable only the header library")
-
-    # Set CUTLASS_REVISION. Used for FetchContent. Also fixes some bogus messages when building.
-    set(CUTLASS_REVISION "v4.4.2")
-
-    # Use the specified CUTLASS source directory for compilation if VLLM_CUTLASS_SRC_DIR is provided
-    if (DEFINED ENV{VLLM_CUTLASS_SRC_DIR})
-      set(VLLM_CUTLASS_SRC_DIR $ENV{VLLM_CUTLASS_SRC_DIR})
-    endif()
-
-    if(VLLM_CUTLASS_SRC_DIR)
-      if(NOT IS_ABSOLUTE VLLM_CUTLASS_SRC_DIR)
-        get_filename_component(VLLM_CUTLASS_SRC_DIR "${VLLM_CUTLASS_SRC_DIR}" ABSOLUTE)
-      endif()
-      message(STATUS "The VLLM_CUTLASS_SRC_DIR is set, using ${VLLM_CUTLASS_SRC_DIR} for compilation")
-      FetchContent_Declare(cutlass SOURCE_DIR ${VLLM_CUTLASS_SRC_DIR})
-    else()
-      FetchContent_Declare(
-          cutlass
-          GIT_REPOSITORY https://github.com/nvidia/cutlass.git
-          # Please keep this in sync with CUTLASS_REVISION line above.
-          GIT_TAG ${CUTLASS_REVISION}
-          GIT_PROGRESS TRUE
-
-          # Speed up CUTLASS download by retrieving only the specified GIT_TAG instead of the history.
-          # Important: If GIT_SHALLOW is enabled then GIT_TAG works only with branch names and tags.
-          # So if the GIT_TAG above is updated to a commit hash, GIT_SHALLOW must be set to FALSE
-          GIT_SHALLOW TRUE
-      )
-    endif()
-    FetchContent_MakeAvailable(cutlass)
-
    list(APPEND VLLM_STABLE_EXT_SRC
      "csrc/libtorch_stable/cuda_view.cu"
      "csrc/libtorch_stable/cuda_utils_kernels.cu"
@@ -541,14 +542,6 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
      SRCS "${VLLM_STABLE_EXT_SRC}"
      CUDA_ARCHS "${CUDA_ARCHS}")

-    if(COOPERATIVE_TOPK_ARCHS)
-      list(APPEND VLLM_STABLE_EXT_SRC
-        "csrc/libtorch_stable/cooperative_topk.cu")
-      set_gencode_flags_for_srcs(
-        SRCS "csrc/libtorch_stable/cooperative_topk.cu"
-        CUDA_ARCHS "${COOPERATIVE_TOPK_ARCHS}")
-    endif()
-
    # Only build Marlin kernels if we are building for at least some compatible archs.
    # Keep building Marlin for 9.0 as there are some group sizes and shapes that
    # are not supported by Machete yet.
@@ -894,9 +887,9 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
  endif()

  if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0)
-    cuda_archs_loose_intersection(SCALED_MM_ARCHS "10.0f;11.0f" "${CUDA_ARCHS}")
+    cuda_archs_loose_intersection(SCALED_MM_ARCHS "10.0f" "${CUDA_ARCHS}")
  else()
-    cuda_archs_loose_intersection(SCALED_MM_ARCHS "10.0a;10.1a;10.3a" "${CUDA_ARCHS}")
+    cuda_archs_loose_intersection(SCALED_MM_ARCHS "10.0a;10.3a" "${CUDA_ARCHS}")
  endif()
  if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8 AND SCALED_MM_ARCHS)
    set(CUTLASS_MOE_SM100_SRCS "csrc/libtorch_stable/quantization/w8a8/cutlass/moe/grouped_mm_c3x_sm100.cu")
@@ -966,6 +959,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
      SRCS "${FP4_SM120_SRCS}"
      CUDA_ARCHS "${FP4_SM120_ARCHS}")
    list(APPEND VLLM_STABLE_EXT_SRC "${FP4_SM120_SRCS}")
+    target_compile_definitions(_C PRIVATE ENABLE_NVFP4_SM120=1)
    list(APPEND VLLM_GPU_FLAGS "-DENABLE_NVFP4_SM120=1")
    list(APPEND VLLM_GPU_FLAGS "-DENABLE_CUTLASS_MOE_SM120=1")
    message(STATUS "Building SM12x NVFP4 for archs: ${FP4_SM120_ARCHS}")
@@ -998,6 +992,7 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
      SRCS "${FP4_SM100_SRCS}"
      CUDA_ARCHS "${FP4_SM100_ARCHS}")
    list(APPEND VLLM_STABLE_EXT_SRC "${FP4_SM100_SRCS}")
+    target_compile_definitions(_C PRIVATE ENABLE_NVFP4_SM100=1)
    list(APPEND VLLM_GPU_FLAGS "-DENABLE_NVFP4_SM100=1")
    list(APPEND VLLM_GPU_FLAGS "-DENABLE_CUTLASS_MOE_SM100=1")
    message(STATUS "Building SM10x/11x NVFP4/MXFP4 for archs: ${FP4_SM100_ARCHS}")
@@ -1100,10 +1095,6 @@ if(VLLM_GPU_LANG STREQUAL "CUDA" OR VLLM_GPU_LANG STREQUAL "HIP")
    target_compile_definitions(_C_stable_libtorch PRIVATE
      TORCH_TARGET_VERSION=0x020B000000000000ULL)
    target_compile_definitions(_C_stable_libtorch PRIVATE USE_CUDA)
-    if(COOPERATIVE_TOPK_ARCHS)
-      target_compile_definitions(_C_stable_libtorch PRIVATE
-        VLLM_ENABLE_COOPERATIVE_TOPK=1)
-    endif()
    # Needed by CUTLASS kernels
    target_compile_definitions(_C_stable_libtorch PRIVATE
      CUTLASS_ENABLE_DIRECT_CUDA_DRIVER_CALL=1)
@@ -53,16 +53,6 @@ from common import (
 from vllm.v1.worker.workspace import init_workspace_manager


-def _str2bool(v) -> bool:
-    if isinstance(v, bool):
-        return v
-    if v.lower() in ("true", "1", "yes", "t"):
-        return True
-    if v.lower() in ("false", "0", "no", "f"):
-        return False
-    raise argparse.ArgumentTypeError(f"expected a boolean, got {v!r}")
-
-
 def run_standard_attention_benchmark(config: BenchmarkConfig) -> BenchmarkResult:
    """Run standard attention benchmark (Flash/Triton/FlashInfer)."""
    from runner import run_attention_benchmark
@@ -495,20 +485,6 @@ def main():
        help="Prefill backends to compare (fa2, fa3, fa4). "
        "Uses the first decode backend for impl construction.",
    )
-    parser.add_argument(
-        "--fp8-output-scale",
-        type=float,
-        help="Static per-tensor scale enabling the MLA prefill FP8-output "
-        "comparison on FA4 (fused write vs standalone post-quant).",
-    )
-    parser.add_argument(
-        "--fuse-quant-op",
-        nargs="+",
-        type=_str2bool,
-        help="FP8-output write path(s) to run: false = bf16 attention + "
-        "standalone static-FP8 quant, true = FA4 writes FP8 directly. "
-        "Default: both.",
-    )

    # Batch specifications
    parser.add_argument(
@@ -642,12 +618,6 @@ def main():
        # Prefill backends (e.g., ["fa3", "fa4"])
        args.prefill_backends = yaml_config.get("prefill_backends", None)

-        # FP8 output benchmark knobs; CLI wins.
-        if args.fp8_output_scale is None:
-            args.fp8_output_scale = yaml_config.get("fp8_output_scale", None)
-        if args.fuse_quant_op is None:
-            args.fuse_quant_op = yaml_config.get("fuse_quant_op", None)
-
        # Check for special modes
        args.mode = yaml_config.get("mode", None)

@@ -817,59 +787,8 @@ def main():
            "skipped (timings are placeholder zeros).[/]"
        )

-    # FA4 fused FP8 output vs standalone post-quant, on the same fa4 kernel:
-    # the delta is the post-quant kernel the fused path removes.
-    fp8_output_scale = getattr(args, "fp8_output_scale", None)
-    if fp8_output_scale is not None:
-        decode_backend = backends[0]
-        fuse_variants = args.fuse_quant_op or [False, True]
-        label_of = {False: "post_quant", True: "fused"}
-        console.print(
-            f"[yellow]FP8 output comparison @ scale={fp8_output_scale} "
-            f"(prefill=fa4, decode impl={decode_backend})[/]"
-        )
-        fp8_results = []
-        total = len(fuse_variants) * len(args.batch_specs)
-        with tqdm(total=total, desc="FP8 output benchmarking") as pbar:
-            for spec in args.batch_specs:
-                for fuse in fuse_variants:
-                    config = BenchmarkConfig(
-                        backend=decode_backend,
-                        batch_spec=spec,
-                        num_layers=args.num_layers,
-                        head_dim=args.head_dim,
-                        num_q_heads=args.num_q_heads,
-                        num_kv_heads=args.num_kv_heads,
-                        block_size=args.block_size,
-                        device=args.device,
-                        repeats=args.repeats,
-                        warmup_iters=args.warmup_iters,
-                        profile_memory=args.profile_memory,
-                        kv_cache_dtype=args.kv_cache_dtype,
-                        use_cuda_graphs=args.cuda_graphs,
-                        prefill_backend="fa4",
-                    )
-                    result = run_benchmark(
-                        config, output_scale=fp8_output_scale, fuse_quant_op=fuse
-                    )
-                    label = label_of[fuse]
-                    labeled_config = replace(result.config, backend=label)
-                    result = replace(result, config=labeled_config)
-                    fp8_results.append(result)
-
-                    if not result.success:
-                        console.print(f"[red]Error {label} {spec}: {result.error}[/]")
-
-                    pbar.update(1)
-
-        console.print("\n[bold green]FP8 Output Results:[/]")
-        formatter = ResultsFormatter(console)
-        labels = [label_of[f] for f in fuse_variants]
-        formatter.print_table(fp8_results, labels, compare_to_fastest=True)
-        all_results = fp8_results
-
    # Handle special mode: decode_vs_prefill comparison
-    elif hasattr(args, "mode") and args.mode == "decode_vs_prefill":
+    if hasattr(args, "mode") and args.mode == "decode_vs_prefill":
        console.print("[yellow]Mode: Decode vs Prefill pipeline comparison[/]")
        console.print(
            "[dim]For each query length, testing both decode and prefill pipelines[/]"
@@ -1,44 +0,0 @@
-# MLA prefill FP8-output microbenchmark (FA4).
-# Compares the fused FP8 write against bf16 attention + a standalone static-FP8
-# quant; the delta is the post-quant kernel the fused path removes.
-# DeepSeek-Coder-V2-Lite dims; FA4 needs SM100/110.
-#
-# Usage:
-#   python benchmark.py --config configs/mla_fa4_fp8_output.yaml
-
-description: "MLA prefill FA4 fused-FP8 output vs post-quant"
-
-model:
-  name: "deepseek-v2-lite"
-  num_layers: 27
-  num_q_heads: 16
-  num_kv_heads: 1
-  head_dim: 576
-  kv_lora_rank: 512
-  qk_nope_head_dim: 128
-  qk_rope_head_dim: 64
-  v_head_dim: 128
-  block_size: 128
-
-# Pure prefill (q_len == kv_len) so every token goes through forward_mha.
-batch_specs:
-  - "q512"
-  - "q1k"
-  - "q2k"
-  - "q4k"
-  - "q8k"
-  - "2q4k"
-  - "4q4k"
-  - "8q4k"
-
-# Only used to construct the MLA impl; the pure-prefill specs skip decode.
-decode_backends:
-  - CUTLASS_MLA
-
-# Sweep the two FP8 write paths (prefill backend is fixed to fa4).
-fp8_output_scale: 0.1
-fuse_quant_op: [false, true]
-
-device: "cuda:0"
-repeats: 50
-warmup_iters: 10
@@ -708,8 +708,6 @@ def _run_single_benchmark(
    device: torch.device,
    indexer=None,
    kv_cache_dtype: str | None = None,
-    output_scale: float | None = None,
-    fuse_quant_op: bool = False,
 ) -> BenchmarkResult:
    """
    Run a single benchmark iteration.
@@ -723,11 +721,6 @@ def _run_single_benchmark(
        mla_dims: MLA dimension configuration
        device: Target device
        indexer: Optional MockIndexer for sparse backends
-        output_scale: Static per-tensor FP8 scale for prefill output. None
-            keeps the plain bf16 output (no quantization).
-        fuse_quant_op: With output_scale set, True lets the prefill kernel write
-            FP8 directly; False runs bf16 attention then a standalone static-FP8
-            quant. The delta isolates the saved post-quant kernel.

    Returns:
        BenchmarkResult with timing statistics
@@ -831,55 +824,23 @@ def _run_single_benchmark(
            num_prefill, mla_dims, query_fmt, device, torch.bfloat16
        )

-    # Prefill FP8 output: fused (kernel writes e4m3) vs separate post-quant.
-    prefill_fp8_output = None
-    prefill_output_scale = None
-    prefill_quant_op = None
-    if has_prefill and output_scale is not None:
-        from vllm.platforms import current_platform
-
-        prefill_output_scale = torch.tensor(
-            [output_scale], device=device, dtype=torch.float32
-        )
-        if fuse_quant_op:
-            prefill_fp8_output = torch.empty_like(
-                prefill_inputs["output"], dtype=current_platform.fp8_dtype()
-            )
-        else:
-            from vllm.model_executor.layers.quantization.input_quant_fp8 import (
-                QuantFP8,
-            )
-            from vllm.model_executor.layers.quantization.utils.quant_utils import (
-                GroupShape,
-            )
-
-            prefill_quant_op = QuantFP8(static=True, group_shape=GroupShape.PER_TENSOR)
-
-    fused_output = output_scale is not None and fuse_quant_op
-
    # Build forward function (runs a single decode/prefill pass)
    def forward_fn():
        results = []
        if has_decode:
            results.append(impl.forward_mqa(decode_inputs, kv_cache, metadata, layer))
        if has_prefill:
-            out = impl.forward_mha(
-                prefill_inputs["q"],
-                prefill_inputs["k_c_normed"],
-                prefill_inputs["k_pe"],
-                kv_cache,
-                metadata,
-                prefill_inputs["k_scale"],
-                prefill_fp8_output if fused_output else prefill_inputs["output"],
-                prefill_output_scale if fused_output else None,
-            )
-            if fused_output:
-                out = prefill_fp8_output
-            elif prefill_quant_op is not None:
-                out, _ = prefill_quant_op(
-                    prefill_inputs["output"], prefill_output_scale
+            results.append(
+                impl.forward_mha(
+                    prefill_inputs["q"],
+                    prefill_inputs["k_c_normed"],
+                    prefill_inputs["k_pe"],
+                    kv_cache,
+                    metadata,
+                    prefill_inputs["k_scale"],
+                    prefill_inputs["output"],
                )
-            results.append(out)
+            )
        return results[0] if len(results) == 1 else tuple(results)

    def benchmark_fn():
@@ -920,8 +881,6 @@ def _run_mla_benchmark_batched(
    configs_with_params: list[tuple],  # [(config, threshold, num_splits), ...]
    index_topk: int = 2048,
    prefill_backend: str | None = None,
-    output_scale: float | None = None,
-    fuse_quant_op: bool = False,
 ) -> list[BenchmarkResult]:
    """
    Unified batched MLA benchmark runner for all backends.
@@ -1061,8 +1020,6 @@ def _run_mla_benchmark_batched(
                    device,
                    indexer=indexer,
                    kv_cache_dtype=kv_cache_dtype,
-                    output_scale=output_scale,
-                    fuse_quant_op=fuse_quant_op,
                )
                results.append(result)

@@ -1090,8 +1047,6 @@ def run_mla_benchmark(
    num_kv_splits: int | None = None,
    index_topk: int = 2048,
    prefill_backend: str | None = None,
-    output_scale: float | None = None,
-    fuse_quant_op: bool = False,
 ) -> BenchmarkResult | list[BenchmarkResult]:
    """
    Unified MLA benchmark runner for all backends.
@@ -1111,9 +1066,6 @@ def run_mla_benchmark(
        index_topk: Topk value for sparse MLA backends (default 2048)
        prefill_backend: Prefill backend name (e.g., "fa3", "fa4").
            When set, forces the specified FlashAttention version for prefill.
-        output_scale: Static per-tensor FP8 scale for prefill output (None = bf16).
-        fuse_quant_op: With output_scale set, fuse the FP8 write into the prefill
-            kernel vs a standalone post-quant kernel. See _run_single_benchmark.

    Returns:
        BenchmarkResult (single mode) or list of BenchmarkResult (batched mode)
@@ -1138,12 +1090,7 @@ def run_mla_benchmark(

    # Use unified batched execution
    results = _run_mla_benchmark_batched(
-        backend,
-        configs_with_params,
-        index_topk,
-        prefill_backend=prefill_backend,
-        output_scale=output_scale,
-        fuse_quant_op=fuse_quant_op,
+        backend, configs_with_params, index_topk, prefill_backend=prefill_backend
    )

    # Return single result or list based on input
@@ -1,358 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""Benchmark and regression-test pinned (page-locked) CPU memory for vLLM.
-
-Verifies that enabling pinned memory does not regress throughput or latency
-compared to unpinned memory.  Each condition runs in an isolated ``spawn``
-subprocess so both start from a cold CUDA context, giving an unbiased
-comparison.
-
-Usage
-----
-Run all tests with the default model::
-
-    python benchmarks/benchmark_pin_memory.py -v
-
-Override the model and optional max-model-len::
-
-    python benchmarks/benchmark_pin_memory.py --model unsloth/Qwen3-1.7B -v
-    python benchmarks/benchmark_pin_memory.py --model unsloth/Qwen3-1.7B \
-        --max-model-len 8192 -v
-
-Run only throughput or latency tests::
-
-    python benchmarks/benchmark_pin_memory.py -v -k test_throughput
-    python benchmarks/benchmark_pin_memory.py -v -k test_latency
-
-Run only the v1 or v2 runner variant::
-
-    python benchmarks/benchmark_pin_memory.py -v -k v1
-    python benchmarks/benchmark_pin_memory.py -v -k v2
-
-Note: on WSL2, v1 runner tests are skipped because pin memory is not available
-for the v1 runner without cpu_offload_gb.  Run on other platforms to exercise v1.
-"""
-
-import argparse
-import json
-import multiprocessing
-import sys
-import tempfile
-
-import pytest
-
-# Allow up to 2% degradation.  Both benchmark runs start from an identical
-# cold CUDA context (separate spawn subprocesses), so the measured difference
-# reflects the genuine pin_memory overhead rather than cold/warm ordering bias.
-_THROUGHPUT_TOLERANCE = 0.98
-_THROUGHPUT_NUM_REQUESTS = 200
-_THROUGHPUT_INPUT_LEN = 128
-_THROUGHPUT_OUTPUT_LEN = 512
-_THROUGHPUT_MAX_NUM_SEQS = 128
-
-# Latency benchmark constants — match latency.py defaults.
-_LATENCY_TOLERANCE = 1.02  # Allow up to 2% latency regression.
-_LATENCY_BATCH_SIZE = 64
-_LATENCY_INPUT_LEN = 32
-_LATENCY_OUTPUT_LEN = 128
-_LATENCY_WARMUP_ITERS = 5
-_LATENCY_BENCH_ITERS = 15
-
-_DEFAULT_MODEL = "unsloth/Qwen3-1.7B"
-_DEFAULT_MAX_MODEL_LEN = 16384
-
-
-def _benchmark_args() -> argparse.Namespace:
-    parser = argparse.ArgumentParser(add_help=False)
-    parser.add_argument("--model", default=_DEFAULT_MODEL)
-    parser.add_argument("--max-model-len", type=int, default=_DEFAULT_MAX_MODEL_LEN)
-    args, _ = parser.parse_known_args()
-    return args
-
-
-@pytest.fixture
-def model() -> str:
-    return _benchmark_args().model
-
-
-@pytest.fixture
-def max_model_len() -> int:
-    return _benchmark_args().max_model_len
-
-
-def _skip_if_pin_memory_not_available(engine_args_kwargs: dict) -> None:
-    """Skip the current pytest test if pin_memory is unavailable for this config."""
-    import vllm.utils.platform_utils as pu
-    from vllm.config import set_current_vllm_config
-    from vllm.engine.arg_utils import EngineArgs
-
-    vllm_config = EngineArgs(**engine_args_kwargs).create_engine_config()
-    with set_current_vllm_config(vllm_config):
-        pu.is_pin_memory_available.cache_clear()
-        if not pu.is_pin_memory_available():
-            import os
-
-            runner = "v2" if os.environ.get("VLLM_USE_V2_MODEL_RUNNER") == "1" else "v1"
-            model = engine_args_kwargs.get("model", "unknown")
-            print(
-                f"\033[33mSKIP: pin_memory not available for "
-                f"{runner} runner, model={model}\033[0m"
-            )
-            pytest.skip("pin_memory not available for this configuration")
-
-
-def _throughput_worker(
-    pin: bool,
-    engine_args_kwargs: dict,
-    q: "multiprocessing.Queue[float]",
-    v2_mode: bool = False,
-) -> None:
-    """Run throughput benchmark in a fresh spawn subprocess.
-
-    Delegates to vllm/benchmarks/throughput.py main() using the random dataset,
-    so the methodology matches the official benchmark.  Results are written to a
-    temp JSON file and forwarded through the queue as tokens/s.
-
-    v2_mode: when True, monkeypatches is_uva_available() to always return True
-    so the v2 model runner's UVA buffers remain functional even when pin=False.
-    This isolates the non-UVA pin_memory paths in v2.
-    """
-    import vllm.utils.platform_utils as pu
-    from vllm.platforms import current_platform
-
-    pu.is_pin_memory_available.cache_clear()
-    pu.is_uva_available.cache_clear()
-    type(current_platform).is_pin_memory_available = classmethod(lambda cls: pin)
-    if v2_mode:
-        pu.is_uva_available = lambda: True
-
-    from vllm.benchmarks.throughput import add_cli_args
-    from vllm.benchmarks.throughput import main as throughput_main
-
-    parser = argparse.ArgumentParser()
-    add_cli_args(parser)
-    args = parser.parse_args([])
-
-    for key, val in engine_args_kwargs.items():
-        setattr(args, key, val)
-    args.max_num_seqs = _THROUGHPUT_MAX_NUM_SEQS
-    args.dataset_name = "random"
-    args.input_len = _THROUGHPUT_INPUT_LEN
-    args.output_len = _THROUGHPUT_OUTPUT_LEN
-    # Nullify defaults that conflict with explicit input/output_len.
-    args.random_input_len = None
-    args.random_output_len = None
-    args.random_prefix_len = None
-    args.num_prompts = _THROUGHPUT_NUM_REQUESTS
-    args.seed = 0
-    args.disable_detokenize = True
-
-    with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
-        tmp_path = f.name
-    args.output_json = tmp_path
-
-    throughput_main(args)
-
-    with open(tmp_path) as f:
-        results = json.load(f)
-    q.put(results["tokens_per_second"])
-
-
-def _run_throughput_benchmark(
-    pin: bool,
-    engine_args_kwargs: dict,
-    v2_mode: bool = False,
-) -> float:
-    ctx = multiprocessing.get_context("spawn")
-    q = ctx.Queue()
-    p = ctx.Process(
-        target=_throughput_worker,
-        args=(pin, engine_args_kwargs, q, v2_mode),
-    )
-    p.start()
-    p.join()
-    if p.exitcode != 0:
-        raise RuntimeError(
-            f"Throughput benchmark subprocess (pin={pin}) exited with code {p.exitcode}"
-        )
-    return q.get()
-
-
-def _latency_worker(
-    pin: bool,
-    engine_args_kwargs: dict,
-    q: "multiprocessing.Queue[dict]",
-    v2_mode: bool = False,
-) -> None:
-    """Run latency benchmark in a fresh spawn subprocess.
-
-    Follows latency.py methodology: fixed batch of dummy token IDs, warmup
-    iterations to reach steady state, then timed iterations reduced to avg
-    and percentiles.  Results are written to a temp JSON file by latency_main
-    and forwarded through the queue.
-    """
-    import vllm.utils.platform_utils as pu
-    from vllm.platforms import current_platform
-
-    pu.is_pin_memory_available.cache_clear()
-    pu.is_uva_available.cache_clear()
-    type(current_platform).is_pin_memory_available = classmethod(lambda cls: pin)
-    if v2_mode:
-        pu.is_uva_available = lambda: True
-
-    from vllm.benchmarks.latency import add_cli_args
-    from vllm.benchmarks.latency import main as latency_main
-
-    parser = argparse.ArgumentParser()
-    add_cli_args(parser)
-    args = parser.parse_args([])
-
-    for key, val in engine_args_kwargs.items():
-        setattr(args, key, val)
-    args.input_len = _LATENCY_INPUT_LEN
-    args.output_len = _LATENCY_OUTPUT_LEN
-    args.batch_size = _LATENCY_BATCH_SIZE
-    args.num_iters_warmup = _LATENCY_WARMUP_ITERS
-    args.num_iters = _LATENCY_BENCH_ITERS
-    args.profile = False
-    args.disable_detokenize = True
-
-    with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f:
-        tmp_path = f.name
-    args.output_json = tmp_path
-
-    latency_main(args)
-
-    with open(tmp_path) as f:
-        results = json.load(f)
-    q.put(results)
-
-
-def _run_latency_benchmark(
-    pin: bool,
-    engine_args_kwargs: dict,
-    v2_mode: bool = False,
-) -> dict:
-    ctx = multiprocessing.get_context("spawn")
-    q = ctx.Queue()
-    p = ctx.Process(
-        target=_latency_worker,
-        args=(pin, engine_args_kwargs, q, v2_mode),
-    )
-    p.start()
-    p.join()
-    if p.exitcode != 0:
-        raise RuntimeError(
-            f"Latency benchmark subprocess (pin={pin}) exited with code {p.exitcode}"
-        )
-    return q.get()
-
-
-@pytest.mark.parametrize(
-    "test_v2_runner",
-    [
-        pytest.param(False, id="v1"),
-        pytest.param(True, id="v2"),
-    ],
-)
-class TestPinnedMemory:
-    """Verify pinned memory yields >= throughput vs unpinned via real vLLM inference."""
-
-    def test_throughput(self, monkeypatch, test_v2_runner, model, max_model_len):
-        """Benchmark throughput with pin_memory forced on then off.
-
-        Delegates to vllm/benchmarks/throughput.py main() with the random
-        dataset.  Each condition runs in an isolated spawn subprocess so both
-        start from a cold CUDA context, giving an unbiased comparison.
-        """
-        monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
-        monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1" if test_v2_runner else "0")
-
-        engine_args_kwargs = dict(
-            model=model,
-            gpu_memory_utilization=0.88,
-            max_model_len=max_model_len,
-            enable_prefix_caching=False,
-        )
-
-        _skip_if_pin_memory_not_available(engine_args_kwargs)
-
-        unpinned_tps = _run_throughput_benchmark(
-            False, engine_args_kwargs, v2_mode=test_v2_runner
-        )
-        pinned_tps = _run_throughput_benchmark(
-            True, engine_args_kwargs, v2_mode=test_v2_runner
-        )
-
-        pct_diff = (pinned_tps - unpinned_tps) / unpinned_tps * 100
-        runner = "v2" if test_v2_runner else "v1"
-        print(
-            f"\n=== Throughput results ({runner} runner, {model}) ==="
-            f"\npin_memory=True:  {pinned_tps:.1f} tok/s"
-            f"\npin_memory=False: {unpinned_tps:.1f} tok/s"
-            f"\nDifference: {pct_diff:+.1f}% (pinned vs unpinned)"
-        )
-
-        assert pinned_tps >= unpinned_tps * _THROUGHPUT_TOLERANCE, (
-            f"Pinned throughput ({pinned_tps:.1f} tok/s) fell more than "
-            f"{(1.0 - _THROUGHPUT_TOLERANCE) * 100:.1f}% below "
-            f"unpinned ({unpinned_tps:.1f} tok/s)."
-        )
-
-    def test_latency(self, monkeypatch, test_v2_runner, model, max_model_len):
-        """Benchmark per-batch latency with pin_memory forced on then off.
-
-        Follows vllm/benchmarks/latency.py: fixed dummy-token batch, warmup
-        iterations to reach steady state, then timed iterations reduced to avg
-        and percentiles.  Subprocesses run serially so each gets a cold CUDA
-        context without GPU memory pressure from the other run.
-        """
-        monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
-        monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1" if test_v2_runner else "0")
-
-        engine_args_kwargs = dict(
-            model=model,
-            gpu_memory_utilization=0.88,
-            max_model_len=max_model_len,
-            enable_prefix_caching=False,
-        )
-
-        _skip_if_pin_memory_not_available(engine_args_kwargs)
-
-        unpinned = _run_latency_benchmark(
-            False, engine_args_kwargs, v2_mode=test_v2_runner
-        )
-        pinned = _run_latency_benchmark(
-            True, engine_args_kwargs, v2_mode=test_v2_runner
-        )
-
-        pct_diff = (
-            (pinned["avg_latency"] - unpinned["avg_latency"])
-            / unpinned["avg_latency"]
-            * 100
-        )
-        runner = "v2" if test_v2_runner else "v1"
-        print(
-            f"\n=== Latency results ({runner} runner, {model}) ==="
-            f"\npin_memory=True:  avg={pinned['avg_latency']:.3f}s"
-            f"  p50={pinned['percentiles']['50']:.3f}s"
-            f"  p99={pinned['percentiles']['99']:.3f}s"
-            f"\npin_memory=False: avg={unpinned['avg_latency']:.3f}s"
-            f"  p50={unpinned['percentiles']['50']:.3f}s"
-            f"  p99={unpinned['percentiles']['99']:.3f}s"
-            f"\nDifference: {pct_diff:+.1f}% (pinned vs unpinned)"
-        )
-
-        assert pinned["avg_latency"] <= unpinned["avg_latency"] * _LATENCY_TOLERANCE, (
-            f"Pinned avg latency ({pinned['avg_latency']:.3f}s) exceeded "
-            f"unpinned ({unpinned['avg_latency']:.3f}s) by more than "
-            f"{(_LATENCY_TOLERANCE - 1.0) * 100:.1f}%."
-        )
-
-
-if __name__ == "__main__":
-    _parser = argparse.ArgumentParser(add_help=False)
-    _parser.add_argument("--model", default=_DEFAULT_MODEL)
-    _parser.add_argument("--max-model-len", type=int, default=_DEFAULT_MAX_MODEL_LEN)
-    _, _remaining = _parser.parse_known_args()
-    sys.exit(pytest.main([__file__] + _remaining))
@@ -33,6 +33,7 @@ from vllm.distributed.device_communicators.custom_all_reduce import CustomAllred
 from vllm.distributed.device_communicators.flashinfer_all_reduce import (
    FlashInferAllReduce,
 )
+from vllm.distributed.device_communicators.push_all_reduce import PushAllReduce
 from vllm.distributed.device_communicators.pynccl import (
    PyNcclCommunicator,
    register_nccl_symmetric_ops,
@@ -80,6 +81,7 @@ class CommunicatorBenchmark:

        # Initialize communicators
        self.custom_allreduce = None
+        self.push_ar_comm = None
        self.pynccl_comm = None
        self.symm_mem_comm = None
        self.symm_mem_comm_multimem = None
@@ -106,6 +108,23 @@ class CommunicatorBenchmark:
            )
            self.custom_allreduce = None

+        try:
+            self.push_ar_comm = PushAllReduce(
+                group=self.cpu_group,
+                device=self.device,
+                max_size=self.max_size_override,
+            )
+            if not self.push_ar_comm.disabled:
+                logger.info("Rank %s: PushAllReduce initialized", self.rank)
+            else:
+                logger.info("Rank %s: PushAllReduce disabled", self.rank)
+                self.push_ar_comm = None
+        except Exception as e:
+            logger.warning(
+                "Rank %s: Failed to initialize PushAllReduce: %s", self.rank, e
+            )
+            self.push_ar_comm = None
+
        try:
            self.pynccl_comm = PyNcclCommunicator(
                group=self.cpu_group, device=self.device
@@ -216,6 +235,19 @@ class CommunicatorBenchmark:
                )
            )

+        if self.push_ar_comm is not None:
+            comm = self.push_ar_comm
+            communicators.append(
+                (
+                    "push_ar",
+                    lambda t, c=comm: c.all_reduce(t),
+                    lambda t, c=comm: c.should_use(t),
+                    comm.capture(),
+                    {},
+                    None,
+                )
+            )
+
        if self.pynccl_comm is not None:
            comm = self.pynccl_comm
            communicators.append(
@@ -80,17 +80,13 @@ _FI_MAX_SIZES = {
    2: 64 * MiB,  # 64MB
    4: 64 * MiB,  # 64MB
    8: 64 * MiB,  # 64MB
-    16: 64 * MiB,  # 64MB (multi-node)
 }

 # Global workspace tensors for FlashInfer (keyed by backend name)
 _FI_WORKSPACES: dict = {}

-# Backends to benchmark. trtllm is single-node only and can hang cross-node, so
-# multi-node sweeps can restrict to mnnvl via FI_BACKENDS=mnnvl.
-FLASHINFER_BACKENDS = [
-    b for b in os.environ.get("FI_BACKENDS", "trtllm,mnnvl").split(",") if b
-]
+# Backends to benchmark
+FLASHINFER_BACKENDS = ["trtllm", "mnnvl"]


 def setup_flashinfer_workspace(
@@ -999,10 +995,7 @@ def main():
    rank = int(os.environ["RANK"])
    world_size = int(os.environ["WORLD_SIZE"])

-    # Use LOCAL_RANK for the device so multi-node runs (global rank >= GPUs per
-    # node) map to a valid local GPU; falls back to global rank single-node.
-    local_rank = int(os.environ.get("LOCAL_RANK", rank))
-    device = torch.device(f"cuda:{local_rank}")
+    device = torch.device(f"cuda:{rank}")
    torch.accelerator.set_device_index(device)
    torch.set_default_device(device)

@@ -391,19 +391,16 @@ def get_configs_compute_bound(use_fp16, block_quant_shape) -> list[dict[str, int
        config = dict(zip(keys, config_values))
        configs.append(config)

-    # Drop configs incompatible with fp8 block quantization. A tile must align
-    # to the quant-block scale grid, i.e. tile and block must divide one
-    # another. The kernel indexes scales per element (offs_bn // group_n,
-    # k_start // group_k), so a tile narrower than the block (e.g. N=64 with
-    # block_n=128) is valid -- and often faster at small batch. An exact
-    # multiple was required before, which dropped those smaller tiles entirely.
+    # Remove configs that are not compatible with fp8 block quantization
+    # BLOCK_SIZE_K must be a multiple of block_k
+    # BLOCK_SIZE_N must be a multiple of block_n
    if block_quant_shape is not None and not use_fp16:
        block_n, block_k = block_quant_shape[0], block_quant_shape[1]
        for config in configs[:]:
-            bn, bk = config["BLOCK_SIZE_N"], config["BLOCK_SIZE_K"]
-            n_aligned = bn % block_n == 0 or block_n % bn == 0
-            k_aligned = bk % block_k == 0 or block_k % bk == 0
-            if not (n_aligned and k_aligned):
+            if (
+                config["BLOCK_SIZE_K"] % block_k != 0
+                or config["BLOCK_SIZE_N"] % block_n != 0
+            ):
                configs.remove(config)
    return configs

@@ -7,7 +7,6 @@ import time
 import numpy as np
 import torch

-from vllm.platforms import CpuArchEnum, current_platform
 from vllm.utils.argparse_utils import FlexibleArgumentParser
 from vllm.utils.torch_utils import set_random_seed

@@ -15,15 +14,17 @@ from vllm.utils.torch_utils import set_random_seed
 try:
    from vllm._custom_ops import cpu_fused_moe, cpu_prepack_moe_weight
 except (ImportError, AttributeError) as e:
+    print("ERROR: CPU fused MoE operations are not available on this platform.")
+    print("This benchmark requires x86 CPU with proper vLLM CPU extensions compiled.")
+    print(
+        "The cpu_fused_moe kernel is typically available on Linux x86_64 "
+        "with AVX2/AVX512."
+    )
    print(f"Import error: {e}")
    sys.exit(1)

 # ISA selection following test_cpu_fused_moe.py pattern
-ISA_CHOICES = ["vec"]
-if torch.cpu._is_amx_tile_supported():
-    ISA_CHOICES.append("amx")
-if current_platform.get_cpu_architecture() == CpuArchEnum.ARM:
-    ISA_CHOICES.append("neon")
+ISA_CHOICES = ["amx", "vec"] if torch.cpu._is_amx_tile_supported() else ["vec"]


@torch.inference_mode()
@@ -144,7 +145,7 @@ if __name__ == "__main__":
        "--isa",
        type=str,
        choices=ISA_CHOICES,
-        default="vec",
+        default=ISA_CHOICES[0],
        help=f"ISA to use (available: {ISA_CHOICES})",
    )
    parser.add_argument("--seed", type=int, default=0)
@@ -24,10 +24,7 @@ set (ENABLE_NUMA TRUE)
 # Check the compile flags
 #
 if(MACOSX_FOUND)
-    # Apple clang needs -Xpreprocessor to enable OpenMP. No runtime link is
-    # needed: _C is a dynamic_lookup bundle and resolves libomp from torch.
    list(APPEND CXX_COMPILE_FLAGS
-        "-Xpreprocessor" "-fopenmp"
        "-DVLLM_CPU_EXTENSION")
 else()
    list(APPEND CXX_COMPILE_FLAGS
@@ -169,13 +166,12 @@ elseif (S390_FOUND)
        "-mtune=native")
 elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "riscv64")
    message(STATUS "RISC-V detected")
-    if(DEFINED VLLM_RVV_VLEN AND VLLM_RVV_VLEN LESS 0)
+    if(DEFINED VLLM_RVV_VLEN AND NOT VLLM_RVV_VLEN GREATER 0)
        message(FATAL_ERROR
-            "VLLM_RVV_VLEN must be zero or a positive integer; got '${VLLM_RVV_VLEN}'")
+            "VLLM_RVV_VLEN must be a positive integer; got '${VLLM_RVV_VLEN}'")
    endif()
    # VLLM_RVV_VLEN selects the target VLEN. Auto-detected from /proc/cpuinfo
-    # by default; set -DVLLM_RVV_VLEN=0 to force scalar RISC-V build.
-    # Override with -DVLLM_RVV_VLEN=128 or -DVLLM_RVV_VLEN=256 for RVV.
+    # by default; override with -DVLLM_RVV_VLEN=128 or -DVLLM_RVV_VLEN=256.
    if(NOT DEFINED VLLM_RVV_VLEN)
        # Auto-detect: find the largest zvl<N>b in /proc/cpuinfo isa line.
        if(EXISTS /proc/cpuinfo)
@@ -427,7 +423,6 @@ if (ASIMD_FOUND AND NOT APPLE_SILICON_FOUND)
    set(VLLM_EXT_SRC
        "csrc/cpu/shm.cpp"
        "csrc/cpu/activation_lut_bf16.cpp"
-        "csrc/cpu/cpu_fused_moe.cpp"
        ${VLLM_EXT_SRC})
 endif()

@@ -8,73 +8,43 @@ if (DEFINED ENV{DEEPGEMM_SRC_DIR})
  set(DEEPGEMM_SRC_DIR $ENV{DEEPGEMM_SRC_DIR})
 endif()

-# Local tree: set deepgemm_SOURCE_DIR directly (no FetchContent download).
-# Upstream git: use FetchContent_Populate with explicit options (CMP0169 NEW
-# disallows one-argument Populate(dep) after Declare; MakeAvailable would run
-# DeepGEMM's top-level CMakeLists.txt, which vLLM must not load).
 if(DEEPGEMM_SRC_DIR)
-  # cmake_path(ABSOLUTE_PATH <var> ...) reads the path from <var>; NORMALIZE is a
-  # flag (no trailing path argument). Resolve relative paths against vLLM root.
-  set(_deepgemm_user_src "${DEEPGEMM_SRC_DIR}")
-  cmake_path(ABSOLUTE_PATH _deepgemm_user_src
-    BASE_DIRECTORY "${CMAKE_SOURCE_DIR}"
-    NORMALIZE)
-  set(DEEPGEMM_SRC_DIR "${_deepgemm_user_src}")
-  if(NOT IS_DIRECTORY "${DEEPGEMM_SRC_DIR}")
-    message(FATAL_ERROR
-      "DEEPGEMM_SRC_DIR is not an existing directory: '${DEEPGEMM_SRC_DIR}'")
-  endif()
-  set(deepgemm_SOURCE_DIR "${DEEPGEMM_SRC_DIR}")
-  message(STATUS "DeepGEMM using local DEEPGEMM_SRC_DIR: ${deepgemm_SOURCE_DIR}")
+  FetchContent_Declare(
+    deepgemm
+    SOURCE_DIR ${DEEPGEMM_SRC_DIR}
+    CONFIGURE_COMMAND ""
+    BUILD_COMMAND ""
+  )
 else()
-  # Keep in sync with tools/install_deepgemm.sh
-  set(_DEEPGEMM_UPSTREAM_REPO "https://github.com/deepseek-ai/DeepGEMM.git")
-  set(_DEEPGEMM_UPSTREAM_TAG "891d57b4db1071624b5c8fa0d1e51cb317fa709f")
-
-  set(_deepgemm_fc_root "${FETCHCONTENT_BASE_DIR}")
-  if(NOT _deepgemm_fc_root)
-    set(_deepgemm_fc_root "${CMAKE_BINARY_DIR}/_deps")
-  endif()
-  set(_deepgemm_src "${_deepgemm_fc_root}/deepgemm-src")
-  set(_deepgemm_bin "${_deepgemm_fc_root}/deepgemm-build")
-  set(_deepgemm_sub "${_deepgemm_fc_root}/deepgemm-subbuild")
-
-  if(EXISTS "${_deepgemm_src}/csrc/python_api.cpp")
-    set(deepgemm_SOURCE_DIR "${_deepgemm_src}")
-    set(deepgemm_BINARY_DIR "${_deepgemm_bin}")
-  else()
-    FetchContent_Populate(
-      deepgemm
-      SUBBUILD_DIR "${_deepgemm_sub}"
-      SOURCE_DIR "${_deepgemm_src}"
-      BINARY_DIR "${_deepgemm_bin}"
-      GIT_REPOSITORY "${_DEEPGEMM_UPSTREAM_REPO}"
-      GIT_TAG "${_DEEPGEMM_UPSTREAM_TAG}"
-      GIT_SUBMODULES "third-party/cutlass" "third-party/fmt"
-      GIT_PROGRESS TRUE
-    )
-  endif()
-  message(STATUS "DeepGEMM is available at ${deepgemm_SOURCE_DIR}")
+  # This ref should be kept in sync with tools/install_deepgemm.sh
+  FetchContent_Declare(
+    deepgemm
+    GIT_REPOSITORY https://github.com/deepseek-ai/DeepGEMM.git
+    GIT_TAG 891d57b4db1071624b5c8fa0d1e51cb317fa709f
+    GIT_SUBMODULES "third-party/cutlass" "third-party/fmt"
+    GIT_PROGRESS TRUE
+    CONFIGURE_COMMAND ""
+    BUILD_COMMAND ""
+  )
 endif()

-# DeepGEMM requires CUDA 12.3+ for SM90, 12.9+ for SM100 (official upstream),
-# and 12.8+ for SM120 / SM12x. CUDA 13+ can use the family-specific SM12x
-# arch; CUDA 12.x builds the arch-specific SM120/SM121 variants.
+# Use FetchContent_Populate (not MakeAvailable) to avoid processing
+# DeepGEMM's own CMakeLists.txt which has incompatible find_package calls.
+FetchContent_GetProperties(deepgemm)
+if(NOT deepgemm_POPULATED)
+  FetchContent_Populate(deepgemm)
+endif()
+message(STATUS "DeepGEMM is available at ${deepgemm_SOURCE_DIR}")
+
+# DeepGEMM requires CUDA 12.3+ for SM90, 12.9+ for SM100
 set(DEEPGEMM_SUPPORT_ARCHS)
 if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.3)
  list(APPEND DEEPGEMM_SUPPORT_ARCHS "9.0a")
 endif()
-if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8)
-  if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.9)
-    list(APPEND DEEPGEMM_SUPPORT_ARCHS "10.0f")
-  else()
-    list(APPEND DEEPGEMM_SUPPORT_ARCHS "10.0a")
-  endif()
-  if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 13.0)
-    list(APPEND DEEPGEMM_SUPPORT_ARCHS "12.0f")
-  else()
-    list(APPEND DEEPGEMM_SUPPORT_ARCHS "12.0a" "12.1a")
-  endif()
+if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.9)
+  list(APPEND DEEPGEMM_SUPPORT_ARCHS "10.0f")
+elseif(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8)
+  list(APPEND DEEPGEMM_SUPPORT_ARCHS "10.0a")
 endif()

 cuda_archs_loose_intersection(DEEPGEMM_ARCHS
@@ -17,7 +17,7 @@ else()
  FetchContent_Declare(
    fmha_sm100
    GIT_REPOSITORY https://github.com/vllm-project/MSA.git
-    GIT_TAG fee783153f3efe57e3e933c5cb7e267a7cebcfb5
+    GIT_TAG 544eee5e09ae2dfa774d5b06739013f9b7402c57
    GIT_PROGRESS TRUE
    CONFIGURE_COMMAND ""
    BUILD_COMMAND ""
@@ -32,42 +32,19 @@ message(STATUS "fmha_sm100 is available at ${fmha_sm100_SOURCE_DIR}")

 add_custom_target(fmha_sm100)

-set(FMHA_SM100_PY_ROOT "${fmha_sm100_SOURCE_DIR}/python/fmha_sm100")
-
 install(FILES
-  "${FMHA_SM100_PY_ROOT}/__init__.py"
-  "${FMHA_SM100_PY_ROOT}/api.py"
-  "${FMHA_SM100_PY_ROOT}/bench_utils.py"
-  "${FMHA_SM100_PY_ROOT}/jit.py"
-  "${FMHA_SM100_PY_ROOT}/sparse.py"
-  "${FMHA_SM100_PY_ROOT}/sparse_fmha_adapter.py"
+  "${fmha_sm100_SOURCE_DIR}/python/fmha_sm100/__init__.py"
+  "${fmha_sm100_SOURCE_DIR}/python/fmha_sm100/sparse.py"
  DESTINATION vllm/third_party/fmha_sm100
  COMPONENT fmha_sm100)

-install(DIRECTORY "${FMHA_SM100_PY_ROOT}/csrc/"
-  DESTINATION vllm/third_party/fmha_sm100/csrc
-  COMPONENT fmha_sm100
-  PATTERN "__pycache__" EXCLUDE
-  PATTERN "*.pyc" EXCLUDE
-  PATTERN ".git*" EXCLUDE)
-
-install(DIRECTORY "${FMHA_SM100_PY_ROOT}/cute/"
+install(DIRECTORY "${fmha_sm100_SOURCE_DIR}/python/fmha_sm100/cute/"
  DESTINATION vllm/third_party/fmha_sm100/cute
  COMPONENT fmha_sm100
-  PATTERN "__pycache__" EXCLUDE
-  PATTERN "*.pyc" EXCLUDE
-  PATTERN ".git*" EXCLUDE)
-
-install(DIRECTORY "${FMHA_SM100_PY_ROOT}/cutlass/include/"
-  DESTINATION vllm/third_party/fmha_sm100/cutlass/include
-  COMPONENT fmha_sm100
-  PATTERN "__pycache__" EXCLUDE
-  PATTERN "*.pyc" EXCLUDE
-  PATTERN ".git*" EXCLUDE)
-
-install(DIRECTORY "${FMHA_SM100_PY_ROOT}/cutlass/tools/util/include/"
-  DESTINATION vllm/third_party/fmha_sm100/cutlass/tools/util/include
-  COMPONENT fmha_sm100
-  PATTERN "__pycache__" EXCLUDE
-  PATTERN "*.pyc" EXCLUDE
-  PATTERN ".git*" EXCLUDE)
+  FILES_MATCHING
+    REGEX "/__pycache__(/.*)?$" EXCLUDE
+    REGEX ".*\\.pyc$" EXCLUDE
+    PATTERN "example.py" EXCLUDE
+    PATTERN "test_*.py" EXCLUDE
+    PATTERN "*.py"
+    PATTERN "build_k2q_csr.cu")
@@ -6,48 +6,26 @@ if(DEFINED ENV{QUTLASS_SRC_DIR})
  set(QUTLASS_SRC_DIR $ENV{QUTLASS_SRC_DIR})
 endif()

-# CMP0169 NEW: one-argument FetchContent_Populate(name) after Declare is invalid.
-# Use explicit Populate(...) for git, or set SOURCE_DIR for local trees.
 if(QUTLASS_SRC_DIR)
-  set(_qutlass_user_src "${QUTLASS_SRC_DIR}")
-  cmake_path(ABSOLUTE_PATH _qutlass_user_src
-    BASE_DIRECTORY "${CMAKE_SOURCE_DIR}"
-    NORMALIZE)
-  set(QUTLASS_SRC_DIR "${_qutlass_user_src}")
-  if(NOT IS_DIRECTORY "${QUTLASS_SRC_DIR}")
-    message(FATAL_ERROR
-      "[QUTLASS] QUTLASS_SRC_DIR is not an existing directory: '${QUTLASS_SRC_DIR}'")
-  endif()
-  set(qutlass_SOURCE_DIR "${QUTLASS_SRC_DIR}")
-  set(qutlass_BINARY_DIR "${CMAKE_BINARY_DIR}/qutlass-binary-dir-unused")
+  FetchContent_Declare(
+    qutlass
+    SOURCE_DIR ${QUTLASS_SRC_DIR}
+    CONFIGURE_COMMAND ""
+    BUILD_COMMAND ""
+  )
 else()
-  set(_QUTLASS_UPSTREAM_REPO "https://github.com/IST-DASLab/qutlass.git")
-  set(_QUTLASS_UPSTREAM_TAG "830d2c4537c7396e14a02a46fbddd18b5d107c65")
-
-  set(_qutlass_fc_root "${FETCHCONTENT_BASE_DIR}")
-  if(NOT _qutlass_fc_root)
-    set(_qutlass_fc_root "${CMAKE_BINARY_DIR}/_deps")
-  endif()
-  set(_qutlass_src "${_qutlass_fc_root}/qutlass-src")
-  set(_qutlass_bin "${_qutlass_fc_root}/qutlass-build")
-  set(_qutlass_sub "${_qutlass_fc_root}/qutlass-subbuild")
-
-  if(EXISTS "${_qutlass_src}/qutlass/csrc/bindings.cpp")
-    set(qutlass_SOURCE_DIR "${_qutlass_src}")
-    set(qutlass_BINARY_DIR "${_qutlass_bin}")
-  else()
-    FetchContent_Populate(
-      qutlass
-      SUBBUILD_DIR "${_qutlass_sub}"
-      SOURCE_DIR "${_qutlass_src}"
-      BINARY_DIR "${_qutlass_bin}"
-      GIT_REPOSITORY "${_QUTLASS_UPSTREAM_REPO}"
-      GIT_TAG "${_QUTLASS_UPSTREAM_TAG}"
-      GIT_PROGRESS TRUE
-    )
-  endif()
+  FetchContent_Declare(
+    qutlass
+    GIT_REPOSITORY https://github.com/IST-DASLab/qutlass.git
+    GIT_TAG 830d2c4537c7396e14a02a46fbddd18b5d107c65
+    GIT_PROGRESS TRUE
+    CONFIGURE_COMMAND ""
+    BUILD_COMMAND ""
+  )
 endif()

+FetchContent_Populate(qutlass)
+
 if(NOT qutlass_SOURCE_DIR)
  message(FATAL_ERROR "[QUTLASS] source directory could not be resolved.")
 endif()
@@ -82,7 +60,6 @@ endif()

 if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8 AND QUTLASS_ARCHS)
  set(QUTLASS_SOURCES
-    csrc/qutlass_registration.cpp
    ${qutlass_SOURCE_DIR}/qutlass/csrc/bindings.cpp
    ${qutlass_SOURCE_DIR}/qutlass/csrc/gemm.cu
    ${qutlass_SOURCE_DIR}/qutlass/csrc/gemm_ada.cu
@@ -101,19 +78,8 @@ if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8 AND QUTLASS_ARCHS)

  if(CUTLASS_INCLUDE_DIR AND EXISTS "${CUTLASS_INCLUDE_DIR}/cutlass/cutlass.h")
    list(APPEND QUTLASS_INCLUDES "${CUTLASS_INCLUDE_DIR}")
-    if(CUTLASS_TOOLS_UTIL_INCLUDE_DIR AND
-       EXISTS "${CUTLASS_TOOLS_UTIL_INCLUDE_DIR}/cutlass/util/packed_stride.hpp")
-      list(APPEND QUTLASS_INCLUDES "${CUTLASS_TOOLS_UTIL_INCLUDE_DIR}")
-    else()
-      get_filename_component(_qutlass_cutlass_root "${CUTLASS_INCLUDE_DIR}" DIRECTORY)
-      if(EXISTS "${_qutlass_cutlass_root}/tools/util/include/cutlass/util/packed_stride.hpp")
-        list(APPEND QUTLASS_INCLUDES "${_qutlass_cutlass_root}/tools/util/include")
-      endif()
-    endif()
  elseif(EXISTS "${qutlass_SOURCE_DIR}/qutlass/third_party/cutlass/include/cutlass/cutlass.h")
-    list(APPEND QUTLASS_INCLUDES
-      "${qutlass_SOURCE_DIR}/qutlass/third_party/cutlass/include"
-      "${qutlass_SOURCE_DIR}/qutlass/third_party/cutlass/tools/util/include")
+    list(APPEND QUTLASS_INCLUDES "${qutlass_SOURCE_DIR}/qutlass/third_party/cutlass/include")
    message(STATUS "[QUTLASS] Using QuTLASS vendored CUTLASS headers (no vLLM CUTLASS detected).")
  else()
    message(FATAL_ERROR "[QUTLASS] CUTLASS headers not found. "
@@ -125,23 +91,12 @@ if(${CMAKE_CUDA_COMPILER_VERSION} VERSION_GREATER_EQUAL 12.8 AND QUTLASS_ARCHS)
    CUDA_ARCHS "${QUTLASS_ARCHS}"
  )

-  # QuTLASS uses legacy ATen headers and cannot be built with TORCH_TARGET_VERSION.
-  # Keep it as its own extension (registers torch.ops._qutlass_C).
-  define_extension_target(
-    _qutlass_C
-    DESTINATION vllm
-    LANGUAGE ${VLLM_GPU_LANG}
-    SOURCES ${QUTLASS_SOURCES}
-    COMPILE_FLAGS ${VLLM_GPU_FLAGS}
-    ARCHITECTURES ${VLLM_GPU_ARCHES}
-    INCLUDE_DIRECTORIES ${QUTLASS_INCLUDES}
-    USE_SABI 3
-    WITH_SOABI)
-
-  target_compile_definitions(_qutlass_C PRIVATE
+  target_sources(_C PRIVATE ${QUTLASS_SOURCES})
+  target_include_directories(_C PRIVATE ${QUTLASS_INCLUDES})
+  target_compile_definitions(_C PRIVATE
    QUTLASS_DISABLE_PYBIND=1
    TARGET_CUDA_ARCH=${QUTLASS_TARGET_CC}
-    CUTLASS_ENABLE_DIRECT_CUDA_DRIVER_CALL=1)
+  )

  set_property(SOURCE ${QUTLASS_SOURCES} APPEND PROPERTY COMPILE_OPTIONS
    $<$<COMPILE_LANGUAGE:CUDA>:--expt-relaxed-constexpr --use_fast_math -O3>
@@ -156,5 +111,4 @@ else()
      "[QUTLASS] Skipping build: no supported arch (12.0f / 10.0f) found in "
      "CUDA_ARCHS='${CUDA_ARCHS}'.")
  endif()
-  add_custom_target(_qutlass_C)
 endif()
@@ -39,7 +39,7 @@ else()
  FetchContent_Declare(
          vllm-flash-attn
          GIT_REPOSITORY https://github.com/vllm-project/flash-attention.git
-          GIT_TAG b3964b1d8b95d8e8447435668ab169a2700bab65
+          GIT_TAG 803020a8fa15407871341d41eba4919ade2ee1ee
          GIT_PROGRESS TRUE
          # Don't share the vllm-flash-attn build between build types
          BINARY_DIR ${CMAKE_BINARY_DIR}/vllm-flash-attn
@@ -11,25 +11,13 @@ static inline cpu_attention::Fp8KVCacheDataType parse_fp8_kv_dtype(
  return cpu_attention::Fp8KVCacheDataType::kAuto;
 }

-bool cpu_attn_has_isa(const std::string& isa) {
-  if (isa == "rvv") {
-#if defined(__riscv) && defined(__riscv_v_min_vlen) && __riscv_v_min_vlen == 128
-    return true;
-#else
-    return false;
-#endif
-  }
-  return false;
-}
-
 torch::Tensor get_scheduler_metadata(
    const int64_t num_req, const int64_t num_heads_q,
    const int64_t num_heads_kv, const int64_t head_dim,
    const torch::Tensor& seq_lens, at::ScalarType dtype,
-    const torch::Tensor& query_start_loc, const bool causal,
+    const torch::Tensor& query_start_loc, const bool casual,
    const int64_t window_size, const std::string& isa_hint,
-    const bool enable_kv_split,
-    const std::optional<torch::Tensor>& dynamic_causal) {
+    const bool enable_kv_split) {
  cpu_attention::ISA isa;
  if (isa_hint == "amx") {
    isa = cpu_attention::ISA::AMX;
@@ -56,13 +44,24 @@ torch::Tensor get_scheduler_metadata(
  input.head_dim = head_dim;
  input.query_start_loc = query_start_loc.data_ptr<int32_t>();
  input.seq_lens = seq_lens.data_ptr<int32_t>();
-
-  input.sliding_window_size = window_size;
-  input.causal = causal;
+  if (window_size != -1) {
+    input.left_sliding_window_size = window_size - 1;
+    if (casual) {
+      input.right_sliding_window_size = 0;
+    } else {
+      input.right_sliding_window_size = window_size - 1;
+    }
+  } else {
+    input.left_sliding_window_size = -1;
+    if (casual) {
+      input.right_sliding_window_size = 0;
+    } else {
+      input.right_sliding_window_size = -1;
+    }
+  }
+  input.casual = casual;
  input.isa = isa;
  input.enable_kv_split = enable_kv_split;
-  input.dynamic_causal =
-      dynamic_causal.has_value() ? dynamic_causal->data_ptr<bool>() : nullptr;

  VLLM_DISPATCH_FLOATING_TYPES(dtype, "get_scheduler_metadata", [&]() {
    CPU_ATTN_DISPATCH(head_dim, isa, 0, [&]() {
@@ -176,11 +175,10 @@ void cpu_attention_with_kv_cache(
    const torch::Tensor& seq_lens,         // [num_tokens]
    const double scale, const bool causal,
    const std::optional<torch::Tensor>& alibi_slopes,  // [num_heads]
-    const int64_t sliding_window,
+    const int64_t sliding_window_left, const int64_t sliding_window_right,
    const torch::Tensor& block_table,  // [num_tokens, max_block_num]
    const double softcap, const torch::Tensor& scheduler_metadata,
-    const std::optional<torch::Tensor>& s_aux,           // [num_heads]
-    const std::optional<torch::Tensor>& dynamic_causal,  // [num_reqs]
+    const std::optional<torch::Tensor>& s_aux,  // [num_heads]
    const double k_scale = 1.0, const double v_scale = 1.0,
    const std::string& kv_cache_dtype = "auto") {
  TORCH_CHECK_EQ(query.dim(), 3);
@@ -222,11 +220,13 @@ void cpu_attention_with_kv_cache(
  input.alibi_slopes =
      alibi_slopes.has_value() ? alibi_slopes->data_ptr<float>() : nullptr;
  input.s_aux = s_aux.has_value() ? s_aux->data_ptr<c10::BFloat16>() : nullptr;
-  input.dynamic_causal =
-      dynamic_causal.has_value() ? dynamic_causal->data_ptr<bool>() : nullptr;
  input.scale = scale;
  input.causal = causal;
-  input.sliding_window_size = sliding_window;
+  input.sliding_window_left = sliding_window_left;
+  input.sliding_window_right = sliding_window_right;
+  if (input.causal) {
+    input.sliding_window_right = 0;
+  }
  input.softcap = static_cast<float>(softcap);

  if (is_fp8) {
@@ -124,7 +124,7 @@ struct AttentionMetadata {
        workitem_group_num(workitem_group_num),
        reduction_item_num(reduction_item_num),
        reduction_split_num(reduction_split_num),
-        thread_num(cpu_utils::get_max_threads()),
+        thread_num(omp_get_max_threads()),
        effective_thread_num(thread_num),
        split_kv_q_token_num_threshold(split_kv_q_token_num_threshold),
        attention_scratchpad_size_per_thread(0),
@@ -388,13 +388,13 @@ class AttentionScheduler {
    int32_t head_dim;
    int32_t* query_start_loc;
    int32_t* seq_lens;
-    int32_t sliding_window_size;
-    bool causal;
+    int32_t left_sliding_window_size;
+    int32_t right_sliding_window_size;
+    bool casual;
    cpu_attention::ISA isa;
    int32_t max_num_q_per_iter;  // max Q head num can be hold in registers
    int32_t kv_block_alignment;  // context length alignment requirement
    bool enable_kv_split;
-    bool* dynamic_causal;
  };

  static constexpr int32_t MaxQTileIterNum = 128;
@@ -403,9 +403,8 @@ class AttentionScheduler {
      : available_cache_size_(cpu_utils::get_available_l2_size()) {}

  torch::Tensor schedule(const ScheduleInput& input) const {
-    const bool causal = input.causal;
-    const bool is_dynamic_causal = input.dynamic_causal != nullptr;
-    const int32_t thread_num = cpu_utils::get_max_threads();
+    const bool casual = input.casual;
+    const int32_t thread_num = omp_get_max_threads();
    const int64_t cache_size = cpu_utils::get_available_l2_size();
    const int32_t max_num_q_per_iter = input.max_num_q_per_iter;
    const int32_t kv_len_alignment = input.kv_block_alignment;
@@ -435,7 +434,8 @@ class AttentionScheduler {
    const int32_t default_tile_token_num = default_tile_size / q_head_per_kv;
    const int32_t split_kv_q_token_num_threshold =
        input.enable_kv_split ? 1 : 0;
-    const int32_t sliding_window_size = input.sliding_window_size;
+    const int32_t left_sliding_window_size = input.left_sliding_window_size;
+    const int32_t right_sliding_window_size = input.right_sliding_window_size;
    TORCH_CHECK_LE(split_kv_q_token_num_threshold * q_head_per_kv, 16);

    // get total kv len
@@ -444,9 +444,7 @@ class AttentionScheduler {
      const int32_t seq_len = input.seq_lens[req_id];
      const int32_t q_token_num =
          input.query_start_loc[req_id + 1] - input.query_start_loc[req_id];
-      const bool req_causal =
-          is_dynamic_causal ? input.dynamic_causal[req_id] : causal;
-      const int32_t q_start_pos = seq_len - q_token_num;
+      const int32_t q_start_pos = (casual ? (seq_len - q_token_num) : 0);
      const int32_t kv_start_pos = 0;
      const int32_t kv_end_pos = seq_len;

@@ -458,7 +456,7 @@ class AttentionScheduler {
        const int32_t q_tile_pos_right = q_tile_pos_left + q_tile_token_num;
        const auto [kv_tile_pos_left, kv_tile_pos_right] = calcu_kv_tile_pos(
            kv_start_pos, kv_end_pos, q_tile_pos_left, q_tile_pos_right,
-            sliding_window_size, req_causal);
+            left_sliding_window_size, right_sliding_window_size);
        const auto [aligned_kv_tile_pos_left, aligned_kv_tile_pos_right] =
            align_kv_tile_pos(kv_tile_pos_left, kv_tile_pos_right,
                              kv_len_alignment);
@@ -486,9 +484,7 @@ class AttentionScheduler {
      const int32_t seq_len = input.seq_lens[req_id];
      const int32_t q_token_num =
          input.query_start_loc[req_id + 1] - input.query_start_loc[req_id];
-      const bool req_causal =
-          is_dynamic_causal ? input.dynamic_causal[req_id] : causal;
-      const int32_t q_start_pos = seq_len - q_token_num;
+      const int32_t q_start_pos = (casual ? (seq_len - q_token_num) : 0);
      const int32_t kv_start_pos = 0;
      const int32_t kv_end_pos = seq_len;
      int32_t local_split_id = 0;
@@ -502,7 +498,7 @@ class AttentionScheduler {
        const int32_t q_tile_pos_right = q_tile_pos_left + q_tile_token_num;
        const auto [kv_tile_pos_left, kv_tile_pos_right] = calcu_kv_tile_pos(
            kv_start_pos, kv_end_pos, q_tile_pos_left, q_tile_pos_right,
-            sliding_window_size, req_causal);
+            left_sliding_window_size, right_sliding_window_size);
        const auto [aligned_kv_tile_pos_left, aligned_kv_tile_pos_right] =
            align_kv_tile_pos(kv_tile_pos_left, kv_tile_pos_right,
                              kv_len_alignment);
@@ -712,41 +708,15 @@ class AttentionScheduler {
    return metadata_tensor;
  }

-  FORCE_INLINE static std::pair<int32_t, int32_t> calcu_sliding_window_size(
-      int32_t window_size, bool causal) {
-    int32_t left_sliding_window_size, right_sliding_window_size;
-    if (window_size != -1) {
-      left_sliding_window_size = window_size - 1;
-      if (causal) {
-        right_sliding_window_size = 0;
-      } else {
-        right_sliding_window_size = window_size - 1;
-      }
-    } else {
-      left_sliding_window_size = -1;
-      if (causal) {
-        right_sliding_window_size = 0;
-      } else {
-        right_sliding_window_size = -1;
-      }
-    }
-
-    return {left_sliding_window_size, right_sliding_window_size};
-  }
-
  FORCE_INLINE static std::pair<int32_t, int32_t> calcu_kv_tile_pos(
      int32_t kv_left_pos, int32_t kv_right_pos, int32_t q_left_pos,
-      int32_t q_right_pos, int32_t window_size, bool causal) {
-    auto [left_sliding_window_size, right_sliding_window_size] =
-        calcu_sliding_window_size(window_size, causal);
-
-    if (left_sliding_window_size != -1) {
-      kv_left_pos =
-          std::max(kv_left_pos, q_left_pos - left_sliding_window_size);
+      int32_t q_right_pos, int32_t sliding_window_left,
+      int32_t sliding_window_right) {
+    if (sliding_window_left != -1) {
+      kv_left_pos = std::max(kv_left_pos, q_left_pos - sliding_window_left);
    }
-    if (right_sliding_window_size != -1) {
-      kv_right_pos =
-          std::min(kv_right_pos, q_right_pos + right_sliding_window_size);
+    if (sliding_window_right != -1) {
+      kv_right_pos = std::min(kv_right_pos, q_right_pos + sliding_window_right);
    }
    return {kv_left_pos, kv_right_pos};
  }
@@ -835,10 +805,10 @@ struct AttentionInput {
  int32_t* block_table;
  float* alibi_slopes;
  c10::BFloat16* s_aux;
-  bool* dynamic_causal;
  float scale;
  bool causal;
-  int32_t sliding_window_size;
+  int32_t sliding_window_left;
+  int32_t sliding_window_right;
  float softcap;
  // FP8 KV cache scales (used by FP8 attention implementations)
  float k_scale_fp8 = 1.0f;
@@ -887,10 +857,12 @@ struct VecTypeTrait<c10::BFloat16> {
  using vec_t = vec_op::BF16Vec16;
 };

+#if !defined(__powerpc__)
 template <>
 struct VecTypeTrait<c10::Half> {
  using vec_t = vec_op::FP16Vec16;
 };
+#endif

 template <typename T>
 void print_logits(const char* name, T* ptr, int32_t row, int32_t col,
@@ -1423,7 +1395,7 @@ class AttentionMainLoop {

 public:
  void operator()(const AttentionInput* input) {
-    const int thread_num = cpu_utils::get_max_threads();
+    const int thread_num = omp_get_max_threads();
    TORCH_CHECK_EQ(input->metadata->thread_num, thread_num);
    std::atomic<int32_t> guard_counter(0);
    std::atomic<int32_t>* guard_counter_ptr = &guard_counter;
@@ -1470,16 +1442,15 @@ class AttentionMainLoop {
      const int64_t q_head_num_stride = input->query_num_heads_stride;
      const int64_t kv_cache_head_num_stride = input->cache_num_kv_heads_stride;
      const int64_t kv_cache_block_num_stride = input->cache_num_blocks_stride;
-      const int32_t sliding_window_size = input->sliding_window_size;
+      const int32_t sliding_window_left = input->sliding_window_left;
+      const int32_t sliding_window_right = input->sliding_window_right;
      const int32_t block_size = input->block_size;
      const float scale = input->scale;
      const float softcap_scale = input->softcap;
      const float* alibi_slopes = input->alibi_slopes;
      const c10::BFloat16* s_aux = input->s_aux;
-      const bool* dynamic_causal = input->dynamic_causal;
-      const bool is_dynamic_causal = dynamic_causal != nullptr;

-      const bool causal = input->causal;
+      const bool casual = input->causal;
      int32_t* const block_table = input->block_table;
      const int64_t block_table_stride = input->blt_num_tokens_stride;

@@ -1562,11 +1533,6 @@ class AttentionMainLoop {
                &curr_workitem_groups[workitem_group_idx];

            const int32_t current_group_idx = current_workitem_group->req_id;
-            const int32_t current_group_causal =
-                is_dynamic_causal ? dynamic_causal[current_group_idx] : causal;
-            auto [sliding_window_left, sliding_window_right] =
-                AttentionScheduler::calcu_sliding_window_size(
-                    sliding_window_size, current_group_causal);
            const int32_t kv_start_pos =
                current_workitem_group->kv_split_pos_start;
            const int32_t kv_end_pos = current_workitem_group->kv_split_pos_end;
@@ -1594,7 +1560,8 @@ class AttentionMainLoop {
            const int32_t q_end = input->query_start_loc[current_group_idx + 1];
            const int32_t q_start = input->query_start_loc[current_group_idx];
            const int32_t seq_len = input->seq_lens[current_group_idx];
-            const int32_t q_start_pos = seq_len - (q_end - q_start);
+            const int32_t q_start_pos =
+                (casual ? seq_len - (q_end - q_start) : 0);
            const int32_t block_num = (seq_len + block_size - 1) / block_size;
            // Only apply sink for the first KV split
            bool use_sink = (s_aux != nullptr &&
@@ -1644,8 +1611,8 @@ class AttentionMainLoop {
              const auto [kv_tile_start_pos, kv_tile_end_pos] =
                  AttentionScheduler::calcu_kv_tile_pos(
                      kv_start_pos, kv_end_pos, q_tile_start_pos,
-                      q_tile_end_pos, sliding_window_size,
-                      current_group_causal);
+                      q_tile_end_pos, sliding_window_left,
+                      sliding_window_right);
              const auto [rounded_kv_tile_start_pos, rounded_kv_tile_end_pos] =
                  AttentionScheduler::align_kv_tile_pos(
                      kv_tile_start_pos, kv_tile_end_pos, blocksize_alignment);
@@ -1758,8 +1725,8 @@ class AttentionMainLoop {
                              actual_kv_tile_pos_right] =
                      AttentionScheduler::calcu_kv_tile_pos(
                          kv_tile_pos_left, kv_tile_pos_right, q_tile_pos_left,
-                          q_tile_pos_right, sliding_window_size,
-                          current_group_causal);
+                          q_tile_pos_right, sliding_window_left,
+                          sliding_window_right);
                  const int32_t q_iter_idx =
                      q_head_tile_token_offset / curr_max_q_token_num_per_iter;

@@ -50,16 +50,7 @@ FORCE_INLINE void load_row8_B_as_f32<c10::BFloat16>(const c10::BFloat16* p,
  b1 = (__vector float)vec_mergel(zeros, raw);
 }

-// [3] Half (FP16) Specialization
-template <>
-FORCE_INLINE void load_row8_B_as_f32<c10::Half>(const c10::Half* p,
-                                                __vector float& b0,
-                                                __vector float& b1) {
-  vec_op::FP16Vec8 fp16_vec(p);
-  vec_op::FP32Vec8 fp32_vec(fp16_vec);
-  b0 = fp32_vec.reg.val[0];
-  b1 = fp32_vec.reg.val[1];
-}
+// Note: c10::Half (FP16) is not supported on PowerPC architecture

 template <int32_t M, typename kv_cache_t>
 FORCE_INLINE void gemm_micro_ppc64le_Mx8_Ku4(
@@ -14,18 +14,6 @@
  #define AMX_DISPATCH(...) case cpu_utils::ISA::AMX:
 #endif

-#if defined(ARM_BF16_SUPPORT)
-  #include "cpu/micro_gemm/cpu_micro_gemm_neon.hpp"
-  #define NEON_DISPATCH(...)                                         \
-    case cpu_utils::ISA::NEON: {                                     \
-      using gemm_t =                                                 \
-          cpu_micro_gemm::MicroGemm<cpu_utils::ISA::NEON, scalar_t>; \
-      return __VA_ARGS__();                                          \
-    }
-#else
-  #define NEON_DISPATCH(...) case cpu_utils::ISA::NEON:
-#endif
-
 #define CPU_ISA_DISPATCH_IMPL(ISA_TYPE, ...)                          \
  [&] {                                                               \
    switch (ISA_TYPE) {                                               \
@@ -35,7 +23,6 @@
            cpu_micro_gemm::MicroGemm<cpu_utils::ISA::VEC, scalar_t>; \
        return __VA_ARGS__();                                         \
      }                                                               \
-        NEON_DISPATCH(__VA_ARGS__)                                    \
      default: {                                                      \
        TORCH_CHECK(false, "Invalid CPU ISA type.");                  \
      }                                                               \
@@ -70,12 +57,10 @@ void swigluoai_and_mul(float* __restrict__ input, scalar_t* __restrict__ output,
                       const int32_t input_stride,
                       const int32_t output_stride) {
  using scalar_vec_t = typename cpu_utils::VecTypeTrait<scalar_t>::vec_t;
-#if !defined(__aarch64__)
  // For GPT-OSS interleaved gate-up weights
  alignas(64) static int32_t index[16] = {0,  2,  4,  6,  8,  10, 12, 14,
                                          16, 18, 20, 22, 24, 26, 28, 30};
  vec_op::INT32Vec16 index_vec(index);
-#endif
  vec_op::FP32Vec16 gate_up_max_vec(7.0);
  vec_op::FP32Vec16 up_min_vec(-7.0);
  vec_op::FP32Vec16 alpha_vec(1.702);
@@ -85,15 +70,8 @@ void swigluoai_and_mul(float* __restrict__ input, scalar_t* __restrict__ output,

  for (int32_t m = 0; m < m_size; ++m) {
    for (int32_t n = 0; n < n_size; n += 32) {
-      // Note: AdvSIMD does not support gather loads
-#if defined(__aarch64__)
-      vec_op::FP32Vec16 gate_vec(vec_op::uninit);
-      vec_op::FP32Vec16 up_vec(vec_op::uninit);
-      vec_op::FP32Vec16::load_even_odd(input + n, gate_vec, up_vec);
-#else
      vec_op::FP32Vec16 gate_vec(input + n, index_vec);
      vec_op::FP32Vec16 up_vec(input + n + 1, index_vec);
-#endif
      gate_vec = gate_vec.min(gate_up_max_vec);
      up_vec = up_vec.clamp(up_min_vec, gate_up_max_vec);
      auto sigmoid_vec = one_vec / (one_vec + fast_exp(-gate_vec * alpha_vec));
@@ -185,6 +163,7 @@ void gelu_tanh_and_mul(float* __restrict__ input, scalar_t* __restrict__ output,
  vec_op::FP32Vec16 w1_vec(0.7978845608028654);
  vec_op::FP32Vec16 w2_vec(0.5);
  vec_op::FP32Vec16 w3_vec(0.044715);
+  alignas(64) float temp[16];

  for (int32_t m = 0; m < m_size; ++m) {
    for (int32_t n = 0; n < dim; n += 16) {
@@ -192,9 +171,12 @@ void gelu_tanh_and_mul(float* __restrict__ input, scalar_t* __restrict__ output,
      vec_op::FP32Vec16 up_vec(up + n);
      auto gate_pow3_vec = gate_vec * gate_vec * gate_vec;
      auto inner_vec = w1_vec * (gate_vec + w3_vec * gate_pow3_vec);
-      // Note: can't use fast_exp form because diffusiongemma will generate
-      // wrong results
-      auto tanh_vec = inner_vec.tanh();
+
+      inner_vec.save(temp);
+      for (int32_t i = 0; i < 16; ++i) {
+        temp[i] = std::tanh(temp[i]);
+      }
+      vec_op::FP32Vec16 tanh_vec(temp);
      auto gelu_tanh = gate_vec * w2_vec * (one_vec + tanh_vec);
      auto gated_output_fp32 = up_vec * gelu_tanh;
      scalar_vec_t gated_output = scalar_vec_t(gated_output_fp32);
@@ -260,14 +242,13 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input,
  constexpr int32_t gemm_n_tile_size = gemm_t::NSize;
  constexpr int32_t gemm_m_tile_size = gemm_t::MaxMSize;
  constexpr int32_t min_w13_n_tile_size = 2 * gemm_n_tile_size;
-  constexpr bool pack_a = gemm_t::PackA;
  static_assert(gemm_n_tile_size % 16 == 0);

  TORCH_CHECK_EQ(output_size_13 % min_w13_n_tile_size, 0);
  TORCH_CHECK_EQ(output_size_2 % gemm_n_tile_size, 0);
  TORCH_CHECK_EQ(output_size_13 / 2, input_size_2);

-  const int32_t thread_num = cpu_utils::get_max_threads();
+  const int32_t thread_num = omp_get_max_threads();

  const int32_t w13_input_buffer_size = cpu_utils::round_up<64>(
      gemm_m_tile_size * input_size_13 * sizeof(scalar_t));
@@ -287,18 +268,12 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input,

  const int32_t w2_input_tile_size = cpu_utils::round_up<64>(
      gemm_m_tile_size * input_size_2 * sizeof(scalar_t));
-  // use w2 input buffer only when we need to pack input
-  const int32_t w2_input_buffer_size =
-      pack_a ? cpu_utils::round_up<64>(gemm_m_tile_size * input_size_2 *
-                                       sizeof(scalar_t))
-             : 0;

  const int32_t w2_n_tile_size = [&]() {
    const int64_t cache_size = cpu_utils::get_available_l2_size();
-    // input tile + optional packed input + weight
+    // input tile + weight
    const int32_t n_size_cache_limit =
-        (cache_size - (pack_a ? w2_input_buffer_size : w2_input_tile_size)) /
-        (input_size_2 * sizeof(scalar_t));
+        (cache_size - w2_input_tile_size) / (input_size_2 * sizeof(scalar_t));
    const int32_t n_size_thread_limit =
        output_size_2 / std::max(1, thread_num / topk_num);
    const int32_t n_size = cpu_utils::round_down<gemm_n_tile_size>(
@@ -351,9 +326,6 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input,
  const int32_t w13_output_buffer_offset = w13_thread_buffer_offset;
  w13_thread_buffer_offset += w13_output_buffer_size;

-  const int32_t w2_input_buffer_offset = w13_thread_buffer_offset;
-  w13_thread_buffer_offset += w2_input_buffer_size;
-
  // Weighted sum thread buffer
  const int32_t ws_output_buffer_size =
      cpu_utils::round_up<64>(output_size_2 * sizeof(float));
@@ -433,8 +405,7 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input,
      gemm_t gemm;

      const int32_t input_size_13_bytes = input_size_13 * sizeof(scalar_t);
-      const int32_t w13_n_group_stride =
-          gemm_t::WeightOCGroupSize * input_size_13;
+      const int32_t w13_n_group_stride = 16 * input_size_13;
      const int32_t w13_n_tile_stride = gemm_n_tile_size * input_size_13;

      for (;;) {
@@ -497,23 +468,8 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input,
             token_idx += gemm_m_tile_size) {
          const int32_t actual_token_num =
              std::min(gemm_m_tile_size, curr_token_num - token_idx);
-
-          scalar_t* __restrict__ curr_w13_gemm_input_buffer = nullptr;
-          if constexpr (pack_a) {
-            // copy and pack inputs
-            curr_w13_gemm_input_buffer = w13_input_buffer;
-            const scalar_t* w13_input_rows[gemm_m_tile_size];
-            for (int32_t i = 0; i < actual_token_num; ++i) {
-              w13_input_rows[i] =
-                  input + curr_expand_token_id_buffer[i] * input_size_13;
-            }
-            gemm_t::pack_input_from_rows(w13_input_rows,
-                                         curr_w13_gemm_input_buffer,
-                                         actual_token_num, input_size_13);
-            curr_expand_token_id_buffer += actual_token_num;
-          } else {
-            // copy inputs
-            curr_w13_gemm_input_buffer = curr_w13_input_buffer;
+          // copy inputs
+          {
            scalar_t* __restrict__ curr_w13_input_buffer_iter =
                curr_w13_input_buffer;
            for (int32_t i = 0; i < actual_token_num; ++i) {
@@ -545,12 +501,14 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input,
            scalar_t* __restrict__ w13_weight_ptr_1_iter = w13_weight_ptr_1;
            scalar_t* __restrict__ w13_bias_ptr_0_iter = w13_bias_ptr_0;
            scalar_t* __restrict__ w13_bias_ptr_1_iter = w13_bias_ptr_1;
+            scalar_t* __restrict__ curr_w13_input_buffer_iter =
+                curr_w13_input_buffer;
            float* __restrict__ w13_output_buffer_0_iter = w13_output_buffer;
            float* __restrict__ w13_output_buffer_1_iter =
                w13_output_buffer + actual_n_tile_size / 2;
            for (int32_t i = 0; i < actual_n_tile_size;
                 i += min_w13_n_tile_size) {
-              gemm.gemm(curr_w13_gemm_input_buffer, w13_weight_ptr_0_iter,
+              gemm.gemm(curr_w13_input_buffer_iter, w13_weight_ptr_0_iter,
                        w13_output_buffer_0_iter, actual_token_num,
                        input_size_13, input_size_13, w13_n_group_stride,
                        actual_n_tile_size, false);
@@ -563,7 +521,7 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input,
                w13_bias_ptr_0_iter += gemm_n_tile_size;
              }

-              gemm.gemm(curr_w13_gemm_input_buffer, w13_weight_ptr_1_iter,
+              gemm.gemm(curr_w13_input_buffer_iter, w13_weight_ptr_1_iter,
                        w13_output_buffer_1_iter, actual_token_num,
                        input_size_13, input_size_13, w13_n_group_stride,
                        actual_n_tile_size, false);
@@ -616,8 +574,7 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input,
      gemm_t gemm;

      const int32_t w2_n_tile_stride = gemm_n_tile_size * input_size_2;
-      const int32_t w2_n_group_stride =
-          gemm_t::WeightOCGroupSize * input_size_2;
+      const int32_t w2_n_group_stride = 16 * input_size_2;

      for (;;) {
        int32_t task_id = counter_ptr->acquire_counter();
@@ -656,30 +613,13 @@ void fused_moe_impl(scalar_t* __restrict__ output, scalar_t* __restrict__ input,
             token_idx += gemm_m_tile_size) {
          const int32_t actual_token_num =
              std::min(gemm_m_tile_size, curr_token_num - token_idx);
-          scalar_t* __restrict__ curr_w2_gemm_input_buffer =
-              curr_w13_gemm_output_buffer;
-          if constexpr (pack_a) {
-            uint8_t* __restrict__ thread_buffer =
-                thread_buffer_start + thread_id * w13_thread_buffer_offset;
-            scalar_t* __restrict__ w2_input_buffer =
-                reinterpret_cast<scalar_t*>(thread_buffer +
-                                            w2_input_buffer_offset);
-            curr_w2_gemm_input_buffer = w2_input_buffer;
-            const scalar_t* w2_input_rows[gemm_m_tile_size];
-            for (int32_t i = 0; i < actual_token_num; ++i) {
-              w2_input_rows[i] = curr_w13_gemm_output_buffer + i * input_size_2;
-            }
-            gemm_t::pack_input_from_rows(w2_input_rows,
-                                         curr_w2_gemm_input_buffer,
-                                         actual_token_num, input_size_2);
-          }

          scalar_t* __restrict__ w2_weight_ptr_iter = w2_weight_ptr;
          scalar_t* __restrict__ w2_bias_ptr_iter = w2_bias_ptr;
          float* __restrict__ curr_w2_gemm_output_buffer_iter =
              curr_w2_gemm_output_buffer;
          for (int32_t i = 0; i < actual_n_tile_size; i += gemm_n_tile_size) {
-            gemm.gemm(curr_w2_gemm_input_buffer, w2_weight_ptr_iter,
+            gemm.gemm(curr_w13_gemm_output_buffer, w2_weight_ptr_iter,
                      curr_w2_gemm_output_buffer_iter, actual_token_num,
                      input_size_2, input_size_2, w2_n_group_stride,
                      output_size_2, false);
@@ -25,20 +25,4 @@
  #include <omp.h>
 #endif

-#include <c10/util/Exception.h>
-
-namespace cpu_utils {
-// Without OpenMP the omp pragmas compile to serial loops, so report 1: kernels
-// that barrier on the thread count would otherwise deadlock.
-inline int get_max_threads() {
-#ifdef _OPENMP
-  return omp_get_max_threads();
-#else
-  TORCH_WARN_ONCE(
-      "vLLM CPU was built without OpenMP; running single-threaded.");
-  return 1;
-#endif
-}
-}  // namespace cpu_utils
-
 #endif
@@ -497,26 +497,6 @@ struct FP32Vec16 : public VectorizedRegWrapper<FP32Vec16, 4, float> {
    reg.val[3] = Vectorized<float>(vcvt_f32_f16(vget_high_f16(v.reg.val[1])));
  };

-  static FORCE_INLINE void load_even_odd(const float* ptr, FP32Vec16& even,
-                                         FP32Vec16& odd) noexcept {
-    const float32x4x2_t x01 = vuzpq_f32(vld1q_f32(ptr), vld1q_f32(ptr + 4));
-    const float32x4x2_t x23 =
-        vuzpq_f32(vld1q_f32(ptr + 8), vld1q_f32(ptr + 12));
-    const float32x4x2_t x45 =
-        vuzpq_f32(vld1q_f32(ptr + 16), vld1q_f32(ptr + 20));
-    const float32x4x2_t x67 =
-        vuzpq_f32(vld1q_f32(ptr + 24), vld1q_f32(ptr + 28));
-
-    even.reg.val[0] = VectorizedT(x01.val[0]);
-    even.reg.val[1] = VectorizedT(x23.val[0]);
-    even.reg.val[2] = VectorizedT(x45.val[0]);
-    even.reg.val[3] = VectorizedT(x67.val[0]);
-    odd.reg.val[0] = VectorizedT(x01.val[1]);
-    odd.reg.val[1] = VectorizedT(x23.val[1]);
-    odd.reg.val[2] = VectorizedT(x45.val[1]);
-    odd.reg.val[3] = VectorizedT(x67.val[1]);
-  }
-
  FORCE_INLINE FP32Vec16 operator+(const FP32Vec16& b) const noexcept {
    FP32Vec16 r(uninit);
    r.reg.val[0] = reg.val[0] + b.reg.val[0];
@@ -535,15 +515,6 @@ struct FP32Vec16 : public VectorizedRegWrapper<FP32Vec16, 4, float> {
    return r;
  }

-  FORCE_INLINE FP32Vec16 operator-() const noexcept {
-    FP32Vec16 r(uninit);
-    r.reg.val[0] = reg.val[0].neg();
-    r.reg.val[1] = reg.val[1].neg();
-    r.reg.val[2] = reg.val[2].neg();
-    r.reg.val[3] = reg.val[3].neg();
-    return r;
-  }
-
  FORCE_INLINE FP32Vec16 operator*(const FP32Vec16& b) const noexcept {
    FP32Vec16 r(uninit);
    r.reg.val[0] = reg.val[0] * b.reg.val[0];
@@ -962,4 +933,4 @@ inline void storeFP32<c10::BFloat16>(float v, c10::BFloat16* ptr) {

 inline void prefetch(const void* addr) { __builtin_prefetch(addr, 0, 1); };

-};  // namespace vec_op
+};  // namespace vec_op
@@ -3,17 +3,13 @@

 // VLEN-to-LMUL mapping for RISC-V Vector extension.
 //
-// LMUL_<N> expands to the LMUL suffix giving N total bits of vector data.
-// LMUL_64 is used by 8-lane int8/uint8 vectors.
-//   VLEN=128:
-//     LMUL_64=mf2, LMUL_128=m1, LMUL_256=m2, LMUL_512=m4, LMUL_1024=m8
-//   VLEN=256:
-//     LMUL_64=mf4, LMUL_128=mf2, LMUL_256=m1, LMUL_512=m2, LMUL_1024=m4
+// LMUL_<N> expands to the LMUL suffix giving N total bits of vector data:
+//   VLEN=128: LMUL_128=m1,  LMUL_256=m2,  LMUL_512=m4,  LMUL_1024=m8
+//   VLEN=256: LMUL_128=mf2, LMUL_256=m1,  LMUL_512=m2,  LMUL_1024=m4

 #include <riscv_vector.h>

 #if __riscv_v_min_vlen == 128
-  #define LMUL_64 mf2
  #define LMUL_128 m1
  #define LMUL_256 m2
  #define LMUL_512 m4
@@ -21,7 +17,6 @@
  #define BOOL_256 b16
  #define BOOL_512 b8
 #elif __riscv_v_min_vlen == 256
-  #define LMUL_64 mf4
  #define LMUL_128 mf2
  #define LMUL_256 m1
  #define LMUL_512 m2
@@ -46,16 +41,6 @@

 // ---- Semantic fixed-vector typedefs (named by element count) ----

-// uint8 / int8
-typedef RVVTYPE(vuint8, LMUL_64, _t) fixed_u8x8_t
-    __attribute__((riscv_rvv_vector_bits(64)));
-typedef RVVTYPE(vint8, LMUL_64, _t) fixed_i8x8_t
-    __attribute__((riscv_rvv_vector_bits(64)));
-
-// int16
-typedef RVVTYPE(vint16, LMUL_128, _t) fixed_i16x8_t
-    __attribute__((riscv_rvv_vector_bits(128)));
-
 // float16
 typedef RVVTYPE(vfloat16, LMUL_128, _t) fixed_fp16x8_t
    __attribute__((riscv_rvv_vector_bits(128)));
@@ -363,13 +363,6 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
    return FP32Vec16(ret);
  }

-  FP32Vec16 tanh() const {
-    f32x16_t ret;
-    unroll_loop<int, VEC_ELEM_NUM>(
-        [&ret, this](int i) { ret.val[i] = std::tanh(reg.val[i]); });
-    return FP32Vec16(ret);
-  }
-
  float reduce_sum() const {
    float result = 0.0f;
    unroll_loop<int, VEC_ELEM_NUM>(
@@ -13,10 +13,10 @@ namespace vec_op {
 struct fp8_e4m3_tag {};
 struct fp8_e5m2_tag {};

-#define VLLM_DISPATCH_CASE_FLOATING_TYPES(...)            \
-  AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__)    \
-  AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__) \
-  AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__)
+// FIXME: FP16 is not fully supported in Torch-CPU
+#define VLLM_DISPATCH_CASE_FLOATING_TYPES(...)         \
+  AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \
+  AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__)

 #define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \
  AT_DISPATCH_SWITCH(TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__))
@@ -34,87 +34,6 @@ struct fp8_e5m2_tag {};
 #define FORCE_INLINE __attribute__((always_inline)) inline

 namespace {
-
-FORCE_INLINE __vector float fp16_to_fp32_bits(__vector unsigned int x) {
-  const __vector unsigned int mask_sign = {0x8000, 0x8000, 0x8000, 0x8000};
-  const __vector unsigned int mask_exp = {0x7C00, 0x7C00, 0x7C00, 0x7C00};
-  const __vector unsigned int mask_mant = {0x03FF, 0x03FF, 0x03FF, 0x03FF};
-  const __vector unsigned int bias_adj = {112, 112, 112, 112};
-  const __vector unsigned int exp_max_fp16 = {0x1F, 0x1F, 0x1F, 0x1F};
-  const __vector unsigned int exp_max_fp32 = {0xFF, 0xFF, 0xFF, 0xFF};
-
-  __vector unsigned int s = (x & mask_sign) << 16;
-  __vector unsigned int e = (x & mask_exp) >> 10;
-  __vector unsigned int m = (x & mask_mant) << 13;
-
-  __vector __bool int is_nan_inf = vec_cmpeq(e, exp_max_fp16);
-
-  __vector unsigned int e_normal = e + bias_adj;
-  e = vec_sel(e_normal, exp_max_fp32, is_nan_inf);
-
-  return (__vector float)(s | (e << 23) | m);
-}
-
-FORCE_INLINE __vector unsigned int fp32_to_fp16_bits(__vector float f_in) {
-  __vector unsigned int in = (__vector unsigned int)f_in;
-
-  const __vector unsigned int mask_sign_32 = {0x80000000, 0x80000000,
-                                              0x80000000, 0x80000000};
-  const __vector unsigned int mask_exp_32 = {0x7F800000, 0x7F800000, 0x7F800000,
-                                             0x7F800000};
-  const __vector unsigned int mask_mant_32 = {0x007FFFFF, 0x007FFFFF,
-                                              0x007FFFFF, 0x007FFFFF};
-
-  const __vector signed int bias_adj = {112, 112, 112, 112};
-  const __vector signed int zero = {0, 0, 0, 0};
-  const __vector signed int max_exp = {31, 31, 31, 31};
-  const __vector unsigned int exp_max_fp32 = {0xFF, 0xFF, 0xFF, 0xFF};
-  const __vector unsigned int exp_max_fp16 = {0x1F, 0x1F, 0x1F, 0x1F};
-
-  __vector unsigned int s = (in & mask_sign_32) >> 16;
-  __vector unsigned int e_u = (in & mask_exp_32) >> 23;
-
-  __vector __bool int is_nan_inf = vec_cmpeq(e_u, exp_max_fp32);
-
-  __vector signed int e_s = (__vector signed int)e_u;
-  e_s = vec_sub(e_s, bias_adj);
-  e_s = vec_max(e_s, zero);
-  e_s = vec_min(e_s, max_exp);
-  __vector unsigned int e_normal = (__vector unsigned int)e_s;
-
-  __vector unsigned int e_final = vec_sel(e_normal, exp_max_fp16, is_nan_inf);
-
-  const __vector unsigned int one_v = {1, 1, 1, 1};
-  const __vector unsigned int mask_sticky = {0xFFF, 0xFFF, 0xFFF, 0xFFF};
-
-  __vector unsigned int round_bit = (in >> 12) & one_v;
-  __vector unsigned int sticky = in & mask_sticky;
-  __vector unsigned int m = (in & mask_mant_32) >> 13;
-  __vector unsigned int lsb = m & one_v;
-
-  // Round up if: round_bit && (sticky || lsb)
-  __vector __bool int sticky_nonzero =
-      vec_cmpgt(sticky, (__vector unsigned int){0, 0, 0, 0});
-  __vector __bool int lsb_set = vec_cmpeq(lsb, one_v);
-  __vector __bool int round_up =
-      vec_and(vec_cmpeq(round_bit, one_v), vec_or(sticky_nonzero, lsb_set));
-
-  m = vec_sel(m, m + one_v, round_up);
-
-  const __vector unsigned int mant_mask = {0x3FF, 0x3FF, 0x3FF, 0x3FF};
-  const __vector unsigned int max_normal_exp = {0x1E, 0x1E, 0x1E, 0x1E};
-  __vector __bool int mant_overflows = vec_cmpgt(m, mant_mask);
-  __vector __bool int would_overflow_to_inf =
-      vec_and(mant_overflows, vec_cmpeq(e_final, max_normal_exp));
-  __vector unsigned int e_inc = vec_min(e_final + one_v, exp_max_fp16);
-  e_final = vec_sel(e_final, e_inc, mant_overflows);
-  m = vec_and(m, mant_mask);
-  e_final = vec_sel(e_final, max_normal_exp, would_overflow_to_inf);
-  m = vec_sel(m, mant_mask, would_overflow_to_inf);
-
-  return s | (e_final << 10) | m;
-}
-
 template <typename T, T... indexes, typename F>
 constexpr void unroll_loop_item(std::integer_sequence<T, indexes...>, F&& f) {
  (f(std::integral_constant<T, indexes>{}), ...);
@@ -170,19 +89,6 @@ struct BF16Vec8 : public Vec<BF16Vec8> {
  }
 };

-struct FP16Vec8 : public Vec<FP16Vec8> {
-  constexpr static int VEC_ELEM_NUM = 8;
-
-  __vector signed short reg;
-
-  explicit FP16Vec8(const void* ptr) : reg(*(__vector signed short*)ptr) {}
-  explicit FP16Vec8(const FP32Vec8&);
-
-  void save(void* ptr) const {
-    *reinterpret_cast<__vector signed short*>(ptr) = reg;
-  }
-};
-
 struct FP16Vec16 : public Vec<FP16Vec16> {
  constexpr static int VEC_ELEM_NUM = 16;
  ss16x8x2_t reg;
@@ -218,11 +124,13 @@ struct BF16Vec16 : public Vec<BF16Vec16> {
  ss16x8x2_t reg;

  explicit BF16Vec16(const void* ptr) {
+    // Load 256 bits in two parts
    reg.val[0] = (__vector signed short)vec_xl(0, (signed short*)ptr);
    reg.val[1] = (__vector signed short)vec_xl(16, (signed short*)ptr);
  }

  explicit BF16Vec16(bool, const void* ptr) : BF16Vec16(ptr) {}
+
  explicit BF16Vec16(const FP32Vec16&);

  void save(void* ptr) const {
@@ -234,16 +142,20 @@ struct BF16Vec16 : public Vec<BF16Vec16> {
  void save(void* ptr, const int elem_num) const {
    const int clamped_elem = std::max(0, std::min(elem_num, 16));

+    // Calculate elements to store in each 128-bit part (8 elements each)
    const int elements_val0 = std::min(clamped_elem, 8);
    const int elements_val1 = std::max(clamped_elem - 8, 0);

+    // Convert elements to bytes (2 bytes per element)
    const size_t bytes_val0 = elements_val0 * sizeof(signed short);
    const size_t bytes_val1 = elements_val1 * sizeof(signed short);

    signed short* dest = static_cast<signed short*>(ptr);
+    // Store the first part using vec_xst_len
    if (bytes_val0 > 0) {
      vec_xst_len(reg.val[0], dest, bytes_val0);
    }
+    // Store the second part if needed
    if (bytes_val1 > 0) {
      vec_xst_len(reg.val[1], dest + elements_val0, bytes_val1);
    }
@@ -326,15 +238,6 @@ struct FP32Vec8 : public Vec<FP32Vec8> {
    reg.val[1] = (__vector float)vec_mergel(zero, v.reg);
  }

-  explicit FP32Vec8(const FP16Vec8& v) {
-    __vector unsigned short raw_u = (__vector unsigned short)v.reg;
-    __vector unsigned int raw_hi =
-        (__vector unsigned int)vec_unpackh((__vector signed short)raw_u);
-    __vector unsigned int raw_lo =
-        (__vector unsigned int)vec_unpackl((__vector signed short)raw_u);
-    reg.val[0] = fp16_to_fp32_bits(raw_hi);
-    reg.val[1] = fp16_to_fp32_bits(raw_lo);
-  }
  float reduce_sum() const {
    AliasReg ar;
    ar.reg = reg;
@@ -507,9 +410,8 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
    reg.val[3] = vec_xl(48, ptr);
  }

-  explicit FP32Vec16(const c10::Half* ptr) : FP32Vec16(FP16Vec16(ptr)) {}
-  explicit FP32Vec16(const FP16Vec16&);
  explicit FP32Vec16(bool, const float* ptr) : FP32Vec16(ptr) {}
+
  explicit FP32Vec16(f32x4x4_t data) : reg(data) {}

  explicit FP32Vec16(const FP32Vec16& data) {
@@ -533,6 +435,7 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
    reg.val[3] = data.reg.val[1];
  }

+  explicit FP32Vec16(const FP16Vec16& v);
  explicit FP32Vec16(const BF16Vec16& v) {
    reg.val[0] = (__vector float)vec_mergeh(zero, v.reg.val[0]);
    reg.val[1] = (__vector float)vec_mergel(zero, v.reg.val[0]);
@@ -599,20 +502,28 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
  FP32Vec16 max(const FP32Vec16& b, int elem_num) const {
    FP32Vec16 result;

+    // Create a vector of element indices for each chunk
    __vector unsigned int indices = {0, 1, 2, 3};
    __vector unsigned int elem_num_vec =
        vec_splats(static_cast<unsigned int>(elem_num));

-    __vector unsigned int chunk_offset0 = {0, 0, 0, 0};
-    __vector unsigned int chunk_offset1 = {4, 4, 4, 4};
-    __vector unsigned int chunk_offset2 = {8, 8, 8, 8};
-    __vector unsigned int chunk_offset3 = {12, 12, 12, 12};
+    // Compute masks for each chunk
+    __vector unsigned int chunk_offset0 = {0, 0, 0,
+                                           0};  // Chunk 0: Elements 0-3
+    __vector unsigned int chunk_offset1 = {4, 4, 4,
+                                           4};  // Chunk 1: Elements 4-7
+    __vector unsigned int chunk_offset2 = {8, 8, 8,
+                                           8};  // Chunk 2: Elements 8-11
+    __vector unsigned int chunk_offset3 = {12, 12, 12,
+                                           12};  // Chunk 3: Elements 12-15

+    // Compute masks for each chunk
    __vector bool int mask0 = vec_cmplt(indices + chunk_offset0, elem_num_vec);
    __vector bool int mask1 = vec_cmplt(indices + chunk_offset1, elem_num_vec);
    __vector bool int mask2 = vec_cmplt(indices + chunk_offset2, elem_num_vec);
    __vector bool int mask3 = vec_cmplt(indices + chunk_offset3, elem_num_vec);

+    // Apply masks to compute the result for each chunk
    result.reg.val[0] = vec_sel(this->reg.val[0],
                                vec_max(this->reg.val[0], b.reg.val[0]), mask0);
    result.reg.val[1] = vec_sel(this->reg.val[1],
@@ -715,16 +626,6 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
    vec_xst(reg.val[3], 48, ptr);
  }

-  void save(c10::Half* ptr) const {
-    FP16Vec16 fp16_vec(*this);
-    fp16_vec.save(ptr);
-  }
-
-  void save(c10::Half* ptr, const int elem_num) const {
-    FP16Vec16 fp16_vec(*this);
-    fp16_vec.save(ptr, elem_num);
-  }
-
  void save(float* ptr, const int elem_num) const {
    const int elements_in_chunk1 =
        (elem_num >= 0) ? ((elem_num >= 4) ? 4 : elem_num) : 0;
@@ -758,7 +659,7 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
 };

 struct INT8Vec16 : public Vec<INT8Vec16> {
-  constexpr static int VEC_NUM_ELEM = 16;
+  constexpr static int VEC_NUM_ELEM = 16;  // 128 bits / 8 bits = 16

  union AliasReg {
    __vector signed char reg;
@@ -806,11 +707,6 @@ struct VecType<c10::BFloat16> {
  using vec_type = BF16Vec8;
 };

-template <>
-struct VecType<c10::Half> {
-  using vec_type = FP16Vec8;
-};
-
 template <typename T>
 void storeFP32(float v, T* ptr) {
  *ptr = v;
@@ -827,15 +723,6 @@ inline void storeFP32<c10::BFloat16>(float v, c10::BFloat16* ptr) {
  *ptr = *(v_ptr + 1);
 }

-template <>
-inline void storeFP32<c10::Half>(float v, c10::Half* ptr) {
-  __vector float v_vec = {v, 0.0f, 0.0f, 0.0f};
-  __vector unsigned int fp16_bits = fp32_to_fp16_bits(v_vec);
-  unsigned short result =
-      (unsigned short)((__vector unsigned short)fp16_bits)[0];
-  *reinterpret_cast<unsigned short*>(ptr) = result;
-}
-
 #ifndef __VEC_CLASS_FP_NAN
  #define __VEC_CLASS_FP_NAN (1 << 6)
 #endif
@@ -882,39 +769,38 @@ inline BF16Vec8::BF16Vec8(const FP32Vec8& v) {
 #endif
 }

-inline FP16Vec8::FP16Vec8(const FP32Vec8& v) {
-  __vector unsigned int fp16_hi = fp32_to_fp16_bits(v.reg.val[0]);
-  __vector unsigned int fp16_lo = fp32_to_fp16_bits(v.reg.val[1]);
-  reg = (__vector signed short)vec_perm((__vector unsigned char)fp16_hi,
-                                        (__vector unsigned char)fp16_lo, omask);
-}
-
 inline FP16Vec16::FP16Vec16(const FP32Vec16& v) {
-  __vector unsigned int fp16_0 = fp32_to_fp16_bits(v.reg.val[0]);
-  __vector unsigned int fp16_1 = fp32_to_fp16_bits(v.reg.val[1]);
-  __vector unsigned int fp16_2 = fp32_to_fp16_bits(v.reg.val[2]);
-  __vector unsigned int fp16_3 = fp32_to_fp16_bits(v.reg.val[3]);
-  reg.val[0] = (__vector signed short)vec_perm(
-      (__vector unsigned char)fp16_0, (__vector unsigned char)fp16_1, omask);
-  reg.val[1] = (__vector signed short)vec_perm(
-      (__vector unsigned char)fp16_2, (__vector unsigned char)fp16_3, omask);
+  alignas(16) float temp_fp32[16];
+  alignas(16) c10::Half temp_fp16[16];
+
+  vec_xst(v.reg.val[0], 0, temp_fp32);
+  vec_xst(v.reg.val[1], 16, temp_fp32);
+  vec_xst(v.reg.val[2], 32, temp_fp32);
+  vec_xst(v.reg.val[3], 48, temp_fp32);
+
+  for (int i = 0; i < 16; i++) {
+    temp_fp16[i] = c10::Half(temp_fp32[i]);
+  }
+
+  reg.val[0] = (__vector signed short)vec_xl(0, (signed short*)temp_fp16);
+  reg.val[1] = (__vector signed short)vec_xl(16, (signed short*)temp_fp16);
 }

 inline FP32Vec16::FP32Vec16(const FP16Vec16& v) {
-  __vector unsigned short raw_u0 = (__vector unsigned short)v.reg.val[0];
-  __vector unsigned short raw_u1 = (__vector unsigned short)v.reg.val[1];
-  __vector unsigned int raw_hi0 =
-      (__vector unsigned int)vec_unpackh((__vector signed short)raw_u0);
-  __vector unsigned int raw_lo0 =
-      (__vector unsigned int)vec_unpackl((__vector signed short)raw_u0);
-  __vector unsigned int raw_hi1 =
-      (__vector unsigned int)vec_unpackh((__vector signed short)raw_u1);
-  __vector unsigned int raw_lo1 =
-      (__vector unsigned int)vec_unpackl((__vector signed short)raw_u1);
-  reg.val[0] = fp16_to_fp32_bits(raw_hi0);
-  reg.val[1] = fp16_to_fp32_bits(raw_lo0);
-  reg.val[2] = fp16_to_fp32_bits(raw_hi1);
-  reg.val[3] = fp16_to_fp32_bits(raw_lo1);
+  alignas(16) c10::Half temp_fp16[16];
+  alignas(16) float temp_fp32[16];
+
+  vec_xst(v.reg.val[0], 0, (signed short*)temp_fp16);
+  vec_xst(v.reg.val[1], 16, (signed short*)temp_fp16);
+
+  for (int i = 0; i < 16; i++) {
+    temp_fp32[i] = float(temp_fp16[i]);
+  }
+
+  reg.val[0] = vec_xl(0, temp_fp32);
+  reg.val[1] = vec_xl(16, temp_fp32);
+  reg.val[2] = vec_xl(32, temp_fp32);
+  reg.val[3] = vec_xl(48, temp_fp32);
 }

 inline BF16Vec16::BF16Vec16(const FP32Vec16& v) {
@@ -978,6 +864,7 @@ inline void prefetch(const void* addr) {

 struct INT8Vec64 {
  __vector signed char data[4];
+
  INT8Vec64() = default;

  explicit INT8Vec64(const int8_t* ptr) {
@@ -1013,4 +900,5 @@ struct INT8Vec64 {
  void nt_save(int8_t* ptr) const { save(ptr); }
 };
 }  // namespace vec_op
+
 #endif
@@ -3,7 +3,6 @@
 #define CPU_TYPES_X86_HPP

 #include <immintrin.h>
-#include <sleef.h>
 #include <torch/all.h>

 #ifndef __AVX2__
@@ -593,8 +592,6 @@ struct FP32Vec16 : public Vec<FP32Vec16> {

  FP32Vec16 abs() const { return FP32Vec16(_mm512_abs_ps(reg)); }

-  FP32Vec16 tanh() const { return FP32Vec16(Sleef_tanhf16_u10(reg)); }
-
  float reduce_sum() const { return _mm512_reduce_add_ps(reg); }

  float reduce_max() const { return _mm512_reduce_max_ps(reg); }
@@ -792,12 +789,6 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
                     _mm256_andnot_ps(sign_mask, reg_high));
  }

-  FP32Vec16 tanh() const {
-    FP32Vec8 low(reg_low);
-    FP32Vec8 high(reg_high);
-    return FP32Vec16(low.tanh().reg, high.tanh().reg);
-  }
-
  FP32Vec16 min(const FP32Vec16& b) const {
    return FP32Vec16(_mm256_min_ps(reg_low, b.reg_low),
                     _mm256_min_ps(reg_high, b.reg_high));
@@ -4,9 +4,6 @@
 #ifdef CPU_CAPABILITY_AMXBF16
  #include "cpu/micro_gemm/cpu_micro_gemm_amx.hpp"
 #endif
-#if defined(__riscv_v)
-  #include "cpu/micro_gemm/cpu_micro_gemm_rvv.hpp"
-#endif
 #include "cpu/micro_gemm/cpu_micro_gemm_vec.hpp"

 #define VLLM_DISPATCH_CASE_16B_TYPES(...)                 \
@@ -155,7 +152,7 @@ void cpu_gemm_wna16_impl(
  constexpr int32_t gemm_m_tile_size = gemm_t::MaxMSize;
  constexpr int32_t n_block_size = 16;
  static_assert(gemm_n_tile_size % n_block_size == 0);
-  const int32_t thread_num = cpu_utils::get_max_threads();
+  const int32_t thread_num = omp_get_max_threads();

  // a simple schedule policy, just to hold more B tiles in L2 and make sure
  // each thread has tasks
@@ -322,8 +319,6 @@ void cpu_gemm_wna16(
      return ISA::AMX;
    } else if (isa_hint == "vec") {
      return ISA::VEC;
-    } else if (isa_hint == "rvv") {
-      return ISA::RVV;
    } else {
      TORCH_CHECK(false, "unsupported isa hint: " + isa_hint);
    }
@@ -402,40 +397,6 @@ void cpu_gemm_wna16(
            pack_factor);
        return;
      }
-    } else if (isa == ISA::RVV) {
-      using gemm_t = cpu_micro_gemm::MicroGemm<ISA::RVV, scalar_t>;
-      if (has_zp) {
-        using dequantizer_t = Dequantizer4b<scalar_t, ISA::RVV, true, false>;
-        cpu_gemm_wna16_impl<scalar_t, dequantizer_t, gemm_t>(
-            input.data_ptr<scalar_t>(), q_weight.data_ptr<int32_t>(),
-            output.data_ptr<scalar_t>(), scales.data_ptr<scalar_t>(), zeros_ptr,
-            g_idx_ptr, bias.has_value() ? bias->data_ptr<scalar_t>() : nullptr,
-            a_m_size, b_n_size, a_k_size, a_m_stride, output_m_stride,
-            scales_group_stride, zeros_group_stride, group_num, group_size,
-            pack_factor);
-        return;
-      }
-      if (use_desc_act) {
-        using dequantizer_t = Dequantizer4b<scalar_t, ISA::RVV, false, true>;
-        cpu_gemm_wna16_impl<scalar_t, dequantizer_t, gemm_t>(
-            input.data_ptr<scalar_t>(), q_weight.data_ptr<int32_t>(),
-            output.data_ptr<scalar_t>(), scales.data_ptr<scalar_t>(), zeros_ptr,
-            g_idx_ptr, bias.has_value() ? bias->data_ptr<scalar_t>() : nullptr,
-            a_m_size, b_n_size, a_k_size, a_m_stride, output_m_stride,
-            scales_group_stride, zeros_group_stride, group_num, group_size,
-            pack_factor);
-        return;
-      } else {
-        using dequantizer_t = Dequantizer4b<scalar_t, ISA::RVV, false, false>;
-        cpu_gemm_wna16_impl<scalar_t, dequantizer_t, gemm_t>(
-            input.data_ptr<scalar_t>(), q_weight.data_ptr<int32_t>(),
-            output.data_ptr<scalar_t>(), scales.data_ptr<scalar_t>(), zeros_ptr,
-            g_idx_ptr, bias.has_value() ? bias->data_ptr<scalar_t>() : nullptr,
-            a_m_size, b_n_size, a_k_size, a_m_stride, output_m_stride,
-            scales_group_stride, zeros_group_stride, group_num, group_size,
-            pack_factor);
-        return;
-      }
    }
  });
 }
@@ -202,7 +202,7 @@ void dynamic_quant_epilogue(const float* input, scalar_t* output,
  using cvt_vec_t = typename KernelVecType<scalar_t>::cvt_vec_type;
  constexpr int vec_elem_num = load_vec_t::VEC_ELEM_NUM;

-  const int64_t thread_num = cpu_utils::get_max_threads();
+  const int64_t thread_num = omp_get_max_threads();
  if (num_tokens > thread_num) {
 #pragma omp parallel for
    for (int64_t i = 0; i < num_tokens; ++i) {
@@ -4,9 +4,8 @@ namespace {
 template <typename scalar_t>
 void rms_norm_impl(scalar_t* __restrict__ out,
                   const scalar_t* __restrict__ input,
-                   const scalar_t* __restrict__ weight, const bool has_weight,
-                   const float epsilon, const int num_tokens,
-                   const int hidden_size) {
+                   const scalar_t* __restrict__ weight, const float epsilon,
+                   const int num_tokens, const int hidden_size) {
  using scalar_vec_t = vec_op::vec_t<scalar_t>;
  constexpr int VEC_ELEM_NUM = scalar_vec_t::get_elem_num();
  TORCH_CHECK(hidden_size % VEC_ELEM_NUM == 0);
@@ -28,15 +27,12 @@ void rms_norm_impl(scalar_t* __restrict__ out,

    for (int j = 0; j < hidden_size; j += VEC_ELEM_NUM) {
      scalar_vec_t x(input_p + j);
+      scalar_vec_t w(weight + j);
+
      vec_op::FP32Vec8 fp32_x(x);
-      vec_op::FP32Vec8 fp32_out;
-      if (has_weight) {
-        scalar_vec_t w(weight + j);
-        vec_op::FP32Vec8 fp32_w(w);
-        fp32_out = fp32_x * fp32_s_variance * fp32_w;
-      } else {
-        fp32_out = fp32_x * fp32_s_variance;
-      }
+      vec_op::FP32Vec8 fp32_w(w);
+
+      vec_op::FP32Vec8 fp32_out = fp32_x * fp32_s_variance * fp32_w;

      scalar_vec_t out(fp32_out);
      out.save(output_p + j);
@@ -48,8 +44,8 @@ template <typename scalar_t>
 void fused_add_rms_norm_impl(scalar_t* __restrict__ input,
                             scalar_t* __restrict__ residual,
                             const scalar_t* __restrict__ weight,
-                             const bool has_weight, const float epsilon,
-                             const int num_tokens, const int hidden_size) {
+                             const float epsilon, const int num_tokens,
+                             const int hidden_size) {
  using scalar_vec_t = vec_op::vec_t<scalar_t>;
  constexpr int VEC_ELEM_NUM = scalar_vec_t::get_elem_num();
  TORCH_CHECK(hidden_size % VEC_ELEM_NUM == 0);
@@ -76,18 +72,13 @@ void fused_add_rms_norm_impl(scalar_t* __restrict__ input,
    vec_op::FP32Vec8 fp32_s_variance(s_variance);

    for (int j = 0; j < hidden_size; j += VEC_ELEM_NUM) {
-      vec_op::FP32Vec8 fp32_out;
-      if (has_weight) {
-        scalar_vec_t w(weight + j);
-        scalar_vec_t res(residual_p + j);
-        vec_op::FP32Vec8 fp32_w(w);
-        vec_op::FP32Vec8 fp32_res(res);
-        fp32_out = fp32_res * fp32_s_variance * fp32_w;
-      } else {
-        scalar_vec_t res(residual_p + j);
-        vec_op::FP32Vec8 fp32_res(res);
-        fp32_out = fp32_res * fp32_s_variance;
-      }
+      scalar_vec_t w(weight + j);
+      scalar_vec_t res(residual_p + j);
+
+      vec_op::FP32Vec8 fp32_w(w);
+      vec_op::FP32Vec8 fp32_res(res);
+
+      vec_op::FP32Vec8 fp32_out = fp32_res * fp32_s_variance * fp32_w;

      scalar_vec_t out(fp32_out);
      out.save(input_p + j);
@@ -96,41 +87,31 @@ void fused_add_rms_norm_impl(scalar_t* __restrict__ input,
 }
 }  // namespace

-void rms_norm(torch::Tensor& out, torch::Tensor& input,
-              std::optional<torch::Tensor> weight, double epsilon) {
+void rms_norm(torch::Tensor& out, torch::Tensor& input, torch::Tensor& weight,
+              double epsilon) {
  int hidden_size = input.size(-1);
  int num_tokens = input.numel() / hidden_size;
-  const bool has_weight = weight.has_value();
-  if (has_weight) {
-    TORCH_CHECK(weight->is_contiguous());
-  }

  VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "rms_norm_impl", [&] {
    CPU_KERNEL_GUARD_IN(rms_norm_impl)
    rms_norm_impl(out.data_ptr<scalar_t>(), input.data_ptr<scalar_t>(),
-                  has_weight ? weight->data_ptr<scalar_t>() : nullptr,
-                  has_weight, epsilon, num_tokens, hidden_size);
+                  weight.data_ptr<scalar_t>(), epsilon, num_tokens,
+                  hidden_size);
    CPU_KERNEL_GUARD_OUT(rms_norm_impl)
  });
 }

 void fused_add_rms_norm(torch::Tensor& input, torch::Tensor& residual,
-                        std::optional<torch::Tensor> weight, double epsilon) {
+                        torch::Tensor& weight, double epsilon) {
  int hidden_size = input.size(-1);
  int num_tokens = input.numel() / hidden_size;
-  const bool has_weight = weight.has_value();
-  if (has_weight) {
-    TORCH_CHECK(weight->scalar_type() == input.scalar_type());
-    TORCH_CHECK(weight->is_contiguous());
-  }

  VLLM_DISPATCH_FLOATING_TYPES(
      input.scalar_type(), "fused_add_rms_norm_impl", [&] {
        CPU_KERNEL_GUARD_IN(fused_add_rms_norm_impl)
        fused_add_rms_norm_impl(
            input.data_ptr<scalar_t>(), residual.data_ptr<scalar_t>(),
-            has_weight ? weight->data_ptr<scalar_t>() : nullptr, has_weight,
-            epsilon, num_tokens, hidden_size);
+            weight.data_ptr<scalar_t>(), epsilon, num_tokens, hidden_size);
        CPU_KERNEL_GUARD_OUT(fused_add_rms_norm_impl)
      });
 }
@@ -213,8 +213,6 @@ class MicroGemm<cpu_utils::ISA::AMX, scalar_t> {
 public:
  static constexpr int32_t MaxMSize = 32;
  static constexpr int32_t NSize = 32;
-  static constexpr int32_t WeightOCGroupSize = 16;
-  static constexpr bool PackA = false;

 public:
  MicroGemm() : curr_m_(-1) {
@@ -21,9 +21,6 @@ class MicroGemm {
 public:
  static constexpr int32_t MaxMSize = 16;
  static constexpr int32_t NSize = 16;
-  static constexpr int32_t WeightOCGroupSize = 16;
-  // callers must pack A matrix before GEMM
-  static constexpr bool PackA = false;

 public:
  void gemm(DEFINE_CPU_MICRO_GEMM_PARAMS) {
@@ -1,503 +0,0 @@
-#ifndef CPU_MICRO_GEMM_NEON_HPP
-#define CPU_MICRO_GEMM_NEON_HPP
-
-#include <algorithm>
-#include <cstdint>
-
-#include "cpu/micro_gemm/cpu_micro_gemm_impl.hpp"
-
-#include <arm_bf16.h>
-#include <arm_neon.h>
-
-namespace cpu_micro_gemm {
-
-namespace {
-
-constexpr int32_t K = 4;
-constexpr int32_t Cols = 2;
-constexpr int32_t TileSize = K * Cols;
-constexpr int32_t Mr = 8;
-constexpr int32_t Nr = 8;
-constexpr int32_t Nr_gemv = 16;
-
-// a = [a0, a1, a2, a3], b = [b0, b1, b2, b3] -> [a0, a1, b0, b1]
-FORCE_INLINE float32x4_t zip1_f32x4(const float32x4_t a, const float32x4_t b) {
-  return vreinterpretq_f32_f64(
-      vzip1q_f64(vreinterpretq_f64_f32(a), vreinterpretq_f64_f32(b)));
-}
-
-// a = [a0, a1, a2, a3], b = [b0, b1, b2, b3] -> [a2, a3, b2, b3]
-FORCE_INLINE float32x4_t zip2_f32x4(const float32x4_t a, const float32x4_t b) {
-  return vreinterpretq_f32_f64(
-      vzip2q_f64(vreinterpretq_f64_f32(a), vreinterpretq_f64_f32(b)));
-}
-
-FORCE_INLINE void init_acc_rowpair(float32x4_t& acc01, float32x4_t& acc23,
-                                   float32x4_t& acc45, float32x4_t& acc67,
-                                   const float* __restrict__ c_ptr,
-                                   const int64_t ldc, const int32_t m_rows,
-                                   const bool accum_c) {
-  if (!accum_c || m_rows == 0) {
-    acc01 = vdupq_n_f32(0.0f);
-    acc23 = vdupq_n_f32(0.0f);
-    acc45 = vdupq_n_f32(0.0f);
-    acc67 = vdupq_n_f32(0.0f);
-    return;
-  }
-
-  const float32x4_t row0_0123 = vld1q_f32(c_ptr);
-  const float32x4_t row0_4567 = vld1q_f32(c_ptr + 4);
-  const float32x4_t row1_0123 =
-      (m_rows == 2) ? vld1q_f32(c_ptr + ldc) : vdupq_n_f32(0.0f);
-  const float32x4_t row1_4567 =
-      (m_rows == 2) ? vld1q_f32(c_ptr + ldc + 4) : vdupq_n_f32(0.0f);
-
-  acc01 = zip1_f32x4(row0_0123, row1_0123);
-  acc23 = zip2_f32x4(row0_0123, row1_0123);
-  acc45 = zip1_f32x4(row0_4567, row1_4567);
-  acc67 = zip2_f32x4(row0_4567, row1_4567);
-}
-
-FORCE_INLINE void store_acc_rowpair(const float32x4_t acc01,
-                                    const float32x4_t acc23,
-                                    const float32x4_t acc45,
-                                    const float32x4_t acc67,
-                                    float* __restrict__ c_ptr,
-                                    const int64_t ldc, const int32_t m_rows) {
-  if (m_rows == 0) {
-    return;
-  }
-
-  vst1q_f32(c_ptr, zip1_f32x4(acc01, acc23));
-  vst1q_f32(c_ptr + 4, zip1_f32x4(acc45, acc67));
-
-  if (m_rows == 2) {
-    vst1q_f32(c_ptr + ldc, zip2_f32x4(acc01, acc23));
-    vst1q_f32(c_ptr + ldc + 4, zip2_f32x4(acc45, acc67));
-  }
-}
-
-FORCE_INLINE void gemm_micro_bfmmla_8x8_packed_a(
-    const bfloat16_t* __restrict__ a_packed,
-    const bfloat16_t* __restrict__ b_packed, float* __restrict__ c_ptr,
-    const int32_t m, const int32_t k_size, const int64_t ldc,
-    const bool accum_c) {
-  float32x4_t acc0101, acc0123, acc0145, acc0167;
-  float32x4_t acc2301, acc2323, acc2345, acc2367;
-  float32x4_t acc4501, acc4523, acc4545, acc4567;
-  float32x4_t acc6701, acc6723, acc6745, acc6767;
-
-  init_acc_rowpair(acc0101, acc0123, acc0145, acc0167, c_ptr, ldc,
-                   std::min(2, m), accum_c);
-  init_acc_rowpair(acc2301, acc2323, acc2345, acc2367, c_ptr + 2 * ldc, ldc,
-                   std::min(2, std::max(0, m - 2)), accum_c);
-  init_acc_rowpair(acc4501, acc4523, acc4545, acc4567, c_ptr + 4 * ldc, ldc,
-                   std::min(2, std::max(0, m - 4)), accum_c);
-  init_acc_rowpair(acc6701, acc6723, acc6745, acc6767, c_ptr + 6 * ldc, ldc,
-                   std::min(2, std::max(0, m - 6)), accum_c);
-
-  const bfloat16_t* __restrict__ a_tile = a_packed;
-  const bfloat16_t* __restrict__ b_tile = b_packed;
-
-#pragma GCC unroll 8
-  for (int32_t k_idx = 0; k_idx < k_size; k_idx += K) {
-    const bfloat16x8_t a_tile01 = vld1q_bf16(a_tile);
-    const bfloat16x8_t a_tile23 = vld1q_bf16(a_tile + TileSize);
-    const bfloat16x8_t a_tile45 = vld1q_bf16(a_tile + 2 * TileSize);
-    const bfloat16x8_t a_tile67 = vld1q_bf16(a_tile + 3 * TileSize);
-
-    const bfloat16x8_t b_tile01 = vld1q_bf16(b_tile);
-    const bfloat16x8_t b_tile23 = vld1q_bf16(b_tile + TileSize);
-    const bfloat16x8_t b_tile45 = vld1q_bf16(b_tile + 2 * TileSize);
-    const bfloat16x8_t b_tile67 = vld1q_bf16(b_tile + 3 * TileSize);
-
-    acc0101 = vbfmmlaq_f32(acc0101, a_tile01, b_tile01);
-    acc2301 = vbfmmlaq_f32(acc2301, a_tile23, b_tile01);
-    acc4501 = vbfmmlaq_f32(acc4501, a_tile45, b_tile01);
-    acc6701 = vbfmmlaq_f32(acc6701, a_tile67, b_tile01);
-
-    acc0123 = vbfmmlaq_f32(acc0123, a_tile01, b_tile23);
-    acc2323 = vbfmmlaq_f32(acc2323, a_tile23, b_tile23);
-    acc4523 = vbfmmlaq_f32(acc4523, a_tile45, b_tile23);
-    acc6723 = vbfmmlaq_f32(acc6723, a_tile67, b_tile23);
-
-    acc0145 = vbfmmlaq_f32(acc0145, a_tile01, b_tile45);
-    acc2345 = vbfmmlaq_f32(acc2345, a_tile23, b_tile45);
-    acc4545 = vbfmmlaq_f32(acc4545, a_tile45, b_tile45);
-    acc6745 = vbfmmlaq_f32(acc6745, a_tile67, b_tile45);
-
-    acc0167 = vbfmmlaq_f32(acc0167, a_tile01, b_tile67);
-    acc2367 = vbfmmlaq_f32(acc2367, a_tile23, b_tile67);
-    acc4567 = vbfmmlaq_f32(acc4567, a_tile45, b_tile67);
-    acc6767 = vbfmmlaq_f32(acc6767, a_tile67, b_tile67);
-
-    a_tile += 4 * TileSize;
-    b_tile += Nr * K;
-  }
-
-  store_acc_rowpair(acc0101, acc0123, acc0145, acc0167, c_ptr, ldc,
-                    std::min(2, m));
-  store_acc_rowpair(acc2301, acc2323, acc2345, acc2367, c_ptr + 2 * ldc, ldc,
-                    std::min(2, std::max(0, m - 2)));
-  store_acc_rowpair(acc4501, acc4523, acc4545, acc4567, c_ptr + 4 * ldc, ldc,
-                    std::min(2, std::max(0, m - 4)));
-  store_acc_rowpair(acc6701, acc6723, acc6745, acc6767, c_ptr + 6 * ldc, ldc,
-                    std::min(2, std::max(0, m - 6)));
-}
-
-FORCE_INLINE void gemm_micro_bfmmla_4x16_packed_a(
-    const bfloat16_t* __restrict__ a_packed,
-    const bfloat16_t* __restrict__ b_packed, float* __restrict__ c_ptr,
-    const int32_t m, const int32_t k_size, const int64_t b_n_group_stride,
-    const int64_t ldc, const bool accum_c) {
-  const int32_t m_rows_01 = std::min(2, m);
-  const int32_t m_rows_23 = std::min(2, std::max(0, m - 2));
-
-  float32x4_t acc0101, acc0123, acc0145, acc0167;
-  float32x4_t acc2301, acc2323, acc2345, acc2367;
-  float32x4_t acc0189, acc011011, acc011213, acc011415;
-  float32x4_t acc2389, acc231011, acc231213, acc231415;
-
-  init_acc_rowpair(acc0101, acc0123, acc0145, acc0167, c_ptr, ldc, m_rows_01,
-                   accum_c);
-  init_acc_rowpair(acc2301, acc2323, acc2345, acc2367, c_ptr + 2 * ldc, ldc,
-                   m_rows_23, accum_c);
-  init_acc_rowpair(acc0189, acc011011, acc011213, acc011415, c_ptr + 8, ldc,
-                   m_rows_01, accum_c);
-  init_acc_rowpair(acc2389, acc231011, acc231213, acc231415,
-                   c_ptr + 2 * ldc + 8, ldc, m_rows_23, accum_c);
-
-  const bfloat16_t* __restrict__ a_tile = a_packed;
-  const bfloat16_t* __restrict__ b_tile0 = b_packed;
-  const bfloat16_t* __restrict__ b_tile1 = b_packed + b_n_group_stride;
-
-#pragma GCC unroll 8
-  for (int32_t k_idx = 0; k_idx < k_size; k_idx += K) {
-    const bfloat16x8_t a_tile01 = vld1q_bf16(a_tile);
-    const bfloat16x8_t a_tile23 = vld1q_bf16(a_tile + TileSize);
-    const bfloat16x8_t b_tile01 = vld1q_bf16(b_tile0);
-    const bfloat16x8_t b_tile23 = vld1q_bf16(b_tile0 + TileSize);
-    const bfloat16x8_t b_tile45 = vld1q_bf16(b_tile0 + 2 * TileSize);
-    const bfloat16x8_t b_tile67 = vld1q_bf16(b_tile0 + 3 * TileSize);
-    const bfloat16x8_t b_tile89 = vld1q_bf16(b_tile1);
-    const bfloat16x8_t b_tile1011 = vld1q_bf16(b_tile1 + TileSize);
-    const bfloat16x8_t b_tile1213 = vld1q_bf16(b_tile1 + 2 * TileSize);
-    const bfloat16x8_t b_tile1415 = vld1q_bf16(b_tile1 + 3 * TileSize);
-
-    acc0101 = vbfmmlaq_f32(acc0101, a_tile01, b_tile01);
-    acc2301 = vbfmmlaq_f32(acc2301, a_tile23, b_tile01);
-    acc0123 = vbfmmlaq_f32(acc0123, a_tile01, b_tile23);
-    acc2323 = vbfmmlaq_f32(acc2323, a_tile23, b_tile23);
-
-    acc0145 = vbfmmlaq_f32(acc0145, a_tile01, b_tile45);
-    acc2345 = vbfmmlaq_f32(acc2345, a_tile23, b_tile45);
-    acc0167 = vbfmmlaq_f32(acc0167, a_tile01, b_tile67);
-    acc2367 = vbfmmlaq_f32(acc2367, a_tile23, b_tile67);
-
-    acc0189 = vbfmmlaq_f32(acc0189, a_tile01, b_tile89);
-    acc2389 = vbfmmlaq_f32(acc2389, a_tile23, b_tile89);
-    acc011011 = vbfmmlaq_f32(acc011011, a_tile01, b_tile1011);
-    acc231011 = vbfmmlaq_f32(acc231011, a_tile23, b_tile1011);
-
-    acc011213 = vbfmmlaq_f32(acc011213, a_tile01, b_tile1213);
-    acc231213 = vbfmmlaq_f32(acc231213, a_tile23, b_tile1213);
-    acc011415 = vbfmmlaq_f32(acc011415, a_tile01, b_tile1415);
-    acc231415 = vbfmmlaq_f32(acc231415, a_tile23, b_tile1415);
-
-    a_tile += 2 * TileSize;
-    b_tile0 += Nr * K;
-    b_tile1 += Nr * K;
-  }
-
-  store_acc_rowpair(acc0101, acc0123, acc0145, acc0167, c_ptr, ldc, m_rows_01);
-  store_acc_rowpair(acc2301, acc2323, acc2345, acc2367, c_ptr + 2 * ldc, ldc,
-                    m_rows_23);
-  store_acc_rowpair(acc0189, acc011011, acc011213, acc011415, c_ptr + 8, ldc,
-                    m_rows_01);
-  store_acc_rowpair(acc2389, acc231011, acc231213, acc231415,
-                    c_ptr + 2 * ldc + 8, ldc, m_rows_23);
-}
-
-}  // namespace
-
-template <typename scalar_t>
-class MicroGemm<cpu_utils::ISA::NEON, scalar_t> {
- public:
-  static constexpr int32_t MaxMSize = 8;
-  static constexpr int32_t NSize = 32;
-  static constexpr int32_t WeightOCGroupSize = Nr;
-  static constexpr bool PackA = false;
-
- public:
-  void gemm(DEFINE_CPU_MICRO_GEMM_PARAMS) {
-    TORCH_CHECK(false, "NEON BFMMLA MicroGemm only supports bfloat16.");
-  }
-
-  static void pack_weight(const scalar_t* __restrict__ /*weight*/,
-                          scalar_t* __restrict__ /*packed_weight*/,
-                          const int32_t /*output_size*/,
-                          const int32_t /*input_size*/) {
-    TORCH_CHECK(false, "NEON BFMMLA MicroGemm only supports bfloat16.");
-  }
-};
-
-template <>
-class MicroGemm<cpu_utils::ISA::NEON, c10::BFloat16> {
- public:
-  using scalar_t = c10::BFloat16;
-
-  static constexpr int32_t MaxMSize = 8;
-  static constexpr int32_t NSize = 32;
-  static constexpr int32_t WeightOCGroupSize = Nr;
-  static constexpr bool PackA = true;
-
- public:
-  // physical layout [
-  //  M / 8; Mr is 8
-  //  K / 4; K for bfmmla is 4
-  //  4,   ; 4 row-pairs for each 8 rows
-  //  2,   ; row-pair is 2 rows
-  //  4    ; 4 elements per row
-  // ]
-
-  static void pack_input_from_rows(const scalar_t* const* __restrict__ rows,
-                                   scalar_t* __restrict__ a_packed,
-                                   const int32_t m, const int32_t k) {
-    TORCH_CHECK(m > 0 && m <= MaxMSize);
-    TORCH_CHECK_EQ(k % K, 0);
-
-    auto* __restrict__ out = reinterpret_cast<bfloat16_t*>(a_packed);
-    const bfloat16x8_t zero_q = vdupq_n_bf16(bfloat16_t{});
-    const bfloat16x4_t zero = vget_low_bf16(zero_q);
-
-    for (int32_t row_base = 0; row_base < m; row_base += Mr) {
-      const int32_t actual_m = std::min(Mr, m - row_base);
-      const bfloat16_t* __restrict__ row[Mr];
-      for (int32_t i = 0; i < actual_m; ++i) {
-        row[i] = reinterpret_cast<const bfloat16_t*>(rows[row_base + i]);
-      }
-
-      if (actual_m == 8) {
-        int32_t k_idx = 0;
-        for (; k_idx + 8 <= k; k_idx += 8) {
-          bfloat16_t* __restrict__ block0 = out;
-          bfloat16_t* __restrict__ block1 = out + 4 * TileSize;
-
-          bfloat16x8_t a0 = vld1q_bf16(row[0] + k_idx);
-          bfloat16x8_t a1 = vld1q_bf16(row[1] + k_idx);
-          vst1q_bf16(block0,
-                     vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1)));
-          vst1q_bf16(block1,
-                     vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1)));
-
-          a0 = vld1q_bf16(row[2] + k_idx);
-          a1 = vld1q_bf16(row[3] + k_idx);
-          vst1q_bf16(block0 + TileSize,
-                     vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1)));
-          vst1q_bf16(block1 + TileSize,
-                     vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1)));
-
-          a0 = vld1q_bf16(row[4] + k_idx);
-          a1 = vld1q_bf16(row[5] + k_idx);
-          vst1q_bf16(block0 + 2 * TileSize,
-                     vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1)));
-          vst1q_bf16(block1 + 2 * TileSize,
-                     vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1)));
-
-          a0 = vld1q_bf16(row[6] + k_idx);
-          a1 = vld1q_bf16(row[7] + k_idx);
-          vst1q_bf16(block0 + 3 * TileSize,
-                     vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1)));
-          vst1q_bf16(block1 + 3 * TileSize,
-                     vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1)));
-
-          out += 8 * TileSize;
-        }
-
-        for (; k_idx < k; k_idx += K) {
-          bfloat16x4_t a0 = vld1_bf16(row[0] + k_idx);
-          bfloat16x4_t a1 = vld1_bf16(row[1] + k_idx);
-          vst1q_bf16(out, vcombine_bf16(a0, a1));
-
-          a0 = vld1_bf16(row[2] + k_idx);
-          a1 = vld1_bf16(row[3] + k_idx);
-          vst1q_bf16(out + TileSize, vcombine_bf16(a0, a1));
-
-          a0 = vld1_bf16(row[4] + k_idx);
-          a1 = vld1_bf16(row[5] + k_idx);
-          vst1q_bf16(out + 2 * TileSize, vcombine_bf16(a0, a1));
-
-          a0 = vld1_bf16(row[6] + k_idx);
-          a1 = vld1_bf16(row[7] + k_idx);
-          vst1q_bf16(out + 3 * TileSize, vcombine_bf16(a0, a1));
-
-          out += 4 * TileSize;
-        }
-        continue;
-      }
-
-      if (actual_m == 4) {
-        int32_t k_idx = 0;
-        for (; k_idx + 8 <= k; k_idx += 8) {
-          bfloat16_t* __restrict__ block0 = out;
-          bfloat16_t* __restrict__ block1 = out + 2 * TileSize;
-
-          bfloat16x8_t a0 = vld1q_bf16(row[0] + k_idx);
-          bfloat16x8_t a1 = vld1q_bf16(row[1] + k_idx);
-          vst1q_bf16(block0,
-                     vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1)));
-          vst1q_bf16(block1,
-                     vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1)));
-
-          a0 = vld1q_bf16(row[2] + k_idx);
-          a1 = vld1q_bf16(row[3] + k_idx);
-          vst1q_bf16(block0 + TileSize,
-                     vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1)));
-          vst1q_bf16(block1 + TileSize,
-                     vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1)));
-
-          out += 4 * TileSize;
-        }
-
-        for (; k_idx < k; k_idx += K) {
-          bfloat16x4_t a0 = vld1_bf16(row[0] + k_idx);
-          bfloat16x4_t a1 = vld1_bf16(row[1] + k_idx);
-          vst1q_bf16(out, vcombine_bf16(a0, a1));
-
-          a0 = vld1_bf16(row[2] + k_idx);
-          a1 = vld1_bf16(row[3] + k_idx);
-          vst1q_bf16(out + TileSize, vcombine_bf16(a0, a1));
-
-          out += 2 * TileSize;
-        }
-        continue;
-      }
-
-      const int32_t row_pair_count = (actual_m <= 4) ? 2 : Mr / 2;
-
-      int32_t k_idx = 0;
-      for (; k_idx + 8 <= k; k_idx += 8) {
-        bfloat16_t* __restrict__ block0 = out;
-        bfloat16_t* __restrict__ block1 = out + row_pair_count * TileSize;
-
-        bfloat16x8_t a0 = vld1q_bf16(row[0] + k_idx);
-        bfloat16x8_t a1 = (actual_m > 1) ? vld1q_bf16(row[1] + k_idx) : zero_q;
-        vst1q_bf16(block0, vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1)));
-        vst1q_bf16(block1,
-                   vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1)));
-
-        a0 = (actual_m > 2) ? vld1q_bf16(row[2] + k_idx) : zero_q;
-        a1 = (actual_m > 3) ? vld1q_bf16(row[3] + k_idx) : zero_q;
-        vst1q_bf16(block0 + TileSize,
-                   vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1)));
-        vst1q_bf16(block1 + TileSize,
-                   vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1)));
-
-        if (actual_m > 4) {
-          a0 = vld1q_bf16(row[4] + k_idx);
-          a1 = (actual_m > 5) ? vld1q_bf16(row[5] + k_idx) : zero_q;
-          vst1q_bf16(block0 + 2 * TileSize,
-                     vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1)));
-          vst1q_bf16(block1 + 2 * TileSize,
-                     vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1)));
-
-          a0 = (actual_m > 6) ? vld1q_bf16(row[6] + k_idx) : zero_q;
-          a1 = (actual_m > 7) ? vld1q_bf16(row[7] + k_idx) : zero_q;
-          vst1q_bf16(block0 + 3 * TileSize,
-                     vcombine_bf16(vget_low_bf16(a0), vget_low_bf16(a1)));
-          vst1q_bf16(block1 + 3 * TileSize,
-                     vcombine_bf16(vget_high_bf16(a0), vget_high_bf16(a1)));
-        }
-
-        out += 2 * row_pair_count * TileSize;
-      }
-
-      for (; k_idx < k; k_idx += K) {
-        bfloat16x4_t a0 = vld1_bf16(row[0] + k_idx);
-        bfloat16x4_t a1 = (actual_m > 1) ? vld1_bf16(row[1] + k_idx) : zero;
-        vst1q_bf16(out, vcombine_bf16(a0, a1));
-
-        a0 = (actual_m > 2) ? vld1_bf16(row[2] + k_idx) : zero;
-        a1 = (actual_m > 3) ? vld1_bf16(row[3] + k_idx) : zero;
-        vst1q_bf16(out + TileSize, vcombine_bf16(a0, a1));
-
-        if (actual_m > 4) {
-          a0 = vld1_bf16(row[4] + k_idx);
-          a1 = (actual_m > 5) ? vld1_bf16(row[5] + k_idx) : zero;
-          vst1q_bf16(out + 2 * TileSize, vcombine_bf16(a0, a1));
-
-          a0 = (actual_m > 6) ? vld1_bf16(row[6] + k_idx) : zero;
-          a1 = (actual_m > 7) ? vld1_bf16(row[7] + k_idx) : zero;
-          vst1q_bf16(out + 3 * TileSize, vcombine_bf16(a0, a1));
-        }
-        out += row_pair_count * TileSize;
-      }
-    }
-  }
-
-  void gemm(DEFINE_CPU_MICRO_GEMM_PARAMS) {
-    (void)lda;  // A is packed, so lda is not needed
-    TORCH_CHECK_EQ(k % K, 0);
-
-    for (int32_t n_idx = 0; n_idx < NSize; n_idx += Nr_gemv) {
-      const bfloat16_t* __restrict__ b_panel =
-          reinterpret_cast<const bfloat16_t*>(b_ptr) + n_idx * k;
-
-      for (int32_t row_base = 0; row_base < m; row_base += Mr) {
-        const int32_t panel_m = std::min(Mr, m - row_base);
-        const bfloat16_t* __restrict__ a_panel =
-            reinterpret_cast<const bfloat16_t*>(a_ptr) + row_base * k;
-        float* __restrict__ c_panel = c_ptr + row_base * ldc + n_idx;
-
-        if (panel_m <= 4) {
-          gemm_micro_bfmmla_4x16_packed_a(a_panel, b_panel, c_panel, panel_m, k,
-                                          b_n_group_stride, ldc, accum_c);
-        } else {
-          gemm_micro_bfmmla_8x8_packed_a(a_panel, b_panel, c_panel, panel_m, k,
-                                         ldc, accum_c);
-          gemm_micro_bfmmla_8x8_packed_a(a_panel, b_panel + b_n_group_stride,
-                                         c_panel + Nr, panel_m, k, ldc,
-                                         accum_c);
-        }
-      }
-    }
-  }
-
-  // physical layout [
-  //  N / 8; Nr is 8
-  //  K / 4; K for bfmmla is 4
-  //  4,   ; 4 col-pairs for each 8 cols
-  //  2,   ; col-pair is 2 cols
-  //  4    ; 4 elements per col
-  // ]
-  static void pack_weight(const c10::BFloat16* __restrict__ weight,
-                          c10::BFloat16* __restrict__ packed_weight,
-                          const int32_t output_size, const int32_t input_size) {
-    TORCH_CHECK_EQ(output_size % NSize, 0);
-    TORCH_CHECK_EQ(input_size % K, 0);
-
-    for (int32_t o_idx = 0; o_idx < output_size; o_idx += Nr) {
-      c10::BFloat16* __restrict__ dst = packed_weight + o_idx * input_size;
-      for (int32_t k_idx = 0; k_idx < input_size; k_idx += K) {
-        for (int32_t pair_idx = 0; pair_idx < Nr; pair_idx += Cols) {
-          const c10::BFloat16* __restrict__ row0 =
-              weight + (o_idx + pair_idx) * input_size;
-          const c10::BFloat16* __restrict__ row1 = row0 + input_size;
-          dst[0] = row0[k_idx + 0];
-          dst[1] = row0[k_idx + 1];
-          dst[2] = row0[k_idx + 2];
-          dst[3] = row0[k_idx + 3];
-          dst[4] = row1[k_idx + 0];
-          dst[5] = row1[k_idx + 1];
-          dst[6] = row1[k_idx + 2];
-          dst[7] = row1[k_idx + 3];
-          dst += TileSize;
-        }
-      }
-    }
-  }
-};
-
-}  // namespace cpu_micro_gemm
-
-#endif
@@ -1,228 +0,0 @@
-#ifndef CPU_MICRO_GEMM_RVV_HPP
-#define CPU_MICRO_GEMM_RVV_HPP
-
-#include "cpu/micro_gemm/cpu_micro_gemm_impl.hpp"
-
-#if defined(__riscv_v)
-
-namespace cpu_micro_gemm {
-namespace {
-
-constexpr int32_t RVV_MGEMM_N8 = 8;
-constexpr int32_t RVV_MGEMM_B_GROUP_STRIDE = 16;
-
-template <typename scalar_t>
-FORCE_INLINE fixed_fp32x8_t load_row8_b_as_f32(const scalar_t* ptr);
-
-template <>
-FORCE_INLINE fixed_fp32x8_t load_row8_b_as_f32<float>(const float* ptr) {
-  return RVVI(__riscv_vle32_v_f32, LMUL_256)(ptr, RVV_MGEMM_N8);
-}
-
-template <>
-FORCE_INLINE fixed_fp32x8_t
-load_row8_b_as_f32<c10::Half>(const c10::Half* ptr) {
-  #if defined(__riscv_zvfh)
-  fixed_fp16x8_t vec = RVVI(__riscv_vle16_v_f16, LMUL_128)(
-      reinterpret_cast<const _Float16*>(ptr), RVV_MGEMM_N8);
-  return RVVI(__riscv_vfwcvt_f_f_v_f32, LMUL_256)(vec, RVV_MGEMM_N8);
-  #else
-  alignas(32) float values[RVV_MGEMM_N8];
-  for (int32_t i = 0; i < RVV_MGEMM_N8; ++i) {
-    values[i] = static_cast<float>(ptr[i]);
-  }
-  return RVVI(__riscv_vle32_v_f32, LMUL_256)(values, RVV_MGEMM_N8);
-  #endif
-}
-
-template <>
-FORCE_INLINE fixed_fp32x8_t
-load_row8_b_as_f32<c10::BFloat16>(const c10::BFloat16* ptr) {
-  #if defined(__riscv_zvfbfmin)
-  fixed_u16x8_t raw = RVVI(__riscv_vle16_v_u16, LMUL_128)(
-      reinterpret_cast<const uint16_t*>(ptr), RVV_MGEMM_N8);
-  fixed_bf16x8_t vec =
-      RVVI4(__riscv_vreinterpret_v_u16, LMUL_128, _bf16, LMUL_128)(raw);
-  return RVVI(__riscv_vfwcvtbf16_f_f_v_f32, LMUL_256)(vec, RVV_MGEMM_N8);
-  #else
-  fixed_u16x8_t raw = RVVI(__riscv_vle16_v_u16, LMUL_128)(
-      reinterpret_cast<const uint16_t*>(ptr), RVV_MGEMM_N8);
-  auto wide = RVVI(__riscv_vzext_vf2_u32, LMUL_256)(raw, RVV_MGEMM_N8);
-  auto shifted = RVVI(__riscv_vsll_vx_u32, LMUL_256)(wide, 16, RVV_MGEMM_N8);
-  return RVVI4(__riscv_vreinterpret_v_u32, LMUL_256, _f32, LMUL_256)(shifted);
-  #endif
-}
-
-// Mx8 RVV kernel. B points at one 8-channel half of a 16-channel packed group,
-// with rows separated by RVV_MGEMM_B_GROUP_STRIDE scalar elements.
-template <int32_t M, typename scalar_t>
-FORCE_INLINE void gemm_micro_rvv_fma_mx8_ku4(const scalar_t* __restrict__ a_ptr,
-                                             const scalar_t* __restrict__ b_ptr,
-                                             float* __restrict__ c_ptr,
-                                             const int64_t lda,
-                                             const int64_t ldc, const int32_t k,
-                                             const bool accum_c) {
-  static_assert(0 < M && M <= 8);
-
-  #define RVV_ROWS_APPLY(OP) OP(0) OP(1) OP(2) OP(3) OP(4) OP(5) OP(6) OP(7)
-  #define RVV_IF_M(i) if constexpr (M > (i))
-
-  #define RVV_DECL_A(i) const scalar_t* __restrict__ a##i = a_ptr + (i) * lda;
-  RVV_ROWS_APPLY(RVV_DECL_A)
-  #undef RVV_DECL_A
-
-  #define RVV_DECL_ACC(i) fixed_fp32x8_t acc##i;
-  RVV_ROWS_APPLY(RVV_DECL_ACC)
-  #undef RVV_DECL_ACC
-
-  #define RVV_INIT_ACC(i)                                                  \
-    RVV_IF_M(i) {                                                          \
-      if (accum_c) {                                                       \
-        acc##i = RVVI(__riscv_vle32_v_f32, LMUL_256)(c_ptr + (i) * ldc,    \
-                                                     RVV_MGEMM_N8);        \
-      } else {                                                             \
-        acc##i = RVVI(__riscv_vfmv_v_f_f32, LMUL_256)(0.0f, RVV_MGEMM_N8); \
-      }                                                                    \
-    }
-  RVV_ROWS_APPLY(RVV_INIT_ACC)
-  #undef RVV_INIT_ACC
-
-  int32_t k_idx = 0;
-  for (; k_idx + 3 < k; k_idx += 4) {
-  #define RVV_FMA_ROW(i, K_OFFSET)                                     \
-    RVV_IF_M(i) {                                                      \
-      acc##i = RVVI(__riscv_vfmacc_vf_f32, LMUL_256)(                  \
-          acc##i, static_cast<float>(*(a##i + k_idx + (K_OFFSET))), b, \
-          RVV_MGEMM_N8);                                               \
-    }
-
-  #define RVV_STEP_K(K_OFFSET)                                      \
-    {                                                               \
-      fixed_fp32x8_t b = load_row8_b_as_f32<scalar_t>(              \
-          b_ptr + (k_idx + (K_OFFSET)) * RVV_MGEMM_B_GROUP_STRIDE); \
-      RVV_FMA_ROW(0, K_OFFSET)                                      \
-      RVV_FMA_ROW(1, K_OFFSET)                                      \
-      RVV_FMA_ROW(2, K_OFFSET)                                      \
-      RVV_FMA_ROW(3, K_OFFSET)                                      \
-      RVV_FMA_ROW(4, K_OFFSET)                                      \
-      RVV_FMA_ROW(5, K_OFFSET)                                      \
-      RVV_FMA_ROW(6, K_OFFSET)                                      \
-      RVV_FMA_ROW(7, K_OFFSET)                                      \
-    }
-
-    RVV_STEP_K(0)
-    RVV_STEP_K(1)
-    RVV_STEP_K(2)
-    RVV_STEP_K(3)
-  #undef RVV_STEP_K
-  #undef RVV_FMA_ROW
-  }
-
-  for (; k_idx < k; ++k_idx) {
-    fixed_fp32x8_t b =
-        load_row8_b_as_f32<scalar_t>(b_ptr + k_idx * RVV_MGEMM_B_GROUP_STRIDE);
-  #define RVV_TAIL_ROW(i)                                                \
-    RVV_IF_M(i) {                                                        \
-      acc##i = RVVI(__riscv_vfmacc_vf_f32, LMUL_256)(                    \
-          acc##i, static_cast<float>(*(a##i + k_idx)), b, RVV_MGEMM_N8); \
-    }
-    RVV_ROWS_APPLY(RVV_TAIL_ROW)
-  #undef RVV_TAIL_ROW
-  }
-
-  #define RVV_STORE_ROW(i)                                           \
-    RVV_IF_M(i) {                                                    \
-      RVVI(__riscv_vse32_v_f32, LMUL_256)(c_ptr + (i) * ldc, acc##i, \
-                                          RVV_MGEMM_N8);             \
-    }
-  RVV_ROWS_APPLY(RVV_STORE_ROW)
-  #undef RVV_STORE_ROW
-
-  #undef RVV_ROWS_APPLY
-  #undef RVV_IF_M
-}
-
-template <int32_t M, typename scalar_t>
-FORCE_INLINE void gemm_micro_rvv_mx32_ku4(DEFINE_CPU_MICRO_GEMM_PARAMS) {
-  static_assert(0 < M && M <= 8);
-  scalar_t* __restrict__ curr_b_0 = b_ptr;
-  scalar_t* __restrict__ curr_b_1 = b_ptr + b_n_group_stride;
-
-  gemm_micro_rvv_fma_mx8_ku4<M>(a_ptr, curr_b_0, c_ptr, lda, ldc, k, accum_c);
-  gemm_micro_rvv_fma_mx8_ku4<M>(a_ptr, curr_b_0 + RVV_MGEMM_N8,
-                                c_ptr + RVV_MGEMM_N8, lda, ldc, k, accum_c);
-  gemm_micro_rvv_fma_mx8_ku4<M>(a_ptr, curr_b_1, c_ptr + 16, lda, ldc, k,
-                                accum_c);
-  gemm_micro_rvv_fma_mx8_ku4<M>(a_ptr, curr_b_1 + RVV_MGEMM_N8, c_ptr + 24, lda,
-                                ldc, k, accum_c);
-}
-
-class TileGemmRVV {
- public:
-  template <typename scalar_t>
-  FORCE_INLINE static void gemm(DEFINE_CPU_MICRO_GEMM_PARAMS) {
-    switch (m) {
-      case 1:
-        gemm_micro_rvv_mx32_ku4<1>(CPU_MICRO_GEMM_PARAMS);
-        break;
-      case 2:
-        gemm_micro_rvv_mx32_ku4<2>(CPU_MICRO_GEMM_PARAMS);
-        break;
-      case 3:
-        gemm_micro_rvv_mx32_ku4<3>(CPU_MICRO_GEMM_PARAMS);
-        break;
-      case 4:
-        gemm_micro_rvv_mx32_ku4<4>(CPU_MICRO_GEMM_PARAMS);
-        break;
-      case 5:
-        gemm_micro_rvv_mx32_ku4<5>(CPU_MICRO_GEMM_PARAMS);
-        break;
-      case 6:
-        gemm_micro_rvv_mx32_ku4<6>(CPU_MICRO_GEMM_PARAMS);
-        break;
-      case 7:
-        gemm_micro_rvv_mx32_ku4<7>(CPU_MICRO_GEMM_PARAMS);
-        break;
-      case 8:
-        gemm_micro_rvv_mx32_ku4<8>(CPU_MICRO_GEMM_PARAMS);
-        break;
-    }
-  }
-};
-
-}  // namespace
-
-template <typename scalar_t>
-class MicroGemm<cpu_utils::ISA::RVV, scalar_t> {
- public:
-  static constexpr int32_t MaxMSize = 8;
-  static constexpr int32_t NSize = 32;
-
- public:
-  void gemm(DEFINE_CPU_MICRO_GEMM_PARAMS) {
-    TileGemmRVV::gemm<scalar_t>(CPU_MICRO_GEMM_PARAMS);
-  }
-
-  static void pack_weight(const scalar_t* __restrict__ weight,
-                          scalar_t* __restrict__ packed_weight,
-                          const int32_t output_size, const int32_t input_size) {
-    TORCH_CHECK_EQ(output_size % 16, 0);
-    for (int32_t o_idx = 0; o_idx < output_size; ++o_idx) {
-      const scalar_t* __restrict__ curr_weight = weight + o_idx * input_size;
-      scalar_t* __restrict__ curr_packed_weight =
-          packed_weight + (o_idx / 16) * (16 * input_size) + o_idx % 16;
-      for (int32_t i_idx = 0; i_idx < input_size; ++i_idx) {
-        *curr_packed_weight = *curr_weight;
-
-        curr_packed_weight += 16;
-        ++curr_weight;
-      }
-    }
-  }
-};
-
-}  // namespace cpu_micro_gemm
-
-#endif  // defined(__riscv_v)
-
-#endif  // CPU_MICRO_GEMM_RVV_HPP
@@ -104,8 +104,6 @@ class MicroGemm<cpu_utils::ISA::VEC, scalar_t> {
 public:
  static constexpr int32_t MaxMSize = 8;
  static constexpr int32_t NSize = 32;
-  static constexpr int32_t WeightOCGroupSize = 16;
-  static constexpr bool PackA = false;

 public:
  void gemm(DEFINE_CPU_MICRO_GEMM_PARAMS) {
@@ -18,9 +18,17 @@ struct KernelVecType<float> {

 template <>
 struct KernelVecType<c10::Half> {
+#if defined(__powerpc64__)
+  // Power specific vector types
+  using qk_load_vec_type = vec_op::FP32Vec16;
+  using qk_vec_type = vec_op::FP32Vec16;
+  using v_load_vec_type = vec_op::FP32Vec16;
+#else
+  // Fallback for other architectures, including x86
  using qk_load_vec_type = vec_op::FP16Vec16;
  using qk_vec_type = vec_op::FP32Vec16;
  using v_load_vec_type = vec_op::FP16Vec16;
+#endif
 };

 #ifdef __AVX512BF16__
@@ -251,7 +259,7 @@ void mla_decode_kvcache_cpu_impl(
  constexpr int QK_NUM_ELEM = qk_vec_type::VEC_ELEM_NUM;

  // shared across threads
-  const int max_threads = cpu_utils::get_max_threads();
+  const int max_threads = omp_get_max_threads();
  const int acc_out_nbytes =
      max_threads * num_heads * V_HEAD_DIM * sizeof(float);
  float* acc_out = static_cast<float*>(std::aligned_alloc(64, acc_out_nbytes));
@@ -1,3 +1,4 @@
+
 #include "cpu_types.hpp"

 namespace {
@@ -96,91 +97,6 @@ void rotary_embedding_impl(
  }
 }

-template <>
-void rotary_embedding_impl<c10::Half>(
-    const int64_t* __restrict__ positions, c10::Half* __restrict__ query,
-    c10::Half* __restrict__ key, const c10::Half* __restrict__ cos_sin_cache,
-    const int rot_dim, const int64_t query_stride, const int64_t key_stride,
-    const int num_heads, const int num_kv_heads, const int head_size,
-    const int num_tokens) {
-  using scalar_vec_t = vec_op::FP16Vec8;
-  constexpr int VEC_ELEM_NUM = scalar_vec_t::get_elem_num();
-
-  const int embed_dim = rot_dim / 2;
-  bool flag = (embed_dim % VEC_ELEM_NUM == 0);
-  const int loop_upper = flag ? embed_dim : embed_dim - VEC_ELEM_NUM;
-
-  auto compute_loop = [&](const int64_t token_head, const c10::Half* cache_ptr,
-                          c10::Half* qk) {
-    int j = 0;
-    for (; j < loop_upper; j += VEC_ELEM_NUM) {
-      const int rot_offset = j;
-      const int x_index = rot_offset;
-      const int y_index = embed_dim + rot_offset;
-
-      const int64_t out_x = token_head + x_index;
-      const int64_t out_y = token_head + y_index;
-
-      const vec_op::FP16Vec8 cos_fp16(cache_ptr + x_index);
-      const vec_op::FP16Vec8 sin_fp16(cache_ptr + y_index);
-      const vec_op::FP16Vec8 q_x_fp16(qk + out_x);
-      const vec_op::FP16Vec8 q_y_fp16(qk + out_y);
-
-      const vec_op::FP32Vec8 fp32_cos(cos_fp16);
-      const vec_op::FP32Vec8 fp32_sin(sin_fp16);
-      const vec_op::FP32Vec8 fp32_q_x(q_x_fp16);
-      const vec_op::FP32Vec8 fp32_q_y(q_y_fp16);
-
-      auto out1 = fp32_q_x * fp32_cos - fp32_q_y * fp32_sin;
-      auto out2 = fp32_q_y * fp32_cos + fp32_q_x * fp32_sin;
-
-      vec_op::FP16Vec8(out1).save(qk + out_x);
-      vec_op::FP16Vec8(out2).save(qk + out_y);
-    }
-    if (!flag) {
-      for (; j < embed_dim; ++j) {
-        const int x_index = j;
-        const int y_index = embed_dim + j;
-
-        const int64_t out_x = token_head + x_index;
-        const int64_t out_y = token_head + y_index;
-
-        const float fp32_cos = static_cast<float>(cache_ptr[x_index]);
-        const float fp32_sin = static_cast<float>(cache_ptr[y_index]);
-        const float fp32_q_x = static_cast<float>(qk[out_x]);
-        const float fp32_q_y = static_cast<float>(qk[out_y]);
-
-        qk[out_x] =
-            static_cast<c10::Half>(fp32_q_x * fp32_cos - fp32_q_y * fp32_sin);
-        qk[out_y] =
-            static_cast<c10::Half>(fp32_q_y * fp32_cos + fp32_q_x * fp32_sin);
-      }
-    }
-  };
-
-#pragma omp parallel for
-  for (int token_idx = 0; token_idx < num_tokens; ++token_idx) {
-    int64_t pos = positions[token_idx];
-    const c10::Half* cache_ptr = cos_sin_cache + pos * rot_dim;
-
-    for (int i = 0; i < num_heads; ++i) {
-      const int head_idx = i;
-      const int64_t token_head =
-          token_idx * query_stride + head_idx * head_size;
-      compute_loop(token_head, cache_ptr, query);
-    }
-
-    if (key != nullptr) {
-      for (int i = 0; i < num_kv_heads; ++i) {
-        const int head_idx = i;
-        const int64_t token_head =
-            token_idx * key_stride + head_idx * head_size;
-        compute_loop(token_head, cache_ptr, key);
-      }
-    }
-  }
-}
-
 template <typename scalar_t>
 void rotary_embedding_gptj_impl(
    const int64_t* __restrict__ positions,  // [batch_size, seq_len] or
@@ -258,75 +174,6 @@ void rotary_embedding_gptj_impl(
    }
  }
 }
-
-template <>
-void rotary_embedding_gptj_impl<c10::Half>(
-    const int64_t* __restrict__ positions, c10::Half* __restrict__ query,
-    c10::Half* __restrict__ key, const c10::Half* __restrict__ cos_sin_cache,
-    const int rot_dim, const int64_t query_stride, const int64_t key_stride,
-    const int num_heads, const int num_kv_heads, const int head_size,
-    const int num_tokens) {
-  const int embed_dim = rot_dim / 2;
-
-#pragma omp parallel for collapse(2)
-  for (int token_idx = 0; token_idx < num_tokens; ++token_idx) {
-    for (int i = 0; i < num_heads; ++i) {
-      int64_t pos = positions[token_idx];
-      const c10::Half* cache_ptr = cos_sin_cache + pos * rot_dim;
-      const c10::Half* cos_cache_ptr = cache_ptr;
-      const c10::Half* sin_cache_ptr = cache_ptr + embed_dim;
-      const int head_idx = i;
-      const int64_t token_head =
-          token_idx * query_stride + head_idx * head_size;
-      c10::Half* head_query = token_head + query;
-      for (int j = 0; j < embed_dim; j += 1) {
-        const int rot_offset = j;
-        const int x_index = 2 * rot_offset;
-        const int y_index = 2 * rot_offset + 1;
-
-        const float cos = static_cast<float>(cos_cache_ptr[rot_offset]);
-        const float sin = static_cast<float>(sin_cache_ptr[rot_offset]);
-
-        const float x = static_cast<float>(head_query[x_index]);
-        const float y = static_cast<float>(head_query[y_index]);
-
-        head_query[x_index] = static_cast<c10::Half>(x * cos - y * sin);
-        head_query[y_index] = static_cast<c10::Half>(y * cos + x * sin);
-      }
-    }
-  }
-
-  if (key == nullptr) {
-    return;
-  }
-
-#pragma omp parallel for collapse(2)
-  for (int token_idx = 0; token_idx < num_tokens; ++token_idx) {
-    for (int i = 0; i < num_kv_heads; ++i) {
-      int64_t pos = positions[token_idx];
-      const c10::Half* cache_ptr = cos_sin_cache + pos * rot_dim;
-      const c10::Half* cos_cache_ptr = cache_ptr;
-      const c10::Half* sin_cache_ptr = cache_ptr + embed_dim;
-      const int head_idx = i;
-      const int64_t token_head = token_idx * key_stride + head_idx * head_size;
-      c10::Half* head_key = key + token_head;
-      for (int j = 0; j < embed_dim; j += 1) {
-        const int rot_offset = j;
-        const int x_index = 2 * rot_offset;
-        const int y_index = 2 * rot_offset + 1;
-
-        const float cos = static_cast<float>(cos_cache_ptr[rot_offset]);
-        const float sin = static_cast<float>(sin_cache_ptr[rot_offset]);
-
-        const float x = static_cast<float>(head_key[x_index]);
-        const float y = static_cast<float>(head_key[y_index]);
-
-        head_key[x_index] = static_cast<c10::Half>(x * cos - y * sin);
-        head_key[y_index] = static_cast<c10::Half>(y * cos + x * sin);
-      }
-    }
-  }
-}
 };  // namespace

 void rotary_embedding(torch::Tensor& positions, torch::Tensor& query,
@@ -289,18 +289,19 @@ void causal_conv1d_fwd_kernel_impl(
  }
 }

-#define LAUNCH_TINYGEMM_VARLEN_KERNEL(K, NB_SIZE)                                                   \
-  tinygemm_kernel<scalar_t, K, NB_SIZE, has_bias, has_silu>::apply(                                 \
-      input + batch_offset * dim + mb_start * dim + nb_start,                                       \
-      weight + nb_start * width,                                                                    \
-      out + batch_offset * dim + mb_start * dim + nb_start,                                         \
-      has_bias ? bias + nb_start : nullptr,                                                         \
-      has_conv_states ? conv_states + conv_state_index * conv_state_slot_stride + nb_start : nullptr, \
-      has_initial_states_value,                                                                     \
-      mb_size,                                                                                      \
-      dim,                                                                                          \
+#define LAUNCH_TINYGEMM_VARLEN_KERNEL(K, NB_SIZE)                   \
+  tinygemm_kernel<scalar_t, K, NB_SIZE, has_bias, has_silu>::apply( \
+      input + batch_offset * dim + mb_start * dim + nb_start,       \
+      weight + nb_start * width,                                    \
+      out + batch_offset * dim + mb_start * dim + nb_start,         \
+      has_bias ? bias + nb_start : nullptr,                         \
+      nullptr,                                                      \
+      false,                                                        \
+      mb_size,                                                      \
+      dim,                                                          \
      mb_start == 0);

+// TODO: add `has_initial_state` support for varlen kernel
 template <typename scalar_t>
 void causal_conv1d_fwd_varlen_kernel_impl(
    scalar_t* __restrict__ out,
@@ -342,9 +343,6 @@ void causal_conv1d_fwd_varlen_kernel_impl(
        int64_t nb_start = nb * BLOCK_N;
        int64_t nb_size = std::min(dim - nb_start, BLOCK_N);

-        const bool has_initial_states_value = has_conv_states ? has_initial_state[bs] : false;
-        int32_t conv_state_index = has_conv_indices ? conv_indices[bs] : bs;
-
        switch (width << 4 | nb_size >> 4) {
          case 0x42:
            LAUNCH_TINYGEMM_VARLEN_KERNEL(4, 32);
@@ -375,7 +373,7 @@ void causal_conv1d_fwd_varlen_kernel_impl(
            width,
            dim,
            seqlen,
-            has_initial_state[bs]);
+            /* has_initial_state */ false);
      }
    });
  }
@@ -285,125 +285,6 @@ inline int32_t load_uint4_vnni(const uint8_t* __restrict__ B, int64_t k, int64_t
  return (n_group % 2 == 0) ? (packed & 0x0f) : ((packed >> 4) & 0x0f);
 }

-#if defined(CPU_CAPABILITY_RVV)
-template <int64_t N, int64_t ldb, int group>
-inline fixed_i8x8_t load_uint4_as_int8_rvv(const uint8_t* __restrict__ B, int64_t k) {
-  constexpr int64_t n_group_size = 8;
-  constexpr int64_t vnni_size = 4;
-  static_assert(N == 32);
-  static_assert(ldb == N / 2);
-  static_assert(group >= 0 && group < N / n_group_size);
-
-  // Unpack: gather 8 packed int4 values from the VNNI4 layout.
-  const int64_t ki = k % vnni_size;
-  const int64_t k_base = k - ki;
-  constexpr int64_t packed_group = group / 2;
-  const uint8_t* packed_ptr = B + k_base * ldb + packed_group * n_group_size * vnni_size + ki;
-
-  fixed_u8x8_t packed = RVVI(__riscv_vlse8_v_u8, LMUL_64)(packed_ptr, vnni_size, n_group_size);
-  if constexpr (group % 2 == 1) {
-    packed = RVVI(__riscv_vsrl_vx_u8, LMUL_64)(packed, 4, n_group_size);
-  }
-  fixed_u8x8_t nibbles = RVVI(__riscv_vand_vx_u8, LMUL_64)(packed, 0x0f, n_group_size);
-  return RVVI4(__riscv_vreinterpret_v_u8, LMUL_64, _i8, LMUL_64)(nibbles);
-}
-
-inline fixed_i32x8_t gemm_accum_uint8_int8_rvv(fixed_i32x8_t acc, uint8_t a, fixed_i8x8_t b) {
-  constexpr int64_t vl = 8;
-  fixed_i16x8_t b_i16 = RVVI(__riscv_vsext_vf2_i16, LMUL_128)(b, vl);
-  return RVVI(__riscv_vwmacc_vx_i32, LMUL_256)(acc, static_cast<int16_t>(a), b_i16, vl);
-}
-
-template <int64_t N, int64_t ldb, int group>
-inline fixed_i32x8_t gemm_accum_uint4_rvv(
-    fixed_i32x8_t acc,
-    const uint8_t* __restrict__ B,
-    const int8_t* __restrict__ qzeros_b,
-    uint8_t a,
-    int64_t k) {
-  constexpr int64_t n_group_size = 8;
-  fixed_i8x8_t b = load_uint4_as_int8_rvv<N, ldb, group>(B, k);
-  fixed_i8x8_t qzeros =
-      RVVI(__riscv_vle8_v_i8, LMUL_64)(qzeros_b + group * n_group_size, n_group_size);
-  b = RVVI(__riscv_vsub_vv_i8, LMUL_64)(b, qzeros, n_group_size);
-  return gemm_accum_uint8_int8_rvv(acc, a, b);
-}
-
-template <int group>
-inline void _dequant_and_store_rvv(
-    float* __restrict__ C,
-    fixed_i32x8_t acc,
-    const float* __restrict__ scales_a,
-    const int32_t* __restrict__ qzeros_a,
-    const float* __restrict__ scales_b,
-    const int32_t* __restrict__ compensation,
-    int64_t m,
-    int64_t ldc) {
-  constexpr int64_t n_group_size = 8;
-  constexpr int64_t n = group * n_group_size;
-  constexpr int64_t vl = n_group_size;
-
-  // Dequant compensation: remove activation zero-point contribution.
-  fixed_i32x8_t comp = RVVI(__riscv_vle32_v_i32, LMUL_256)(compensation + n, vl);
-  fixed_i32x8_t zp_comp = RVVI(__riscv_vmul_vx_i32, LMUL_256)(comp, qzeros_a[m], vl);
-  acc = RVVI(__riscv_vsub_vv_i32, LMUL_256)(acc, zp_comp, vl);
-
-  // Scale: convert int32 accumulators to fp32 and apply activation/weight scales.
-  fixed_fp32x8_t acc_f = RVVI(__riscv_vfcvt_f_x_v_f32, LMUL_256)(acc, vl);
-  acc_f = RVVI(__riscv_vfmul_vf_f32, LMUL_256)(acc_f, scales_a[m], vl);
-  fixed_fp32x8_t scale_b = RVVI(__riscv_vle32_v_f32, LMUL_256)(scales_b + n, vl);
-  acc_f = RVVI(__riscv_vfmul_vv_f32, LMUL_256)(acc_f, scale_b, vl);
-
-  // Store: accumulate into the float scratch buffer that already holds bias/zero.
-  float* c_ptr = C + m * ldc + n;
-  fixed_fp32x8_t c_old = RVVI(__riscv_vle32_v_f32, LMUL_256)(c_ptr, vl);
-  fixed_fp32x8_t c_new = RVVI(__riscv_vfadd_vv_f32, LMUL_256)(c_old, acc_f, vl);
-  RVVI(__riscv_vse32_v_f32, LMUL_256)(c_ptr, c_new, vl);
-}
-
-template <int64_t N, int64_t ldb>
-void _dequant_gemm_accum_rvv(
-    float* __restrict__ C,
-    const uint8_t* __restrict__ A,
-    const float* __restrict__ scales_a,
-    const int32_t* __restrict__ qzeros_a,
-    const uint8_t* __restrict__ B,
-    const float* __restrict__ scales_b,
-    const int8_t* __restrict__ qzeros_b,
-    const int32_t* __restrict__ compensation,
-    int64_t M,
-    int64_t K,
-    int64_t lda,
-    int64_t ldc) {
-  static_assert(N == 32);
-  static_assert(ldb == N / 2);
-  constexpr int64_t vl = 8;
-
-  // Accumulate one C row over the 32-column block.
-  for (int64_t m = 0; m < M; ++m) {
-    fixed_i32x8_t acc0 = RVVI(__riscv_vmv_v_x_i32, LMUL_256)(0, vl);
-    fixed_i32x8_t acc1 = RVVI(__riscv_vmv_v_x_i32, LMUL_256)(0, vl);
-    fixed_i32x8_t acc2 = RVVI(__riscv_vmv_v_x_i32, LMUL_256)(0, vl);
-    fixed_i32x8_t acc3 = RVVI(__riscv_vmv_v_x_i32, LMUL_256)(0, vl);
-    // A[m][k] @ B[k][0:32] -> acc[m][0:32]
-    for (int64_t k = 0; k < K; ++k) {
-      // GEMM K step: one scalar activation updates four 8-column RVV tiles.
-      const uint8_t a = A[m * lda + k];
-      acc0 = gemm_accum_uint4_rvv<N, ldb, 0>(acc0, B, qzeros_b, a, k);
-      acc1 = gemm_accum_uint4_rvv<N, ldb, 1>(acc1, B, qzeros_b, a, k);
-      acc2 = gemm_accum_uint4_rvv<N, ldb, 2>(acc2, B, qzeros_b, a, k);
-      acc3 = gemm_accum_uint4_rvv<N, ldb, 3>(acc3, B, qzeros_b, a, k);
-    }
-
-    // Dequant/scale/store each 8-column group back into C.
-    _dequant_and_store_rvv<0>(C, acc0, scales_a, qzeros_a, scales_b, compensation, m, ldc);
-    _dequant_and_store_rvv<1>(C, acc1, scales_a, qzeros_a, scales_b, compensation, m, ldc);
-    _dequant_and_store_rvv<2>(C, acc2, scales_a, qzeros_a, scales_b, compensation, m, ldc);
-    _dequant_and_store_rvv<3>(C, acc3, scales_a, qzeros_a, scales_b, compensation, m, ldc);
-  }
-}
-#endif
-
 template <int64_t N, int64_t ldb, bool sym_quant_act>
 void _dequant_gemm_accum(
    float* C,
@@ -455,11 +336,6 @@ void _dequant_gemm_accum(
    _dequant_and_store<true, N, sym_quant_act>(
        C, C_i32, scales_a, qzeros_a, scales_b, compensation, M, N /*ldi*/, ldc, 1 /*ldsa*/);
  } else
-#elif defined(CPU_CAPABILITY_RVV)
-  if constexpr (!sym_quant_act && N == BLOCK_N && ldb == BLOCK_N / 2) {
-    _dequant_gemm_accum_rvv<N, ldb>(C, A, scales_a, qzeros_a, B, scales_b, qzeros_b, compensation, M, K, lda, ldc);
-    return;
-  } else
 #endif
  {
    for (int64_t m = 0; m < M; ++m) {
@@ -9,19 +9,11 @@
 #define CPU_CAPABILITY_AVX512
 #endif

-#if defined(__riscv_v_min_vlen) && (__riscv_v_min_vlen == 128 || __riscv_v_min_vlen == 256)
-#define CPU_CAPABILITY_RVV
-#endif
-
 #include <ATen/cpu/vec/functional.h>
 #include <ATen/cpu/vec/vec.h>
 #if defined(CPU_CAPABILITY_AVX512)
 #include <immintrin.h>
 #endif
-
-#if defined(CPU_CAPABILITY_RVV)
-#include "../cpu_types_riscv_defs.hpp"
-#endif
 namespace {

 using namespace at::vec;
@@ -208,89 +208,6 @@ void copy_and_expand_eagle_inputs_kernel_impl(
  }
 }

-void copy_and_expand_dflash_inputs_kernel_impl(
-    const torch::Tensor& next_token_ids, const torch::Tensor& target_positions,
-    torch::Tensor& out_input_ids, torch::Tensor& out_context_positions,
-    torch::Tensor& out_query_positions, torch::Tensor& out_context_slot_mapping,
-    torch::Tensor& out_query_slot_mapping, torch::Tensor& out_token_indices,
-    const torch::Tensor& block_table, const torch::Tensor& query_start_loc,
-    const std::optional<torch::Tensor>& num_rejected_tokens,
-    const int64_t parallel_drafting_token_id, const int64_t block_size,
-    const int64_t num_query_per_req, const int64_t num_speculative_tokens,
-    const int64_t total_input_tokens, const bool has_num_rejected) {
-  const int64_t num_reqs = query_start_loc.size(0) - 1;
-
-  const int64_t* next_ids_ptr = next_token_ids.data_ptr<int64_t>();
-  const int64_t* target_pos_ptr = target_positions.data_ptr<int64_t>();
-  const int32_t* block_table_ptr = block_table.data_ptr<int32_t>();
-  const int32_t* query_start_ptr = query_start_loc.data_ptr<int32_t>();
-  const int64_t* rejected_ptr =
-      has_num_rejected && num_rejected_tokens.has_value()
-          ? num_rejected_tokens.value().data_ptr<int64_t>()
-          : nullptr;
-
-  int64_t* out_ids_ptr = out_input_ids.data_ptr<int64_t>();
-  int64_t* out_ctx_pos_ptr = out_context_positions.data_ptr<int64_t>();
-  int64_t* out_query_pos_ptr = out_query_positions.data_ptr<int64_t>();
-  int64_t* out_ctx_slot_ptr = out_context_slot_mapping.data_ptr<int64_t>();
-  int64_t* out_query_slot_ptr = out_query_slot_mapping.data_ptr<int64_t>();
-  int32_t* out_token_idx_ptr = out_token_indices.data_ptr<int32_t>();
-
-  const int64_t block_table_stride = block_table.stride(0);
-
-#pragma omp parallel for
-  for (int64_t req_idx = 0; req_idx < num_reqs; ++req_idx) {
-    int32_t ctx_start = query_start_ptr[req_idx];
-    int32_t ctx_end = query_start_ptr[req_idx + 1];
-    int64_t num_ctx = ctx_end - ctx_start;
-    int64_t valid_ctx_end = ctx_end;
-    if (rejected_ptr != nullptr) {
-      valid_ctx_end -= rejected_ptr[req_idx];
-    }
-    // Guard against out-of-bounds: ensure valid_ctx_end > ctx_start so that
-    // valid_ctx_end - 1 never reads before the request's context range.
-    valid_ctx_end =
-        std::max(valid_ctx_end, static_cast<int64_t>(ctx_start + 1));
-
-    int64_t last_pos = target_pos_ptr[valid_ctx_end - 1];
-
-    for (int64_t j = 0; j < num_ctx; ++j) {
-      int64_t ctx_idx = ctx_start + j;
-      int64_t ctx_pos_idx = std::min(ctx_idx, total_input_tokens - 1);
-      int64_t position = target_pos_ptr[ctx_pos_idx];
-      int64_t block_num = position / block_size;
-      block_num = std::min(block_num, block_table_stride - 1);
-      int32_t block_id =
-          block_table_ptr[req_idx * block_table_stride + block_num];
-      int64_t slot = block_id * block_size + (position % block_size);
-
-      out_ctx_pos_ptr[ctx_idx] = position;
-      out_ctx_slot_ptr[ctx_idx] = slot;
-    }
-
-    for (int64_t query_off = 0; query_off < num_query_per_req; ++query_off) {
-      int64_t query_out = req_idx * num_query_per_req + query_off;
-      int64_t position = last_pos + 1 + query_off;
-      int64_t block_num = position / block_size;
-      block_num = std::min(block_num, block_table_stride - 1);
-      int32_t block_id =
-          block_table_ptr[req_idx * block_table_stride + block_num];
-      int64_t slot = block_id * block_size + (position % block_size);
-
-      out_query_pos_ptr[query_out] = position;
-      out_query_slot_ptr[query_out] = slot;
-      out_ids_ptr[query_out] =
-          query_off == 0 ? next_ids_ptr[req_idx] : parallel_drafting_token_id;
-
-      if (query_off > 0) {
-        int64_t sample_out_idx =
-            req_idx * num_speculative_tokens + (query_off - 1);
-        out_token_idx_ptr[sample_out_idx] = query_out;
-      }
-    }
-  }
-}
-
 void rejection_greedy_sample_kernel_impl(
    torch::Tensor& output_token_ids, const torch::Tensor& cu_num_draft_tokens,
    const torch::Tensor& draft_token_ids, const torch::Tensor& target_argmax,
@@ -146,16 +146,13 @@ at::Tensor causal_conv1d_update_cpu(
 void activation_lut_bf16(torch::Tensor& out, torch::Tensor& input,
                         const std::string& activation);

-bool cpu_attn_has_isa(const std::string& isa);
-
 torch::Tensor get_scheduler_metadata(
    const int64_t num_req, const int64_t num_heads_q,
    const int64_t num_heads_kv, const int64_t head_dim,
    const torch::Tensor& seq_lens, at::ScalarType dtype,
    const torch::Tensor& query_start_loc, const bool casual,
    const int64_t window_size, const std::string& isa_hint,
-    const bool enable_kv_split,
-    const std::optional<torch::Tensor>& dynamic_causal);
+    const bool enable_kv_split);

 void cpu_attn_reshape_and_cache(const torch::Tensor& key,
                                const torch::Tensor& value,
@@ -172,10 +169,10 @@ void cpu_attention_with_kv_cache(
    const torch::Tensor& query_start_loc, const torch::Tensor& seq_lens,
    const double scale, const bool causal,
    const std::optional<torch::Tensor>& alibi_slopes,
-    const int64_t sliding_window_left, const torch::Tensor& block_table,
-    const double softcap, const torch::Tensor& scheduler_metadata,
-    const std::optional<torch::Tensor>& s_aux,
-    const std::optional<torch::Tensor>& dynamic_causal, const double k_scale,
+    const int64_t sliding_window_left, const int64_t sliding_window_right,
+    const torch::Tensor& block_table, const double softcap,
+    const torch::Tensor& scheduler_metadata,
+    const std::optional<torch::Tensor>& s_aux, const double k_scale,
    const double v_scale, const std::string& kv_cache_dtype);

 // Note: just for avoiding importing errors
@@ -237,16 +234,6 @@ void copy_and_expand_eagle_inputs_kernel_impl(
    const int64_t padding_token_id, const int64_t parallel_drafting_token_id,
    const int64_t total_input_tokens,
    const int64_t num_padding_slots_per_request, const bool shift_input_ids);
-void copy_and_expand_dflash_inputs_kernel_impl(
-    const torch::Tensor& next_token_ids, const torch::Tensor& target_positions,
-    torch::Tensor& out_input_ids, torch::Tensor& out_context_positions,
-    torch::Tensor& out_query_positions, torch::Tensor& out_context_slot_mapping,
-    torch::Tensor& out_query_slot_mapping, torch::Tensor& out_token_indices,
-    const torch::Tensor& block_table, const torch::Tensor& query_start_loc,
-    const std::optional<torch::Tensor>& num_rejected_tokens,
-    const int64_t parallel_drafting_token_id, const int64_t block_size,
-    const int64_t num_query_per_req, const int64_t num_speculative_tokens,
-    const int64_t total_input_tokens, const bool has_num_rejected);
 void rejection_greedy_sample_kernel_impl(
    torch::Tensor& output_token_ids, const torch::Tensor& cu_num_draft_tokens,
    const torch::Tensor& draft_token_ids, const torch::Tensor& target_argmax,
@@ -322,13 +309,13 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
  // Layernorm
  // Apply Root Mean Square (RMS) Normalization to the input tensor.
  ops.def(
-      "rms_norm(Tensor! out, Tensor input, Tensor? weight, float epsilon) -> "
+      "rms_norm(Tensor! out, Tensor input, Tensor weight, float epsilon) -> "
      "()");
  ops.impl("rms_norm", torch::kCPU, &rms_norm);

  // In-place fused Add and RMS Normalization.
  ops.def(
-      "fused_add_rms_norm(Tensor! input, Tensor! residual, Tensor? weight, "
+      "fused_add_rms_norm(Tensor! input, Tensor! residual, Tensor weight, "
      "float epsilon) -> ()");
  ops.impl("fused_add_rms_norm", torch::kCPU, &fused_add_rms_norm);

@@ -509,12 +496,11 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
  ops.impl("fused_gdn_gating_cpu", torch::kCPU, &fused_gdn_gating_cpu);

  // CPU attention kernels
-  ops.def("cpu_attn_has_isa(str isa) -> bool", &cpu_attn_has_isa);
  ops.def(
      "get_scheduler_metadata(int num_req, int num_heads_q, int num_heads_kv, "
      "int head_dim, Tensor seq_lens, ScalarType dtype, Tensor "
      "query_start_loc, bool casual, int window_size, str isa_hint, bool "
-      "enable_kv_split, Tensor? dynamic_causal) -> Tensor",
+      "enable_kv_split) -> Tensor",
      &get_scheduler_metadata);
  ops.def(
      "cpu_attn_reshape_and_cache(Tensor key, Tensor value, Tensor(a2!) "
@@ -526,9 +512,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
      "cpu_attention_with_kv_cache(Tensor query, Tensor key_cache, Tensor "
      "value_cache, Tensor(a3!) output, Tensor query_start_loc, Tensor "
      "seq_lens, float scale, bool causal, Tensor? alibi_slopes, SymInt "
-      "sliding_window_size, Tensor block_table, "
-      "float softcap, Tensor scheduler_metadata, Tensor? s_aux, Tensor? "
-      "dynamic_causal, "
+      "sliding_window_left, SymInt sliding_window_right, Tensor block_table, "
+      "float softcap, Tensor scheduler_metadata, Tensor? s_aux, "
      "float k_scale=1.0, float v_scale=1.0, str kv_cache_dtype=\"auto\") -> "
      "()",
      &cpu_attention_with_kv_cache);
@@ -548,7 +533,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
 #endif

  // fused moe
-#if defined(__AVX512F__) || (defined(ARM_BF16_SUPPORT))
+#if defined(__AVX512F__)
  ops.def(
      "prepack_moe_weight(Tensor weight, Tensor(a1!) packed_weight, str isa) "
      "-> ()");
@@ -559,7 +544,7 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
      "bool skip_weighted, "
      "str act, str isa) -> ()");
  ops.impl("cpu_fused_moe", torch::kCPU, &cpu_fused_moe);
-#endif  // #if defined(__AVX512F__) || (defined(ARM_BF16_SUPPORT))
+#endif
  ops.def(
      "mla_decode_kvcache("
      "   Tensor! out, Tensor query, Tensor kv_cache,"
@@ -609,19 +594,6 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
      "SymInt total_input_tokens, SymInt num_padding_slots_per_request, "
      "bool shift_input_ids) -> ()",
      &cpu_utils::copy_and_expand_eagle_inputs_kernel_impl);
-  ops.def(
-      "copy_and_expand_dflash_inputs_kernel_impl("
-      "Tensor next_token_ids, Tensor target_positions, "
-      "Tensor(a2!) out_input_ids, Tensor(a3!) out_context_positions, "
-      "Tensor(a4!) out_query_positions, "
-      "Tensor(a5!) out_context_slot_mapping, "
-      "Tensor(a6!) out_query_slot_mapping, "
-      "Tensor(a7!) out_token_indices, Tensor block_table, "
-      "Tensor query_start_loc, Tensor? num_rejected_tokens, "
-      "SymInt parallel_drafting_token_id, SymInt block_size, "
-      "SymInt num_query_per_req, SymInt num_speculative_tokens, "
-      "SymInt total_input_tokens, bool has_num_rejected) -> ()",
-      &cpu_utils::copy_and_expand_dflash_inputs_kernel_impl);
  ops.def(
      "rejection_greedy_sample_kernel_impl("
      "Tensor(a0!) output_token_ids, Tensor cu_num_draft_tokens, "
@@ -2,24 +2,19 @@
 #define UTILS_HPP

 #include <atomic>
-#include <string>
 #include <unistd.h>
 #include <ATen/cpu/Utils.h>

 #include "cpu/cpu_types.hpp"

 namespace cpu_utils {
-enum class ISA { AMX, VEC, RVV, NEON };
+enum class ISA { AMX, VEC };

 inline ISA get_isa(const std::string& isa) {
  if (isa == "amx") {
    return ISA::AMX;
  } else if (isa == "vec") {
    return ISA::VEC;
-  } else if (isa == "rvv") {
-    return ISA::RVV;
-  } else if (isa == "neon") {
-    return ISA::NEON;
  } else {
    TORCH_CHECK(false, "Invalid isa type: " + isa);
  }
@@ -48,8 +48,8 @@ static inline unsigned long long my_min(unsigned long long a,
 }

 static CUresult reserve_rocm_address(CUdeviceptr* d_mem, size_t size,
-                                     size_t alignment, CUdeviceptr addr = 0) {
-  CUresult status = cuMemAddressReserve(d_mem, size, alignment, addr, 0);
+                                     size_t alignment) {
+  CUresult status = cuMemAddressReserve(d_mem, size, alignment, 0, 0);
  if (status == CUresult(0) || alignment == 0) {
    return status;
  }
@@ -58,7 +58,7 @@ static CUresult reserve_rocm_address(CUdeviceptr* d_mem, size_t size,
  // alignment even when physical VRAM is free. Let HIP choose the default
  // alignment, then verify that the returned address still satisfies the
  // requested alignment before accepting it.
-  status = cuMemAddressReserve(d_mem, size, 0, addr, 0);
+  status = cuMemAddressReserve(d_mem, size, 0, 0, 0);
  if (status != CUresult(0)) {
    return status;
  }
@@ -535,14 +535,7 @@ void my_free(void* ptr, ssize_t size, int device, CUstream stream) {
  Py_DECREF(py_result);
  PyGILState_Release(gstate);

-  // An empty chunk list means this allocation is asleep: its physical chunks
-  // were already unmapped and released by sleep(), but the virtual address is
-  // still held as a placeholder reservation. Skip unmap/release (freeing the
-  // placeholder address happens below).
-  if (num_chunks > 0) {
-    unmap_and_release(device, size, d_mem, p_memHandle, chunk_sizes,
-                      num_chunks);
-  }
+  unmap_and_release(device, size, d_mem, p_memHandle, chunk_sizes, num_chunks);
 #else
  // Non-ROCm path: simple integer handle already extracted; drop temporary
  // Python refs while still holding the GIL, then release it.
@@ -555,13 +548,11 @@ void my_free(void* ptr, ssize_t size, int device, CUstream stream) {
  unmap_and_release(device, size, d_mem, p_memHandle);
 #endif

-  // Free the virtual address. On ROCm this also covers an asleep allocation,
-  // whose placeholder reservation made by sleep() is still held here.
+  // free address and the handle
  CUDA_CHECK(cuMemAddressFree(d_mem, size));
 #ifndef USE_ROCM
  free(p_memHandle);
 #else
-  // Only awake allocations have per-chunk handles to free.
  for (auto i = 0; i < num_chunks; ++i) {
    free(p_memHandle[i]);
  }
@@ -681,29 +672,6 @@ static PyObject* python_unmap_and_release(PyObject* self, PyObject* args) {
  unmap_and_release(recv_device, recv_size, d_mem_ptr, p_memHandle, chunk_sizes,
                    num_chunks);

-  // On ROCm/Linux, physical VRAM is only reclaimed once the virtual address
-  // range is freed; hipMemUnmap + hipMemRelease alone leave the memory
-  // resident (see ROCm#6021). Free the address to release physical memory,
-  // then immediately re-reserve the SAME address as an empty placeholder so
-  // the regular allocator cannot hand it out while we sleep. wake_up remaps
-  // physical chunks into this placeholder.
-  if (error_code == no_error) {
-    CUDA_CHECK(cuMemAddressFree(d_mem_ptr, recv_size));
-    if (error_code == no_error) {
-      CUdeviceptr reserved = 0;
-      CUDA_CHECK(reserve_rocm_address(&reserved, recv_size, /*alignment=*/0,
-                                      d_mem_ptr));
-      if (error_code == no_error && reserved != d_mem_ptr) {
-        (void)cuMemAddressFree(reserved, recv_size);
-        snprintf(error_msg, sizeof(error_msg),
-                 "failed to re-reserve placeholder address on sleep "
-                 "(requested %#llx, got %#llx)",
-                 (unsigned long long)d_mem_ptr, (unsigned long long)reserved);
-        error_code = CUresult(1);
-      }
-    }
-  }
-
  free(p_memHandle);
  free(chunk_sizes);
 #endif
@@ -768,7 +736,6 @@ static PyObject* python_create_and_map(PyObject* self, PyObject* args) {
    chunk_sizes[i] = PyLong_AsUnsignedLongLong(size_py);
  }

-  // Address already reserved as a placeholder by sleep(); just remap chunks.
  create_and_map(recv_device, recv_size, d_mem_ptr, p_memHandle, chunk_sizes,
                 num_chunks);

@@ -0,0 +1,361 @@
+/**
+ * This is a standalone test for custom allreduce.
+ * To compile, make sure you have MPI and NCCL installed in your system.
+ * export MPI_HOME=XXX
+ * nvcc -O2 -arch=native -std=c++17 custom_all_reduce_test.cu -o
+ * custom_all_reduce_test -lnccl -I${MPI_HOME}/include -lmpi
+ *
+ * Warning: this C++ test is not designed to be very readable and was used
+ * during the rapid prototyping process.
+ *
+ * To run:
+ * mpirun --allow-run-as-root -np 8 ./custom_all_reduce_test
+ */
+#include <cuda.h>
+#include <curand_kernel.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <limits>
+#include <vector>
+
+#include "cuda_profiler_api.h"
+#include "custom_all_reduce.cuh"
+#include "mpi.h"
+#ifdef USE_ROCM
+  #include <hip/hip_bf16.h>
+typedef __hip_bfloat16 nv_bfloat16;
+  #include "rccl/rccl.h"
+  #include "custom_all_reduce_hip.cuh"
+#else
+  #include "nccl.h"
+  #include "custom_all_reduce.cuh"
+#endif
+
+#define MPICHECK(cmd)                                                  \
+  do {                                                                 \
+    int e = cmd;                                                       \
+    if (e != MPI_SUCCESS) {                                            \
+      printf("Failed: MPI error %s:%d '%d'\n", __FILE__, __LINE__, e); \
+      exit(EXIT_FAILURE);                                              \
+    }                                                                  \
+  } while (0)
+
+#define NCCLCHECK(cmd)                                              \
+  do {                                                              \
+    ncclResult_t r = cmd;                                           \
+    if (r != ncclSuccess) {                                         \
+      printf("Failed, NCCL error %s:%d '%s'\n", __FILE__, __LINE__, \
+             ncclGetErrorString(r));                                \
+      exit(EXIT_FAILURE);                                           \
+    }                                                               \
+  } while (0)
+
+#ifdef USE_ROCM
+__global__ void dummy_kernel() {
+  for (int i = 0; i < 100; i++) {
+    uint64_t start = wall_clock64();
+    uint64_t cycles_elapsed;
+    do {
+      cycles_elapsed = wall_clock64() - start;
+    } while (cycles_elapsed < 100);
+  }
+  for (int i = 0; i < 100; i++) __nanosleep(1000000);  // 100ms
+}
+#else
+__global__ void dummy_kernel() {
+  #if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 700
+  for (int i = 0; i < 100; i++) __nanosleep(1000000);  // 100ms
+  #else
+  for (int i = 0; i < 100; i++) {
+    long long int start = clock64();
+    while (clock64() - start < 150000000);  // approximately 98.4ms on P40
+  }
+  #endif
+}
+#endif
+
+template <typename T>
+__global__ void set_data(T* data, int size, int myRank) {
+  for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < size;
+       idx += gridDim.x * blockDim.x) {
+    data[idx] = myRank * 0.11f;
+  }
+}
+
+template <typename T>
+__global__ void convert_data(const T* data1, const T* data2, double* fdata1,
+                             double* fdata2, int size) {
+  for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < size;
+       idx += gridDim.x * blockDim.x) {
+    fdata1[idx] = data1[idx];
+    fdata2[idx] = data2[idx];
+  }
+}
+
+__global__ void init_rand(curandState_t* state, int size, int nRanks) {
+  for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < size;
+       idx += gridDim.x * blockDim.x) {
+    for (int i = 0; i < nRanks; i++) {
+      curand_init(i + 1, idx, 0, &state[idx * nRanks + i]);
+    }
+  }
+}
+
+template <typename T>
+__global__ void gen_data(curandState_t* state, T* data, double* ground_truth,
+                         int myRank, int nRanks, int size) {
+  for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < size;
+       idx += gridDim.x * blockDim.x) {
+    double sum = 0.0;
+    for (int i = 0; i < nRanks; i++) {
+      double val = curand_uniform_double(&state[idx * nRanks + i]) * 4;
+      T hval = val;  // downcast first
+      sum += static_cast<double>(hval);
+      if (i == myRank) data[idx] = hval;
+    }
+    ground_truth[idx] = sum;
+  }
+}
+
+template <typename T>
+void run(int myRank, int nRanks, ncclComm_t& comm, int threads, int block_limit,
+         int data_size, bool performance_test) {
+  T* result;
+  cudaStream_t stream;
+  CUDACHECK(cudaStreamCreateWithFlags(&stream, cudaStreamNonBlocking));
+  CUDACHECK(cudaMalloc(&result, data_size * sizeof(T)));
+  CUDACHECK(cudaMemset(result, 0, data_size * sizeof(T)));
+
+  cudaIpcMemHandle_t self_data_handle;
+  cudaIpcMemHandle_t data_handles[8];
+  vllm::Signal* buffer;
+  T* self_data_copy;
+  /**
+   * Allocate IPC buffer
+   *
+   * The first section is a temporary buffer for storing intermediate allreduce
+   * results, if a particular algorithm requires it. The second section is for
+   * the input to the allreduce. The actual API takes the input pointer as an
+   * argument (that is, they can and usually should be allocated separately).
+   * But since the input pointers and the temporary buffer all require IPC
+   * registration, they are allocated and registered together in the test for
+   * convenience.
+   */
+#ifdef USE_ROCM
+  CUDACHECK(hipExtMallocWithFlags(
+      (void**)&buffer, 2 * data_size * sizeof(T) + sizeof(vllm::Signal),
+      hipDeviceMallocUncached));
+#else
+  CUDACHECK(
+      cudaMalloc(&buffer, 2 * data_size * sizeof(T) + sizeof(vllm::Signal)));
+#endif
+  CUDACHECK(
+      cudaMemset(buffer, 0, 2 * data_size * sizeof(T) + sizeof(vllm::Signal)));
+  CUDACHECK(cudaMalloc(&self_data_copy, data_size * sizeof(T)));
+  CUDACHECK(cudaIpcGetMemHandle(&self_data_handle, buffer));
+
+  MPICHECK(MPI_Allgather(&self_data_handle, sizeof(cudaIpcMemHandle_t),
+                         MPI_BYTE, data_handles, sizeof(cudaIpcMemHandle_t),
+                         MPI_BYTE, MPI_COMM_WORLD));
+
+  void* rank_data;
+  size_t rank_data_sz = 16 * 1024 * 1024;
+  CUDACHECK(cudaMalloc(&rank_data, rank_data_sz));
+  vllm::Signal* ipc_ptrs[8];
+  for (int i = 0; i < nRanks; i++) {
+    if (i == myRank)
+      ipc_ptrs[i] = buffer;
+    else
+      CUDACHECK(cudaIpcOpenMemHandle((void**)&ipc_ptrs[i], data_handles[i],
+                                     cudaIpcMemLazyEnablePeerAccess));
+  }
+  vllm::CustomAllreduce fa(ipc_ptrs, rank_data, rank_data_sz, myRank, nRanks);
+  auto* self_data =
+      reinterpret_cast<T*>(reinterpret_cast<char*>(buffer) +
+                           sizeof(vllm::Signal) + data_size * sizeof(T));
+  // hack buffer registration
+  {
+    void* data[8];
+    for (int i = 0; i < nRanks; i++) {
+      data[i] =
+          ((char*)ipc_ptrs[i]) + sizeof(vllm::Signal) + data_size * sizeof(T);
+    }
+    fa.register_buffer(data);
+  }
+
+  double* ground_truth;
+  CUDACHECK(cudaMallocHost(&ground_truth, data_size * sizeof(double)));
+  curandState_t* states;
+  CUDACHECK(cudaMalloc(&states, sizeof(curandState_t) * nRanks * data_size));
+  init_rand<<<108, 1024, 0, stream>>>(states, data_size, nRanks);
+  gen_data<T><<<108, 1024, 0, stream>>>(states, self_data, ground_truth, myRank,
+                                        nRanks, data_size);
+  CUDACHECK(cudaMemcpyAsync(self_data_copy, self_data, data_size * sizeof(T),
+                            cudaMemcpyDeviceToDevice, stream));
+  cudaEvent_t start, stop;
+  CUDACHECK(cudaEventCreate(&start));
+  CUDACHECK(cudaEventCreate(&stop));
+
+  ncclDataType_t ncclDtype;
+  if (std::is_same<T, half>::value) {
+    ncclDtype = ncclFloat16;
+  } else if (std::is_same<T, nv_bfloat16>::value) {
+    ncclDtype = ncclBfloat16;
+  } else {
+    ncclDtype = ncclFloat;
+  }
+  double *nccl_result, *my_result;
+  CUDACHECK(cudaMallocHost(&nccl_result, data_size * sizeof(double)));
+  CUDACHECK(cudaMallocHost(&my_result, data_size * sizeof(double)));
+  if (performance_test) {
+    dummy_kernel<<<1, 1, 0, stream>>>();
+    constexpr int warmup_iters = 5;
+    constexpr int num_iters = 100;
+    // warmup
+    for (int i = 0; i < warmup_iters; i++) {
+      NCCLCHECK(ncclAllReduce(result, result, data_size, ncclDtype, ncclSum,
+                              comm, stream));
+    }
+    CUDACHECK(cudaEventRecord(start, stream));
+    for (int i = 0; i < num_iters; i++) {
+      NCCLCHECK(ncclAllReduce(result, result, data_size, ncclDtype, ncclSum,
+                              comm, stream));
+    }
+    CUDACHECK(cudaEventRecord(stop, stream));
+    CUDACHECK(cudaStreamSynchronize(stream));
+    float allreduce_ms = 0;
+    cudaEventElapsedTime(&allreduce_ms, start, stop);
+
+    dummy_kernel<<<1, 1, 0, stream>>>();
+    // warm up
+    for (int i = 0; i < warmup_iters; i++) {
+      fa.allreduce<T>(stream, self_data, result, data_size, threads,
+                      block_limit);
+    }
+    CUDACHECK(cudaEventRecord(start, stream));
+    for (int i = 0; i < num_iters; i++) {
+      fa.allreduce<T>(stream, self_data, result, data_size, threads,
+                      block_limit);
+    }
+    CUDACHECK(cudaEventRecord(stop, stream));
+    CUDACHECK(cudaStreamSynchronize(stream));
+
+    float duration_ms = 0;
+    cudaEventElapsedTime(&duration_ms, start, stop);
+    if (myRank == 0)
+      printf(
+          "Rank %d done, nGPUs:%d, sz (kb): %d, %d, %d, my time:%.2fus, nccl "
+          "time:%.2fus\n",
+          myRank, nRanks, data_size * sizeof(T) / 1024, threads, block_limit,
+          duration_ms * 1e3 / num_iters, allreduce_ms * 1e3 / num_iters);
+
+    // And wait for all the queued up work to complete
+    CUDACHECK(cudaStreamSynchronize(stream));
+
+    NCCLCHECK(ncclAllReduce(self_data_copy, self_data, data_size, ncclDtype,
+                            ncclSum, comm, stream));
+
+    convert_data<T><<<108, 1024, 0, stream>>>(self_data, result, nccl_result,
+                                              my_result, data_size);
+    CUDACHECK(cudaStreamSynchronize(stream));
+
+    for (unsigned long j = 0; j < data_size; j++) {
+      auto diff = abs(nccl_result[j] - my_result[j]);
+      if (diff >= 4e-2) {
+        printf("Rank %d: Verification mismatch at %lld: %f != (my) %f, gt=%f\n",
+               myRank, j, nccl_result[j], my_result[j], ground_truth[j]);
+        break;
+      }
+    }
+    long double nccl_diffs = 0.0;
+    long double my_diffs = 0.0;
+    for (int j = 0; j < data_size; j++) {
+      nccl_diffs += abs(nccl_result[j] - ground_truth[j]);
+      my_diffs += abs(my_result[j] - ground_truth[j]);
+    }
+    if (myRank == 0)
+      std::cout << "average abs diffs: nccl: " << nccl_diffs / data_size
+                << " me: " << my_diffs / data_size << std::endl;
+  } else {
+    for (int i = 0; i < 100; i++) {
+      fa.allreduce<T>(stream, self_data, result, data_size, threads,
+                      block_limit);
+      CUDACHECK(cudaStreamSynchronize(stream));
+      NCCLCHECK(ncclAllReduce(self_data, self_data_copy, data_size, ncclDtype,
+                              ncclSum, comm, stream));
+      convert_data<T><<<108, 1024, 0, stream>>>(
+          self_data_copy, result, nccl_result, my_result, data_size);
+      CUDACHECK(cudaStreamSynchronize(stream));
+
+      for (unsigned long j = 0; j < data_size; j++) {
+        auto diff = abs(nccl_result[j] - my_result[j]);
+        if (diff >= 4e-2) {
+          printf(
+              "Rank %d: Verification mismatch at %lld: %f != (my) %f, gt=%f\n",
+              myRank, j, nccl_result[j], my_result[j], ground_truth[j]);
+          break;
+        }
+      }
+    }
+    if (myRank == 0)
+      printf("Test passed: nGPUs:%d, sz (kb): %d, %d, %d\n", nRanks,
+             data_size * sizeof(T) / 1024, threads, block_limit);
+    // long double nccl_diffs = 0.0;
+    // long double my_diffs = 0.0;
+    // for (int j = 0; j < data_size; j++) {
+    //   nccl_diffs += abs(nccl_result[j] - ground_truth[j]);
+    //   my_diffs += abs(my_result[j] - ground_truth[j]);
+    // }
+    // if (myRank == 0)
+    //   std::cout << "average abs diffs: nccl: " << nccl_diffs / data_size
+    //             << " me: " << my_diffs / data_size << std::endl;
+  }
+
+  CUDACHECK(cudaFree(result));
+  CUDACHECK(cudaFree(self_data_copy));
+  CUDACHECK(cudaFree(rank_data));
+  CUDACHECK(cudaFree(buffer));
+  CUDACHECK(cudaFree(states));
+  CUDACHECK(cudaFreeHost(ground_truth));
+  CUDACHECK(cudaFreeHost(nccl_result));
+  CUDACHECK(cudaFreeHost(my_result));
+  CUDACHECK(cudaStreamDestroy(stream));
+}
+
+int main(int argc, char** argv) {
+  int nRanks, myRank;
+  MPICHECK(MPI_Init(&argc, &argv));
+  MPICHECK(MPI_Comm_rank(MPI_COMM_WORLD, &myRank));
+  MPICHECK(MPI_Comm_size(MPI_COMM_WORLD, &nRanks));
+  CUDACHECK(cudaSetDevice(myRank));
+  ncclUniqueId id;
+  ncclComm_t comm;
+  if (myRank == 0) ncclGetUniqueId(&id);
+  MPICHECK(MPI_Bcast(static_cast<void*>(&id), sizeof(id), MPI_BYTE, 0,
+                     MPI_COMM_WORLD));
+  NCCLCHECK(ncclCommInitRank(&comm, nRanks, id, myRank));
+
+  bool performance_test = true;
+  cudaProfilerStart();
+// Uncomment to scan through different block size configs.
+// for (int threads : {256, 512, 1024}) {
+//   for (int block_limit = 16; block_limit < 112; block_limit += 4) {
+//     run<half>(myRank, nRanks, comm, threads, block_limit, 1024 * 1024,
+//     performance_test);
+//   }
+// }
+#ifdef USE_ROCM
+  const int block_limit = 16;
+#else
+  const int block_limit = 36;
+#endif
+  // Scan through different sizes to test performance.
+  for (int sz = 512; sz <= (8 << 20); sz *= 2) {
+    run<half>(myRank, nRanks, comm, 512, 36, sz + 8 * 47, performance_test);
+  }
+
+  cudaProfilerStop();
+  MPICHECK(MPI_Finalize());
+  return EXIT_SUCCESS;
+}
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Alexander MatveevandClaude Opus 4.6	fa6b6a83ec	Apply pre-commit formatting and use torch.accelerator API - Apply ruff, clang-format formatting fixes - Replace torch.cuda.set_device/device_count/synchronize with torch.accelerator equivalents per project convention Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Signed-off-by: Alexander Matveev <alexm-redhat@dgx-b200-02.mgmt.accl-001.lab.rdu2.dc.redhat.com> Signed-off-by: <>	2026-06-15 18:21:59 -04:00
Alexander MatveevandClaude Opus 4.6	fd44100bb0	Address review: add push_ar to benchmark and backend logging - Add PushAllReduce to benchmark_device_communicators.py for comparing against other allreduce implementations - Add PUSH_AR to _log_all_reduce_backend_selection in cuda_communicator.py for visibility in dispatch logging Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Signed-off-by: Alexander Matveev <alexm-redhat@dgx-b200-02.mgmt.accl-001.lab.rdu2.dc.redhat.com>	2026-06-15 18:09:24 -04:00
Alexander MatveevandClaude Opus 4.6	bc50e5fc2e	Address review: use registered envs.VLLM_DISABLE_PUSH_ALLREDUCE Replace direct os.environ.get(_DISABLE_ENV_VAR) == "1" check with envs.VLLM_DISABLE_PUSH_ALLREDUCE to use the centrally registered env var from envs.py, which provides validation and caching. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Signed-off-by: Alexander Matveev <alexm-redhat@dgx-b200-02.mgmt.accl-001.lab.rdu2.dc.redhat.com>	2026-06-15 18:09:24 -04:00
Alexander MatveevandClaude Opus 4.6	625bea3939	Address review: register VLLM_DISABLE_PUSH_ALLREDUCE in envs.py Register the push allreduce feature toggle env var in the central envs.py registry so it is validated on startup and follows the standard vllm env var pattern. Default is False (push allreduce enabled); set to 1 to disable. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Signed-off-by: Alexander Matveev <alexm-redhat@dgx-b200-02.mgmt.accl-001.lab.rdu2.dc.redhat.com>	2026-06-15 18:09:24 -04:00
Alexander MatveevandClaude Opus 4.6	17488a743f	Address review: add architecture-specific threshold selection The push threshold map was labeled as sm100-specific but applied unconditionally to all architectures. Now: - PUSH_THRESHOLD_SM100 is only used on Blackwell (compute capability 10.x) - PUSH_THRESHOLD_DEFAULT provides conservative 512 KB thresholds for architectures without tuned values - _THRESHOLD_BY_ARCH maps GPU major compute capability to threshold tables - A log message is emitted when falling back to conservative defaults Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Signed-off-by: Alexander Matveev <alexm-redhat@dgx-b200-02.mgmt.accl-001.lab.rdu2.dc.redhat.com>	2026-06-15 18:09:24 -04:00
Alexander MatveevandClaude Opus 4.6	54b43935a8	Address review: add type annotation and cleanup for push_ar_comm - Add PushAllReduce \| None type annotation on push_ar_comm to be consistent with other communicator fields (ca_comm, qr_comm, etc.) - Add push_ar_comm.close() + None assignment in destroy() method to match the cleanup pattern for other communicators - Add lazy import of PushAllReduce alongside other communicator imports Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Signed-off-by: Alexander Matveev <alexm-redhat@dgx-b200-02.mgmt.accl-001.lab.rdu2.dc.redhat.com>	2026-06-15 18:09:24 -04:00
Alexander MatveevandClaude Opus 4.6	e0601e1b94	Address review: bind test sockets to localhost instead of all interfaces Fix CodeQL security warning by binding test helper sockets to "localhost" instead of "" (all interfaces). These sockets are only used for finding a free port for torch distributed init in tests. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Signed-off-by: Alexander Matveev <alexm-redhat@dgx-b200-02.mgmt.accl-001.lab.rdu2.dc.redhat.com>	2026-06-15 18:09:24 -04:00
Alexander MatveevandClaude Opus 4.6	c55216ebbf	Address review: add CUDA error checking and runtime buffer overflow guard - Add PUSH_AR_CUDACHECK macro wrapping all CUDA API calls (cudaGetDevice, cudaDeviceGetAttribute, cudaMalloc, cudaMemset, cudaIpcGetMemHandle, cudaIpcOpenMemHandle) to match the CUDACHECK pattern in custom_all_reduce.cuh - Replace assert(input_bytes <= push_buffer_bytes_) with a runtime std::runtime_error check that is not compiled out under -DNDEBUG - Add #include <stdexcept> and #include <string> for the runtime check Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Signed-off-by: Alexander Matveev <alexm-redhat@dgx-b200-02.mgmt.accl-001.lab.rdu2.dc.redhat.com>	2026-06-15 18:09:24 -04:00
Alexander MatveevandAlexander Matveev	5fd9bd4f04	perf: add push-based allreduce for small tensor reductions Port SGLang's push-based 2-buffer allreduce protocol into vLLM as a new communicator backend for small-message reductions. The push protocol eliminates the two explicit cross-GPU NVLink barrier round-trips used by the existing barrier-based CustomAllreduce, replacing them with a sentinel-based data arrival detection mechanism and double-buffered epoch alternation. Key advantages over the barrier-based approach: - Zero barriers: data arrival IS the synchronization (positive-zero sentinel) - Single NVLink round-trip instead of two barrier exchanges + remote reads - All SMs active (SM_count CTAs vs 2 CTAs) for higher NVLink bandwidth - No cudaMemcpy to IPC staging buffer in eager mode - PDL (griddepcontrol) support for kernel overlap on sm_90+ The new PushAllReduce is inserted in the CudaCommunicator dispatch chain above the existing CustomAllreduce for messages below a size threshold (~720 KB at TP=8). Larger messages continue to use the barrier-based path. The existing CustomAllreduce code is not modified. Measured results on DeepSeek-V4-Pro (61 layers, TP=8, 8x NVIDIA B200, BS=1, decode with ISL=4, OSL=33024): - Throughput: +2.14% (84.06 vs 82.30 tokens/s) - TPOT: -2.09% (11.90 vs 12.15 ms/token) Correctness verified via lm_eval gsm8k 5-shot with no regression (exact_match delta within statistical noise). The feature can be disabled at runtime via VLLM_DISABLE_PUSH_ALLREDUCE=1 to fall back to the barrier-based path. Signed-off-by: Alexander Matveev <amatveev@redhat.com> Signed-off-by: Alexander Matveev <alexm-redhat@dgx-b200-02.mgmt.accl-001.lab.rdu2.dc.redhat.com>	2026-06-15 18:09:24 -04:00