[Misc] Add EPLB debug logging for balance diagnostics

Add comprehensive debug logging to the EPLB system to help diagnose expert load balancing issues in wideEP deployments: - Per-step balance breakdown: worst/best layer indices, min/max rank token counts for the worst layer, replica distribution stats - Pre-rearrange diagnostics: window utilization, load distribution across logical experts, top-5 hottest experts - Post-rearrange diagnostics: number of changed slots, replica count stats, predicted post-rearrange balancedness (simulates expected balance with the new mapping applied to current load data) - Warning when window_size > step_interval (stale data risk) Signed-off-by: Travis Shears <travis@neuralmagic.com> Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com> Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
[BugFix] Fix EPLB balancedness metric using wrong dimension
2026-03-06 22:20:28 -05:00 · 2026-03-06 22:18:49 -05:00
4718 changed files with 196431 additions and 851497 deletions
@@ -8,9 +8,8 @@ run_all_patterns:
  - "CMakeLists.txt"
  - "requirements/common.txt"
  - "requirements/cuda.txt"
-  - "requirements/kv_connectors.txt"
+  - "requirements/build.txt"
-  - "requirements/build/cuda.txt"
+  - "requirements/test.txt"
  - "requirements/test/cuda.txt"
  - "setup.py"
  - "csrc/"
  - "cmake/"
@@ -1,23 +0,0 @@
 name: vllm_intel_ci
 job_dirs:
  - ".buildkite/intel_jobs"
 run_all_patterns:
  - ".buildkite/ci_config_intel.yaml"
  - ".buildkite/scripts/hardware_ci/run-intel-test.sh"
  - "docker/Dockerfile"
  - "docker/Dockerfile.xpu"
  - "CMakeLists.txt"
  - "requirements/common.txt"
  - "requirements/xpu.txt"
  - "setup.py"
  - "csrc/"
  - "cmake/"
 run_all_exclude_patterns:
  - "csrc/cpu/"
  - "csrc/rocm/"
  - "cmake/hipify.py"
  - "cmake/cpu_extension.cmake"
 registries: public.ecr.aws/q9t5s3a7
 repositories:
  main: "vllm-ci-test-repo"
  premerge: "vllm-ci-test-repo"
@@ -1,24 +0,0 @@
 name: vllm_rocm_ci
 job_dirs:
  - ".buildkite/hardware_tests"
 run_all_patterns:
  - "docker/Dockerfile.rocm"
  - "docker/Dockerfile.rocm_base"
  - "docker/ci-rocm.hcl"
  - "docker/docker-bake-rocm.hcl"
  - ".buildkite/hardware_tests/amd.yaml"
  - ".buildkite/scripts/ci-bake-rocm.sh"
  - ".buildkite/scripts/rocm/"
  - ".buildkite/scripts/hardware_ci/run-amd-test.py"
  - ".buildkite/scripts/hardware_ci/run-amd-test.sh"
  - "CMakeLists.txt"
  - "requirements/common.txt"
  - "requirements/rocm.txt"
  - "requirements/build/rocm.txt"
  - "requirements/test/rocm.txt"
  - "setup.py"
  - "csrc/"
  - "cmake/"
 run_all_exclude_patterns:
  - "csrc/cpu/"
  - "cmake/cpu_extension.cmake"
@@ -1,84 +1,30 @@
-group: Hardware - AMD Build
+group: Hardware - AMD Build 
 # ROCm image flow:
 # 1. Refresh the long-lived ROCm base image only when Dockerfile.rocm_base changes.
 # 2. Build ci_base from either the stable base or the freshly refreshed base.
 # 3. Build the per-commit ROCm CI image and smoke-test it before GPU jobs run.
 steps:
-  - label: "AMD: :docker: refresh ROCm base"
+  - label: "AMD: :docker: build image"
-    key: refresh-rocm-base-amd
+    key: image-build-amd
    depends_on: []
    device: amd_cpu
    no_plugin: true
    commands:
-      - bash .buildkite/scripts/rocm/refresh-base-image.sh
+    - >
      docker build
      --build-arg max_jobs=16
      --build-arg REMOTE_VLLM=1
      --build-arg ARG_PYTORCH_ROCM_ARCH='gfx942;gfx950'
      --build-arg VLLM_BRANCH=$BUILDKITE_COMMIT
      --tag "rocm/vllm-ci:${BUILDKITE_COMMIT}"
      -f docker/Dockerfile.rocm
      --target test
      --no-cache
      --progress plain .
    - docker push "rocm/vllm-ci:${BUILDKITE_COMMIT}"
    env:
      DOCKER_BUILDKIT: "1"
      BUILDKIT_PROGRESS: "tty"
      TERM: "xterm-256color"
    retry:
      automatic:
        - exit_status: 1  # Transient Docker/BuildKit failure
          limit: 1
        - exit_status: -1  # Agent was lost
          limit: 1
        - exit_status: -10  # Agent was lost
          limit: 1
-
+        - exit_status: 1  # Machine occasionally fail
  # Ensure ci_base is up-to-date before building the test image.
  # Compares a content hash of ci_base-affecting files against the remote
  # image label. If hashes match the build is skipped (< 30 s); if they
  # differ ci_base is rebuilt and pushed automatically.
  - label: "AMD: :docker: ensure ci_base"
    key: ensure-ci-base-amd
    soft_fail: false
    depends_on:
      - refresh-rocm-base-amd
    device: amd_cpu
    no_plugin: true
    commands:
      - bash .buildkite/scripts/rocm/build-ci-base.sh
    env:
      DOCKER_BUILDKIT: "1"
      BUILDKIT_PROGRESS: "tty"
      TERM: "xterm-256color"
      VLLM_BAKE_FILE: "docker/docker-bake-rocm.hcl"
      PYTORCH_ROCM_ARCH: "gfx90a;gfx942;gfx950"
      REMOTE_VLLM: "1"
      VLLM_BRANCH: "$BUILDKITE_COMMIT"
    retry:
      automatic:
        - exit_status: 1  # Transient Docker/BuildKit failure
          limit: 1
        - exit_status: -1  # Agent was lost
          limit: 1
        - exit_status: -10  # Agent was lost
          limit: 1
  - label: "AMD: :docker: build test image and artifacts"
    key: image-build-amd
    soft_fail: false
    depends_on:
      - ensure-ci-base-amd
    device: amd_cpu
    no_plugin: true
    commands:
      - bash .buildkite/scripts/rocm/build-test-image.sh
      - bash .buildkite/scripts/rocm/smoke-test-image.sh
    env:
      DOCKER_BUILDKIT: "1"
      BUILDKIT_PROGRESS: "tty"
      TERM: "xterm-256color"
      VLLM_BAKE_FILE: "docker/docker-bake-rocm.hcl"
      PYTORCH_ROCM_ARCH: "gfx90a;gfx942;gfx950"
      IMAGE_TAG: "rocm/vllm-ci:$BUILDKITE_COMMIT"
      REMOTE_VLLM: "1"
      VLLM_BRANCH: "$BUILDKITE_COMMIT"
    retry:
      automatic:
        - exit_status: 1  # Transient Docker/BuildKit failure
          limit: 1
        - exit_status: -1  # Agent was lost
          limit: 1
        - exit_status: -10  # Agent was lost
          limit: 1
@@ -3,6 +3,7 @@ depends_on: []
 steps:
 - label: CPU-Kernel Tests
  depends_on: []
  soft_fail: true
  device: intel_cpu
  no_plugin: true
  source_file_dependencies:
@@ -12,44 +13,17 @@ steps:
  - vllm/_custom_ops.py
  - tests/kernels/attention/test_cpu_attn.py
  - tests/kernels/moe/test_cpu_fused_moe.py
  - tests/kernels/moe/test_cpu_quant_fused_moe.py
  - tests/kernels/test_onednn.py
  - tests/kernels/test_awq_int4_to_int8.py
  - tests/kernels/quantization/test_cpu_fp8_scaled_mm.py
  - tests/kernels/mamba/cpu/test_cpu_gdn_ops.py
  - tests/kernels/mamba/test_cpu_short_conv.py
  - tests/kernels/mamba/test_causal_conv1d.py
  - tests/kernels/mamba/test_mamba_ssm.py
  commands:
    - |
-      bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 30m "
+      bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 20m "
      pytest -x -v -s tests/kernels/attention/test_cpu_attn.py
      pytest -x -v -s tests/kernels/moe/test_cpu_fused_moe.py
-      pytest -x -v -s tests/kernels/moe/test_cpu_quant_fused_moe.py
+      pytest -x -v -s tests/kernels/test_onednn.py"
      pytest -x -v -s tests/kernels/mamba/test_cpu_short_conv.py
      pytest -x -v -s tests/kernels/test_onednn.py
      pytest -x -v -s tests/kernels/test_awq_int4_to_int8.py
      pytest -x -v -s tests/kernels/quantization/test_cpu_fp8_scaled_mm.py
      pytest -x -v -s tests/kernels/mamba/cpu/test_cpu_gdn_ops.py
      pytest -x -v -s tests/kernels/mamba/test_causal_conv1d.py
      pytest -x -v -s tests/kernels/mamba/test_mamba_ssm.py"
 # Note: SDE can't be downloaded from CI host because of AWS WAF
 # - label: CPU-Compatibility Tests
 #   depends_on: []
 #   device: intel_cpu
 #   no_plugin: true
 #   source_file_dependencies:
 #   - cmake/cpu_extension.cmake
 #   - setup.py
 #   - vllm/platforms/cpu.py
 #   commands:
 #     - |
 #       bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 20m "
 #       bash .buildkite/scripts/hardware_ci/run-cpu-compatibility-test.sh"
 - label: CPU-Language Generation and Pooling Model Tests
  depends_on: []
  soft_fail: true
  device: intel_cpu
  no_plugin: true
  source_file_dependencies:
@@ -59,57 +33,36 @@ steps:
  - tests/models/language/pooling/
  commands:
    - |
-      bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 50m "
+      bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 30m "
      pytest -x -v -s tests/models/language/generation -m cpu_model
      pytest -x -v -s tests/models/language/pooling -m cpu_model"
 - label: CPU-ModelRunnerV2 Tests
  depends_on: []
  device: intel_cpu
  no_plugin: true
  soft_fail: true
  source_file_dependencies:
  - vllm/v1/worker/cpu/
  - vllm/v1/worker/gpu/
  - vllm/v1/sample/ops/topk_topp_triton.py
  - vllm/v1/sample/ops/topk_topp_sampler.py
  - tests/v1/sample/test_topk_topp_sampler.py
  - tests/v1/e2e/test_cpu_linear_attn_chunked_prefix.py
  commands:
    - |
      bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 45m "
      uv pip install git+https://github.com/triton-lang/triton-cpu.git@270e696d
      VLLM_USE_V2_MODEL_RUNNER=1 pytest -x -v -s tests/models/language/generation/test_granite.py -m cpu_model
      # TODO: move to CPU-Kernel Tests once triton-cpu has a pre-built wheel
      pytest -x -v -s tests/v1/sample/test_topk_topp_sampler.py::TestTritonTopkTopp
      pytest -x -v -s tests/v1/e2e/test_cpu_linear_attn_chunked_prefix.py"
 - label: CPU-Quantization Model Tests
  depends_on: []
  soft_fail: true
  device: intel_cpu
  no_plugin: true
  source_file_dependencies:
  - csrc/cpu/
-  - vllm/model_executor/layers/quantization/auto_gptq.py
+  - vllm/model_executor/layers/quantization/cpu_wna16.py
  - vllm/model_executor/layers/quantization/gptq_marlin.py
  - vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py
-  - vllm/model_executor/kernels/linear/mixed_precision/cpu.py
+  - vllm/model_executor/layers/quantization/kernels/scaled_mm/cpu.py
-  - vllm/model_executor/kernels/linear/scaled_mm/cpu.py
+  - vllm/model_executor/layers/quantization/kernels/mixed_precision/cpu.py
  - vllm/model_executor/layers/fused_moe/experts/cpu_moe.py
  - tests/quantization/test_compressed_tensors.py
  - tests/quantization/test_cpu_wna16.py
  - tests/quantization/test_cpu_w8a8.py
  commands:
    - |
-      bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 45m "
+      bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 20m "
      pytest -x -v -s tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_logprobs
-      pytest -x -v -s tests/quantization/test_cpu_wna16.py
+      pytest -x -v -s tests/quantization/test_cpu_wna16.py"
      pytest -x -v -s tests/quantization/test_cpu_w8a8.py"
- label: CPU-Distributed Tests (PP+TP)
+- label: CPU-Distributed Tests
  depends_on: []
  soft_fail: true
  device: intel_cpu
  no_plugin: true
-  source_file_dependencies: &cpu_distributed_deps
+  source_file_dependencies:
  - csrc/cpu/shm.cpp
  - vllm/v1/worker/cpu_worker.py
  - vllm/v1/worker/gpu_worker.py
@@ -118,24 +71,14 @@ steps:
  - vllm/platforms/cpu.py
  - vllm/distributed/parallel_state.py
  - vllm/distributed/device_communicators/cpu_communicator.py
  - .buildkite/scripts/hardware_ci/run-cpu-distributed-smoke-test.sh
  commands:
    - |
      bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 10m "
-      bash .buildkite/scripts/hardware_ci/run-cpu-distributed-smoke-test.sh tp_pp"
+      bash .buildkite/scripts/hardware_ci/run-cpu-distributed-smoke-test.sh"
 - label: CPU-Distributed Tests (DP+TP)
  depends_on: []
  device: intel_cpu
  no_plugin: true
  source_file_dependencies: *cpu_distributed_deps
  commands:
    - |
      bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 10m "
      bash .buildkite/scripts/hardware_ci/run-cpu-distributed-smoke-test.sh dp_tp"
 - label: CPU-Multi-Modal Model Tests %N
  depends_on: []
  soft_fail: true
  device: intel_cpu
  no_plugin: true
  source_file_dependencies:
@@ -145,25 +88,12 @@ steps:
  commands:
    - |
      bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 45m "
-      pytest -x -v -s tests/models/multimodal/generation --ignore=tests/models/multimodal/generation/test_pixtral.py --ignore=tests/models/multimodal/generation/test_qwen2_5_vl.py -m cpu_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
+      pytest -x -v -s tests/models/multimodal/generation --ignore=tests/models/multimodal/generation/test_pixtral.py -m cpu_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
-  parallelism: 4
+  parallelism: 2
 - label: CPU-Qwen2.5-VL Multimodal Tests
  depends_on: []
  device: intel_cpu
  no_plugin: true
  source_file_dependencies:
  # - vllm/
  - vllm/model_executor/layers/rotary_embedding
  - tests/models/multimodal/generation/
  commands:
    - |
      bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 40m "
      VLLM_CI_ENV=0 pytest -x -v -s tests/models/multimodal/generation/test_qwen2_5_vl.py"
 - label: "Arm CPU Test"
  depends_on: []
-  soft_fail: false
+  soft_fail: true
  device: arm_cpu
  no_plugin: true
  commands: 
@@ -8,3 +8,10 @@ steps:
    commands: 
    - bash .buildkite/scripts/hardware_ci/run-hpu-test.sh
  - label: "Intel GPU Test"
    depends_on: []
    soft_fail: true
    device: intel_gpu
    no_plugin: true
    commands: 
    - bash .buildkite/scripts/hardware_ci/run-xpu-test.sh
@@ -1,80 +0,0 @@
 group: Intel
 steps:
  - label: ":docker: Build XPU image"
    soft_fail: true
    optional: true
    depends_on: []
    key: image-build-xpu
    commands:
      - bash -lc '.buildkite/image_build/image_build_xpu.sh "public.ecr.aws/q9t5s3a7" "vllm-ci-test-repo" "$BUILDKITE_COMMIT"'
    env:
      DOCKER_BUILDKIT: "1"
    retry:
      automatic:
        - exit_status: -1  # Agent was lost
          limit: 2
        - exit_status: -10  # Agent was lost
          limit: 2
  - label: "XPU example Test"
    depends_on:
      - image-build-xpu
    timeout_in_minutes: 50
    optional: true
    device: intel_gpu
    agent_tags:
      label: production
      gpu: 2+
      mem: 24+
    no_plugin: true
    env:
      REGISTRY: "public.ecr.aws/q9t5s3a7"
      REPO: "vllm-ci-test-repo"
    source_file_dependencies:
      - .buildkite/hardware_tests/intel_xpu_ci/test-intel.yaml
      - .buildkite/scripts/hardware_ci/run-intel-ci-test.sh
    commands:
      - >-
        bash .buildkite/scripts/hardware_ci/run-intel-test.sh
        'bash .buildkite/scripts/hardware_ci/run-intel-ci-test.sh example'
  - label: "XPU V1 test"
    depends_on:
      - image-build-xpu
    timeout_in_minutes: 70
    optional: true
    device: intel_gpu
    agent_tags:
      label: production
      gpu: 1+
      mem: 24+
    no_plugin: true
    env:
      REGISTRY: "public.ecr.aws/q9t5s3a7"
      REPO: "vllm-ci-test-repo"
    source_file_dependencies:
      - .buildkite/hardware_tests/intel_xpu_ci/test-intel.yaml
      - .buildkite/scripts/hardware_ci/run-intel-ci-test.sh
    commands:
      - >-
        bash .buildkite/scripts/hardware_ci/run-intel-test.sh
        'bash .buildkite/scripts/hardware_ci/run-intel-ci-test.sh v1'
  - label: "XPU server test"
    depends_on:
      - image-build-xpu
    timeout_in_minutes: 45
    optional: true
    device: intel_gpu
    agent_tags:
      label: production
      gpu: 1+
      mem: 16+
    no_plugin: true
    env:
      REGISTRY: "public.ecr.aws/q9t5s3a7"
      REPO: "vllm-ci-test-repo"
    source_file_dependencies:
      - .buildkite/hardware_tests/intel_xpu_ci/test-intel.yaml
      - .buildkite/scripts/hardware_ci/run-intel-ci-test.sh
    commands:
      - >-
        bash .buildkite/scripts/hardware_ci/run-intel-test.sh
        'bash .buildkite/scripts/hardware_ci/run-intel-ci-test.sh server'
@@ -79,18 +79,12 @@ setup_buildx_builder() {
    docker buildx ls | grep -E '^\*|^NAME' || docker buildx ls
 }
 annotate_image_tags() {
    .buildkite/scripts/annotate-image-build.sh \
        "${IMAGE_TAG:-}" "${IMAGE_TAG_LATEST:-}"
 }
 check_and_skip_if_image_exists() {
    if [[ -n "${IMAGE_TAG:-}" ]]; then
        echo "--- :mag: Checking if image exists"
        if docker manifest inspect "${IMAGE_TAG}" >/dev/null 2>&1; then
            echo "Image already exists: ${IMAGE_TAG}"
            echo "Skipping build"
            annotate_image_tags
            exit 0
        fi
        echo "Image not found, proceeding with build"
@@ -98,8 +92,8 @@ check_and_skip_if_image_exists() {
 }
 ecr_login() {
-    aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true
+    aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY"
-    aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 936637512419.dkr.ecr.us-east-1.amazonaws.com || true
+    aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 936637512419.dkr.ecr.us-east-1.amazonaws.com
 }
 prepare_cache_tags() {
@@ -177,18 +171,6 @@ BRANCH=$4
 IMAGE_TAG=$5
 IMAGE_TAG_LATEST=${6:-} # only used for main branch, optional
 # When TORCH_NIGHTLY=1, build the base CI image against PyTorch nightly so the
 # entire existing pipeline runs on nightly torch (CUDA/GPU lane only). Delegate
 # to the dedicated nightly build (PYTORCH_NIGHTLY=1, CUDA 13.0) and tag it at the
 # normal IMAGE_TAG that every test step already pulls -- no separate image tag,
 # no duplicate "vLLM Against PyTorch Nightly" pipeline section.
 if [[ "${TORCH_NIGHTLY:-0}" == "1" ]]; then
    echo "--- :warning: TORCH_NIGHTLY=1 -- building base image on PyTorch nightly"
    SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
    exec "${SCRIPT_DIR}/image_build_torch_nightly.sh" \
        "${REGISTRY}" "${REPO}" "${BUILDKITE_COMMIT}" "${BRANCH}" "${IMAGE_TAG}"
 fi
 # build config
 TARGET="test-ci"
 VLLM_BAKE_FILE_PATH="${VLLM_BAKE_FILE_PATH:-docker/docker-bake.hcl}"
@@ -210,7 +192,6 @@ export BUILDKITE_COMMIT
 export PARENT_COMMIT
 export IMAGE_TAG
 export IMAGE_TAG_LATEST
 export COMMIT="${COMMIT:-${BUILDKITE_COMMIT}}"
 export CACHE_FROM
 export CACHE_FROM_BASE_BRANCH
 export CACHE_FROM_MAIN
@@ -272,5 +253,3 @@ echo "--- :docker: Building ${TARGET}"
 docker --debug buildx bake -f "${VLLM_BAKE_FILE_PATH}" -f "${CI_HCL_PATH}" --progress plain "${TARGET}"
 echo "--- :white_check_mark: Build complete"
 annotate_image_tags
@@ -13,60 +13,6 @@ steps:
        - exit_status: -10  # Agent was lost
          limit: 2
  - label: ":docker: :smoking: Non-root smoke tests"
    key: image-build-smoke-test
    depends_on:
      - image-build
    commands:
    # Smoke 1: the default (root) image must still be importable
    # under a non-root UID via `--user 2000:0`. Validates the `vllm` passwd
    # entry + group-0-writable /home/vllm + uv path cleanup from #31959.
    # Uses `import vllm` rather than `vllm serve --help` because the latter
    # instantiates `VllmConfig` which requires a GPU attached to the
    # container.
    - docker run --rm --user 2000:0 --entrypoint python3 "$IMAGE_TAG" -c "import vllm; print(vllm.__version__)"
    # Smoke 2: assert the non-root enabling invariants are baked
    # into the image. Runs as UID 2000:0 via a shell so we can verify
    # filesystem perms + passwd/group file state + wrapper presence without
    # triggering vLLM's GPU-requiring config-init path. The opt-in
    # `vllm-openai-nonroot` target adds only `USER vllm`, `WORKDIR
    # /home/vllm`, and an `ENTRYPOINT` override on top of these invariants;
    # its build correctness is reviewed at the Dockerfile level. Wrapper
    # logic is covered separately by the pre-commit hook
    # `test-nonroot-entrypoint` (see .pre-commit-config.yaml).
    - |
      docker run --rm --user 2000:0 --entrypoint /bin/sh "$IMAGE_TAG" -ec '
        if ! getent passwd 2000 | grep -q ^vllm:; then
          echo FAIL: UID 2000 != vllm
          exit 1
        fi
        if ! id -gn 2>/dev/null | grep -qx root; then
          echo FAIL: GID 0 not root group
          exit 1
        fi
        touch /home/vllm/.smoke && rm /home/vllm/.smoke
        touch /opt/uv/cache/.smoke && rm /opt/uv/cache/.smoke
        if ! test -x /usr/local/bin/vllm-nonroot-entrypoint.sh; then
          echo FAIL: wrapper missing
          exit 1
        fi
        if ! test -w /etc/passwd; then
          echo FAIL: /etc/passwd not group-writable
          exit 1
        fi
        if ! test -w /etc/group; then
          echo FAIL: /etc/group not group-writable
          exit 1
        fi
        echo non-root invariants OK
      '
    retry:
      automatic:
        - exit_status: -1  # Agent was lost
          limit: 2
        - exit_status: -10  # Agent was lost
          limit: 2
  - label: ":docker: Build CPU image"
    key: image-build-cpu
    depends_on: []
@@ -110,21 +56,3 @@ steps:
          limit: 2
        - exit_status: -10  # Agent was lost
          limit: 2
  - label: ":docker: Build arm64 image"
    key: arm64-image-build
    depends_on: []
    source_file_dependencies:
      - ".buildkite/image_build/image_build.yaml"
      - ".buildkite/image_build/image_build_arm64.sh"
      - "docker/Dockerfile"
    commands:
    - .buildkite/image_build/image_build_arm64.sh $REGISTRY $REPO $BUILDKITE_COMMIT
    env:
      DOCKER_BUILDKIT: "1"
    retry:
      automatic:
        - exit_status: -1  # Agent was lost
          limit: 2
        - exit_status: -10  # Agent was lost
          limit: 2
@@ -1,39 +0,0 @@
 #!/bin/bash
 set -e
 if [[ $# -lt 3 ]]; then
  echo "Usage: $0 <registry> <repo> <commit>"
  exit 1
 fi
 REGISTRY=$1
 REPO=$2
 BUILDKITE_COMMIT=$3
 IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-arm64"
 # authenticate with AWS ECR
 aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true
 # skip build if image already exists
 if docker manifest inspect "$IMAGE" >/dev/null 2>&1; then
  echo "Image found"
 else
  echo "Image not found, proceeding with build..."
  # build for arm64 GPU targets: Grace/GH200 (sm_90),
  # Blackwell/Thor (sm_100/sm_103/sm_110), and DGX Spark/GB10
  # (sm_121, family-covered by 12.0 under CUDA 13)
  docker build --file docker/Dockerfile \
    --platform linux/arm64 \
    --build-arg max_jobs=16 \
    --build-arg nvcc_threads=4 \
    --build-arg torch_cuda_arch_list="9.0 10.0 11.0 12.0" \
    --build-arg USE_SCCACHE=1 \
    --build-arg buildkite_commit="$BUILDKITE_COMMIT" \
    --tag "$IMAGE" \
    --target test \
    --progress plain .
  # push
  docker push "$IMAGE"
 fi
 .buildkite/scripts/annotate-image-build.sh "$IMAGE"
@@ -9,26 +9,28 @@ fi
 REGISTRY=$1
 REPO=$2
 BUILDKITE_COMMIT=$3
 IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-cpu"
 # authenticate with AWS ECR
-aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true
+aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY"
 # skip build if image already exists
-if docker manifest inspect "$IMAGE" >/dev/null 2>&1; then
+if [[ -z $(docker manifest inspect "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-cpu) ]]; then
  echo "Image found"
 else
  echo "Image not found, proceeding with build..."
-  # build
+else
-  docker build --file docker/Dockerfile.cpu \
+  echo "Image found"
-    --build-arg max_jobs=16 \
+  exit 0
    --build-arg buildkite_commit="$BUILDKITE_COMMIT" \
    --build-arg VLLM_CPU_X86=true \
    --tag "$IMAGE" \
    --target vllm-test \
    --progress plain .
  # push
  docker push "$IMAGE"
 fi
-.buildkite/scripts/annotate-image-build.sh "$IMAGE"
+# build
 docker build --file docker/Dockerfile.cpu \
  --build-arg max_jobs=16 \
  --build-arg buildkite_commit="$BUILDKITE_COMMIT" \
  --build-arg VLLM_CPU_AVX512BF16=true \
  --build-arg VLLM_CPU_AVX512VNNI=true \
  --build-arg VLLM_CPU_AMXBF16=true \
  --tag "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-cpu \
  --target vllm-test \
  --progress plain .
 # push
 docker push "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-cpu
@@ -9,25 +9,25 @@ fi
 REGISTRY=$1
 REPO=$2
 BUILDKITE_COMMIT=$3
 IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-arm64-cpu"
 # authenticate with AWS ECR
-aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true
+aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY"
 # skip build if image already exists
-if docker manifest inspect "$IMAGE" >/dev/null 2>&1; then
+if [[ -z $(docker manifest inspect "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64-cpu) ]]; then
  echo "Image found"
 else
  echo "Image not found, proceeding with build..."
-  # build
+else
-  docker build --file docker/Dockerfile.cpu \
+  echo "Image found"
-    --build-arg max_jobs=16 \
+  exit 0
    --build-arg buildkite_commit="$BUILDKITE_COMMIT" \
    --tag "$IMAGE" \
    --target vllm-test \
    --progress plain .
  # push
  docker push "$IMAGE"
 fi
-.buildkite/scripts/annotate-image-build.sh "$IMAGE"
+# build
 docker build --file docker/Dockerfile.cpu \
  --build-arg max_jobs=16 \
  --build-arg buildkite_commit="$BUILDKITE_COMMIT" \
  --tag "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64-cpu \
  --target vllm-test \
  --progress plain .
 # push
 docker push "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64-cpu
@@ -9,26 +9,26 @@ fi
 REGISTRY=$1
 REPO=$2
 BUILDKITE_COMMIT=$3
 IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-hpu"
 # authenticate with AWS ECR
-aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true
+aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY"
 # skip build if image already exists
-if docker manifest inspect "$IMAGE" >/dev/null 2>&1; then
+if [[ -z $(docker manifest inspect "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-hpu) ]]; then
  echo "Image found"
 else
  echo "Image not found, proceeding with build..."
-  # build
+else
-  docker build \
+  echo "Image found"
-    --file tests/pytorch_ci_hud_benchmark/Dockerfile.hpu \
+  exit 0
    --build-arg max_jobs=16 \
    --build-arg buildkite_commit="$BUILDKITE_COMMIT" \
    --tag "$IMAGE" \
    --progress plain \
    https://github.com/vllm-project/vllm-gaudi.git
  # push
  docker push "$IMAGE"
 fi
-.buildkite/scripts/annotate-image-build.sh "$IMAGE"
+# build
 docker build \
  --file tests/pytorch_ci_hud_benchmark/Dockerfile.hpu \
  --build-arg max_jobs=16 \
  --build-arg buildkite_commit="$BUILDKITE_COMMIT" \
  --tag "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-hpu \
  --progress plain \
  https://github.com/vllm-project/vllm-gaudi.git
 # push
 docker push "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-hpu
@@ -1,71 +0,0 @@
 #!/bin/bash
 set -euo pipefail
 # Build a vLLM test image with PyTorch nightly installed.
 # Called by the pipeline generator's "vLLM Against PyTorch Nightly" group.
 if [[ $# -lt 5 ]]; then
  echo "Usage: $0 <registry> <repo> <commit> <branch> <image_tag>"
  exit 1
 fi
 REGISTRY=$1
 REPO=$2
 BUILDKITE_COMMIT=$3
 BRANCH=$4
 IMAGE_TAG=$5
 # --- Arguments ---
 echo "--- :mag: Arguments"
 echo "REGISTRY: ${REGISTRY}"
 echo "REPO: ${REPO}"
 echo "BUILDKITE_COMMIT: ${BUILDKITE_COMMIT}"
 echo "BRANCH: ${BRANCH}"
 echo "IMAGE_TAG: ${IMAGE_TAG}"
 # --- ECR login ---
 echo "--- :key: ECR login"
 aws ecr-public get-login-password --region us-east-1 \
  | docker login --username AWS --password-stdin "$REGISTRY"
 aws ecr get-login-password --region us-east-1 \
  | docker login --username AWS --password-stdin 936637512419.dkr.ecr.us-east-1.amazonaws.com
 # --- Set up buildx ---
 echo "--- :docker: Setting up buildx"
 docker buildx create --name vllm-builder --driver docker-container --use || true
 docker buildx inspect --bootstrap
 docker buildx ls
 # --- Skip if image already exists ---
 echo "--- :mag: Checking if image already exists"
 if docker manifest inspect "$IMAGE_TAG" >/dev/null 2>&1; then
  echo "Image found: $IMAGE_TAG — skipping build"
  .buildkite/scripts/annotate-image-build.sh "$IMAGE_TAG"
  exit 0
 fi
 echo "Image not found, proceeding with build..."
 # --- CUDA 13.0 for nightly builds ---
 # Nightly CI uses CUDA 13.0 while regular CI stays on CUDA 12.9
 NIGHTLY_CUDA_VERSION="13.0.2"
 NIGHTLY_BUILD_BASE_IMAGE="nvidia/cuda:${NIGHTLY_CUDA_VERSION}-devel-ubuntu22.04"
 NIGHTLY_FINAL_BASE_IMAGE="nvidia/cuda:${NIGHTLY_CUDA_VERSION}-base-ubuntu22.04"
 echo "--- :docker: Building torch nightly image (CUDA ${NIGHTLY_CUDA_VERSION})"
 docker buildx build --file docker/Dockerfile \
  --build-arg max_jobs=16 \
  --build-arg buildkite_commit="$BUILDKITE_COMMIT" \
  --build-arg USE_SCCACHE=1 \
  --build-arg PYTORCH_NIGHTLY=1 \
  --build-arg CUDA_VERSION="${NIGHTLY_CUDA_VERSION}" \
  --build-arg BUILD_BASE_IMAGE="${NIGHTLY_BUILD_BASE_IMAGE}" \
  --build-arg FINAL_BASE_IMAGE="${NIGHTLY_FINAL_BASE_IMAGE}" \
  --build-arg torch_cuda_arch_list="8.0 8.9 9.0 10.0 12.0" \
  --tag "$IMAGE_TAG" \
  --push \
  --target test \
  --progress plain .
 echo "--- :white_check_mark: Torch nightly image build complete: $IMAGE_TAG"
 .buildkite/scripts/annotate-image-build.sh "$IMAGE_TAG"
@@ -1,34 +0,0 @@
 #!/bin/bash
 set -e
 if [[ $# -lt 3 ]]; then
  echo "Usage: $0 <registry> <repo> <commit>"
  exit 1
 fi
 REGISTRY=$1
 REPO=$2
 BUILDKITE_COMMIT=$3
 IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-xpu"
 # authenticate with AWS ECR
 aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true
 aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 936637512419.dkr.ecr.us-east-1.amazonaws.com || true
 # skip build if image already exists
 if docker manifest inspect "$IMAGE" &> /dev/null; then
  echo "Image found"
 else
  echo "Image not found, proceeding with build..."
  # build
  docker build \
    --file docker/Dockerfile.xpu \
    --build-arg max_jobs=16 \
    --build-arg buildkite_commit="$BUILDKITE_COMMIT" \
    --tag "$IMAGE" \
    --progress plain .
  # push
  docker push "$IMAGE"
 fi
 .buildkite/scripts/annotate-image-build.sh "$IMAGE"
@@ -1,27 +0,0 @@
 group: Basic Correctness
 depends_on:
  - image-build-xpu
 steps:
 - label: XPU Sleep Mode
  timeout_in_minutes: 45
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
  - vllm/
  - tests/basic_correctness/test_cumem.py
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'cd tests &&
      export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
      pytest -v -s basic_correctness/test_cpu_offload.py &&
      pytest -v -s basic_correctness/test_mem.py::test_end_to_end'
@@ -1,25 +0,0 @@
 group: Engine Intel
 depends_on:
  - image-build-xpu
 steps:
 - label: Engine (1 GPU)
  timeout_in_minutes: 30
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
    - vllm/v1/engine/
    - tests/v1/engine/
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'cd tests &&
      pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py'
@@ -1,27 +0,0 @@
 group: Expert Parallelism
 depends_on:
  - image-build-xpu
 steps:
 - label: EPLB Algorithm
  key: eplb-algorithm
  timeout_in_minutes: 45
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
  - vllm/distributed/eplb
  - tests/distributed/test_eplb_algo.py
  - tests/distributed/test_eplb_utils.py
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'cd tests &&
      pytest -v -s distributed/test_eplb_algo.py'
@@ -1,25 +0,0 @@
 group: Kernels Intel
 depends_on: 
  - image-build-xpu
 steps:
 - label: vLLM IR Tests
  timeout_in_minutes: 30
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
    - vllm/ir
    - vllm/kernels
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'cd tests &&
      pytest -v -s kernels/ir'
@@ -1,161 +0,0 @@
 group: LoRA Intel
 depends_on:
  - image-build-xpu
 steps:
 - label: LoRA Runtime + Utils
  timeout_in_minutes: 45
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 24+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
    - vllm/lora
    - tests/lora
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'cd tests &&
      export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
      pytest -v -s lora/test_layers.py &&
      pytest -v -s lora/test_lora_checkpoints.py &&
      pytest -v -s lora/test_lora_functions.py &&
      pytest -v -s lora/test_lora_huggingface.py &&
      pytest -v -s lora/test_lora_manager.py &&
      pytest -v -s lora/test_lora_utils.py &&
      pytest -v -s lora/test_peft_helper.py &&
      pytest -v -s lora/test_resolver.py &&
      pytest -v -s lora/test_utils.py &&
      pytest -v -s lora/test_add_lora.py  &&
      pytest -v -s lora/test_worker.py'
 - label: LoRA Fused/MoE Kernels
  timeout_in_minutes: 45
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
    - vllm/lora
    - tests/lora
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'cd tests &&
      export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
      pytest -v -s lora/test_fused_moe_lora_kernel.py && 
      pytest -v -s lora/test_moe_lora_align_sum.py --deselect="tests/lora/test_moe_lora_align_sum.py::test_moe_lora_align_block_size_mixed_base_and_lora[1]"'
 - label: LoRA Punica Kernels
  timeout_in_minutes: 45
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
    - vllm/lora
    - tests/lora
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'cd tests &&
      export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
      set -o pipefail &&
      pytest -v -s lora/test_punica_ops.py::test_kernels &&
      pytest -v -s lora/test_punica_ops.py::test_kernels_hidden_size &&
      pytest -v -s lora/test_punica_ops.py::test_add_lora_fused_moe_early_exit'
 - label: LoRA Punica FP8/XPU Ops
  timeout_in_minutes: 60
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
    - vllm/lora
    - tests/lora
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'cd tests &&
      export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
      pytest -v -s lora/test_punica_ops_fp8.py &&
      pytest -v -s lora/test_punica_xpu_ops.py'
 - label: LoRA Models
  timeout_in_minutes: 45
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 2+
    mem: 24+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
    - vllm/lora
    - tests/lora
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'cd tests &&
      export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
      pytest -v -s lora/test_quant_model.py --deselect="tests/lora/test_quant_model.py::test_quant_model_lora[model0]" --deselect="tests/lora/test_quant_model.py::test_quant_model_lora[model1]" --deselect="tests/lora/test_quant_model.py::test_quant_model_tp_equality[model0]" &&
      pytest -v -s lora/test_transformers_model.py &&
      pytest -v -s lora/test_chatglm3_tp.py &&
      pytest -v -s lora/test_llama_tp.py::test_llama_lora &&
      pytest -s -v lora/test_minicpmv_tp.py'
 - label: LoRA Multimodal
  timeout_in_minutes: 45
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
    - vllm/lora
    - tests/lora
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'cd tests &&
      export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
      pytest -v -s lora/test_default_mm_loras.py && 
      pytest -v -s lora/test_whisper.py'
@@ -1,261 +0,0 @@
 group: Miscellaneous Intel
 depends_on:
  - image-build-xpu
 steps:
 - label: V1 Core + KV + Metrics
  timeout_in_minutes: 45
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
    - vllm/
    - tests/v1/core
    - tests/v1/executor
    - tests/v1/kv_offload
    - tests/v1/worker
    - tests/v1/kv_connector/unit
    - tests/v1/metrics
    - tests/entrypoints/openai/correctness/test_lmeval.py
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'pip install -r requirements/kv_connectors.txt &&
      export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
      cd tests &&
      pytest -v -s v1/executor'
 - label: V1 Sample + Logits
  timeout_in_minutes: 90
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 24+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
    - vllm/config/
    - vllm/distributed/
    - vllm/engine/
    - vllm/inputs/
    - vllm/logger.py
    - vllm/model_executor/
    - vllm/platforms/
    - vllm/sampling_params.py
    - vllm/transformers_utils/
    - vllm/utils/
    - vllm/v1/
    - tests/v1/sample
    - tests/v1/logits_processors
    - tests/v1/test_oracle.py
    - tests/v1/test_request.py
    - tests/v1/test_outputs.py
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'pip install lm_eval[api]>=0.4.12 &&
      export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
      cd tests &&
      pytest -v -s v1/logits_processors --ignore=v1/logits_processors/test_custom_online.py --ignore=v1/logits_processors/test_custom_offline.py &&
      pytest -v -s v1/test_oracle.py &&
      pytest -v -s v1/test_request.py &&
      pytest -v -s v1/test_outputs.py &&
      pytest -v -s v1/sample'
 - label: Basic Models Tests (Initialization)
  timeout_in_minutes: 60
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
    - vllm/
    - tests/models/test_initialization.py
    - tests/models/registry.py
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'export VLLM_XPU_FUSED_MOE_USE_REF=1 &&
      cd tests &&
      pytest -v -s models/test_initialization.py::test_can_initialize_large_subset[Eagle3MiniMaxM2ForCausalLM]'
 - label: XPU CPU Offload
  timeout_in_minutes: 60
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
    - vllm/
    - vllm/v1/kv_offload/
    - vllm/v1/kv_connector/
    - tests/v1/kv_offload/
    - tests/v1/kv_connector/unit/test_offloading_connector.py
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
      cd tests &&
      pytest -v -s v1/kv_offload &&
      pytest -v -s v1/kv_connector/unit/test_offloading_connector.py'
 - label: NixlConnector PD accuracy (2 GPUs)
  timeout_in_minutes: 60
  num_devices: 2
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 2+
    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
    - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
    - vllm/v1/worker/kv_connector_model_runner_mixin.py
    - tests/v1/kv_connector/nixl_integration/
    - vllm/platforms/xpu.py
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'cd tests &&
      bash v1/kv_connector/nixl_integration/run_xpu_disagg_accuracy_test.sh'
 - label: Regression
  key: regression
  timeout_in_minutes: 50
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
  - vllm/config/
  - vllm/distributed/
  - vllm/engine/
  - vllm/inputs/
  - vllm/model_executor/
  - vllm/multimodal/
  - vllm/platforms/
  - vllm/sampling_params.py
  - vllm/transformers_utils/
  - vllm/utils/
  - vllm/v1/
  - tests/test_regression
  commands:
  - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'pip install modelscope\<1.38 &&
      cd tests &&
      pytest -v -s test_regression.py'
 - label: Metrics, Tracing (2 GPUs)
  key: metrics-tracing-2-gpus
  timeout_in_minutes: 45
  num_devices: 2
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 2+
    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
  - vllm/config/
  - vllm/distributed/
  - vllm/engine/
  - vllm/inputs/
  - vllm/model_executor/
  - vllm/multimodal/
  - vllm/platforms/
  - vllm/sampling_params.py
  - vllm/tracing/
  - vllm/transformers_utils/
  - vllm/utils/
  - vllm/v1/
  - tests/v1/tracing
  commands:
  - >-
    bash .buildkite/scripts/hardware_ci/run-intel-test.sh
    'pip install opentelemetry-sdk\>=1.26.0 opentelemetry-api\>=1.26.0 opentelemetry-exporter-otlp\>=1.26.0 opentelemetry-semantic-conventions-ai\>=0.4.1 &&
    cd tests &&
    pytest -v -s v1/tracing'
 - label: Async Engine, Inputs, Utils, Worker
  key: async-engine-inputs-utils-worker
  timeout_in_minutes: 55
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 24+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
  - vllm/assets/
  - vllm/config/
  - vllm/distributed/
  - vllm/engine/
  - vllm/inputs/
  - vllm/model_executor/
  - vllm/multimodal/
  - vllm/platforms/
  - vllm/sampling_params.py
  - vllm/tokenizers/
  - vllm/transformers_utils/
  - vllm/utils/
  - vllm/v1/
  - tests/detokenizer
  - tests/multimodal
  - tests/utils_
  commands:
  - >-
    bash .buildkite/scripts/hardware_ci/run-intel-test.sh
    'cd tests &&
    pip install av &&
    pytest -v -s detokenizer &&
    pytest -v -s -m "not cpu_test" ./multimodal &&
    pytest -v -s utils_ --ignore=utils_/test_mem_utils.py'
@@ -1,62 +0,0 @@
 group: Model Runner V2 Intel
 depends_on:
  - image-build-xpu
 steps:
 - label: Model Runner V2 Core Tests (Intel)
  timeout_in_minutes: 45
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 2+
    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
    - vllm/v1/worker/gpu/
    - vllm/v1/worker/gpu_worker.py
    - vllm/v1/core/sched/
    - vllm/v1/attention/
    - tests/v1/engine/test_llm_engine.py
    - tests/v1/e2e/
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'export VLLM_USE_V2_MODEL_RUNNER=1 &&
      cd tests &&
      pytest -v -s v1/engine/test_llm_engine.py -k "not test_engine_metrics" &&
      ENFORCE_EAGER=1 pytest -v -s v1/e2e/general/test_async_scheduling.py -k "not ngram" &&
      pytest -v -s v1/e2e/general/test_min_tokens.py'
 - label: Model Runner V2 Examples (Intel)
  timeout_in_minutes: 45
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 24+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
    - vllm/v1/worker/gpu/
    - vllm/v1/core/sched/
    - vllm/v1/worker/gpu_worker.py
    - examples/basic/offline_inference/
    - examples/generate/multimodal/
    - examples/features/
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'export VLLM_USE_V2_MODEL_RUNNER=1 &&
      cd examples &&
      python3 basic/offline_inference/chat.py &&
      python3 basic/offline_inference/generate.py --model facebook/opt-125m &&
      python3 generate/multimodal/vision_language_offline.py --seed 0 &&
      python3 features/automatic_prefix_caching/prefix_caching_offline.py'
@@ -1,27 +0,0 @@
 group: Models - Distributed
 depends_on: 
  - image-build-xpu
 steps:
 - label: Distributed Model Tests (2 GPUs)
  key: distributed-model-tests-2-gpus
  timeout_in_minutes: 65
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 2+
    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
  - vllm/model_executor/model_loader/sharded_state_loader.py
  - vllm/model_executor/models/
  - tests/model_executor/model_loader/test_sharded_state_loader.py
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'cd tests &&
      pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m "not slow_test"'
@@ -1,127 +0,0 @@
 group: Models - Multimodal
 depends_on:
  - image-build-xpu
 steps:
 - label: "Multi-Modal Models (Standard) 1: qwen2"
  key: multi-modal-models-standard-1-qwen2
  timeout_in_minutes: 70
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 24+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
  - vllm/
  - tests/models/multimodal
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'pip install av &&
      cd tests &&
      pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2" &&
      pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model'
 - label: "Multi-Modal Models (Standard) 2: qwen3 + gemma"
  key: multi-modal-models-standard-2-qwen3-gemma
  timeout_in_minutes: 70
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
  - vllm/
  - tests/models/multimodal
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'cd tests &&
      pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model'
 - label: "Multi-Modal Models (Standard) 3: llava + qwen2_vl"
  key: multi-modal-models-standard-3-llava-qwen2-vl
  timeout_in_minutes: 65
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 24+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
  - vllm/
  - tests/models/multimodal
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'cd tests &&
      pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma" &&
      pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model'
 - label: "Multi-Modal Models (Standard) 4: other + whisper"
  key: multi-modal-models-standard-4-other-whisper
  timeout_in_minutes: 45
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 24+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
  - vllm/
  - tests/models/multimodal
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'pip install av &&
      cd tests &&
      pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py  --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/processing'
 - label: Multi-Modal Processor # 44min
  key: multi-modal-processor
  timeout_in_minutes: 60
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 24+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
  - vllm/
  - tests/models/multimodal
  - tests/models/registry.py
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'pip install av matplotlib ftfy &&
      pip install open-clip-torch --no-deps &&
      cd tests &&
      pytest -v -s models/multimodal/processing/test_tensor_schema.py
      --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB'
  parallelism: 4
@@ -1,28 +0,0 @@
 group: Quantization
 depends_on:
  - image-build-xpu
 steps:
 - label: Quantization
  key: quantization
  timeout_in_minutes: 30
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  no_plugin: true
  working_dir: "."
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 16+
  source_file_dependencies:
  - csrc/
  - vllm/model_executor/layers/quantization
  - tests/quantization
  commands:
  # - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py
  - >-
    bash .buildkite/scripts/hardware_ci/run-intel-test.sh
    'VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s tests/quantization/test_per_token_kv_cache.py --deselect="tests/quantization/test_per_token_kv_cache.py::test_triton_unified_attention_per_token_head_scale[int4-16-128-num_heads0-seq_lens1]"'
@@ -1,171 +0,0 @@
 group: Intel
 steps:
  - label: ":docker: Build XPU image"
    soft_fail: true
    depends_on: []
    key: image-build-xpu
    commands:
      - bash -lc '.buildkite/image_build/image_build_xpu.sh "public.ecr.aws/q9t5s3a7" "vllm-ci-test-repo" "$BUILDKITE_COMMIT"'
    env:
      DOCKER_BUILDKIT: "1"
    retry:
      automatic:
        - exit_status: -1  # Agent was lost
          limit: 2
        - exit_status: -10  # Agent was lost
          limit: 2
  - label: "XPU example Test"
    depends_on:
      - image-build-xpu
    timeout_in_minutes: 50
    device: intel_gpu
    agent_tags:
      label: production
      gpu: 2+
      mem: 24+
    no_plugin: true
    env:
      REGISTRY: "public.ecr.aws/q9t5s3a7"
      REPO: "vllm-ci-test-repo"
    source_file_dependencies:
      - vllm/
      - .buildkite/intel_jobs/test-intel.yaml 
    commands:
      - >-
        bash .buildkite/scripts/hardware_ci/run-intel-test.sh
        'pip install tblib==3.1.0 &&
        python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager &&
        python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 -O3 -cc.cudagraph_mode=NONE &&
        python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager -tp 2 --distributed-executor-backend mp &&
        python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --attention-backend=TRITON_ATTN &&
        python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --quantization fp8 &&
        python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --kv-cache-dtype fp8 &&
        python3 examples/basic/offline_inference/generate.py --model nvidia/Llama-3.1-8B-Instruct-FP8 --block-size 64 --enforce-eager --quantization modelopt --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --max-model-len 4096 &&
        python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --block-size 64 --enforce-eager --max-model-len 8192 &&
        python3 examples/basic/offline_inference/generate.py --model TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ --block-size 64 --enforce-eager &&
        python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 &&
        python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 --enable-expert-parallel &&
        python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --max-model-len 8192 &&
        VLLM_XPU_FUSED_MOE_USE_REF=1 python3 examples/basic/offline_inference/generate.py --model Qwen/Qwen3-30B-A3B-Instruct-2507-FP8 --enforce-eager -tp 2 --max-model-len 8192 &&
        python3 examples/basic/offline_inference/generate.py --model INCModel/Qwen3-30B-A3B-Instruct-2507-MXFP4-LLMC --enforce-eager -tp 2 --max-model-len 8192
        '
  - label: "XPU W8A8 FP8 Linear Examples"
    depends_on:
      - image-build-xpu
    timeout_in_minutes: 60
    device: intel_gpu
    agent_tags:
      label: production
      gpu: 1+
      mem: 24+
    no_plugin: true
    env:
      REGISTRY: "public.ecr.aws/q9t5s3a7"
      REPO: "vllm-ci-test-repo"
      VLLM_TEST_DEVICE: "xpu"
    source_file_dependencies:
      - vllm/
      - .buildkite/intel_jobs/test-intel.yaml
    commands:
      - >-
        bash .buildkite/scripts/hardware_ci/run-intel-test.sh
        'python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8 --enforce-eager --max-model-len 4096 &&
        python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model neuralmagic/Llama-3.2-1B-Instruct-FP8-dynamic --enforce-eager --max-model-len 4096 &&
        python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8 --enforce-eager --max-model-len 4096
        '
  - label: "XPU V1 test"
    depends_on:
      - image-build-xpu
    timeout_in_minutes: 70
    device: intel_gpu
    agent_tags:
      label: production
      gpu: 1+
      mem: 24+
    no_plugin: true
    env:
      REGISTRY: "public.ecr.aws/q9t5s3a7"
      REPO: "vllm-ci-test-repo"
    source_file_dependencies:
      - vllm/
      - .buildkite/intel_jobs/test-intel.yaml 
    commands:
      - >-
        bash .buildkite/scripts/hardware_ci/run-intel-test.sh
        'cd tests &&
        pytest -v -s v1/core --ignore=v1/core/test_reset_prefix_cache_e2e.py --ignore=v1/core/test_scheduler_e2e.py &&
        pytest -v -s v1/engine --ignore=v1/engine/test_output_processor.py &&
        pytest -v -s v1/sample --ignore=v1/sample/test_logprobs.py --ignore=v1/sample/test_logprobs_e2e.py -k "not test_topk_only and not test_topp_only and not test_topk_and_topp" &&
        pytest -v -s v1/worker --ignore=v1/worker/test_gpu_model_runner.py --ignore=v1/worker/test_worker_memory_snapshot.py &&
        pytest -v -s v1/structured_output &&
        pytest -v -s v1/test_serial_utils.py &&
        pytest -v -s v1/e2e/general/test_correctness_sliding_window.py --deselect="tests/v1/e2e/general/test_correctness_sliding_window.py::test_sliding_window_retrieval[True-1-5-google/gemma-3-1b-it]" &&
        pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py --ignore=v1/spec_decode/test_speculators_correctness.py &&
        pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_multi_connector.py --ignore=v1/kv_connector/unit/test_example_connector.py --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py --ignore=v1/kv_connector/unit/test_offloading_connector.py'
  - label: "XPU server test"
    depends_on:
      - image-build-xpu
    timeout_in_minutes: 45
    device: intel_gpu
    agent_tags:
      label: production
      gpu: 1+
      mem: 16+
    no_plugin: true
    env:
      REGISTRY: "public.ecr.aws/q9t5s3a7"
      REPO: "vllm-ci-test-repo"
    source_file_dependencies:
      - vllm/
      - .buildkite/intel_jobs/test-intel.yaml 
    commands:
      - >-
        bash .buildkite/scripts/hardware_ci/run-intel-test.sh
        'pip install av &&
        cd tests &&
        pytest -v -s entrypoints/multimodal/openai/chat_completion/test_audio_in_video.py &&
        pytest -v -s benchmarks/test_serve_cli.py'
  - label: "XPU quantization test"
    depends_on:
      - image-build-xpu
    timeout_in_minutes: 30
    device: intel_gpu
    agent_tags:
      label: production
      gpu: 1+
      mem: 16+
    no_plugin: true
    env:
      REGISTRY: "public.ecr.aws/q9t5s3a7"
      REPO: "vllm-ci-test-repo"
    source_file_dependencies:
      - vllm/
      - .buildkite/intel_jobs/test-intel.yaml 
    commands:
      - >-
        bash .buildkite/scripts/hardware_ci/run-intel-test.sh
        'cd tests &&
        pytest -v -s quantization/test_auto_round.py'
  - label: "XPU compressed tensors FP8 test"
    depends_on:
      - image-build-xpu
    timeout_in_minutes: 60
    device: intel_gpu
    agent_tags:
      label: production
      gpu: 1+
      mem: 16+
    no_plugin: true
    env:
      REGISTRY: "public.ecr.aws/q9t5s3a7"
      REPO: "vllm-ci-test-repo"
      VLLM_TEST_DEVICE: "xpu"
    source_file_dependencies:
      - vllm/
      - tests/quantization/test_compressed_tensors.py
      - .buildkite/intel_jobs/test-intel.yaml
    commands:
      - >-
        bash .buildkite/scripts/hardware_ci/run-intel-test.sh
        'cd tests &&
        pytest -v -s quantization/test_compressed_tensors.py::test_compressed_tensors_fp8'
@@ -1,10 +1,6 @@
 # For hf script, without -t option (tensor parallel size).
 # bash .buildkite/lm-eval-harness/run-lm-eval-mmlupro-vllm-baseline.sh -m meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 -l 250 -t 8 -f 5
 model_name: "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8"
 rocm_safetensors_load_strategy: lazy
 required_gpu_arch:
  - gfx942
  - gfx950
 tasks:
 - name: "mmlu_pro"
  metrics:
@@ -6,7 +6,9 @@ tasks:
    value: 0.7142
  - name: "exact_match,flexible-extract"
    value: 0.4579
-moe_backend: "flashinfer_cutlass"
+env_vars:
  VLLM_USE_FLASHINFER_MOE_FP8: "1"
  VLLM_FLASHINFER_MOE_BACKEND: "throughput"
 limit: 1319
 num_fewshot: 5
 max_model_len: 262144
@@ -1,9 +1,6 @@
 # For vllm script, with -t option (tensor parallel size)
 # bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m RedHatAI/Qwen2.5-VL-3B-Instruct-FP8-Dynamic -l 1319 -t 1
 model_name: "RedHatAI/Qwen2.5-VL-3B-Instruct-FP8-Dynamic"
 required_gpu_arch:
  - gfx942
  - gfx950
 tasks:
 - name: "gsm8k"
  metrics:
@@ -1,7 +1,4 @@
 model_name: "Qwen/Qwen3-235B-A22B-Instruct-2507-FP8"
 required_gpu_arch:
  - gfx942
  - gfx950
 tasks:
  - name: "mmlu_pro"
    metrics:
@@ -0,0 +1,12 @@
 # For vllm script, with -t option (tensor parallel size).
 # bash ./run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/SparseLlama-3.1-8B-gsm8k-pruned.2of4-chnl_wts_per_tok_dyn_act_fp8-BitM -b "auto" -t 2
 model_name: "nm-testing/SparseLlama-3.1-8B-gsm8k-pruned.2of4-chnl_wts_per_tok_dyn_act_fp8-BitM"
 tasks:
 - name: "gsm8k"
  metrics:
  - name: "exact_match,strict-match"
    value: 0.6353
  - name: "exact_match,flexible-extract"
    value: 0.637
 limit: null
 num_fewshot: null 
@@ -1 +0,0 @@
 Qwen3-235B-A22B-Instruct-2507-FP8.yaml
@@ -1,6 +1,5 @@
 Qwen2.5-1.5B-Instruct.yaml
 Meta-Llama-3.2-1B-Instruct-INT8-compressed-tensors.yaml
 Meta-Llama-3-8B-Instruct-INT8-compressed-tensors-asym.yaml
 Meta-Llama-3-8B-Instruct-nonuniform-compressed-tensors.yaml
 Qwen2.5-VL-3B-Instruct-FP8-dynamic.yaml
 Qwen1.5-MoE-W4A16-compressed-tensors.yaml
@@ -2,7 +2,7 @@
 # We can use this script to compute baseline accuracy on chartqa for vllm.
 #
 # Make sure you have lm-eval-harness installed:
-#   pip install "lm-eval[api]>=0.4.12"
+#   pip install "lm-eval[api]>=0.4.11"
 usage() {
    echo``
@@ -2,7 +2,7 @@
 # We can use this script to compute baseline accuracy on GSM for transformers.
 #
 # Make sure you have lm-eval-harness installed:
-#   pip install "lm-eval[api]>=0.4.12"
+#   pip install "lm-eval[api]>=0.4.11"
 usage() {
    echo``
@@ -3,7 +3,7 @@
 # We use this for fp8, which HF does not support.
 #
 # Make sure you have lm-eval-harness installed:
-#   pip install "lm-eval[api]>=0.4.12"
+#   pip install "lm-eval[api]>=0.4.11"
 usage() {
    echo``
@@ -3,7 +3,7 @@
 # We use this for fp8, which HF does not support.
 #
 # Make sure you have lm-eval-harness installed:
-#   pip install "lm-eval[api]>=0.4.12"
+#   pip install "lm-eval[api]>=0.4.11"
 usage() {
    echo``
@@ -13,11 +13,9 @@ import os
 from contextlib import contextmanager
 import lm_eval
-import pytest
+import numpy as np
 import yaml
 from vllm.platforms import current_platform
 DEFAULT_RTOL = 0.08
@@ -65,18 +63,6 @@ def launch_lm_eval(eval_config, tp_size):
        "allow_deprecated_quantization=True,"
    )
    if current_platform.is_rocm() and "Nemotron-3" in eval_config["model_name"]:
        model_args += "attention_backend=TRITON_ATTN"
    moe_backend = eval_config.get("moe_backend", None)
    if moe_backend is not None:
        model_args += f"moe_backend={moe_backend},"
    if current_platform.is_rocm():
        rocm_load_strategy = eval_config.get("rocm_safetensors_load_strategy")
        if rocm_load_strategy is not None:
            model_args += f"safetensors_load_strategy={rocm_load_strategy},"
    env_vars = eval_config.get("env_vars", None)
    with scoped_env_vars(env_vars):
        results = lm_eval.simple_evaluate(
@@ -99,40 +85,9 @@ def launch_lm_eval(eval_config, tp_size):
    return results
 def _check_rocm_gpu_arch_requirement(eval_config):
    """Skip the test if the model requires a ROCm GPU arch not present.
    Model YAML configs can specify::
        required_gpu_arch:
          - gfx942
          - gfx950
    The check only applies on ROCm.  On other platforms (e.g. CUDA) the
    field is ignored so that shared config files work for both NVIDIA and
    AMD CI pipelines.
    """
    required_archs = eval_config.get("required_gpu_arch")
    if not required_archs:
        return
    if not current_platform.is_rocm():
        return
    from vllm.platforms.rocm import _GCN_ARCH  # noqa: E402
    if not any(arch in _GCN_ARCH for arch in required_archs):
        pytest.skip(
            f"Model requires GPU arch {required_archs}, "
            f"but detected arch is '{_GCN_ARCH}'"
        )
 def test_lm_eval_correctness_param(config_filename, tp_size):
    eval_config = yaml.safe_load(config_filename.read_text(encoding="utf-8"))
    _check_rocm_gpu_arch_requirement(eval_config)
    results = launch_lm_eval(eval_config, tp_size)
    rtol = eval_config.get("rtol", DEFAULT_RTOL)
@@ -147,8 +102,6 @@ def test_lm_eval_correctness_param(config_filename, tp_size):
                f"ground_truth={ground_truth:.3f} | "
                f"measured={measured_value:.3f} | rtol={rtol}"
            )
-
+            success = success and np.isclose(ground_truth, measured_value, rtol=rtol)
            min_acceptable = ground_truth * (1 - rtol)
            success = success and measured_value >= min_acceptable
    assert success
@@ -83,6 +83,7 @@ We test the throughput by using `vllm bench serve` with request rate = inf to co
        "server_parameters": {
            "model": "meta-llama/Meta-Llama-3-8B",
            "tensor_parallel_size": 1,
            "swap_space": 16,
            "disable_log_stats": "",
            "load_format": "dummy"
        },
@@ -7,12 +7,12 @@ import argparse
 import html as _html
 import json
 import os
 from contextlib import nullcontext
 from dataclasses import dataclass
 from importlib import util
 from pathlib import Path
 import pandas as pd
 import regex as re
 pd.options.display.float_format = "{:.2f}".format
 plotly_found = util.find_spec("plotly.express") is not None
@@ -33,45 +33,6 @@ pd.set_option("display.precision", 2)
 pd.set_option("display.float_format", lambda x: f"{x:.2f}")
 # -----------------------------
 # Concurrency normalization (NEW, small)
 # -----------------------------
 def _find_concurrency_col(df: pd.DataFrame) -> str:
    for c in [
        "# of max concurrency.",
        "# of max concurrency",
        "Max Concurrency",
        "max_concurrency",
        "Concurrency",
    ]:
        if c in df.columns:
            return c
    for c in df.columns:
        if "concurr" in str(c).lower():
            s = df[c]
            if s.dtype.kind in "iu" and s.nunique() > 1 and s.min() >= 1:
                return c
    raise ValueError(
        "Cannot infer concurrency column. "
        "Please rename the column to one of the known names "
        "or add an explicit override (e.g., --concurrency-col)."
    )
 def _normalize_concurrency_in_df(
    df: pd.DataFrame, canonical: str = "# of max concurrency."
 ) -> pd.DataFrame:
    if canonical in df.columns:
        return df
    detected = _find_concurrency_col(df)
    if detected in df.columns and detected != canonical:
        return df.rename(columns={detected: canonical})
    df[canonical] = pd.NA
    return df
 # -----------------------------
 # Core data compare
 # -----------------------------
@@ -91,25 +52,19 @@ def compare_data_columns(
    - Concat along axis=1 (indexes align), then reset_index so callers can
      group by columns.
    - If --debug, add a <file_label>_name column per file.
    Minimal fix to support different max_concurrency lists across files:
      - normalize concurrency column naming to "# of max concurrency."
      - align on UNION of keys (missing points become NaN)
      - BUGFIX: don't drop throughput rows based on P99/Median presence
    """
    print("\ncompare_data_column:", data_column)
    frames = []
    raw_data_cols: list[str] = []
    compare_frames = []
    # Determine key cols after normalizing concurrency
    cols_per_file: list[set] = []
    for f in files:
        try:
            df_tmp = pd.read_json(f, orient="records")
        except Exception as err:
            raise ValueError(f"Failed to read {f}") from err
        df_tmp = _normalize_concurrency_in_df(df_tmp, canonical="# of max concurrency.")
        cols_per_file.append(set(df_tmp.columns))
    key_cols = [c for c in info_cols if all(c in cset for cset in cols_per_file)]
@@ -120,25 +75,12 @@ def compare_data_columns(
            "No common key columns found from info_cols across the input files."
        )
-    union_index = None
+    meta_added = False
    metas: list[pd.DataFrame] = []
    staged: list[tuple[str, pd.Series, pd.Series | None]] = []
    for file in files:
        df = pd.read_json(file, orient="records")
        df = _normalize_concurrency_in_df(df, canonical="# of max concurrency.")
-        # BUGFIX: only drop rows for latency-like metrics; throughput rows may have
+        if drop_column in df.columns:
        # NaN in P99/Median columns even if the column exists in the JSON.
        metric_lc = str(data_column).lower()
        is_latency_metric = (
            "ttft" in metric_lc
            or "tpot" in metric_lc
            or "p99" in metric_lc
            or "median" in metric_lc
            or metric_lc.strip() in {"p99", "median"}
        )
        if is_latency_metric and drop_column in df.columns:
            df = df.dropna(subset=[drop_column], ignore_index=True)
        for c in (
@@ -163,61 +105,35 @@ def compare_data_columns(
            meta = meta.groupby(level=key_cols, dropna=False).first()
        file_label = "/".join(file.split("/")[:-1]) or os.path.basename(file)
-
+        s = df_idx[data_column]
-        if data_column in df_idx.columns:
+        if not s.index.is_unique:
-            s = df_idx[data_column]
+            s = s.groupby(level=key_cols, dropna=False).mean()
            if not s.index.is_unique:
                s = s.groupby(level=key_cols, dropna=False).mean()
        else:
            # keep NA series to preserve meta keys for union_index
            s = pd.Series(pd.NA, index=meta.index)
        s.name = file_label
-        name_s = None
+        if not meta_added:
            frames.append(meta)
            meta_added = True
        if debug and name_column in df_idx.columns:
            name_s = df_idx[name_column]
            if not name_s.index.is_unique:
                name_s = name_s.groupby(level=key_cols, dropna=False).first()
            name_s.name = f"{file_label}_name"
            frames.append(name_s)
-        if union_index is None:
+        frames.append(s)
            union_index = meta.index
        else:
            union_index = union_index.union(meta.index)
        metas.append(meta)
        staged.append((file_label, s, name_s))
    if union_index is None:
        raise ValueError("No data found after loading inputs.")
    # meta first (union-aligned): build UNION meta across all files
    if metas:
        meta_union = pd.concat(metas, axis=0)
        # Collapse duplicates on the MultiIndex; keep first non-null per column
        meta_union = meta_union.groupby(level=key_cols, dropna=False).first()
        frames.append(meta_union.reindex(union_index))
    # values + ratios (union-aligned)
    metric_series_aligned: list[pd.Series] = []
    for file_label, s, name_s in staged:
        s_aligned = s.reindex(union_index)
        frames.append(s_aligned)
        raw_data_cols.append(file_label)
-        metric_series_aligned.append(s_aligned)
+        compare_frames.append(s)
-        if debug and name_s is not None:
+        if len(compare_frames) >= 2:
-            frames.append(name_s.reindex(union_index))
+            base = compare_frames[0]
-
+            current = compare_frames[-1]
-        if len(metric_series_aligned) >= 2:
+            if "P99" in data_column or "Median" in data_column:
            base = metric_series_aligned[0]
            current = metric_series_aligned[-1]
            if "P99" in str(data_column) or "Median" in str(data_column):
                ratio = base / current
            else:
                ratio = current / base
            ratio = ratio.mask(base == 0)
-            ratio.name = f"Ratio 1 vs {len(metric_series_aligned)}"
+            ratio.name = f"Ratio 1 vs {len(compare_frames)}"
            frames.append(ratio)
    concat_df = pd.concat(frames, axis=1).reset_index(drop=True)
@@ -288,10 +204,24 @@ def split_json_by_tp_pp(
 # -----------------------------
 # Styling helpers
 # -----------------------------
 def _find_concurrency_col(df: pd.DataFrame) -> str:
    for c in [
        "# of max concurrency.",
        "# of max concurrency",
        "Max Concurrency",
        "max_concurrency",
        "Concurrency",
    ]:
        if c in df.columns:
            return c
    for c in df.columns:
        if df[c].dtype.kind in "iu" and df[c].nunique() > 1 and df[c].min() >= 1:
            return c
    return "# of max concurrency."
 def _highlight_threshold(
-    df: pd.DataFrame,
+    df: pd.DataFrame, threshold: float
    threshold: float,
    slack_pct: float = 0.0,
 ) -> pd.io.formats.style.Styler:
    conc_col = _find_concurrency_col(df)
    key_cols = [
@@ -304,24 +234,12 @@ def _highlight_threshold(
    ]
    conf_cols = [c for c in conf_cols if pd.api.types.is_numeric_dtype(df[c])]
-    try:
+    return df.style.map(
-        slack_pct = float(slack_pct or 0.0)
+        lambda v: "background-color:#e6ffe6;font-weight:bold;"
-    except Exception:
+        if pd.notna(v) and v <= threshold
-        slack_pct = 0.0
+        else "",
-    slack_limit = threshold * (1.0 + slack_pct / 100.0)
+        subset=conf_cols,
-
+    )
    def _cell(v):
        if pd.isna(v):
            return ""
        if v <= threshold:
            # Strict SLA
            return "background-color:#e6ffe6;font-weight:bold;"
        if v <= slack_limit:
            # Within slack range
            return "background-color:#ffe5cc;font-weight:bold;"
        return ""
    return df.style.map(_cell, subset=conf_cols)
 def highlight_ratio_columns(styler: pd.io.formats.style.Styler):
@@ -368,30 +286,11 @@ def _sanitize_sheet_name(name: str) -> str:
      - max 31 chars
      - cannot contain: : \ / ? * [ ]
      - cannot be empty
    NOTE: Use fast, non-regex operations here to avoid the third-party `regex`
    module's compile overhead/edge-cases on some systems.
    """
    name = "sheet" if name is None else str(name)
-
+    name = re.sub(r"[:\\/?*\[\]]", "_", name)
    # Replace illegal characters with underscore.
    trans = str.maketrans(
        {
            ":": "_",
            "\\": "_",
            "/": "_",
            "?": "_",
            "*": "_",
            "[": "_",
            "]": "_",
        }
    )
    name = name.translate(trans)
    # Strip quotes/spaces and collapse whitespace.
    name = name.strip().strip("'")
-    name = " ".join(name.split())
+    name = re.sub(r"\s+", " ", name)
    if not name:
        name = "sheet"
    return name[:31]
@@ -399,57 +298,30 @@ def _sanitize_sheet_name(name: str) -> str:
 def _group_to_sheet_base(group_cols: list[str], gkey_tuple) -> str:
    d = dict(zip(group_cols, gkey_tuple))
-
+    model = d.get("Model", "model")
-    # Always keep input/output lengths (these are important).
+    model_short = str(model).split("/")[-1]
    ilen = d.get("Input Len", "")
    olen = d.get("Output Len", "")
    lens = f"_{ilen}x{olen}" if ilen != "" and olen != "" else ""
    # Shorten model name aggressively to make room for lens.
    model = d.get("Model", "model")
    leaf = str(model).split("/")[-1]
    max_model_len = max(1, 31 - len(lens))
    model_short = leaf[:max_model_len]
    return _sanitize_sheet_name(f"{model_short}{lens}")
 def _write_tables_to_excel_sheet(
    writer: pd.ExcelWriter, sheet: str, blocks: list[tuple[str, pd.DataFrame]]
 ):
-    """Write all blocks to a sheet with a single to_excel() call.
+    startrow = 0
    Pandas+openpyxl can be extremely slow when called many times per sheet.
    We flatten blocks into one table with a 'Section' column to keep structure
    while making Excel generation fast and deterministic.
    """
    if not blocks:
        pd.DataFrame().to_excel(writer, sheet_name=sheet, index=False)
        return
    combined_parts: list[pd.DataFrame] = []
    for title, df in blocks:
-        df2 = df.copy()
+        pd.DataFrame([[title]]).to_excel(
-        # Put the section label as the first column for readability.
+            writer, sheet_name=sheet, index=False, header=False, startrow=startrow
-        df2.insert(0, "Section", title)
+        )
-        combined_parts.append(df2)
+        startrow += 1
-
+        df.to_excel(writer, sheet_name=sheet, index=False, startrow=startrow)
-    combined = pd.concat(combined_parts, axis=0, ignore_index=True, sort=False)
+        startrow += len(df) + 3
    combined.to_excel(writer, sheet_name=sheet, index=False)
 def _safe_filename(s: str) -> str:
-    # Fast path without the third-party `regex` module.
+    s = re.sub(r"[^\w\-.]+", "_", str(s).strip())
-    s = " ".join(str(s).strip().split())
+    return s[:180] if len(s) > 180 else s
    allowed = []
    for ch in s:
        if ch.isalnum() or ch in "._-":
            allowed.append(ch)
        else:
            allowed.append("_")
    out = "".join(allowed)
    return out[:180] if len(out) > 180 else out
 # -----------------------------
@@ -556,11 +428,7 @@ def _config_value_columns(df: pd.DataFrame, conc_col: str) -> list[str]:
 def _max_concurrency_ok(
-    df: pd.DataFrame,
+    df: pd.DataFrame, conc_col: str, cfg_col: str, threshold: float
    conc_col: str,
    cfg_col: str,
    threshold: float,
    slack_pct: float = 0.0,
 ):
    if df is None or conc_col not in df.columns or cfg_col not in df.columns:
        return pd.NA
@@ -573,14 +441,7 @@ def _max_concurrency_ok(
    if d.empty:
        return pd.NA
-    # Accept values up to (1 + slack_pct%) above the SLA.
+    ok = d[d[cfg_col] <= threshold]
    try:
        slack_pct = float(slack_pct or 0.0)
    except Exception:
        slack_pct = 0.0
    effective_limit = float(threshold) * (1.0 + slack_pct / 100.0)
    ok = d[d[cfg_col] <= effective_limit]
    if ok.empty:
        return pd.NA
@@ -646,25 +507,15 @@ def build_valid_max_concurrency_summary_html(
    if not cfg_cols:
        cfg_cols = sorted(set(ttft_cols) | set(tpot_cols) | set(tput_cols), key=str)
    # Display SLA ranges in the table header (SLA .. SLA*(1+slack))
    ttft_hi = args.ttft_max_ms * (1.0 + args.ttft_slack_pct / 100.0)
    tpot_hi = args.tpot_max_ms * (1.0 + args.tpot_slack_pct / 100.0)
    ttft_range = f"{args.ttft_max_ms:g}–{ttft_hi:g} ms (+{args.ttft_slack_pct:g}%)"
    tpot_range = f"{args.tpot_max_ms:g}–{tpot_hi:g} ms (+{args.tpot_slack_pct:g}%)"
    rows = []
    for cfg in cfg_cols:
        ttft_max = (
-            _max_concurrency_ok(
+            _max_concurrency_ok(ttft_group_df, conc_col, cfg, args.ttft_max_ms)
                ttft_group_df, conc_col, cfg, args.ttft_max_ms, args.ttft_slack_pct
            )
            if ttft_group_df is not None
            else pd.NA
        )
        tpot_max = (
-            _max_concurrency_ok(
+            _max_concurrency_ok(tpot_group_df, conc_col, cfg, args.tpot_max_ms)
                tpot_group_df, conc_col, cfg, args.tpot_max_ms, args.tpot_slack_pct
            )
            if tpot_group_df is not None
            else pd.NA
        )
@@ -693,8 +544,8 @@ def build_valid_max_concurrency_summary_html(
        rows.append(
            {
                "Configuration": cfg,
-                f"Max {conc_col} (TTFT ≤ {ttft_range})": ttft_max,
+                f"Max {conc_col} (TTFT ≤ {args.ttft_max_ms:g} ms)": ttft_max,
-                f"Max {conc_col} (TPOT ≤ {tpot_range})": tpot_max,
+                f"Max {conc_col} (TPOT ≤ {args.tpot_max_ms:g} ms)": tpot_max,
                f"Max {conc_col} (Both)": both,
                "Output Tput @ Both (tok/s)": tput_at_both,
                "TTFT @ Both (ms)": ttft_at_both,
@@ -769,24 +620,15 @@ def build_valid_max_concurrency_summary_df(
    if not cfg_cols:
        cfg_cols = sorted(set(ttft_cols) | set(tpot_cols) | set(tput_cols), key=str)
    ttft_hi = args.ttft_max_ms * (1.0 + args.ttft_slack_pct / 100.0)
    tpot_hi = args.tpot_max_ms * (1.0 + args.tpot_slack_pct / 100.0)
    ttft_range = f"{args.ttft_max_ms:g}–{ttft_hi:g} ms (+{args.ttft_slack_pct:g}%)"
    tpot_range = f"{args.tpot_max_ms:g}–{tpot_hi:g} ms (+{args.tpot_slack_pct:g}%)"
    rows = []
    for cfg in cfg_cols:
        ttft_max = (
-            _max_concurrency_ok(
+            _max_concurrency_ok(ttft_group_df, conc_col, cfg, args.ttft_max_ms)
                ttft_group_df, conc_col, cfg, args.ttft_max_ms, args.ttft_slack_pct
            )
            if ttft_group_df is not None
            else pd.NA
        )
        tpot_max = (
-            _max_concurrency_ok(
+            _max_concurrency_ok(tpot_group_df, conc_col, cfg, args.tpot_max_ms)
                tpot_group_df, conc_col, cfg, args.tpot_max_ms, args.tpot_slack_pct
            )
            if tpot_group_df is not None
            else pd.NA
        )
@@ -815,8 +657,8 @@ def build_valid_max_concurrency_summary_df(
        rows.append(
            {
                "Configuration": cfg,
-                f"Max {conc_col} (TTFT ≤ {ttft_range})": ttft_max,
+                f"Max {conc_col} (TTFT ≤ {args.ttft_max_ms:g} ms)": ttft_max,
-                f"Max {conc_col} (TPOT ≤ {tpot_range})": tpot_max,
+                f"Max {conc_col} (TPOT ≤ {args.tpot_max_ms:g} ms)": tpot_max,
                f"Max {conc_col} (Both)": both,
                "Output Tput @ Both (tok/s)": tput_at_both,
                "TTFT @ Both (ms)": ttft_at_both,
@@ -909,21 +751,7 @@ def build_parser() -> argparse.ArgumentParser:
        help="Reference limit for TPOT plots (ms)",
    )
-    # ---- SLA tolerance (slack) options ----
+    # ---- NEW: export options ----
    parser.add_argument(
        "--ttft-slack-pct",
        type=float,
        default=5.0,
        help="Allowed percentage above TTFT SLA (default: 5).",
    )
    parser.add_argument(
        "--tpot-slack-pct",
        type=float,
        default=5.0,
        help="Allowed percentage above TPOT SLA (default: 5).",
    )
    # ---- export options ----
    parser.add_argument(
        "--excel-out",
        type=str,
@@ -1015,13 +843,9 @@ def render_metric_table_html(
    metric_name = metric_label.lower()
    if "ttft" in metric_name:
-        styler = _highlight_threshold(
+        styler = _highlight_threshold(display_group, args.ttft_max_ms)
            display_group, args.ttft_max_ms, args.ttft_slack_pct
        )
    elif ("tpot" in metric_name) or ("median" in metric_name) or ("p99" in metric_name):
-        styler = _highlight_threshold(
+        styler = _highlight_threshold(display_group, args.tpot_max_ms)
            display_group, args.tpot_max_ms, args.tpot_slack_pct
        )
    else:
        styler = display_group.style
@@ -1138,46 +962,22 @@ def write_report_group_first(
        csv_dir.mkdir(parents=True, exist_ok=True)
    excel_path = args.excel_out or "perf_comparison.xlsx"
-    disable_excel = os.getenv("VLLM_COMPARE_DISABLE_EXCEL", "0") == "1"
+    with pd.ExcelWriter(excel_path, engine="openpyxl") as xw:
    # Prefer xlsxwriter for speed; fallback to openpyxl if unavailable.
    excel_engine = (
        os.getenv("VLLM_COMPARE_EXCEL_ENGINE", "xlsxwriter").strip() or "xlsxwriter"
    )
    if excel_engine == "xlsxwriter" and util.find_spec("xlsxwriter") is None:
        excel_engine = "openpyxl"
    excel_engine_kwargs = {}
    if excel_engine == "xlsxwriter":
        # Reduce memory pressure & usually faster writes.
        excel_engine_kwargs = {"options": {"constant_memory": True}}
    xw_ctx = (
        nullcontext(None)
        if disable_excel
        else pd.ExcelWriter(
            excel_path, engine=excel_engine, engine_kwargs=excel_engine_kwargs
        )
    )
    with xw_ctx as xw:
        used_sheets: set[str] = set()
        # ---- Environment sheet (first) ----
        env_sheet = _sanitize_sheet_name("Environment")
        env_df = _load_env_df_for_inputs(args, files)
-        if xw is not None:
+        if env_df is None or env_df.empty:
-            if env_df is None or env_df.empty:
+            pd.DataFrame(
-                pd.DataFrame(
+                [
-                    [
+                    {
-                        {
+                        "Section": "Environment",
-                            "Section": "Environment",
+                        "Key": "vllm_env.txt",
-                            "Key": "vllm_env.txt",
+                        "Value": "NOT FOUND (or empty)",
-                            "Value": "NOT FOUND (or empty)",
+                    }
-                        }
+                ]
-                    ]
+            ).to_excel(xw, sheet_name=env_sheet, index=False)
-                ).to_excel(xw, sheet_name=env_sheet, index=False)
+        else:
-            else:
+            env_df.to_excel(xw, sheet_name=env_sheet, index=False)
                env_df.to_excel(xw, sheet_name=env_sheet, index=False)
            used_sheets.add(env_sheet)
        with open("perf_comparison.html", "w", encoding="utf-8") as main_fh:
            main_fh.write('<meta charset="utf-8">\n')
            for gkey in group_keys:
@@ -1193,19 +993,12 @@ def write_report_group_first(
                main_fh.write(group_header)
                do_excel = xw is not None
                sheet = _group_to_sheet_base(group_cols_canonical, gkey_tuple)
                sheet_base = sheet
-                if do_excel:
+                dedup_i = 1
-                    dedup_i = 1
+                while sheet in xw.sheets:
-                    while sheet in used_sheets:
+                    dedup_i += 1
-                        dedup_i += 1
+                    sheet = _sanitize_sheet_name(f"{sheet_base}_{dedup_i}")
                        suffix = f"_{dedup_i}"
                        # Ensure uniqueness even when sheet names are truncated.
                        base = str(sheet_base)
                        keep = max(1, 31 - len(suffix))
                        sheet = _sanitize_sheet_name(base[:keep] + suffix)
                    used_sheets.add(sheet)
                excel_blocks: list[tuple[str, pd.DataFrame]] = []
@@ -1266,7 +1059,7 @@ def write_report_group_first(
                        )
                        excel_blocks.append(
-                            (metric_label, group_df.reset_index(drop=True))
+                            (metric_label, display_group.reset_index(drop=True))
                        )
                        if csv_dir:
                            fn = _safe_filename(
@@ -1274,7 +1067,7 @@ def write_report_group_first(
                                    "/", "_"
                                )
                            )
-                            group_df.to_csv(csv_dir / f"{fn}.csv", index=False)
+                            display_group.to_csv(csv_dir / f"{fn}.csv", index=False)
                    summary_html = build_valid_max_concurrency_summary_html(
                        tput_group_df=tput_group_df,
@@ -1304,13 +1097,9 @@ def write_report_group_first(
                            )
                            summary_df.to_csv(csv_dir / f"{fn}.csv", index=False)
-                if do_excel:
+                _write_tables_to_excel_sheet(xw, sheet, excel_blocks)
                    _write_tables_to_excel_sheet(xw, sheet, excel_blocks)
-    if disable_excel:
+    print(f"Wrote Excel: {excel_path}")
        print("Skipped Excel generation (VLLM_COMPARE_DISABLE_EXCEL=1).")
    else:
        print(f"Wrote Excel: {excel_path}")
    if csv_dir:
        print(f"Wrote CSVs under: {csv_dir}")
@@ -12,13 +12,6 @@ DRY_RUN="${DRY_RUN:-0}"
 MODEL_FILTER="${MODEL_FILTER:-}"
 DTYPE_FILTER="${DTYPE_FILTER:-}"
 # Adaptive search controls
 ENABLE_ADAPTIVE_CONCURRENCY="${ENABLE_ADAPTIVE_CONCURRENCY:-0}"
 SLA_TTFT_MS="${SLA_TTFT_MS:-3000}"
 SLA_TPOT_MS="${SLA_TPOT_MS:-100}"
 ADAPTIVE_MAX_PROBES="${ADAPTIVE_MAX_PROBES:-8}"
 ADAPTIVE_MAX_CONCURRENCY="${ADAPTIVE_MAX_CONCURRENCY:-1024}"
 check_gpus() {
  if command -v nvidia-smi; then
    # check the number of GPUs and GPU type.
@@ -190,304 +183,6 @@ upload_to_buildkite() {
  $BUILDKITE_AGENT_COMMAND artifact upload "$RESULTS_FOLDER/*"
 }
 # -------------------------------
 # Adaptive concurrency helpers
 # -------------------------------
 result_json_path_for_serving() {
  local test_name=$1
  local qps=$2
  local max_concurrency=$3
  echo "$RESULTS_FOLDER/${test_name}_qps_${qps}_concurrency_${max_concurrency}.json"
 }
 extract_metric_ms() {
  local metric_name=$1
  local json_file=$2
  [[ -f "$json_file" ]] || return 0
  if [[ "$metric_name" == "ttft" ]]; then
    jq -r '
      [
        .ttft_ms.p99?,
        .metrics.ttft_ms.p99?,
        .ttft.p99?,
        .metrics.ttft.p99?,
        .p99_ttft_ms?,
        .ttft_ms.mean?,
        .metrics.ttft_ms.mean?,
        .ttft.mean?,
        .metrics.ttft.mean?,
        .mean_ttft_ms?
      ] | map(select(. != null)) | .[0] // empty
    ' "$json_file"
  else
    jq -r '
      [
        .tpot_ms.p99?,
        .metrics.tpot_ms.p99?,
        .tpot.p99?,
        .metrics.tpot.p99?,
        .p99_tpot_ms?,
        .itl_ms.p99?,
        .metrics.itl_ms.p99?,
        .inter_token_latency_ms.p99?,
        .tpot_ms.mean?,
        .metrics.tpot_ms.mean?,
        .tpot.mean?,
        .metrics.tpot.mean?,
        .itl_ms.mean?,
        .metrics.itl_ms.mean?,
        .mean_tpot_ms?,
        .mean_itl_ms?
      ] | map(select(. != null)) | .[0] // empty
    ' "$json_file"
  fi
 }
 evaluate_sla_from_json() {
  local json_file=$1
  local ttft
  local tpot
  local pass
  [[ -f "$json_file" ]] || return 2
  ttft=$(extract_metric_ms ttft "$json_file")
  tpot=$(extract_metric_ms tpot "$json_file")
  [[ -n "$ttft" && -n "$tpot" ]] || return 2
  pass=$(jq -n \
    --argjson ttft "$ttft" \
    --argjson tpot "$tpot" \
    --argjson sla_ttft "$SLA_TTFT_MS" \
    --argjson sla_tpot "$SLA_TPOT_MS" \
    '($ttft <= $sla_ttft) and ($tpot <= $sla_tpot)')
  [[ "$pass" == "true" ]]
 }
 write_adaptive_summary_json() {
  local summary_file=$1
  local test_name=$2
  local qps=$3
  local static_last_pass=$4
  local static_first_fail=$5
  local final_last_pass=$6
  local final_first_fail=$7
  jq -n \
    --arg test_name "$test_name" \
    --arg qps "$qps" \
    --argjson sla_ttft "$SLA_TTFT_MS" \
    --argjson sla_tpot "$SLA_TPOT_MS" \
    --arg static_last_pass "${static_last_pass:-}" \
    --arg static_first_fail "${static_first_fail:-}" \
    --arg final_last_pass "${final_last_pass:-}" \
    --arg final_first_fail "${final_first_fail:-}" \
    '{
      test_name: $test_name,
      qps: $qps,
      sla_ttft_ms: $sla_ttft,
      sla_tpot_ms: $sla_tpot,
      static_last_pass: (if $static_last_pass == "" then null else ($static_last_pass | tonumber) end),
      static_first_fail: (if $static_first_fail == "" then null else ($static_first_fail | tonumber) end),
      final_last_pass: (if $final_last_pass == "" then null else ($final_last_pass | tonumber) end),
      final_first_fail: (if $final_first_fail == "" then null else ($final_first_fail | tonumber) end)
    }' > "$summary_file"
 }
 run_single_serving_probe() {
  local test_name=$1
  local qps=$2
  local max_concurrency=$3
  local tp=$4
  local compilation_config_mode=$5
  local optimization_level=$6
  local client_args_effective=$7
  local client_remote_args=$8
  local server_command=$9
  local new_test_name="${test_name}_qps_${qps}_concurrency_${max_concurrency}"
  local result_json
  local num_prompts_arg=""
  local client_command
  result_json=$(result_json_path_for_serving "$test_name" "$qps" "$max_concurrency")
  if [[ -f "$result_json" ]]; then
    evaluate_sla_from_json "$result_json"
    return $?
  fi
  if [[ -n "${PROMPTS_PER_CONCURRENCY}" ]]; then
    num_prompts=$(( max_concurrency * PROMPTS_PER_CONCURRENCY ))
    if (( num_prompts < MIN_NUM_PROMPTS )); then num_prompts=$MIN_NUM_PROMPTS; fi
    if (( num_prompts > MAX_NUM_PROMPTS )); then num_prompts=$MAX_NUM_PROMPTS; fi
    num_prompts_arg="--num-prompts $num_prompts"
  fi
  client_command="vllm bench serve \
    --save-result \
    --result-dir $RESULTS_FOLDER \
    --result-filename ${new_test_name}.json \
    --request-rate $qps \
    --max-concurrency $max_concurrency \
    $num_prompts_arg \
    --metadata tensor_parallel_size=$tp compilation_config.mode=$compilation_config_mode optimization_level=$optimization_level adaptive_search=1 \
    $client_args_effective $client_remote_args "
  echo "Adaptive probe: $client_command"
  if [[ "${DRY_RUN:-0}" != "1" ]]; then
    bash -c "$client_command"
  fi
  jq_output=$(jq -n \
    --arg server "$server_command" \
    --arg client "$client_command" \
    --arg gpu "$gpu_type" \
    '{
      server_command: $server,
      client_command: $client,
      gpu_type: $gpu,
      adaptive_search: true
    }')
  echo "$jq_output" > "$RESULTS_FOLDER/${new_test_name}.commands"
  evaluate_sla_from_json "$result_json"
 }
 adaptive_refine_from_static_results() {
  local test_name=$1
  local qps=$2
  local max_concurrency_list_raw=$3
  local tp=$4
  local compilation_config_mode=$5
  local optimization_level=$6
  local client_args_effective=$7
  local client_remote_args=$8
  local server_command=$9
  local sorted_points
  local point
  local rc
  local static_last_pass=""
  local static_first_fail=""
  local largest_static=""
  local step_hint=1
  local previous_point=""
  local low
  local high
  local mid
  local probes=0
  local summary_file="$RESULTS_FOLDER/${test_name}_qps_${qps}_sla_summary.json"
  [[ "${ENABLE_ADAPTIVE_CONCURRENCY}" == "1" ]] || return 0
  [[ "${DRY_RUN:-0}" != "1" ]] || return 0
  sorted_points=$(for point in $max_concurrency_list_raw; do printf '%s\n' "$point"; done | tr -d "'" | awk '/^[0-9]+$/' | sort -n | uniq)
  [[ -n "$sorted_points" ]] || return 0
  while read -r point; do
    [[ -z "$point" ]] && continue
    largest_static="$point"
    evaluate_sla_from_json "$(result_json_path_for_serving "$test_name" "$qps" "$point")"
    rc=$?
    if (( rc == 0 )); then
      static_last_pass="$point"
    elif (( rc == 1 )); then
      if [[ -n "$static_last_pass" ]]; then
        static_first_fail="$point"
        break
      fi
    fi
    if [[ -n "$previous_point" ]]; then
      step_hint=$(( point - previous_point ))
      if (( step_hint < 1 )); then step_hint=1; fi
    fi
    previous_point="$point"
  done <<< "$sorted_points"
  if [[ -z "$static_last_pass" ]]; then
    write_adaptive_summary_json "$summary_file" "$test_name" "$qps" "" "$static_first_fail" "" "$static_first_fail"
    return 0
  fi
  if [[ -n "$static_first_fail" ]]; then
    low=$static_last_pass
    high=$static_first_fail
    while (( low + 1 < high )) && (( probes < ADAPTIVE_MAX_PROBES )); do
      mid=$(( (low + high) / 2 ))
      probes=$(( probes + 1 ))
      run_single_serving_probe \
        "$test_name" "$qps" "$mid" "$tp" \
        "$compilation_config_mode" "$optimization_level" \
        "$client_args_effective" "$client_remote_args" "$server_command"
      rc=$?
      if (( rc == 0 )); then
        low=$mid
      elif (( rc == 1 )); then
        high=$mid
      else
        break
      fi
    done
    write_adaptive_summary_json "$summary_file" "$test_name" "$qps" "$static_last_pass" "$static_first_fail" "$low" "$high"
    return 0
  fi
  low=$largest_static
  high=""
  while (( probes < ADAPTIVE_MAX_PROBES )); do
    point=$(( low + step_hint ))
    if (( point > ADAPTIVE_MAX_CONCURRENCY )); then
      point=$ADAPTIVE_MAX_CONCURRENCY
    fi
    (( point > low )) || break
    probes=$(( probes + 1 ))
    run_single_serving_probe \
      "$test_name" "$qps" "$point" "$tp" \
      "$compilation_config_mode" "$optimization_level" \
      "$client_args_effective" "$client_remote_args" "$server_command"
    rc=$?
    if (( rc == 0 )); then
      low=$point
      (( point == ADAPTIVE_MAX_CONCURRENCY )) && break
      step_hint=$(( step_hint * 2 ))
      if (( step_hint < 1 )); then step_hint=1; fi
    elif (( rc == 1 )); then
      high=$point
      break
    else
      break
    fi
  done
  if [[ -n "$high" ]]; then
    while (( low + 1 < high )) && (( probes < ADAPTIVE_MAX_PROBES )); do
      mid=$(( (low + high) / 2 ))
      probes=$(( probes + 1 ))
      run_single_serving_probe \
        "$test_name" "$qps" "$mid" "$tp" \
        "$compilation_config_mode" "$optimization_level" \
        "$client_args_effective" "$client_remote_args" "$server_command"
      rc=$?
      if (( rc == 0 )); then
        low=$mid
      elif (( rc == 1 )); then
        high=$mid
      else
        break
      fi
    done
  fi
  write_adaptive_summary_json "$summary_file" "$test_name" "$qps" "$static_last_pass" "" "$low" "$high"
 }
 run_benchmark_tests() {
  # run benchmark tests using `vllm bench <test_type>` command
  # $1: test type (latency or throughput)
@@ -652,48 +347,10 @@ run_serving_tests() {
    server_envs=$(echo "$params" | jq -r '.server_environment_variables')
    client_params=$(echo "$params" | jq -r '.client_parameters')
-    # vLLM serve CLI: model must be positional (no --model). Convert server_parameters accordingly.
+    server_args=$(json2args "$server_params")
    server_model=$(echo "$server_params" | jq -r '.model // empty')
    if [[ -z "$server_model" || "$server_model" == "null" ]]; then
      echo "Error: serving test '$test_name' is missing server_parameters.model" >&2
      exit 1
    fi
    server_params_no_model=$(echo "$server_params" | jq -c 'del(.model)')
    server_args=$(json2args "$server_params_no_model")
    server_envs=$(json2envs "$server_envs")
    client_args=$(json2args "$client_params")
    # ------------------------------------------------------------
    # Option 1: Dynamic num-prompts scaling based on max_concurrency
    #
    # If PROMPTS_PER_CONCURRENCY is set, override JSON num_prompts with:
    #   num_prompts = max_concurrency * PROMPTS_PER_CONCURRENCY
    #
    # If PROMPTS_PER_CONCURRENCY is NOT set, keep JSON num_prompts behavior
    # unchanged (i.e., whatever is in serving-tests-*.json).
    # ------------------------------------------------------------
    PROMPTS_PER_CONCURRENCY="${PROMPTS_PER_CONCURRENCY-}"  # no default on purpose
    MIN_NUM_PROMPTS="${MIN_NUM_PROMPTS:-1}"
    MAX_NUM_PROMPTS="${MAX_NUM_PROMPTS:-1000000}"
    if [[ -n "${PROMPTS_PER_CONCURRENCY}" ]]; then
      # Remove any fixed --num-prompts from JSON-derived args (avoid duplicates)
      # Remove any fixed --num-prompts from JSON-derived args (avoid duplicates)
      # Handles: --num-prompts 123   and   --num-prompts=123
      client_args_no_np="$(
        printf ' %s ' "$client_args" \
        | sed -E \
          -e 's/[[:space:]]--num-prompts=([^[:space:]]+)([[:space:]]|$)/ /g' \
          -e 's/[[:space:]]--num-prompts[[:space:]]+([^[:space:]]+)([[:space:]]|$)/ /g'
      )"
      # normalize whitespace
      client_args_no_np="$(echo "$client_args_no_np" | tr -s ' ' | sed -E 's/^ //; s/ $//')"
      client_args_no_np="$(echo "$client_args_no_np" | xargs)"
      client_args_effective="$client_args_no_np"
    else
      client_args_effective="$client_args"
    fi
    # qps_list
    qps_list=$(echo "$params" | jq -r '.qps_list')
    qps_list=$(echo "$qps_list" | jq -r '.[] | @sh')
@@ -725,13 +382,14 @@ run_serving_tests() {
    fi
    # check if server model and client model is aligned
    server_model=$(echo "$server_params" | jq -r '.model')
    client_model=$(echo "$client_params" | jq -r '.model')
    if [[ $server_model != "$client_model" ]]; then
      echo "Server model and client model must be the same. Skip testcase $test_name."
      continue
    fi
-    server_command="$server_envs vllm serve $server_model \
+    server_command="$server_envs vllm serve \
      $server_args"
    # run the server
@@ -778,14 +436,6 @@ run_serving_tests() {
      for max_concurrency in $max_concurrency_list; do
        new_test_name="${test_name}_qps_${qps}_concurrency_${max_concurrency}"
        echo " new test name $new_test_name"
        # If PROMPTS_PER_CONCURRENCY is set, compute per-concurrency --num-prompts.
        num_prompts_arg=""
        if [[ -n "${PROMPTS_PER_CONCURRENCY}" ]]; then
          num_prompts=$(( max_concurrency * PROMPTS_PER_CONCURRENCY ))
          if (( num_prompts < MIN_NUM_PROMPTS )); then num_prompts=$MIN_NUM_PROMPTS; fi
          if (( num_prompts > MAX_NUM_PROMPTS )); then num_prompts=$MAX_NUM_PROMPTS; fi
          num_prompts_arg="--num-prompts $num_prompts"
        fi
        # pass the tensor parallel size, the compilation mode, and the optimization
        # level to the client so that they can be used on the benchmark dashboard
        client_command="vllm bench serve \
@@ -794,9 +444,8 @@ run_serving_tests() {
          --result-filename ${new_test_name}.json \
          --request-rate $qps \
          --max-concurrency $max_concurrency \
          $num_prompts_arg \
          --metadata tensor_parallel_size=$tp compilation_config.mode=$compilation_config_mode optimization_level=$optimization_level \
-          $client_args_effective $client_remote_args "
+          $client_args $client_remote_args "
        echo "Running test case $test_name with qps $qps"
        echo "Client command: $client_command"
@@ -818,11 +467,6 @@ run_serving_tests() {
        echo "$jq_output" >"$RESULTS_FOLDER/${new_test_name}.commands"
      done
      adaptive_refine_from_static_results \
        "$test_name" "$qps" "$max_concurrency_list" "$tp" \
        "$compilation_config_mode" "$optimization_level" \
        "$client_args_effective" "$client_remote_args" "$server_command"
    done
    # clean up
@@ -888,7 +532,6 @@ main() {
  # postprocess benchmarking results
  pip install tabulate pandas
  python3 $QUICK_BENCHMARK_ROOT/scripts/convert-results-json-to-markdown.py
  python3 $QUICK_BENCHMARK_ROOT/scripts/compare-json-results.py -f $RESULTS_FOLDER/benchmark_results.json
  upload_to_buildkite
 }
@@ -2,6 +2,7 @@
    {
        "test_name": "latency_llama8B_tp1",
        "environment_variables": {
            "VLLM_RPC_TIMEOUT": 100000,
            "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
            "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
            "VLLM_CPU_KVCACHE_SPACE": 40
@@ -2,6 +2,7 @@
    {
        "test_name": "latency_llama8B_tp2",
        "environment_variables": {
            "VLLM_RPC_TIMEOUT": 100000,
 	    "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
 	    "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
 	    "VLLM_CPU_SGL_KERNEL": 1,
@@ -13,6 +13,7 @@
      200
    ],
    "server_environment_variables": {
      "VLLM_RPC_TIMEOUT": 100000,
      "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
      "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
      "VLLM_CPU_SGL_KERNEL": 1,
@@ -35,7 +36,6 @@
      "model": "meta-llama/Llama-3.1-8B-Instruct",
      "backend": "vllm",
      "ignore-eos": "",
      "temperature": 0,
      "num_prompts": 200
    }
  },
@@ -127,4 +127,4 @@
      }
    }
  ]
-}
+}
@@ -1,37 +0,0 @@
 {
  "defaults": {
    "qps_list": [
      "inf"
    ],
    "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
    "server_environment_variables": {
      "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120
    },
    "server_parameters": {
      "dtype": "bfloat16",
      "model": "openai/whisper-large-v3-turbo"
    },
    "client_parameters": {
      "model": "openai/whisper-large-v3-turbo",
      "backend": "openai-audio",
      "endpoint": "/v1/audio/transcriptions",
      "dataset_name": "hf",
      "dataset_path": "openslr/librispeech_asr",
      "hf_subset": "clean",
      "hf_split": "test",
      "no_stream": "",
      "no_oversample": "",
      "temperature": 0,
      "num_prompts": 200
    }
  },
  "tests": [
    {
      "test_name": "serving_whisper_large_v3_turbo_librispeech_clean_tp1",
      "server_parameters": {
        "tensor_parallel_size": 1
      },
      "client_parameters": {}
    }
  ]
 }
@@ -9,6 +9,7 @@
      128
    ],
    "server_environment_variables": {
      "VLLM_RPC_TIMEOUT": 100000,
      "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
      "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
      "VLLM_CPU_SGL_KERNEL": 1,
@@ -5,6 +5,7 @@
    ],
    "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
    "server_environment_variables": {
      "VLLM_RPC_TIMEOUT": 100000,
      "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
      "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
      "VLLM_CPU_SGL_KERNEL": 1,
@@ -25,14 +26,34 @@
      "model": "meta-llama/Llama-3.1-8B-Instruct",
      "backend": "vllm",
      "ignore-eos": "",
      "temperature": 0,
      "num_prompts": 200
    }
  },
  "tests": [
    {
      "test_name": "serving_llama8B_tp1_sharegpt",
      "server_parameters": {
        "tensor_parallel_size": 1
      },
      "client_parameters": {
        "dataset_name": "sharegpt",
        "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json"
      }
    },
    {
      "test_name": "serving_llama8B_tp2_sharegpt",
      "server_parameters": {
        "tensor_parallel_size": 2
      },
      "client_parameters": {
        "dataset_name": "sharegpt",
        "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json"
      }
    },
    {
      "test_name": "serving_llama8B_tp1_random_128_128",
      "server_parameters": {
        "tensor_parallel_size": 1
      },
      "client_parameters": {
        "dataset_name": "random",
@@ -40,10 +61,99 @@
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_llama8B_tp2_random_128_128",
      "server_parameters": {
        "tensor_parallel_size": 2
      },
      "client_parameters": {
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_llama8B_tp4_random_128_128",
      "server_parameters": {
        "tensor_parallel_size": 4
      },
      "client_parameters": {
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_llama8B_tp1_random_128_2048",
      "server_parameters": {
        "tensor_parallel_size": 1
      },
      "client_parameters": {
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 2048
      }
    },
    {
      "test_name": "serving_llama8B_tp2_random_128_2048",
      "server_parameters": {
        "tensor_parallel_size": 2
      },
      "client_parameters": {
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 2048
      }
    },
    {
      "test_name": "serving_llama8B_tp4_random_128_2048",
      "server_parameters": {
        "tensor_parallel_size": 4
      },
      "client_parameters": {
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 2048
      }
    },
    {
      "test_name": "serving_llama8B_tp1_random_2048_128",
      "server_parameters": {
        "tensor_parallel_size": 1
      },
      "client_parameters": {
        "dataset_name": "random",
        "random-input-len": 2048,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_llama8B_tp2_random_2048_128",
      "server_parameters": {
        "tensor_parallel_size": 2
      },
      "client_parameters": {
        "dataset_name": "random",
        "random-input-len": 2048,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_llama8B_tp4_random_2048_128",
      "server_parameters": {
        "tensor_parallel_size": 4
      },
      "client_parameters": {
        "dataset_name": "random",
        "random-input-len": 2048,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_llama8B_int4_tp1_random_128_128",
      "server_parameters": {
-        "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4"
+        "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
        "tensor_parallel_size": 1
      },
      "client_parameters": {
        "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
@@ -53,24 +163,26 @@
      }
    },
    {
-      "test_name": "serving_llama8B_int8_tp1_random_128_128",
+      "test_name": "serving_llama8B_int4_tp2_random_128_128",
      "server_parameters": {
-        "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8"
+        "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
        "tensor_parallel_size": 2
      },
      "client_parameters": {
-        "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
+        "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    },
    {
-      "test_name": "serving_llama1B_tp1_random_128_128",
+      "test_name": "serving_llama8B_int4_tp4_random_128_128",
      "server_parameters": {
-        "model": "meta-llama/Llama-3.2-1B"
+        "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
        "tensor_parallel_size": 4
      },
      "client_parameters": {
-        "model": "meta-llama/Llama-3.2-1B",
+        "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
@@ -79,7 +191,8 @@
    {
      "test_name": "serving_llama3B_tp1_random_128_128",
      "server_parameters": {
-        "model": "meta-llama/Llama-3.2-3B-Instruct"
+        "model": "meta-llama/Llama-3.2-3B-Instruct",
        "tensor_parallel_size": 1
      },
      "client_parameters": {
        "model": "meta-llama/Llama-3.2-3B-Instruct",
@@ -88,22 +201,11 @@
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_llama70B_tp1_random_128_128",
      "server_parameters": {
        "model": "meta-llama/Llama-3.3-70B-Instruct"
      },
      "client_parameters": {
        "model": "meta-llama/Llama-3.3-70B-Instruct",
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_granite2B_tp1_random_128_128",
      "server_parameters": {
-        "model": "ibm-granite/granite-3.2-2b-instruct"
+        "model": "ibm-granite/granite-3.2-2b-instruct",
        "tensor_parallel_size": 1
      },
      "client_parameters": {
        "model": "ibm-granite/granite-3.2-2b-instruct",
@@ -115,7 +217,8 @@
    {
      "test_name": "serving_qwen1.7B_tp1_random_128_128",
      "server_parameters": {
-        "model": "Qwen/Qwen3-1.7B"
+        "model": "Qwen/Qwen3-1.7B",
        "tensor_parallel_size": 1
      },
      "client_parameters": {
        "model": "Qwen/Qwen3-1.7B",
@@ -127,7 +230,8 @@
    {
      "test_name": "serving_qwen4B_tp1_random_128_128",
      "server_parameters": {
-        "model": "Qwen/Qwen3-4B"
+        "model": "Qwen/Qwen3-4B",
        "tensor_parallel_size": 1
      },
      "client_parameters": {
        "model": "Qwen/Qwen3-4B",
@@ -139,7 +243,8 @@
    {
      "test_name": "serving_qwen8B_tp1_random_128_128",
      "server_parameters": {
-        "model": "Qwen/Qwen3-8B"
+        "model": "Qwen/Qwen3-8B",
        "tensor_parallel_size": 1
      },
      "client_parameters": {
        "model": "Qwen/Qwen3-8B",
@@ -148,34 +253,11 @@
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_qwen14B_tp1_random_128_128",
      "server_parameters": {
        "model": "Qwen/Qwen3-14B"
      },
      "client_parameters": {
        "model": "Qwen/Qwen3-14B",
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_qwen30B_tp1_random_128_128",
      "server_parameters": {
        "model": "Qwen/Qwen3-30B-A3B"
      },
      "client_parameters": {
        "model": "Qwen/Qwen3-30B-A3B",
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_glm9B_tp1_random_128_128",
      "server_parameters": {
-        "model": "zai-org/glm-4-9b-hf"
+        "model": "zai-org/glm-4-9b-hf",
        "tensor_parallel_size": 1
      },
      "client_parameters": {
        "model": "zai-org/glm-4-9b-hf",
@@ -187,7 +269,8 @@
    {
      "test_name": "serving_gemma7B_tp1_random_128_128",
      "server_parameters": {
-        "model": "google/gemma-7b"
+        "model": "google/gemma-7b",
        "tensor_parallel_size": 1
      },
      "client_parameters": {
        "model": "google/gemma-7b",
@@ -195,94 +278,6 @@
        "random-input-len": 128,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_gemma3-4b_tp1_random_128_128",
      "server_environment_variables": {
 	"VLLM_CPU_SGL_KERNEL": 0
      },
      "server_parameters": {
        "model": "google/gemma-3-4b-it"
      },
      "client_parameters": {
        "model": "google/gemma-3-4b-it",
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_gemma3-12b_tp1_random_128_128",
      "server_environment_variables": {
 	"VLLM_CPU_SGL_KERNEL": 0
      },
      "server_parameters": {
        "model": "google/gemma-3-12b-it"
      },
      "client_parameters": {
        "model": "google/gemma-3-12b-it",
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_gemma4-4b_tp1_random_128_128",
      "server_environment_variables": {
 	"VLLM_CPU_SGL_KERNEL": 0
      },
      "server_parameters": {
        "model": "google/gemma-4-E4B-it"
      },
      "client_parameters": {
        "model": "google/gemma-4-E4B-it",
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_gemma4-2b_tp1_random_128_128",
      "server_environment_variables": {
 	"VLLM_CPU_SGL_KERNEL": 0
      },
      "server_parameters": {
        "model": "google/gemma-4-E2B-it"
      },
      "client_parameters": {
        "model": "google/gemma-4-E2B-it",
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_gemma4-26b_tp1_random_128_128",
      "server_environment_variables": {
 	"VLLM_CPU_SGL_KERNEL": 0,
        "VLLM_CPU_ATTN_SPLIT_KV": 0
      },
      "server_parameters": {
        "model": "google/gemma-4-26B-A4B-it"
      },
      "client_parameters": {
        "model": "google/gemma-4-26B-A4B-it",
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_phi4_tp1_random_128_128",
      "server_parameters": {
        "model": "microsoft/Phi-4-reasoning"
      },
      "client_parameters": {
        "model": "microsoft/Phi-4-reasoning",
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    }
  ]
 }
@@ -5,6 +5,7 @@
    ],
    "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
    "server_environment_variables": {
      "VLLM_RPC_TIMEOUT": 100000,
      "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
      "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
      "VLLM_CPU_SGL_KERNEL": 1,
@@ -25,7 +26,6 @@
      "model": "meta-llama/Llama-3.1-8B-Instruct",
      "backend": "vllm",
      "ignore-eos": "",
      "temperature": 0,
      "num_prompts": 200
    }
  },
@@ -72,6 +72,17 @@
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_llama8B_tp4_random_128_128",
      "server_parameters": {
        "tensor_parallel_size": 4
      },
      "client_parameters": {
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_llama8B_tp1_random_128_2048",
      "server_parameters": {
@@ -94,6 +105,17 @@
        "random-output-len": 2048
      }
    },
    {
      "test_name": "serving_llama8B_tp4_random_128_2048",
      "server_parameters": {
        "tensor_parallel_size": 4
      },
      "client_parameters": {
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 2048
      }
    },
    {
      "test_name": "serving_llama8B_tp1_random_2048_128",
      "server_parameters": {
@@ -117,25 +139,14 @@
      }
    },
    {
-      "test_name": "serving_llama8B_tp1_random_2048_2048",
+      "test_name": "serving_llama8B_tp4_random_2048_128",
      "server_parameters": {
-        "tensor_parallel_size": 1
+        "tensor_parallel_size": 4
      },
      "client_parameters": {
        "dataset_name": "random",
        "random-input-len": 2048,
-        "random-output-len": 2048
+        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_llama8B_tp2_random_2048_2048",
      "server_parameters": {
        "tensor_parallel_size": 2
      },
      "client_parameters": {
        "dataset_name": "random",
        "random-input-len": 2048,
        "random-output-len": 2048
      }
    }
  ]
@@ -10,6 +10,7 @@
        "server_parameters": {
            "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
            "tensor_parallel_size": 1,
            "swap_space": 16,
            "disable_log_stats": "",
            "load_format": "dummy",
            "max-model-len": 2048,
@@ -21,7 +22,6 @@
            "backend": "vllm",
            "dataset_name": "sharegpt",
            "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
            "temperature": 0,
            "num_prompts": 200
        }
    },
@@ -37,6 +37,7 @@
        "server_parameters": {
            "model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
            "tensor_parallel_size": 4,
            "swap_space": 16,
            "disable_log_stats": "",
            "load_format": "dummy",
            "max-model-len": 2048,
@@ -48,7 +49,6 @@
            "backend": "vllm",
            "dataset_name": "sharegpt",
            "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
            "temperature": 0,
            "num_prompts": 200
        }
    },
@@ -64,6 +64,7 @@
        "server_parameters": {
            "model": "mistralai/Mixtral-8x7B-Instruct-v0.1",
            "tensor_parallel_size": 2,
            "swap_space": 16,
            "disable_log_stats": "",
            "load_format": "dummy",
            "max-model-len": 2048,
@@ -75,7 +76,6 @@
            "backend": "vllm",
            "dataset_name": "sharegpt",
            "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
            "temperature": 0,
            "num_prompts": 200
        }
    },
@@ -91,6 +91,7 @@
        "server_parameters": {
            "model": "deepseek-ai/DeepSeek-R1",
            "tensor_parallel_size": 8,
            "swap_space": 16,
            "disable_log_stats": "",
            "load_format": "dummy",
            "max-model-len": 2048,
@@ -103,7 +104,6 @@
            "backend": "vllm",
            "dataset_name": "sharegpt",
            "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
            "temperature": 0,
            "num_prompts": 200
        }
    },
@@ -131,7 +131,6 @@
            "backend": "vllm",
            "dataset_name": "sharegpt",
            "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
            "temperature": 0,
            "num_prompts": 200
        }
    },
@@ -156,7 +155,6 @@
            "backend": "vllm",
            "dataset_name": "sharegpt",
            "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
            "temperature": 0,
            "num_prompts": 200
        }
    }
@@ -1,112 +1,77 @@
-{
+[
-  "defaults": {
+    {
-    "qps_list": [
+        "test_name": "serving_llama8B_tp1_sharegpt",
-      "inf"
+        "qps_list": [1, 4, 16, "inf"],
-    ],
+        "server_parameters": {
-    "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
+            "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
-    "server_parameters": {
+            "tensor_parallel_size": 1,
-      "model": "meta-llama/Llama-3.1-8B-Instruct",
+            "swap_space": 16,
-      "tensor_parallel_size": 1,
+            "disable_log_stats": "",
-      "dtype": "bfloat16"
+            "load_format": "dummy"
        },
        "client_parameters": {
            "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
            "backend": "vllm",
            "dataset_name": "sharegpt",
            "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
            "num_prompts": 200
        }
    },
-    "client_parameters": {
+    {
-      "model": "meta-llama/Llama-3.1-8B-Instruct",
+        "test_name": "serving_llama70B_tp4_sharegpt",
-      "backend": "vllm",
+        "qps_list": [1, 4, 16, "inf"],
-      "ignore-eos": "",
+        "server_parameters": {
-      "temperature": 0,
+            "model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
-      "num_prompts": 200
+            "tensor_parallel_size": 4,
            "swap_space": 16,
            "disable_log_stats": "",
            "load_format": "dummy"
        },
        "client_parameters": {
            "model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
            "backend": "vllm",
            "dataset_name": "sharegpt",
            "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
            "num_prompts": 200
        }
    },
    {
        "test_name": "serving_mixtral8x7B_tp2_sharegpt",
        "qps_list": [1, 4, 16, "inf"],
        "server_parameters": {
            "model": "mistralai/Mixtral-8x7B-Instruct-v0.1",
            "tensor_parallel_size": 2,
            "swap_space": 16,
            "disable_log_stats": "",
            "load_format": "dummy"
        },
        "client_parameters": {
            "model": "mistralai/Mixtral-8x7B-Instruct-v0.1",
            "backend": "vllm",
            "dataset_name": "sharegpt",
            "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
            "num_prompts": 200
        }
    },
    {
        "test_name": "serving_llama70B_tp4_sharegpt_specdecode",
        "qps_list": [2],
        "server_parameters": {
            "model": "meta-llama/Meta-Llama-3.1-70B-Instruct", 
            "tensor_parallel_size": 4,
            "swap_space": 16,
            "speculative_config": {
                "model": "turboderp/Qwama-0.5B-Instruct",
                "num_speculative_tokens": 4,
                "draft_tensor_parallel_size": 1
            }
        },
        "client_parameters": {
            "model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
            "backend": "vllm",
            "dataset_name": "sharegpt",
            "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
            "num_prompts": 200 
        }
    }
-  },
+]
  "tests": [
    {
      "test_name": "serving_llama8B_tp1_sharegpt",
      "server_parameters": {
        "tensor_parallel_size": 1
      },
      "client_parameters": {
        "dataset_name": "sharegpt",
        "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json"
      }
    },
    {
        "dataset_name": "sharegpt",
        "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json"
      }
    },
    {
      "test_name": "serving_llama8B_tp1_random_128_128",
      "server_parameters": {
        "tensor_parallel_size": 1
      },
      "client_parameters": {
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_llama8B_tp1_random_128_2048",
      "server_parameters": {
        "tensor_parallel_size": 1
      },
      "client_parameters": {
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 2048
      }
    },
    {
      "test_name": "serving_llama8B_tp1_random_2048_128",
      "server_parameters": {
        "tensor_parallel_size": 1
      },
      "client_parameters": {
        "dataset_name": "random",
        "random-input-len": 2048,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_llama8B_tp1_random_2048_2048",
      "server_parameters": {
        "tensor_parallel_size": 1
      },
      "client_parameters": {
        "dataset_name": "random",
        "random-input-len": 2048,
        "random-output-len": 2048
      }
    },
    {
      "test_name": "serving_llama70B_tp4_random_128_128",
      "server_parameters": {
        "model": "meta-llama/Llama-3.3-70B-Instruct",
        "async_scheduling": "",
 	"no_enable_prefix_caching": "",
        "max_num_batched_tokens": 8192
      },
      "client_parameters": {
        "model": "meta-llama/Llama-3.3-70B-Instruct",
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_gemma4-e4b_tp1_random_128_128",
      "server_parameters": {
        "model": "google/gemma-4-E4B-it",
 	"enable_auto_tool_choice": "",
 	"tool_call_parser": "gemma4",
 	"chat_template": "examples/tool_chat_template_gemma4.jinja",
 	"reasoning_parser": "gemma4"
      },
      "client_parameters": {
        "model": "google/gemma-4-E4B-it",
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    }
  ]
 }
@@ -2,6 +2,7 @@
    {
        "test_name": "throughput_llama8B_tp1",
        "environment_variables": {
            "VLLM_RPC_TIMEOUT": 100000,
            "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
            "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
            "VLLM_CPU_KVCACHE_SPACE": 40
@@ -2,6 +2,7 @@
    {
        "test_name": "throughput_llama8B_tp2",
        "environment_variables": {
            "VLLM_RPC_TIMEOUT": 100000,
 	    "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
 	    "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
 	    "VLLM_CPU_SGL_KERNEL": 1,
@@ -1,10 +0,0 @@
 #!/bin/bash
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 #
 # Append a build artifact line to the Buildkite annotation.
 # Usage: annotate-build-artifact.sh <label> <value> <context>
 set -e
 echo "- **${1}**: \`${2}\`" | \
  buildkite-agent annotate --append --style 'info' \
    --context "${3:?context is required}"
@@ -1,36 +0,0 @@
 #!/bin/bash
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 #
 # Append the Docker image tag(s) an image-build step pushed to a Buildkite
 # annotation, so the built image tags show up on the build page instead of
 # being buried in the job logs.
 #
 # Usage: annotate-image-build.sh <image_tag> [<image_tag> ...]
 set -euo pipefail
 # buildkite-agent only exists on Buildkite agents; no-op elsewhere so the
 # image build scripts stay runnable locally.
 if ! command -v buildkite-agent >/dev/null 2>&1; then
    echo "buildkite-agent not found; skipping image tag annotation"
    exit 0
 fi
 label="${BUILDKITE_LABEL:-Image build}"
 content=""
 for image in "$@"; do
    [[ -n "$image" ]] || continue
    content+="- **${label}**: \`${image}\`"$'\n'
 done
 if [[ -z "$content" ]]; then
    echo "No image tags provided; nothing to annotate"
    exit 0
 fi
 # Best-effort: a flaky annotation must never fail an otherwise successful
 # (and expensive) image build.
 if ! printf '%s' "$content" | \
    buildkite-agent annotate --append --style 'info' --context 'docker-images'; then
    echo "warning: failed to annotate build with image tags"
 fi
@@ -0,0 +1,117 @@
 #!/bin/bash
 set -ex
 # Get release version, default to 1.0.0.dev for nightly/per-commit builds
 RELEASE_VERSION=$(buildkite-agent meta-data get release-version 2>/dev/null | sed 's/^v//')
 if [ -z "${RELEASE_VERSION}" ]; then
  RELEASE_VERSION="1.0.0.dev"
 fi
 buildkite-agent annotate --style 'info' --context 'release-workflow' << EOF
 To download the wheel (by commit):
 \`\`\`
 aws s3 cp s3://vllm-wheels/${BUILDKITE_COMMIT}/vllm-${RELEASE_VERSION}-cp38-abi3-manylinux_2_31_x86_64.whl .
 aws s3 cp s3://vllm-wheels/${BUILDKITE_COMMIT}/vllm-${RELEASE_VERSION}-cp38-abi3-manylinux_2_31_aarch64.whl .
 (Optional) For CUDA 13.0:
 aws s3 cp s3://vllm-wheels/${BUILDKITE_COMMIT}/vllm-${RELEASE_VERSION}+cu130-cp38-abi3-manylinux_2_35_x86_64.whl .
 aws s3 cp s3://vllm-wheels/${BUILDKITE_COMMIT}/vllm-${RELEASE_VERSION}+cu130-cp38-abi3-manylinux_2_35_aarch64.whl .
 (Optional) For CPU:
 aws s3 cp s3://vllm-wheels/${BUILDKITE_COMMIT}/vllm-${RELEASE_VERSION}+cpu-cp38-abi3-manylinux_2_35_x86_64.whl .
 aws s3 cp s3://vllm-wheels/${BUILDKITE_COMMIT}/vllm-${RELEASE_VERSION}+cpu-cp38-abi3-manylinux_2_35_aarch64.whl .
 \`\`\`
 To download and upload the image:
 \`\`\`
 # Download images:
 docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-x86_64
 docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-aarch64
 docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-x86_64-cu130
 docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-aarch64-cu130
 docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-rocm-base
 docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-rocm
 docker pull public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:v${RELEASE_VERSION}
 docker pull public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:v${RELEASE_VERSION}
 # Tag and push images:
 ## CUDA
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-x86_64 vllm/vllm-openai:x86_64
 docker tag vllm/vllm-openai:x86_64 vllm/vllm-openai:latest-x86_64
 docker tag vllm/vllm-openai:x86_64 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64
 docker push vllm/vllm-openai:latest-x86_64
 docker push vllm/vllm-openai:v${RELEASE_VERSION}-x86_64
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-x86_64-cu130 vllm/vllm-openai:x86_64-cu130
 docker tag vllm/vllm-openai:x86_64-cu130 vllm/vllm-openai:latest-x86_64-cu130
 docker tag vllm/vllm-openai:x86_64-cu130 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu130
 docker push vllm/vllm-openai:latest-x86_64-cu130
 docker push vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu130
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-aarch64 vllm/vllm-openai:aarch64
 docker tag vllm/vllm-openai:aarch64 vllm/vllm-openai:latest-aarch64
 docker tag vllm/vllm-openai:aarch64 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64
 docker push vllm/vllm-openai:latest-aarch64
 docker push vllm/vllm-openai:v${RELEASE_VERSION}-aarch64
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-aarch64-cu130 vllm/vllm-openai:aarch64-cu130
 docker tag vllm/vllm-openai:aarch64-cu130 vllm/vllm-openai:latest-aarch64-cu130
 docker tag vllm/vllm-openai:aarch64-cu130 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu130
 docker push vllm/vllm-openai:latest-aarch64-cu130
 docker push vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu130
 ## ROCm
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-rocm vllm/vllm-openai-rocm:${BUILDKITE_COMMIT}
 docker tag vllm/vllm-openai-rocm:${BUILDKITE_COMMIT} vllm/vllm-openai-rocm:latest
 docker tag vllm/vllm-openai-rocm:${BUILDKITE_COMMIT} vllm/vllm-openai-rocm:v${RELEASE_VERSION}
 docker push vllm/vllm-openai-rocm:latest
 docker push vllm/vllm-openai-rocm:v${RELEASE_VERSION}
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-rocm-base vllm/vllm-openai-rocm:${BUILDKITE_COMMIT}-base
 docker tag vllm/vllm-openai-rocm:${BUILDKITE_COMMIT}-base vllm/vllm-openai-rocm:latest-base
 docker tag vllm/vllm-openai-rocm:${BUILDKITE_COMMIT}-base vllm/vllm-openai-rocm:v${RELEASE_VERSION}-base
 docker push vllm/vllm-openai-rocm:latest-base
 docker push vllm/vllm-openai-rocm:v${RELEASE_VERSION}-base
 ## CPU
 docker tag public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:v${RELEASE_VERSION} vllm/vllm-openai-cpu:x86_64
 docker tag vllm/vllm-openai-cpu:x86_64 vllm/vllm-openai-cpu:latest-x86_64
 docker tag vllm/vllm-openai-cpu:x86_64 vllm/vllm-openai-cpu:v${RELEASE_VERSION}-x86_64
 docker push vllm/vllm-openai-cpu:latest-x86_64
 docker push vllm/vllm-openai-cpu:v${RELEASE_VERSION}-x86_64
 docker tag public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:v${RELEASE_VERSION} vllm/vllm-openai-cpu:arm64
 docker tag vllm/vllm-openai-cpu:arm64 vllm/vllm-openai-cpu:latest-arm64
 docker tag vllm/vllm-openai-cpu:arm64 vllm/vllm-openai-cpu:v${RELEASE_VERSION}-arm64
 docker push vllm/vllm-openai-cpu:latest-arm64
 docker push vllm/vllm-openai-cpu:v${RELEASE_VERSION}-arm64
 # Create multi-arch manifest:
 docker manifest rm vllm/vllm-openai:latest
 docker manifest create vllm/vllm-openai:latest vllm/vllm-openai:latest-x86_64 vllm/vllm-openai:latest-aarch64
 docker manifest create vllm/vllm-openai:v${RELEASE_VERSION} vllm/vllm-openai:v${RELEASE_VERSION}-x86_64 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64
 docker manifest push vllm/vllm-openai:latest
 docker manifest push vllm/vllm-openai:v${RELEASE_VERSION}
 docker manifest rm vllm/vllm-openai:latest-cu130
 docker manifest create vllm/vllm-openai:latest-cu130 vllm/vllm-openai:latest-x86_64-cu130 vllm/vllm-openai:latest-aarch64-cu130
 docker manifest create vllm/vllm-openai:v${RELEASE_VERSION}-cu130 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu130 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu130
 docker manifest push vllm/vllm-openai:latest-cu130
 docker manifest push vllm/vllm-openai:v${RELEASE_VERSION}-cu130
 docker manifest rm vllm/vllm-openai-cpu:latest || true
 docker manifest create vllm/vllm-openai-cpu:latest vllm/vllm-openai-cpu:latest-x86_64 vllm/vllm-openai-cpu:latest-arm64
 docker manifest create vllm/vllm-openai-cpu:v${RELEASE_VERSION} vllm/vllm-openai-cpu:v${RELEASE_VERSION}-x86_64 vllm/vllm-openai-cpu:v${RELEASE_VERSION}-arm64
 docker manifest push vllm/vllm-openai-cpu:latest
 docker manifest push vllm/vllm-openai-cpu:v${RELEASE_VERSION}
 \`\`\`
 EOF
@@ -5,21 +5,20 @@
 # Generate Buildkite annotation for ROCm wheel release
 set -ex
-# Extract build configuration from Dockerfile.rocm_base (single source of truth)
+# Get build configuration from meta-data
 # Extract ROCm version dynamically from Dockerfile.rocm_base
 # BASE_IMAGE format: rocm/dev-ubuntu-22.04:7.0-complete -> extracts "7.0"
 ROCM_VERSION=$(grep -E '^ARG BASE_IMAGE=' docker/Dockerfile.rocm_base | sed -E 's/.*:([0-9]+\.[0-9]+).*/\1/' || echo "unknown")
-PYTHON_VERSION=$(grep '^ARG PYTHON_VERSION=' docker/Dockerfile.rocm_base | sed 's/^ARG PYTHON_VERSION=//')
+PYTHON_VERSION=$(buildkite-agent meta-data get rocm-python-version 2>/dev/null || echo "3.12")
-PYTORCH_ROCM_ARCH=$(grep '^ARG PYTORCH_ROCM_ARCH=' docker/Dockerfile.rocm_base | sed 's/^ARG PYTORCH_ROCM_ARCH=//')
+PYTORCH_ROCM_ARCH=$(buildkite-agent meta-data get rocm-pytorch-rocm-arch 2>/dev/null || echo "gfx90a;gfx942;gfx950;gfx1100;gfx1101;gfx1200;gfx1201;gfx1150;gfx1151")
 # TODO: Enable the nightly build for ROCm
 # Get release version, default to 1.0.0.dev for nightly/per-commit builds
 RELEASE_VERSION=$(buildkite-agent meta-data get release-version 2>/dev/null || echo "")
 if [ -z "${RELEASE_VERSION}" ]; then
  RELEASE_VERSION="1.0.0.dev"
 fi
 ROCM_BASE_CACHE_KEY=$(.buildkite/scripts/cache-rocm-base-wheels.sh key)
 # S3 URLs
 S3_BUCKET="${S3_BUCKET:-vllm-wheels}"
 S3_REGION="${AWS_DEFAULT_REGION:-us-west-2}"
@@ -97,7 +96,7 @@ To download and upload the image:
 docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-rocm-base
 docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-rocm
-docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${ROCM_BASE_CACHE_KEY}-rocm-base vllm/vllm-openai-rocm:${BUILDKITE_COMMIT}-base
+docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-rocm-base vllm/vllm-openai-rocm:${BUILDKITE_COMMIT}-base
 docker tag vllm/vllm-openai-rocm:${BUILDKITE_COMMIT}-base vllm/vllm-openai-rocm:latest-base
 docker tag vllm/vllm-openai-rocm:${BUILDKITE_COMMIT}-base vllm/vllm-openai-rocm:v${RELEASE_VERSION}-base
 docker push vllm/vllm-openai-rocm:latest-base
@@ -1,32 +0,0 @@
 #!/usr/bin/env bash
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 #
 # Build the macOS arm64 CPU wheel natively on a macOS agent (the `macmini`
 # queue) into artifacts/dist/ for upload-nightly-wheels.sh.
 set -euo pipefail
 # The Rust frontend build needs protoc.
 if ! command -v protoc >/dev/null 2>&1; then
  brew install protobuf
 fi
 # upload-nightly-wheels.sh expects exactly one wheel.
 rm -rf artifacts/dist
 mkdir -p artifacts/dist
 export VLLM_TARGET_DEVICE=cpu
 export VLLM_REQUIRE_RUST_FRONTEND=1
 export MACOSX_DEPLOYMENT_TARGET=11.0
 # uv's CPython is universal2; force an arm64-only build and tag so the wheel
 # isn't mislabelled universal2 and installed on Intel Macs where import fails.
 export ARCHFLAGS="-arch arm64"
 export _PYTHON_HOST_PLATFORM="macosx-11.0-arm64"
 export CMAKE_BUILD_PARALLEL_LEVEL="${CMAKE_BUILD_PARALLEL_LEVEL:-4}"
 uv venv --python 3.12
 uv pip install -r requirements/build/cpu.txt --index-strategy unsafe-best-match
 uv build --wheel --no-build-isolation -o artifacts/dist
 ls -l artifacts/dist/*.whl
@@ -15,6 +15,8 @@
 #
 # Environment variables:
 #   S3_BUCKET          - S3 bucket name (default: vllm-wheels)
 #   PYTHON_VERSION     - Python version (affects cache key)
 #   PYTORCH_ROCM_ARCH  - GPU architectures (affects cache key)
 #
 # Note: ROCm version is determined by BASE_IMAGE in Dockerfile.rocm_base,
 #       so changes to ROCm version are captured by the Dockerfile hash.
@@ -34,7 +36,13 @@ generate_cache_key() {
    fi
    local dockerfile_hash=$(sha256sum "$DOCKERFILE" | cut -c1-16)
-    echo "${dockerfile_hash}"
+    # Include key build args that affect the output
    # These should match the ARGs in Dockerfile.rocm_base that change the build output
    # Note: ROCm version is determined by BASE_IMAGE in the Dockerfile, so it's captured by dockerfile_hash
    local args_string="${PYTHON_VERSION:-}|${PYTORCH_ROCM_ARCH:-}"
    local args_hash=$(echo "$args_string" | sha256sum | cut -c1-8)
    echo "${dockerfile_hash}-${args_hash}"
 }
 CACHE_KEY=$(generate_cache_key)
@@ -44,6 +52,9 @@ case "${1:-}" in
    check)
        echo "Checking cache for key: ${CACHE_KEY}" >&2
        echo "Cache path: ${CACHE_PATH}" >&2
        echo "Variables used in cache key:" >&2
        echo "  PYTHON_VERSION: ${PYTHON_VERSION:-<not set>}" >&2
        echo "  PYTORCH_ROCM_ARCH: ${PYTORCH_ROCM_ARCH:-<not set>}" >&2
        # Check if cache exists by listing objects
        # We look for at least one .whl file
@@ -93,16 +104,14 @@ case "${1:-}" in
        echo "Cache key: ${CACHE_KEY}"
        echo "Cache path: ${CACHE_PATH}"
        echo ""
        mkdir -p artifacts/rocm-base-wheels
-        
+        aws s3 cp --recursive "${CACHE_PATH}" artifacts/rocm-base-wheels/
-        # Use sync with include/exclude to only download .whl files
+
        aws s3 sync "${CACHE_PATH}" artifacts/rocm-base-wheels/ \
            --exclude "*" \
            --include "*.whl"
        echo ""
        echo "Downloaded wheels:"
        find artifacts/rocm-base-wheels -maxdepth 1 -name '*.whl' -exec ls -lh {} \;
        WHEEL_COUNT=$(find artifacts/rocm-base-wheels -maxdepth 1 -name '*.whl' 2>/dev/null | wc -l)
        echo ""
        echo "Total: $WHEEL_COUNT wheels"
@@ -16,27 +16,6 @@ RAY_BASE_URL="https://raw.githubusercontent.com/ray-project/ray/master/python"
 WORK_DIR=$(mktemp -d)
 trap 'rm -rf "$WORK_DIR"' EXIT
 # ── Detect PyTorch index URL ─────────────────────────────────────────────
 if python3 -c "import torch; assert torch.version.hip" 2>/dev/null; then
    ROCM_VER=$(python3 -c "import torch; print(torch.version.hip.rsplit('.', 1)[0])")
    CANDIDATE_URL="https://download.pytorch.org/whl/rocm${ROCM_VER}"
    if curl -fsSL --head "${CANDIDATE_URL}/" >/dev/null 2>&1; then
        TORCH_INDEX_URL="${CANDIDATE_URL}"
    else
        echo ">>> WARNING: ROCm ${ROCM_VER} wheel index not found at ${CANDIDATE_URL}"
        echo ">>>          Falling back to default PyPI (resolution may be incomplete)"
        TORCH_INDEX_URL=""
    fi
 else
    if [ "${TORCH_NIGHTLY:-0}" = "1" ]; then
        TORCH_INDEX_URL="https://download.pytorch.org/whl/nightly/cu130"
    else
        TORCH_INDEX_URL="https://download.pytorch.org/whl/cu130"
    fi
 fi
 echo ">>> Using PyTorch index: ${TORCH_INDEX_URL:-PyPI default}"
 # Fetch all Ray requirement files used in the LLM depset pipeline
 echo ">>> Fetching Ray requirement files"
 RAY_FILES=(
@@ -137,11 +116,6 @@ echo "============================================================"
 echo ">>> Resolving: Can Ray generate compatible lock files?"
 echo "============================================================"
 EXTRA_INDEX_ARGS=()
 if [[ -n "${TORCH_INDEX_URL}" ]]; then
    EXTRA_INDEX_ARGS+=(--extra-index-url "${TORCH_INDEX_URL}")
 fi
 set +e
 uv pip compile \
    "${WORK_DIR}/requirements.txt" \
@@ -152,7 +126,7 @@ uv pip compile \
    -c "${WORK_DIR}/vllm-constraints.txt" \
    --python-version 3.12 \
    --python-platform x86_64-manylinux_2_31 \
-    "${EXTRA_INDEX_ARGS[@]}" \
+    --extra-index-url https://download.pytorch.org/whl/cu129 \
    --index-strategy unsafe-best-match \
    --unsafe-package setuptools \
    --unsafe-package ray \
@@ -13,8 +13,5 @@ INPUT_FILE="$1"
 # Strip timestamps
 sed -i 's/^\[[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}T[0-9]\{2\}:[0-9]\{2\}:[0-9]\{2\}Z\] //' "$INPUT_FILE"
 # Strip Buildkite inline timestamp markers (ESC _bk;t=<ms> BEL)
 sed -i 's/\x1B_bk;t=[0-9]*\x07//g' "$INPUT_FILE"
 # Strip colorization
 sed -i -r 's/\x1B\[[0-9;]*[mK]//g' "$INPUT_FILE"
@@ -1,178 +0,0 @@
 #!/bin/bash
 # Fetch vLLM Buildkite CI logs (public; no login required).
 #
 # Usage:
 #   ci-fetch-log.sh [--soft|--all] --pr [<PR>]  failed jobs in the PR's latest
 #                                               build (current branch if omitted)
 #   ci-fetch-log.sh [--soft|--all] <build_url>  failed jobs in that build
 #   ci-fetch-log.sh <job_url> [output]          one job; both #<job_uuid> and
 #                                               ?sid=<id> URL forms work
 #   ci-fetch-log.sh <build> <job_uuid> [output]
 #
 # --soft also fetches soft-failed jobs; --all fetches every finished job.
 # Saves each log as ci-<build>-<job-name>.log (ANSI/timestamps stripped) and
 # prints "<file>\t<job name>" per job. [output] is single-job only; "-"
 # streams to stdout. Existing files are kept; CI_FETCH_LOG_FORCE=1 refetches.
 set -euo pipefail
 ORG="vllm"
 PIPELINE="ci"
 UA="vllm-ci-fetch-log"
 UUID_RE='[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'
 usage() {
    sed -n '2,15p' "$0" | sed 's/^# \{0,1\}//'
    exit 1
 }
 die() {
    echo "$1" >&2
    exit 1
 }
 BUILD="" JOB="" SID="" OUT=""
 SCOPE="failed"
 while :; do
    case "${1:-}" in
    --soft) SCOPE="soft" ;;
    --all) SCOPE="all" ;;
    *) break ;;
    esac
    shift
 done
 case "${1:-}" in
 --pr)
    PR="${2:-}"
    # gh pr checks exits non-zero when checks are failing; that is the
    # expected case here.
    URL=$(gh pr checks ${PR:+"$PR"} --repo vllm-project/vllm 2>/dev/null |
        grep -oE "https://buildkite.com/${ORG}/${PIPELINE}/builds/[0-9]+" |
        sort -t/ -k7 -n | tail -1 || true)
    [ -n "$URL" ] || die "No Buildkite build found via: gh pr checks ${PR:-<current branch>}"
    BUILD="${URL##*/}"
    ;;
 https://*)
    BUILD=$(echo "$1" | sed -nE 's#.*/builds/([0-9]+).*#\1#p')
    JOB=$(echo "$1" | grep -oE "#${UUID_RE}" | head -n 1 | cut -c2- || true)
    SID=$(echo "$1" | grep -oE "[?&]sid=${UUID_RE}" | head -n 1 | sed 's/.*sid=//' || true)
    OUT="${2:-}"
    [ -n "$BUILD" ] || die "Could not parse build number from: $1"
    ;;
 [0-9]*)
    [ $# -ge 2 ] || usage
    BUILD="$1"
    JOB="$2"
    OUT="${3:-}"
    ;;
 *)
    usage
    ;;
 esac
 COOKIES=$(mktemp)
 JOBS_TSV=$(mktemp)
 trap 'rm -f "$COOKIES" "$JOBS_TSV"' EXIT
 # Buildkite issues a session cookie on first hit; later requests need it.
 curl -fsSL -c "$COOKIES" -A "$UA" \
    "https://buildkite.com/${ORG}/${PIPELINE}/builds/${BUILD}" -o /dev/null
 # The build's job list (id, step uuid, state, name) is served as JSON from
 # the user-facing /data/jobs endpoint. Flatten it to TSV for easy filtering:
 #   job_id  step_uuid  failed  soft_failed  finished  slug  name
 curl -fsSL -b "$COOKIES" -A "$UA" \
    "https://buildkite.com/${ORG}/${PIPELINE}/builds/${BUILD}/data/jobs" |
    python3 -c '
 import json, re, sys
 data = json.load(sys.stdin)
 if data.get("has_next_page"):
    print("warning: job list is paginated; some jobs not shown", file=sys.stderr)
 for r in data["records"]:
    if r.get("type") != "script":
        continue
    name = (r.get("name") or "").replace("\t", " ").replace("\n", " ")
    slug = re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-")[:60]
    print("\t".join([
        r["id"],
        r.get("step_uuid") or "",
        str(r.get("passed") is False),
        str(bool(r.get("soft_failed"))),
        str(bool(r.get("finished_at"))),
        slug,
        name,
    ]))
 ' >"$JOBS_TSV" || die "Could not list jobs for build ${BUILD}"
 if [ -n "$SID" ] && [ -z "$JOB" ]; then
    # The ?sid= in builds/<N>/list URLs is the *step* uuid, not the job uuid.
    JOB=$(awk -F'\t' -v s="$SID" '$1 == s || $2 == s {print $1; exit}' "$JOBS_TSV")
    [ -n "$JOB" ] || die "No job matching sid=${SID} in build ${BUILD}"
 fi
 fetch_job() { # <job_uuid> <output_file>
    curl -fsSL -b "$COOKIES" -A "$UA" \
        "https://buildkite.com/organizations/${ORG}/pipelines/${PIPELINE}/builds/${BUILD}/jobs/$1/download" \
        -o "$2"
    bash "$(dirname "$0")/ci-clean-log.sh" "$2"
 }
 if [ -n "$JOB" ]; then
    # Single-job mode.
    NAME=$(awk -F'\t' -v j="$JOB" '$1 == j {print $7; exit}' "$JOBS_TSV")
    SLUG=$(awk -F'\t' -v j="$JOB" '$1 == j {print $6; exit}' "$JOBS_TSV")
    [ -n "$OUT" ] || OUT="ci-${BUILD}-${SLUG:-${JOB:0:13}}.log"
    if [ "$OUT" = "-" ]; then
        TMP=$(mktemp)
        fetch_job "$JOB" "$TMP"
        cat "$TMP"
        rm -f "$TMP"
        exit 0
    fi
    if [ -e "$OUT" ] && [ -z "${CI_FETCH_LOG_FORCE:-}" ]; then
        die "Refusing to overwrite existing ${OUT} (set CI_FETCH_LOG_FORCE=1 or pass an output path)."
    fi
    fetch_job "$JOB" "$OUT"
    printf '%s\t%s\n' "$OUT" "${NAME:-$JOB}"
    exit 0
 fi
 # Build-wide mode: fetch finished jobs matching $SCOPE.
 [ -z "$OUT" ] || die "[output_file] is only valid when fetching a single job."
 case "$SCOPE" in
 failed) FILTER='$3 == "True" && $4 == "False" && $5 == "True"' ;;
 soft) FILTER='$3 == "True" && $5 == "True"' ;;
 all) FILTER='$5 == "True"' ;;
 esac
 if [ "$SCOPE" = "failed" ]; then
    SOFT=$(awk -F'\t' '$3 == "True" && $4 == "True"' "$JOBS_TSV" | wc -l)
    [ "$SOFT" -eq 0 ] || echo "Skipping ${SOFT} soft-failed job(s); use --soft to include them." >&2
 fi
 FOUND=0
 EMITTED=" "
 while IFS=$'\t' read -r job_id _ _ _ _ slug name; do
    FOUND=$((FOUND + 1))
    out="ci-${BUILD}-${slug:-${job_id:0:13}}.log"
    # Retries share a name with the original job; disambiguate by uuid.
    case "$EMITTED" in
    *" $out "*) out="ci-${BUILD}-${slug:-job}-${job_id:0:13}.log" ;;
    esac
    EMITTED="${EMITTED}${out} "
    if [ -e "$out" ] && [ -z "${CI_FETCH_LOG_FORCE:-}" ]; then
        echo "Keeping existing ${out} (set CI_FETCH_LOG_FORCE=1 to refetch)." >&2
    elif ! fetch_job "$job_id" "$out"; then
        echo "Failed to download log for job ${job_id} (${name})." >&2
        continue
    fi
    printf '%s\t%s\n' "$out" "$name"
 done < <(awk -F'\t' "$FILTER" "$JOBS_TSV")
 if [ "$FOUND" -eq 0 ]; then
    echo "No matching jobs in build ${BUILD} (scope: ${SCOPE})." >&2
 fi
@@ -4,19 +4,16 @@ set -ex
 # Clean up old nightly builds from DockerHub, keeping only the last 14 builds
 # This script uses DockerHub API to list and delete old tags with specified prefix
-# Usage: cleanup-nightly-builds.sh [TAG_PREFIX] [REPO]
+# Usage: cleanup-nightly-builds.sh [TAG_PREFIX]
-# Example: cleanup-nightly-builds.sh "nightly-"
+# Example: cleanup-nightly-builds.sh "nightly-" or cleanup-nightly-builds.sh "cu130-nightly-"
 # Example: cleanup-nightly-builds.sh "cu130-nightly-"
 # Example: cleanup-nightly-builds.sh "nightly-" "vllm/vllm-openai-rocm"
-# Get tag prefix and repo from arguments
+# Get tag prefix from argument, default to "nightly-" if not provided
 TAG_PREFIX="${1:-nightly-}"
 REPO="${2:-vllm/vllm-openai}"
-echo "Cleaning up tags with prefix: $TAG_PREFIX in repository: $REPO"
+echo "Cleaning up tags with prefix: $TAG_PREFIX"
-# DockerHub API endpoint for the repository
+# DockerHub API endpoint for vllm/vllm-openai repository
-REPO_API_URL="https://hub.docker.com/v2/repositories/${REPO}/tags"
+REPO_API_URL="https://hub.docker.com/v2/repositories/vllm/vllm-openai/tags"
 # Get DockerHub credentials from environment
 if [ -z "$DOCKERHUB_TOKEN" ]; then
@@ -73,7 +70,7 @@ delete_tag() {
    local tag_name="$1"
    echo "Deleting tag: $tag_name"
-    local delete_url="https://hub.docker.com/v2/repositories/${REPO}/tags/$tag_name"
+    local delete_url="https://hub.docker.com/v2/repositories/vllm/vllm-openai/tags/$tag_name"
    set +x
    local response=$(curl -s -X DELETE -H "Authorization: Bearer $BEARER_TOKEN" "$delete_url")
    set -x
@@ -1,142 +0,0 @@
 #!/usr/bin/env python3
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 """Detect the manylinux platform tag for a wheel and rename it in place.
 vLLM's build images produce wheels with the generic ``linux_<arch>`` platform
 tag, which installers like ``pip`` won't accept off PyPI/our index. We need to
 rewrite the platform tag to the appropriate ``manylinux_<major>_<minor>_<arch>``
 before uploading.
 Historically the tag was hard-coded per build (``manylinux_2_31`` for the
 Ubuntu 20.04-based image, ``manylinux_2_35`` for the Ubuntu 22.04-based
 images). That is brittle: bumping the base image silently produces wheels
 labelled with the wrong glibc requirement. This script asks ``auditwheel``
 to derive the tag from the symbol versions actually referenced by the
 binaries inside the wheel, so the label tracks reality.
 We can't simply call ``auditwheel repair`` -- it tries to graft external
 shared libraries into the wheel and fails on vLLM's CUDA/cuBLAS dependencies.
 Instead we use ``auditwheel.wheel_abi.analyze_wheel_abi`` directly, which is
 the same call that powers ``auditwheel show``, and read off
 ``winfo.sym_policy.name``.
 Usage:
    detect-manylinux-tag.py <wheel_path>
 The wheel is renamed in place; the new path is printed on stdout. All
 diagnostics go to stderr so callers can capture stdout safely.
 """
 from __future__ import annotations
 import argparse
 import sys
 from pathlib import Path
 from auditwheel.error import (
    AuditwheelError,
    NonPlatformWheelError,
    WheelToolsError,
 )
 from auditwheel.wheel_abi import analyze_wheel_abi
 from auditwheel.wheeltools import get_wheel_architecture, get_wheel_libc
 def detect_platform_tag(wheel_path: Path) -> str:
    """Return the most precise platform tag the wheel is consistent with.
    Mirrors ``auditwheel show`` but returns ``sym_policy`` rather than
    ``overall_policy``: we only care about the glibc symbol versions used,
    not about other policy axes (ISA extensions, blacklist, etc.) that
    ``overall_policy`` folds in.
    """
    fn = wheel_path.name
    try:
        arch = get_wheel_architecture(fn)
    except (WheelToolsError, NonPlatformWheelError):
        # Architecture isn't deducible from the filename; let auditwheel
        # infer it from the ELF binaries inside the wheel.
        arch = None
    try:
        libc = get_wheel_libc(fn)
    except WheelToolsError:
        # An unrepaired wheel uses ``linux_<arch>``, which doesn't encode
        # libc. Let auditwheel infer it from the ELF binaries.
        libc = None
    winfo = analyze_wheel_abi(
        libc,
        arch,
        wheel_path,
        frozenset(),
        disable_isa_ext_check=False,
        allow_graft=False,
    )
    return winfo.sym_policy.name
 def rename_wheel(wheel_path: Path, new_platform_tag: str) -> Path:
    """Rename the wheel in place, replacing only its platform tag."""
    # Wheel filename per PEP 427:
    #   {distribution}-{version}(-{build})?-{python}-{abi}-{platform}.whl
    # The platform tag is always the last ``-``-separated token before
    # ``.whl``. Compound tags like ``manylinux_2_31_x86_64`` use ``_`` as the
    # internal separator, so ``-``-splitting is unambiguous.
    parts = wheel_path.stem.split("-")
    if len(parts) < 5:
        raise ValueError(f"Unrecognised wheel filename: {wheel_path.name}")
    parts[-1] = new_platform_tag
    new_path = wheel_path.with_name("-".join(parts) + ".whl")
    if new_path != wheel_path:
        wheel_path.rename(new_path)
    return new_path
 def main() -> int:
    parser = argparse.ArgumentParser(
        description="Detect a wheel's manylinux platform tag with "
        "auditwheel and rename the wheel in place."
    )
    parser.add_argument(
        "wheel",
        type=Path,
        help="Path to the wheel to inspect and rename.",
    )
    args = parser.parse_args()
    wheel_path: Path = args.wheel
    if not wheel_path.is_file():
        print(f"error: {wheel_path} is not a file", file=sys.stderr)
        return 1
    # Catch the things that ``analyze_wheel_abi`` and ``rename_wheel`` can
    # raise: any subclass of ``AuditwheelError`` (pure-Python wheels,
    # invalid libc, malformed wheels), filesystem errors, or our own
    # ``ValueError`` for an unrecognised wheel filename. Print a single
    # ``ERROR_TYPE: message`` line to stderr instead of a Python
    # traceback, which is much friendlier in CI logs.
    try:
        new_tag = detect_platform_tag(wheel_path)
        print(f"detected platform tag: {new_tag}", file=sys.stderr)
        new_path = rename_wheel(wheel_path, new_tag)
    except (AuditwheelError, ValueError, OSError) as e:
        print(
            f"error: failed to retag {wheel_path.name}: {type(e).__name__}: {e}",
            file=sys.stderr,
        )
        return 2
    if new_path != wheel_path:
        print(f"renamed {wheel_path.name} -> {new_path.name}", file=sys.stderr)
    else:
        print(f"wheel already tagged {new_tag}", file=sys.stderr)
    print(new_path)
    return 0
 if __name__ == "__main__":
    sys.exit(main())
@@ -1,54 +0,0 @@
 #!/bin/bash
 # Emit docker build flags for release image provenance metadata.
 # Keep this helper best-effort: missing Buildkite metadata should fall back to
 # local/default values instead of blocking the Docker build.
 # Variant examples: "", "cu129", "ubuntu2404", "cu129-ubuntu2404".
 variant="${1:-}"
 variant_suffix="${variant:+-${variant}}"
 image_name="${VLLM_DOCKER_IMAGE_NAME:-vllm/vllm-openai}"
 staging_repo="${VLLM_STAGING_IMAGE_REPO:-public.ecr.aws/q9t5s3a7/vllm-release-repo}"
 build_commit="${VLLM_BUILD_COMMIT:-${BUILDKITE_COMMIT:-unknown}}"
 build_pipeline="${VLLM_BUILD_PIPELINE:-${BUILDKITE_PIPELINE_ID:-${BUILDKITE_PIPELINE_SLUG:-local}}}"
 build_url="${VLLM_BUILD_URL:-${BUILDKITE_BUILD_URL:-}}"
 tag_commit="${BUILDKITE_COMMIT:-${build_commit}}"
 if [[ -n "${BUILDKITE:-}" || -n "${BUILDKITE_COMMIT:-}" ]]; then
  release_version="${RELEASE_VERSION:-}"
  if command -v buildkite-agent >/dev/null 2>&1; then
    release_version="${release_version:-$(buildkite-agent meta-data get release-version 2>/dev/null)}"
  fi
  release_version="${release_version#v}"
  release_version="${release_version:-${tag_commit}}"
  staging_image_ref="${staging_repo}:${tag_commit}-$(uname -m)${variant_suffix}"
  if [[ "${NIGHTLY:-}" == "1" ]]; then
    if [[ -z "${variant}" ]]; then
      image_tag="${image_name}:nightly-${tag_commit}"
    elif [[ "${variant}" == cu* ]]; then
      cuda_variant="${variant%%-*}"
      remaining_variant="${variant#${cuda_variant}}"
      image_tag="${image_name}:${cuda_variant}-nightly-${tag_commit}${remaining_variant}"
    else
      image_tag="${image_name}:nightly-${tag_commit}${variant_suffix}"
    fi
  else
    image_tag="${image_name}:v${release_version}${variant_suffix}"
  fi
 else
  image_tag="${VLLM_IMAGE_TAG:-local/vllm-openai:dev}"
  staging_image_ref="${image_tag}"
 fi
 emit_arg() {
  printf -- "--build-arg %s=%s " "$1" "$2"
 }
 emit_arg VLLM_BUILD_COMMIT "${build_commit}"
 emit_arg VLLM_BUILD_PIPELINE "${build_pipeline}"
 emit_arg VLLM_BUILD_URL "${build_url}"
 # This is the intended public tag. The final digest is only known after push.
 emit_arg VLLM_IMAGE_TAG "${image_tag}"
 printf -- "--tag %s " "${staging_image_ref}"
@@ -1,79 +0,0 @@
 #!/usr/bin/env bash
 set -ex
 # Generate and upload wheel indices for all wheels in the commit directory.
 # This script should run once after all wheels have been built and uploaded.
 # ======== setup ========
 BUCKET="vllm-wheels"
 INDICES_OUTPUT_DIR="indices"
 DEFAULT_VARIANT_ALIAS="cu130" # align with vLLM_MAIN_CUDA_VERSION in vllm/envs.py
 SUBPATH=$BUILDKITE_COMMIT
 S3_COMMIT_PREFIX="s3://$BUCKET/$SUBPATH/"
 # Select python3 (>= 3.12) -- local if available, else a docker fallback.
 # shellcheck source=lib/select-python.sh
 source .buildkite/scripts/lib/select-python.sh
 select_python
 # ======== generate and upload indices ========
 # list all wheels in the commit directory
 echo "Existing wheels on S3:"
 aws s3 ls "$S3_COMMIT_PREFIX"
 obj_json="objects.json"
 aws s3api list-objects-v2 --bucket "$BUCKET" --prefix "$SUBPATH/" --delimiter / --output json > "$obj_json"
 mkdir -p "$INDICES_OUTPUT_DIR"
 # call script to generate indices for all existing wheels
 # these indices have relative paths that work as long as they are next to the wheel directory in s3
 # i.e., the wheels are always in s3://vllm-wheels/<commit>/
 # and indices can be placed in /<commit>/, or /nightly/, or /<version>/
 alias_args=()
 if [[ -n "$DEFAULT_VARIANT_ALIAS" ]]; then
    alias_args=(--alias-to-default "$DEFAULT_VARIANT_ALIAS")
 fi
 # HACK: we do not need regex module here, but it is required by pre-commit hook
 # To avoid any external dependency, we simply replace it back to the stdlib re module
 sed -i 's/import regex as re/import re/g' .buildkite/scripts/generate-nightly-index.py
 $PYTHON .buildkite/scripts/generate-nightly-index.py --version "$SUBPATH" --current-objects "$obj_json" --output-dir "$INDICES_OUTPUT_DIR" --comment "commit $BUILDKITE_COMMIT" "${alias_args[@]}"
 # copy indices to /<commit>/ unconditionally
 echo "Uploading indices to $S3_COMMIT_PREFIX"
 aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "$S3_COMMIT_PREFIX"
 # copy to /nightly/ only when enabled for a main branch build that is not a PR
 if [[ "${UPDATE_NIGHTLY_INDEX:-1}" == "1" && \
      "$BUILDKITE_BRANCH" == "main" && \
      "$BUILDKITE_PULL_REQUEST" == "false" ]]; then
    echo "Uploading indices to overwrite /nightly/"
    aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "s3://$BUCKET/nightly/"
 fi
 # detect version from any wheel in the commit directory
 # download the first wheel we find to extract version metadata
 first_wheel_key=$($PYTHON -c "import json; obj=json.load(open('$obj_json')); print(next((c['Key'] for c in obj.get('Contents', []) if c['Key'].endswith('.whl')), ''))")
 if [[ -z "$first_wheel_key" ]]; then
    echo "Error: No wheels found in $S3_COMMIT_PREFIX"
    exit 1
 fi
 first_wheel=$(basename "$first_wheel_key")
 aws s3 cp "s3://$BUCKET/${first_wheel_key}" "/tmp/${first_wheel}"
 version=$(unzip -p "/tmp/${first_wheel}" '**/METADATA' | grep '^Version: ' | cut -d' ' -f2)
 rm -f "/tmp/${first_wheel}"
 echo "Version in wheel: $version"
 pure_version="${version%%+*}"
 echo "Pure version (without variant): $pure_version"
 # re-generate and copy to /<pure_version>/ only if it does not have "dev" in the version
 if [[ "${UPDATE_VERSION_INDEX:-1}" == "1" && "$version" != *"dev"* ]]; then
    echo "Re-generating indices for /$pure_version/"
    rm -rf "${INDICES_OUTPUT_DIR:?}"
    mkdir -p "$INDICES_OUTPUT_DIR"
    # wheel-dir is overridden to be the commit directory, so that the indices point to the correct wheel path
    $PYTHON .buildkite/scripts/generate-nightly-index.py --version "$pure_version" --wheel-dir "$SUBPATH" --current-objects "$obj_json" --output-dir "$INDICES_OUTPUT_DIR" --comment "version $pure_version" "${alias_args[@]}"
    aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "s3://$BUCKET/$pure_version/"
 fi
@@ -1,7 +1,7 @@
 #!/bin/bash
-# This script runs ROCm tests either directly in a native CI pod or inside the
+# This script runs tests inside the corresponding ROCm docker container.
-# corresponding Docker container. Multi-node tests continue to use Docker.
+# It handles both single-node and multi-node test configurations.
 #
 # Multi-node detection: Instead of matching on fragile group names, we detect
 # multi-node jobs structurally by looking for the bracket command syntax
@@ -28,68 +28,49 @@
 ###############################################################################
 set -o pipefail
-: "${BUILDKIT_PROGRESS:=plain}"
+# Export Python path
-: "${TERM:=xterm-256color}"
+export PYTHONPATH=".."
 : "${FORCE_COLOR:=1}"
 : "${CLICOLOR_FORCE:=1}"
 : "${PY_COLORS:=1}"
 : "${ROCM_DOCKER_TTY:=1}"
 : "${PYTHONFAULTHANDLER:=1}"
 : "${PYTEST_TIMEOUT:=2100}"
 if [[ " ${PYTEST_ADDOPTS:-} " != *" --color"* ]]; then
  PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--color=yes"
 fi
 if [[ " ${PYTEST_ADDOPTS:-} " != *" --durations="* ]]; then
  PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--durations=25"
 fi
 if [[ " ${PYTEST_ADDOPTS:-} " != *" --durations-min="* ]]; then
  PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--durations-min=1.0"
 fi
 # Dump stacks after 15 minutes, then stop an individual test after 35 minutes.
 if [[ " ${PYTEST_ADDOPTS:-} " != *" faulthandler_timeout="* ]]; then
  PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }-o faulthandler_timeout=900"
 fi
 if [[ " ${PYTEST_ADDOPTS:-} " != *" --timeout-method="* &&
  " ${PYTEST_ADDOPTS:-} " != *" --timeout-method "* ]]; then
  PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--timeout-method=thread"
 fi
 export BUILDKIT_PROGRESS TERM FORCE_COLOR CLICOLOR_FORCE PY_COLORS PYTEST_ADDOPTS PYTEST_TIMEOUT ROCM_DOCKER_TTY
 export PYTHONFAULTHANDLER
 # Export Python path for commands that run directly on the host. Containerized
 # tests set this to /vllm-workspace below so spawned Python processes do not
 # depend on their current working directory.
 export PYTHONPATH="${PYTHONPATH:-..}"
 ###############################################################################
 # Helper Functions
 ###############################################################################
-report_docker_usage() {
+wait_for_clean_gpus() {
-  echo "--- Docker usage"
+  local timeout=${1:-300}
-  docker system df || true
+  local start=$SECONDS
  echo "--- Waiting for clean GPU state (timeout: ${timeout}s)"
  while true; do
    if grep -q clean /opt/amdgpu/etc/gpu_state; then
      echo "GPUs state is \"clean\""
      return
    fi
    if (( SECONDS - start >= timeout )); then
      echo "Error: GPUs did not reach clean state within ${timeout}s" >&2
      exit 1
    fi
    sleep 3
  done
 }
-clear_ci_orchestration_env() {
+cleanup_docker() {
-  unset -v \
+  # Get Docker's root directory
-    VLLM_TEST_GROUP_NAME \
+  docker_root=$(docker info -f '{{.DockerRootDir}}')
-    VLLM_CI_REQUIRE_PERSISTENT_HF_CACHE \
+  if [ -z "$docker_root" ]; then
-    VLLM_CI_ARTIFACT_STEP \
+    echo "Failed to determine Docker root directory."
-    VLLM_TEST_CACHE \
+    exit 1
-    VLLM_CI_EXECUTION_MODE \
+  fi
-    VLLM_CI_WORKSPACE \
+  echo "Docker root directory: $docker_root"
-    VLLM_CI_REQUIRE_WORKSPACE_MOUNT \
+
-    VLLM_TEST_COMMANDS \
+  disk_usage=$(df "$docker_root" | tail -1 | awk '{print $5}' | sed 's/%//')
-    VLLM_CI_BRANCH \
+  threshold=70
-    VLLM_CI_BASE_IMAGE \
+  if [ "$disk_usage" -gt "$threshold" ]; then
-    VLLM_CI_FALLBACK_IMAGE \
+    echo "Disk usage is above $threshold%. Cleaning up Docker images and volumes..."
-    VLLM_CI_DOCKER_DISABLED \
+    docker image prune -f
-    VLLM_CI_ARTIFACT_GLOB \
+    docker volume prune -f && docker system prune --force --filter "until=72h" --all
-    VLLM_CI_ARTIFACT_CHECKSUM_GLOB \
+    echo "Docker images and volumes cleanup completed."
-    VLLM_CI_EXPECTED_GPU_COUNT \
+  else
-    VLLM_CI_USE_ARTIFACTS \
+    echo "Disk usage is below $threshold%. No cleanup needed."
-    VLLM_CI_RESULTS_ROOT \
+  fi
    VLLM_ALLOW_DEPRECATED_BEAM_SEARCH
 }
 cleanup_network() {
@@ -104,364 +85,6 @@ cleanup_network() {
  fi
 }
 prepare_artifact_image() {
  if [[ "${VLLM_CI_USE_ARTIFACTS:-0}" != "1" ]]; then
    return 1
  fi
  if ! command -v buildkite-agent >/dev/null 2>&1; then
    echo "buildkite-agent not found; cannot download ROCm wheel artifact"
    return 1
  fi
  local artifact_glob="${VLLM_CI_ARTIFACT_GLOB:-artifacts/vllm-rocm-install/vllm-rocm-install.tar.gz}"
  local archive=""
  local metadata_file=""
  local base_image="${VLLM_CI_BASE_IMAGE:-rocm/vllm-dev:ci_base}"
  local artifact_image=""
  local artifact_key=""
  local base_digest=""
  local wheel_dir=""
  local context_dir=""
  local workspace_dir=""
  artifact_work_dir=$(mktemp -d -t vllm-rocm-artifact.XXXXXX)
  wheel_dir="${artifact_work_dir}/wheels"
  context_dir="${artifact_work_dir}/context"
  workspace_dir="${context_dir}/workspace"
  mkdir -p "${wheel_dir}" "${context_dir}/wheels" "${workspace_dir}"
  echo "--- Downloading ROCm wheel artifact"
  if ! buildkite-agent artifact download "${artifact_glob}" "${artifact_work_dir}"; then
    echo "Failed to download ${artifact_glob}"
    return 1
  fi
  buildkite-agent artifact download \
    "artifacts/vllm-rocm-install/ci-base-image.txt" \
    "${artifact_work_dir}" >/dev/null 2>&1 || true
  archive=$(find "${artifact_work_dir}" -name "vllm-rocm-install.tar.gz" -type f | head -1)
  if [[ -z "${archive}" || ! -f "${archive}" ]]; then
    echo "ROCm wheel artifact archive was not found"
    return 1
  fi
  metadata_file=$(find "${artifact_work_dir}" -name "ci-base-image.txt" -type f | head -1)
  if [[ -n "${metadata_file}" && -s "${metadata_file}" ]]; then
    base_image=$(tr -d '[:space:]' < "${metadata_file}")
  fi
  echo "--- Preparing local ROCm test image"
  echo "Base image: ${base_image}"
  docker pull "${base_image}" || return 1
  base_digest=$(
    docker image inspect \
      --format='{{if .RepoDigests}}{{index .RepoDigests 0}}{{else}}{{.Id}}{{end}}' \
      "${base_image}" 2>/dev/null || printf '%s' "${base_image}"
  )
  artifact_key=$(
    {
      printf 'base-image:%s\n' "${base_digest}"
      sha256sum "${archive}"
    } | sha256sum | cut -c1-24
  )
  artifact_image="rocm/vllm-ci-artifact:${artifact_key}"
  if docker image inspect "${artifact_image}" >/dev/null 2>&1; then
    echo "Using existing local ROCm artifact image: ${artifact_image}"
    image_name="${artifact_image}"
    return 0
  fi
  tar -xzf "${archive}" -C "${wheel_dir}" || return 1
  if ! ls "${wheel_dir}"/*.whl >/dev/null 2>&1; then
    echo "ROCm wheel artifact did not contain a wheel"
    return 1
  fi
  if [[ ! -d "${wheel_dir}/tests" ]]; then
    echo "ROCm wheel artifact did not contain the test workspace"
    return 1
  fi
  cp "${wheel_dir}"/*.whl "${context_dir}/wheels/" || return 1
  tar -C "${wheel_dir}" \
    --exclude='*.whl' \
    --exclude='.vllm-ci-artifact' \
    --exclude='./.vllm-ci-artifact' \
    -cf - . \
    | tar -C "${workspace_dir}" -xf - || return 1
  cat > "${context_dir}/Dockerfile" <<'EOF'
 ARG BASE_IMAGE
 FROM ${BASE_IMAGE}
 COPY wheels/ /tmp/vllm-wheels/
 COPY workspace/ /vllm-workspace/
 RUN python3 -m pip install --no-deps --force-reinstall /tmp/vllm-wheels/*.whl \
    && rm -rf /tmp/vllm-wheels
 WORKDIR /vllm-workspace
 EOF
  echo "--- Building local ROCm test image"
  docker build \
    --pull=false \
    --progress "${BUILDKIT_PROGRESS}" \
    --build-arg "BASE_IMAGE=${base_image}" \
    -t "${artifact_image}" \
    "${context_dir}" || return 1
  image_name="${artifact_image}"
  return 0
 }
 is_native_runtime() {
  [[ "${AMD_CI_RUNTIME:-}" == "native" || "${NATIVE_CI:-}" == "true" ]]
 }
 validate_native_workspace() {
  local workspace_dir="${VLLM_CI_WORKSPACE:-/vllm-workspace}"
  local workspace_real=""
  local checkout_real=""
  local workspace_mount=""
  mkdir -p "${workspace_dir}" || return 1
  workspace_real=$(readlink -m "${workspace_dir}") || return 1
  if [[ -n "${BUILDKITE_BUILD_CHECKOUT_PATH:-}" ]]; then
    checkout_real=$(readlink -m "${BUILDKITE_BUILD_CHECKOUT_PATH}") || return 1
    if [[ "${checkout_real}" == "${workspace_real}" \
      || "${checkout_real}" == "${workspace_real}/"* \
      || "${workspace_real}" == "${checkout_real}/"* ]]; then
      echo "Refusing to replace ${workspace_real}; it overlaps the Buildkite checkout ${checkout_real}" >&2
      return 1
    fi
  fi
  if [[ "${VLLM_CI_REQUIRE_WORKSPACE_MOUNT:-1}" == "1" ]]; then
    if ! command -v findmnt >/dev/null 2>&1; then
      echo "findmnt is required to verify the native workspace mount" >&2
      return 1
    fi
    workspace_mount=$(findmnt -n -T "${workspace_real}" -o TARGET 2>/dev/null || true)
    if [[ "$(readlink -m "${workspace_mount:-/}")" != "${workspace_real}" ]]; then
      echo "Native CI requires a dedicated volume mounted at ${workspace_real}" >&2
      return 1
    fi
  fi
 }
 prepare_native_workspace() {
  if [[ "${VLLM_CI_USE_ARTIFACTS:-0}" != "1" ]]; then
    echo "Native CI requires VLLM_CI_USE_ARTIFACTS=1"
    return 1
  fi
  if ! command -v buildkite-agent >/dev/null 2>&1; then
    echo "buildkite-agent not found; cannot download ROCm wheel artifact"
    return 1
  fi
  validate_native_workspace || return 1
  local artifact_glob="${VLLM_CI_ARTIFACT_GLOB:-artifacts/vllm-rocm-install/vllm-rocm-install.tar.gz}"
  local artifact_checksum_glob="${VLLM_CI_ARTIFACT_CHECKSUM_GLOB:-${artifact_glob}.sha256}"
  local artifact_step="${VLLM_CI_ARTIFACT_STEP:-image-build-amd}"
  local archive=""
  local checksum=""
  local download_dir=""
  local metadata_dir=""
  local recorded_base=""
  local recorded_commit=""
  local recorded_wheel=""
  local workspace_dir="${VLLM_CI_WORKSPACE:-/vllm-workspace}"
  local wheel_dir=""
  local attempt=0
  local attempt_dir=""
  local -a archives=()
  local -a checksums=()
  local -a wheels=()
  artifact_work_dir=$(mktemp -d -t vllm-rocm-artifact.XXXXXX) || return 1
  wheel_dir="${artifact_work_dir}/wheels"
  mkdir -p "${wheel_dir}" || return 1
  echo "--- Downloading ROCm wheel artifact from ${artifact_step} (native in-pod)"
  for attempt in 1 2 3; do
    attempt_dir="${artifact_work_dir}/download-${attempt}"
    rm -rf "${attempt_dir}" || return 1
    mkdir -p "${attempt_dir}" || return 1
    if buildkite-agent artifact download \
      "${artifact_glob}" "${attempt_dir}" --step "${artifact_step}" \
      && buildkite-agent artifact download \
        "${artifact_checksum_glob}" "${attempt_dir}" --step "${artifact_step}"; then
      download_dir="${attempt_dir}"
      break
    fi
    echo "Artifact download attempt ${attempt}/3 failed"
    if [[ "${attempt}" -lt 3 ]]; then
      sleep $((attempt * 2))
    fi
  done
  if [[ -z "${download_dir}" ]]; then
    echo "Failed to download ${artifact_glob} and ${artifact_checksum_glob} from ${artifact_step}"
    return 1
  fi
  mapfile -t archives < <(
    find "${download_dir}" -name "vllm-rocm-install.tar.gz" -type f -print
  )
  mapfile -t checksums < <(
    find "${download_dir}" -name "vllm-rocm-install.tar.gz.sha256" -type f -print
  )
  if [[ ${#archives[@]} -ne 1 || ${#checksums[@]} -ne 1 ]]; then
    echo "Expected exactly one ROCm archive and checksum; found ${#archives[@]} archive(s) and ${#checksums[@]} checksum(s)" >&2
    return 1
  fi
  archive="${archives[0]}"
  checksum="${checksums[0]}"
  if [[ "$(dirname "${archive}")" != "$(dirname "${checksum}")" ]]; then
    echo "ROCm archive and checksum were downloaded to different directories" >&2
    return 1
  fi
  (
    cd "$(dirname "${archive}")"
    sha256sum -c "$(basename "${checksum}")"
  ) || return 1
  tar --no-same-owner -xzf "${archive}" -C "${wheel_dir}" || return 1
  mapfile -t wheels < <(
    find "${wheel_dir}" -maxdepth 1 -type f -name '*.whl' -print
  )
  if [[ ${#wheels[@]} -ne 1 ]]; then
    echo "ROCm artifact must contain exactly one top-level wheel; found ${#wheels[@]}" >&2
    return 1
  fi
  metadata_dir="${wheel_dir}/.vllm-ci-artifact"
  for metadata_file in commit.txt native-base-image.txt wheel-filename.txt; do
    if [[ ! -s "${metadata_dir}/${metadata_file}" ]]; then
      echo "ROCm artifact metadata is missing ${metadata_file}" >&2
      return 1
    fi
  done
  for metadata_file in ci-base-image.txt fallback-image.txt; do
    if [[ ! -f "${metadata_dir}/${metadata_file}" ]]; then
      echo "ROCm artifact metadata is missing ${metadata_file}" >&2
      return 1
    fi
  done
  recorded_commit=$(tr -d '\r\n' < "${metadata_dir}/commit.txt")
  recorded_base=$(tr -d '\r\n' < "${metadata_dir}/native-base-image.txt")
  recorded_wheel=$(tr -d '\r\n' < "${metadata_dir}/wheel-filename.txt")
  if [[ -z "${BUILDKITE_COMMIT:-}" || "${recorded_commit}" != "${BUILDKITE_COMMIT}" ]]; then
    echo "ROCm artifact commit ${recorded_commit} does not match ${BUILDKITE_COMMIT:-unset}" >&2
    return 1
  fi
  if [[ -z "${VLLM_CI_BASE_IMAGE:-}" || "${recorded_base}" != "${VLLM_CI_BASE_IMAGE}" ]]; then
    echo "ROCm artifact base ${recorded_base} does not match ${VLLM_CI_BASE_IMAGE:-unset}" >&2
    return 1
  fi
  if [[ "${recorded_wheel}" != "$(basename "${wheels[0]}")" ]]; then
    echo "ROCm artifact wheel manifest ${recorded_wheel} does not match $(basename "${wheels[0]}")" >&2
    return 1
  fi
  for required_dir in tests .buildkite requirements; do
    if [[ ! -d "${wheel_dir}/${required_dir}" ]]; then
      echo "ROCm wheel artifact did not contain ${required_dir}/" >&2
      return 1
    fi
  done
  echo "--- Installing ROCm wheel into pod environment"
  python3 -m pip install --no-deps --force-reinstall "${wheels[0]}" || return 1
  echo "--- Preparing ${workspace_dir} from artifact"
  find "${workspace_dir}" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + || return 1
  tar -C "${wheel_dir}" \
    --exclude='*.whl' \
    --exclude='.vllm-ci-artifact' \
    --exclude='./.vllm-ci-artifact' \
    -cf - . | tar --no-same-owner -C "${workspace_dir}" -xf - || return 1
  if [[ ! -d "${workspace_dir}/tests" ]]; then
    echo "Failed to stage the native test workspace" >&2
    return 1
  fi
  return 0
 }
 initialize_native_environment() {
  local job_id="${BUILDKITE_JOB_ID:-${BUILDKITE_PARALLEL_JOB:-local}}"
  local job_id_suffix=""
  local native_root=""
  local hf_mount=""
  if [[ "$(id -u)" -ne 0 ]]; then
    echo "Native ROCm CI currently requires the ci_base container to run as root" >&2
    return 1
  fi
  job_id="${job_id//[^A-Za-z0-9_.-]/_}"
  job_id_suffix="${job_id##*-}"
  job_id_suffix="${job_id_suffix:0:12}"
  native_root="/tmp/vllm-native-${job_id}"
  TMPDIR="/tmp/vllm-${job_id_suffix}/tmp"
  VLLM_RPC_BASE_PATH="/tmp"
  : "${TORCHINDUCTOR_CACHE_DIR:=${native_root}/cache/torchinductor}"
  : "${TRITON_CACHE_DIR:=${native_root}/cache/triton}"
  : "${VLLM_CACHE_ROOT:=${native_root}/cache/vllm}"
  : "${XDG_CACHE_HOME:=${native_root}/cache/xdg}"
  : "${HF_HOME:=/home/buildkite-agent/huggingface}"
  : "${HF_HUB_DOWNLOAD_TIMEOUT:=300}"
  : "${HF_HUB_ETAG_TIMEOUT:=60}"
  export TMPDIR VLLM_RPC_BASE_PATH
  export TORCHINDUCTOR_CACHE_DIR TRITON_CACHE_DIR VLLM_CACHE_ROOT XDG_CACHE_HOME
  export HF_HOME HF_HUB_DOWNLOAD_TIMEOUT HF_HUB_ETAG_TIMEOUT
  export PYTORCH_ROCM_ARCH=""
  mkdir -p "${TMPDIR}" \
    "${TORCHINDUCTOR_CACHE_DIR}" \
    "${TRITON_CACHE_DIR}" \
    "${VLLM_CACHE_ROOT}" \
    "${XDG_CACHE_HOME}" \
    "${HF_HOME}" || return 1
  if [[ "${VLLM_CI_REQUIRE_PERSISTENT_HF_CACHE:-0}" == "1" ]]; then
    if ! command -v findmnt >/dev/null 2>&1; then
      echo "findmnt is required to verify the native Hugging Face cache mount" >&2
      return 1
    fi
    hf_mount=$(findmnt -n -T "${HF_HOME}" -o TARGET 2>/dev/null || true)
    if [[ -z "${hf_mount}" || "${hf_mount}" == "/" ]]; then
      echo "Native CI requires a persistent volume mounted at or above ${HF_HOME}" >&2
      return 1
    fi
  fi
 }
 run_native_preflight() {
  local expected_gpus="${VLLM_CI_EXPECTED_GPU_COUNT:-1}"
  if [[ ! "${expected_gpus}" =~ ^[0-9]+$ ]]; then
    echo "Invalid VLLM_CI_EXPECTED_GPU_COUNT=${expected_gpus}" >&2
    return 1
  fi
  python3 -c "import encodings, importlib.metadata as im, importlib.util as iu; [im.version(d) for d in ('transformers', 'torch', 'ray', 'sympy', 'markupsafe', 'vllm')]; missing=[m for m in ('torch.utils.model_zoo', 'transformers.models.nomic_bert', 'ray.dag', 'sympy.physics', 'markupsafe._speedups') if iu.find_spec(m) is None]; assert not missing, missing" || return 1
  if [[ "${expected_gpus}" == "0" ]]; then
    echo "Native CPU-only AMD job: skipping ROCm device validation"
    return 0
  fi
  echo "--- ROCm info"
  rocminfo || return 1
  VLLM_CI_EXPECTED_GPU_COUNT="${expected_gpus}" python3 - <<'PY'
 import os
 import torch
 expected = int(os.environ["VLLM_CI_EXPECTED_GPU_COUNT"])
 assert torch.version.hip, "PyTorch is not a ROCm build"
 assert torch.cuda.is_available(), "ROCm GPU is not available to PyTorch"
 actual = torch.cuda.device_count()
 assert actual == expected, f"Expected {expected} ROCm GPU(s), found {actual}"
 PY
 }
 is_multi_node() {
  local cmds="$1"
  # Primary signal: NUM_NODES environment variable set by the pipeline
@@ -508,7 +131,8 @@ handle_pytest_exit() {
 # unquoted since they have no spaces and work fine.
 #
 # Already-quoted expressions (containing literal single quotes) are passed
-# through untouched to avoid double-quoting well-formed shell fragments.
+# through untouched to avoid double-quoting values injected by
 # apply_rocm_test_overrides.
 #
 # NOTE: This ONLY fixes -m/-k flags. It cannot recover arbitrary inner
 # double-quotes stripped by the calling shell (see header comment).
@@ -581,13 +205,6 @@ re_quote_pytest_markers() {
      esac
      if $is_boundary; then
        # Strip surrounding double quotes if present (from upstream
        # single-to-double conversion); without this, wrapping below
        # would produce '"expr"' with literal double-quote characters.
        if [[ "$marker_buf" == '"'*'"' ]]; then
          marker_buf="${marker_buf#\"}"
          marker_buf="${marker_buf%\"}"
        fi
        # Flush the collected marker expression
        if [[ "$marker_buf" == *" "* || "$marker_buf" == *"("* ]]; then
          output+="'${marker_buf}' "
@@ -625,11 +242,6 @@ re_quote_pytest_markers() {
  # Flush any trailing marker expression (marker at end of command)
  if $collecting && [[ -n "$marker_buf" ]]; then
    # Strip surrounding double quotes (see mid-stream flush comment)
    if [[ "$marker_buf" == '"'*'"' ]]; then
      marker_buf="${marker_buf#\"}"
      marker_buf="${marker_buf%\"}"
    fi
    if [[ "$marker_buf" == *" "* || "$marker_buf" == *"("* ]]; then
      output+="'${marker_buf}'"
    else
@@ -640,109 +252,129 @@ re_quote_pytest_markers() {
  echo "${output% }"
 }
 ###############################################################################
 # ROCm-specific pytest command rewrites
 #
 # These apply ignore flags and environment overrides for tests that are not
 # yet supported or behave differently on ROCm hardware. Kept as a single
 # function so new exclusions are easy to add in one place.
 ###############################################################################
 apply_rocm_test_overrides() {
  local cmds="$1"
  # --- Model registry filter ---
  if [[ $cmds == *"pytest -v -s models/test_registry.py"* ]]; then
    cmds=${cmds//"pytest -v -s models/test_registry.py"/"pytest -v -s models/test_registry.py -k 'not BambaForCausalLM and not GritLM and not Mamba2ForCausalLM and not Zamba2ForCausalLM'"}
  fi
  # --- LoRA: disable custom paged attention ---
  if [[ $cmds == *"pytest -v -s lora"* ]]; then
    cmds=${cmds//"pytest -v -s lora"/"VLLM_ROCM_CUSTOM_PAGED_ATTN=0 pytest -v -s lora"}
  fi
  # --- Kernel ignores ---
  if [[ $cmds == *" kernels/core"* ]]; then
    cmds="${cmds} \
    --ignore=kernels/core/test_fused_quant_layernorm.py \
    --ignore=kernels/core/test_permute_cols.py"
  fi
  if [[ $cmds == *" kernels/attention"* ]]; then
    cmds="${cmds} \
    --ignore=kernels/attention/test_attention_selector.py \
    --ignore=kernels/attention/test_encoder_decoder_attn.py \
    --ignore=kernels/attention/test_flash_attn.py \
    --ignore=kernels/attention/test_flashinfer.py \
    --ignore=kernels/attention/test_prefix_prefill.py \
    --ignore=kernels/attention/test_cascade_flash_attn.py \
    --ignore=kernels/attention/test_mha_attn.py \
    --ignore=kernels/attention/test_lightning_attn.py \
    --ignore=kernels/attention/test_attention.py"
  fi
  if [[ $cmds == *" kernels/quantization"* ]]; then
    cmds="${cmds} \
    --ignore=kernels/quantization/test_int8_quant.py \
    --ignore=kernels/quantization/test_machete_mm.py \
    --ignore=kernels/quantization/test_block_fp8.py \
    --ignore=kernels/quantization/test_block_int8.py \
    --ignore=kernels/quantization/test_marlin_gemm.py \
    --ignore=kernels/quantization/test_cutlass_scaled_mm.py \
    --ignore=kernels/quantization/test_int8_kernel.py"
  fi
  if [[ $cmds == *" kernels/mamba"* ]]; then
    cmds="${cmds} \
    --ignore=kernels/mamba/test_mamba_mixer2.py \
    --ignore=kernels/mamba/test_causal_conv1d.py \
    --ignore=kernels/mamba/test_mamba_ssm_ssd.py"
  fi
  if [[ $cmds == *" kernels/moe"* ]]; then
    cmds="${cmds} \
    --ignore=kernels/moe/test_moe.py \
    --ignore=kernels/moe/test_cutlass_moe.py \
    --ignore=kernels/moe/test_triton_moe_ptpc_fp8.py"
  fi
  # --- Entrypoint ignores ---
  if [[ $cmds == *" entrypoints/openai "* ]]; then
    cmds=${cmds//" entrypoints/openai "/" entrypoints/openai \
    --ignore=entrypoints/openai/test_audio.py \
    --ignore=entrypoints/openai/test_shutdown.py \
    --ignore=entrypoints/openai/test_completion.py \
    --ignore=entrypoints/openai/test_models.py \
    --ignore=entrypoints/openai/test_lora_adapters.py \
    --ignore=entrypoints/openai/test_return_tokens_as_ids.py \
    --ignore=entrypoints/openai/test_root_path.py \
    --ignore=entrypoints/openai/test_tokenization.py \
    --ignore=entrypoints/openai/test_prompt_validation.py "}
  fi
  if [[ $cmds == *" entrypoints/llm "* ]]; then
    cmds=${cmds//" entrypoints/llm "/" entrypoints/llm \
    --ignore=entrypoints/llm/test_chat.py \
    --ignore=entrypoints/llm/test_accuracy.py \
    --ignore=entrypoints/llm/test_init.py \
    --ignore=entrypoints/llm/test_prompt_validation.py "}
  fi
  # Clean up escaped newlines from --ignore appends
  cmds=$(echo "$cmds" | sed 's/ \\ / /g')
  echo "$cmds"
 }
 ###############################################################################
 # Main
 ###############################################################################
-if is_native_runtime; then
+# --- GPU initialization ---
-  echo "--- Native in-pod ROCm CI (AMD_CI_RUNTIME=${AMD_CI_RUNTIME:-unset}, NATIVE_CI=${NATIVE_CI:-unset})"
+echo "--- Confirming Clean Initial State"
-  artifact_work_dir=""
+wait_for_clean_gpus
  cleanup_native_workspace() {
    if [[ -n "${artifact_work_dir}" ]]; then
      rm -rf "${artifact_work_dir}"
    fi
  }
  trap cleanup_native_workspace EXIT
  if [[ -n "${VLLM_TEST_COMMANDS:-}" ]]; then
    commands="${VLLM_TEST_COMMANDS}"
    commands_source="env"
  else
    commands="$*"
    commands_source="argv"
    if [[ -z "$commands" ]]; then
      echo "Error: No test commands provided for native CI." >&2
      exit 1
    fi
  fi
  if [[ "$commands_source" == "argv" ]]; then
    commands=$(re_quote_pytest_markers "$commands")
  fi
  if is_multi_node "$commands"; then
    echo "Native CI does not support multi-node jobs yet."
    exit 1
  fi
  if ! initialize_native_environment; then
    echo "Failed to initialize the native test environment"
    exit 1
  fi
  if ! prepare_native_workspace; then
    echo "Failed to prepare native test workspace"
    exit 1
  fi
  export PYTHONPATH="${VLLM_CI_WORKSPACE:-/vllm-workspace}"
  echo "Native test commands: $commands"
  run_native_preflight || exit 1
  # Keep AMD CI orchestration variables out of vLLM's runtime environment.
  clear_ci_orchestration_env
  /bin/bash -o pipefail -c "${commands}"
  handle_pytest_exit "$?"
 fi
 # --- GPU initialization for legacy Docker execution ---
 echo "--- ROCm info"
 rocminfo
-# --- Docker status ---
+# --- Docker housekeeping ---
-report_docker_usage
+cleanup_docker
 echo "--- Resetting GPUs"
 echo "reset" > /opt/amdgpu/etc/gpu_state
 wait_for_clean_gpus
 # --- Pull test image ---
 echo "--- Pulling container"
-image_name="${VLLM_CI_FALLBACK_IMAGE:-rocm/vllm-ci:${BUILDKITE_COMMIT:-local}}"
+image_name="rocm/vllm-ci:${BUILDKITE_COMMIT}"
 artifact_work_dir=""
 container_name="rocm_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10; echo)"
 docker pull "${image_name}"
 remove_docker_container() {
-  if docker container inspect "${container_name}" >/dev/null 2>&1; then
+  docker rm -f "${container_name}" || docker image rm -f "${image_name}" || true
    docker rm -f "${container_name}" || true
  fi
  if [[ "${VLLM_CI_REMOVE_TEST_IMAGE:-0}" == "1" ]]; then
    docker image rm -f "${image_name}" || true
  else
    # Keep images by default so later jobs on the same AMD node can reuse layers.
    echo "Keeping ROCm test image locally: ${image_name}"
  fi
  if [[ -n "${artifact_work_dir}" ]]; then
    rm -rf "${artifact_work_dir}"
  fi
 }
 trap remove_docker_container EXIT
 # python_only_compile.sh runs `python setup.py develop` and needs the full repo tree
 # under /vllm-workspace (Dockerfile.rocm test stage: mkdir src && mv vllm).
 # The ROCm wheel artifact tarball only ships a thin tree (tests, etc.), so
 # artifact images cannot satisfy that test — use the full rocm/vllm-ci image.
 _cmd_probe="${VLLM_TEST_COMMANDS:-}"
 if [[ -z "${_cmd_probe}" ]]; then
  _cmd_probe="$*"
 fi
 if [[ "${VLLM_CI_USE_ARTIFACTS:-0}" == "1" && "${_cmd_probe}" == *python_only_compile.sh* ]]; then
  echo "INFO: disabling VLLM_CI_USE_ARTIFACTS for python_only_compile (requires full /vllm-workspace tree)"
  export VLLM_CI_USE_ARTIFACTS=0
 fi
 unset -v _cmd_probe
 if ! prepare_artifact_image; then
  echo "Using full ROCm CI image: ${image_name}"
  docker pull "${image_name}" || exit 1
 fi
 # --- Prepare commands ---
 echo "--- Running container"
@@ -750,25 +382,15 @@ HF_CACHE="$(realpath ~)/huggingface"
 mkdir -p "${HF_CACHE}"
 HF_MOUNT="/root/.cache/huggingface"
 # Hugging Face Hub defaults to 10s request/download timeouts, while the ROCm
 # CI image currently raises downloads to 60s. AMD model-test jobs routinely
 # start from a cold or partially-populated shared cache, and the 60s read cap
 # has still timed out before pytest reached the vLLM behavior under test.
 # Keep the CI default explicit and overridable from the Buildkite environment.
 : "${HF_HUB_DOWNLOAD_TIMEOUT:=300}"
 : "${HF_HUB_ETAG_TIMEOUT:=60}"
 # ---- Command source selection ----
 # Prefer VLLM_TEST_COMMANDS (preserves all inner quoting intact).
 # Fall back to $* for backward compatibility, but warn that inner
 # double-quotes will have been stripped by the calling shell.
 if [[ -n "${VLLM_TEST_COMMANDS:-}" ]]; then
  commands="${VLLM_TEST_COMMANDS}"
  commands_source="env"
  echo "Commands sourced from VLLM_TEST_COMMANDS (quoting preserved)"
 else
  commands="$*"
  commands_source="argv"
  if [[ -z "$commands" ]]; then
    echo "Error: No test commands provided." >&2
    echo "Usage:" >&2
@@ -785,48 +407,14 @@ fi
 echo "Raw commands: $commands"
-# Only try to repair stripped pytest -m/-k quoting in legacy argv mode.
+# Fix quoting before ROCm overrides (so overrides see correct structure)
-# VLLM_TEST_COMMANDS preserves inner quoting already, and re-quoting that path
+commands=$(re_quote_pytest_markers "$commands")
-# can corrupt embedded echo strings or otherwise well-formed shell fragments.
+echo "After re-quoting: $commands"
 if [[ "$commands_source" == "argv" ]]; then
  commands=$(re_quote_pytest_markers "$commands")
  echo "After re-quoting: $commands"
 else
  echo "Skipping re-quoting for VLLM_TEST_COMMANDS input"
 fi
 commands=$(apply_rocm_test_overrides "$commands")
 echo "Final commands: $commands"
-standalone_merge_base_env=()
+MYPYTHONPATH=".."
 if [[ "$commands" == *python_only_compile.sh* ]]; then
  # The ROCm test image often ships /vllm-workspace without .git. Resolve the
  # wheels.vllm.ai commit from the agent checkout for this test only.
  vllm_standalone_merge_base=""
  checkout="${BUILDKITE_BUILD_CHECKOUT_PATH:-}"
  if [[ -z "${checkout}" || ! -d "${checkout}" ]]; then
    checkout="."
  fi
  # Pass safe.directory per-command because Buildkite uses mixed user IDs.
  if git -c "safe.directory=${checkout}" -C "${checkout}" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
    vllm_standalone_merge_base="$(
      git -c "safe.directory=${checkout}" -C "${checkout}" merge-base HEAD origin/main 2>/dev/null || true
    )"
  fi
  if [[ -z "${vllm_standalone_merge_base}" ]]; then
    vllm_standalone_merge_base="${BUILDKITE_COMMIT:-}"
  fi
  echo "INFO: passing CI_STANDALONE_MERGE_BASE into container: ${vllm_standalone_merge_base}"
  standalone_merge_base_env=(-e "CI_STANDALONE_MERGE_BASE=${vllm_standalone_merge_base}")
 fi
 MYPYTHONPATH="/vllm-workspace"
 container_job_id="${BUILDKITE_JOB_ID:-${BUILDKITE_PARALLEL_JOB:-0}}"
 container_job_id="${container_job_id//[^A-Za-z0-9_.-]/_}"
 container_job_id_short="${container_job_id:0:8}"
 CONTAINER_TMPDIR="/tmp/vllm-${container_job_id_short}"
 CONTAINER_CACHE_ROOT="/tmp/vllm-buildkite-${container_job_id}/cache"
 CONTAINER_PREFLIGHT="mkdir -p \"\$TMPDIR\" \"\$TORCHINDUCTOR_CACHE_DIR\" \"\$TRITON_CACHE_DIR\" \"\$VLLM_CACHE_ROOT\" \"\$XDG_CACHE_HOME\" && python -c \"import encodings, importlib.metadata as im, importlib.util as iu; [im.version(d) for d in ('transformers', 'torch', 'ray', 'sympy', 'markupsafe', 'vllm')]; missing=[m for m in ('torch.utils.model_zoo', 'transformers.models.nomic_bert', 'ray.dag', 'sympy.physics', 'markupsafe._speedups') if iu.find_spec(m) is None]; assert not missing, missing\""
 # Verify GPU access
 render_gid=$(getent group render | cut -d: -f3)
@@ -848,7 +436,6 @@ else
 fi
 # --- Route: multi-node vs single-node ---
 clear_ci_orchestration_env
 if is_multi_node "$commands"; then
  echo "--- Multi-node job detected"
  export DCKR_VER=$(docker --version | sed 's/Docker version \(.*\), build .*/\1/')
@@ -895,64 +482,22 @@ if is_multi_node "$commands"; then
 else
  echo "--- Single-node job"
  echo "Render devices: $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES"
  docker_run_terminal_args=(-i)
  if [[ "${ROCM_DOCKER_TTY}" == "1" ]]; then
    docker_run_terminal_args+=(-t)
    echo "Docker interactive stdin: enabled; TTY allocation: enabled"
  else
    echo "Docker interactive stdin: enabled; TTY allocation: disabled"
  fi
  ulimit_core_hard=$(ulimit -H -c)
  if [[ "$ulimit_core_hard" == "unlimited" ]]; then
    # docker run can't pass "unlimited" to --ulimit
    ulimit_core_hard="-1"
  fi
   # Disable core dumps in the ROCm test container unless the ROCm debug agent is enabled
  coredump_flags="--ulimit core=0:$ulimit_core_hard"
  if [[ "$commands" == *"ROCm debug agent enabled"* ]]; then
    # Works around https://github.com/rocm/rocm-systems/issues/6206
    coredump_flags='-e HSA_COREDUMP_PATTERN="/tmp/gpucore.%p"'
  else
    echo "ROCm debug agent not enabled, coredumps are disabled in the test container."
  fi
  docker run \
    "${docker_run_terminal_args[@]}" \
    --device /dev/kfd $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES \
    $RDMA_FLAGS \
    --network=host \
    --shm-size=16gb \
    --group-add "$render_gid" \
    --rm \
    $coredump_flags \
    -e HF_TOKEN \
    -e "HF_HUB_DOWNLOAD_TIMEOUT=${HF_HUB_DOWNLOAD_TIMEOUT}" \
    -e "HF_HUB_ETAG_TIMEOUT=${HF_HUB_ETAG_TIMEOUT}" \
    -e AWS_ACCESS_KEY_ID \
    -e AWS_SECRET_ACCESS_KEY \
    -e BUILDKITE_PARALLEL_JOB \
    -e BUILDKITE_PARALLEL_JOB_COUNT \
    -e TERM \
    -e FORCE_COLOR \
    -e CLICOLOR_FORCE \
    -e PY_COLORS \
    -e PYTHONFAULTHANDLER \
    -e PYTEST_ADDOPTS \
    -e PYTEST_TIMEOUT \
    -v "${HF_CACHE}:${HF_MOUNT}" \
    -e "HF_HOME=${HF_MOUNT}" \
    -e "PYTHONPATH=${MYPYTHONPATH}" \
    -e "TMPDIR=${CONTAINER_TMPDIR}/tmp" \
    -e "TORCHINDUCTOR_CACHE_DIR=${CONTAINER_CACHE_ROOT}/torchinductor" \
    -e "TRITON_CACHE_DIR=${CONTAINER_CACHE_ROOT}/triton" \
    -e "VLLM_CACHE_ROOT=${CONTAINER_CACHE_ROOT}/vllm" \
    -e "XDG_CACHE_HOME=${CONTAINER_CACHE_ROOT}/xdg" \
    -e "PYTORCH_ROCM_ARCH=" \
    "${standalone_merge_base_env[@]}" \
    --name "${container_name}" \
    "${image_name}" \
-    /bin/bash -c "${CONTAINER_PREFLIGHT} && ${commands}"
+    /bin/bash -c "${commands}"
  exit_code=$?
  handle_pytest_exit "$exit_code"
@@ -1,65 +0,0 @@
 #!/bin/bash
 set -euox pipefail
 export VLLM_CPU_KVCACHE_SPACE=1 
 export VLLM_CPU_CI_ENV=1
 # Reduce sub-processes for acceleration
 export TORCH_COMPILE_DISABLE=1 
 export VLLM_ENABLE_V1_MULTIPROCESSING=0
 SDE_ARCHIVE="sde-external-10.7.0-2026-02-18-lin.tar.xz"
 SDE_CHECKSUM="CA3D4086DE4ACB3FAEDF9F57B541C6936B7D5E19AE2BF763B6EA933573A0A217"
 wget "https://downloadmirror.intel.com/913594/${SDE_ARCHIVE}"
 echo "${SDE_CHECKSUM}  ${SDE_ARCHIVE}" | sha256sum --check
 mkdir -p sde
 tar -xvf "./${SDE_ARCHIVE}" --strip-components=1 -C ./sde/
 wait_for_pid_and_check_log() {
    local pid="$1"
    local log_file="$2"
    local exit_status
    if [ -z "$pid" ] || [ -z "$log_file" ]; then
        echo "Usage: wait_for_pid_and_check_log <PID> <LOG_FILE>"
        return 1
    fi
    echo "Waiting for process $pid to finish..."
    # Use the 'wait' command to pause the script until the specific PID exits.
    # The 'wait' command's own exit status will be that of the waited-for process.
    if wait "$pid"; then
        exit_status=$?
        echo "Process $pid finished with exit status $exit_status (Success)."
    else
        exit_status=$?
        echo "Process $pid finished with exit status $exit_status (Failure)."
    fi
    if [ "$exit_status" -ne 0 ]; then
        echo "Process exited with a non-zero status."
        echo "--- Last few lines of log file: $log_file ---"
        tail -n 50 "$log_file"
        echo "---------------------------------------------"
        return 1 # Indicate failure based on exit status
    fi
    echo "No errors detected in log file and process exited successfully."
    return 0
 }
 # Test Sky Lake (AVX512F)
 ./sde/sde64 -skl -- python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --dtype bfloat16 > test_0.log 2>&1 &
 PID_TEST_0=$!
 # Test Cascade Lake (AVX512F + VNNI)
 ./sde/sde64 -clx -- python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --dtype bfloat16 > test_1.log 2>&1 &
 PID_TEST_1=$!
 # Test Cooper Lake (AVX512F + VNNI + BF16)
 ./sde/sde64 -cpx -- python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --dtype bfloat16 > test_2.log 2>&1 &
 PID_TEST_2=$!
 wait_for_pid_and_check_log $PID_TEST_0 test_0.log
 wait_for_pid_and_check_log $PID_TEST_1 test_1.log
 wait_for_pid_and_check_log $PID_TEST_2 test_2.log
@@ -1,39 +1,43 @@
 #!/bin/bash
 set -euox pipefail
 export VLLM_CPU_CI_ENV=0
 export VLLM_CPU_KVCACHE_SPACE=1 # avoid OOM
-MODE=${1:-all}
+echo "--- PP+TP"
 vllm serve meta-llama/Llama-3.2-3B-Instruct -tp=2 -pp=2 &
 server_pid=$!
 timeout 600 bash -c "until curl localhost:8000/v1/models > /dev/null 2>&1; do sleep 1; done" || exit 1
 vllm bench serve \
    --backend vllm \
    --dataset-name random \
    --model meta-llama/Llama-3.2-3B-Instruct \
    --num-prompts 20 \
    --result-dir ./test_results \
    --result-filename tp_pp.json \
    --save-result \
    --endpoint /v1/completions
 kill -s SIGTERM $server_pid; wait $server_pid || true
 failed_req=$(jq '.failed' ./test_results/tp_pp.json)
 if [ "$failed_req" -ne 0 ]; then
  echo "Some requests were failed!"
  exit 1
 fi
-run_scenario() {
+echo "--- DP+TP"
-    local label="$1" result_file="$2"
+vllm serve meta-llama/Llama-3.2-3B-Instruct -tp=2 -dp=2 &
-    shift 2
+server_pid=$!
-    echo "--- $label"
+timeout 600 bash -c "until curl localhost:8000/v1/models > /dev/null 2>&1; do sleep 1; done" || exit 1
-    vllm serve meta-llama/Llama-3.2-3B-Instruct "$@" --max-model-len=4096 &
+vllm bench serve \
-    local server_pid=$!
+    --backend vllm \
-    timeout 600 bash -c "until curl localhost:8000/v1/models > /dev/null 2>&1; do sleep 1; done" || exit 1
+    --dataset-name random \
-    vllm bench serve \
+    --model meta-llama/Llama-3.2-3B-Instruct \
-        --backend vllm \
+    --num-prompts 20 \
-        --dataset-name random \
+    --result-dir ./test_results \
-        --model meta-llama/Llama-3.2-3B-Instruct \
+    --result-filename dp_pp.json \
-        --num-prompts 20 \
+    --save-result \
-        --result-dir ./test_results \
+    --endpoint /v1/completions
-        --result-filename "$result_file" \
+kill -s SIGTERM $server_pid; wait $server_pid || true
-        --save-result \
+failed_req=$(jq '.failed' ./test_results/dp_pp.json)
-        --endpoint /v1/completions
+if [ "$failed_req" -ne 0 ]; then
-    kill -s SIGTERM "$server_pid"; wait "$server_pid" || true
+  echo "Some requests were failed!"
-    if [ "$(jq '.failed' "./test_results/$result_file")" -ne 0 ]; then
+  exit 1
-        echo "Some requests were failed in $label!"
+fi
        exit 1
    fi
 }
 case "$MODE" in
    tp_pp) run_scenario "PP+TP" tp_pp.json -tp=2 -pp=2 ;;
    dp_tp) run_scenario "DP+TP" dp_tp.json -tp=2 -dp=2 ;;
    all)
        run_scenario "PP+TP" tp_pp.json -tp=2 -pp=2
        run_scenario "DP+TP" dp_tp.json -tp=2 -dp=2
        ;;
    *) echo "ERROR: unknown mode '$MODE' (expected: tp_pp | dp_tp | all)" >&2; exit 1 ;;
 esac
@@ -5,10 +5,10 @@
 set -ex
 # allow to bind to different cores
-CORE_RANGE=${CORE_RANGE:-0-31}
+CORE_RANGE=${CORE_RANGE:-0-16}
-OMP_CORE_RANGE=${OMP_CORE_RANGE:-0-31}
+OMP_CORE_RANGE=${OMP_CORE_RANGE:-0-16}
-export CMAKE_BUILD_PARALLEL_LEVEL=32
+export CMAKE_BUILD_PARALLEL_LEVEL=16
 # Setup cleanup
 remove_docker_container() {
@@ -31,40 +31,22 @@ function cpu_tests() {
    set -e
    pip list"
  # Run kernel tests
  docker exec cpu-test bash -c "
    set -e
    pytest -x -v -s tests/kernels/test_onednn.py
    pytest -x -v -s tests/kernels/attention/test_cpu_attn.py
    pytest -x -v -s tests/kernels/core/test_cpu_activation.py
    pytest -x -v -s tests/kernels/moe/test_cpu_fused_moe.py
    pytest -x -v -s tests/kernels/mamba/cpu/test_cpu_gdn_ops.py
    pytest -x -v -s tests/kernels/moe/test_cpu_int4_moe.py
    pytest -x -v -s tests/kernels/mamba/test_cpu_short_conv.py
    pytest -x -v -s tests/kernels/mamba/test_causal_conv1d.py
    pytest -x -v -s tests/kernels/mamba/test_mamba_ssm.py"
  # skip tests requiring model downloads if HF_TOKEN is not set
  # due to rate-limits
  if [ -z "$HF_TOKEN" ]; then
    echo "Warning: HF_TOKEN is not set. Skipping tests that require model downloads."
    return
  fi
  # offline inference
  docker exec cpu-test bash -c "
    set -e
-    python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m"
+    python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m"
  # Run model tests
  docker exec cpu-test bash -c "
    set -e
    pytest -x -v -s tests/models/multimodal/generation/test_whisper.py -m cpu_model"
-  # Run quantized model tests
+  # Run kernel tests
  docker exec cpu-test bash -c "
    set -e
-    pytest -x -v -s tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_logprobs"
+    pytest -x -v -s tests/kernels/test_onednn.py
    pytest -x -v -s tests/kernels/attention/test_cpu_attn.py
    pytest -x -v -s tests/kernels/moe/test_moe.py -k test_cpu_fused_moe_basic"
  # basic online serving
  docker exec cpu-test bash -c '
@@ -79,24 +61,8 @@ function cpu_tests() {
      --num-prompts 20 \
      --endpoint /v1/completions
    kill -s SIGTERM $server_pid &'
  # smoke test for Gated DeltaNet
  docker exec cpu-test bash -c '
    set -e
    VLLM_CPU_OMP_THREADS_BIND=$E2E_OMP_THREADS vllm serve Qwen/Qwen3.5-0.8B --max-model-len 2048 &
    server_pid=$!
    timeout 600 bash -c "until curl localhost:8000/v1/models; do sleep 1; done" || exit 1
    vllm bench serve \
      --backend vllm \
      --dataset-name random \
      --model Qwen/Qwen3.5-0.8B \
      --num-prompts 20 \
      --endpoint /v1/completions
    kill -s SIGTERM $server_pid &'
 }
 # All of CPU tests are expected to be finished less than 40 mins.
 export -f cpu_tests
 timeout 2h bash -c cpu_tests
@@ -27,7 +27,7 @@ function cpu_tests() {
  podman exec -it "$container_id" bash -c "
    export TORCH_COMPILE_DISABLE=1
    set -xve
-    python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m" >> "$HOME"/test_basic.log
+    python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m" >> "$HOME"/test_basic.log
  # Run basic model test
  podman exec -it "$container_id" bash -c "
@@ -7,53 +7,14 @@ set -euox pipefail
 # allow to bind to different cores
 CORE_RANGE=${CORE_RANGE:-48-95}
 NUMA_NODE=${NUMA_NODE:-1}
-AGENT_SLOT=${AGENT_SLOT:-}
+IMAGE_NAME="cpu-test-$NUMA_NODE"
 IMAGE_NAME="cpu-test-${NUMA_NODE}${AGENT_SLOT:+-${AGENT_SLOT}}"
 TIMEOUT_VAL=$1
 TEST_COMMAND=$2
 # Disk hygiene knobs. Reclaim space only once the Docker root filesystem crosses
 # DISK_USAGE_THRESHOLD percent, and cap the shared BuildKit cache at
 # BUILDKIT_CACHE_MAX so subsequent builds keep reusing the hottest layers.
 DISK_USAGE_THRESHOLD=${DISK_USAGE_THRESHOLD:-70}
 BUILDKIT_CACHE_MAX=${BUILDKIT_CACHE_MAX:-80GB}
 # Reclaim disk only when the host is under pressure. We trim (not purge) the
 # shared BuildKit cache so cross-job/cross-agent reuse stays intact, and only
 # touch dangling images; other agents' uniquely tagged images are left alone.
 prune_if_disk_pressure() {
    local docker_root disk_usage
    docker_root=$(docker info -f '{{.DockerRootDir}}' 2>/dev/null || true)
    if [ -z "$docker_root" ]; then
        return 0
    fi
    disk_usage=$(df "$docker_root" 2>/dev/null | tail -1 | awk '{print $5}' | tr -d '%')
    if [ "${disk_usage:-0}" -gt "$DISK_USAGE_THRESHOLD" ]; then
        echo "--- :broom: Disk usage ${disk_usage}% exceeds ${DISK_USAGE_THRESHOLD}%, reclaiming space"
        docker image prune -f || true
        docker builder prune -f --keep-storage="$BUILDKIT_CACHE_MAX" || true
    else
        echo "Disk usage ${disk_usage:-unknown}% within ${DISK_USAGE_THRESHOLD}% threshold; skipping prune"
    fi
 }
 # Always drop this agent's image once the job ends (the default builder never
 # uses it as a cache source, so removing it costs no rebuild speed), then
 # reclaim space if needed. Guard every docker call with `|| true` so the trap
 # never overrides the test's exit code.
 cleanup() {
    docker image rm -f "$IMAGE_NAME" || true
    prune_if_disk_pressure
 }
 trap cleanup EXIT
 # Free space up front so a nearly-full host doesn't fail the build.
 prune_if_disk_pressure
 # building the docker image
 echo "--- :docker: Building Docker image"
 docker build --progress plain --tag "$IMAGE_NAME" --target vllm-test -f docker/Dockerfile.cpu .
 # Run the image, setting --shm-size=4g for tensor parallel.
-docker run --rm --cpuset-cpus="$CORE_RANGE" --cpuset-mems="$NUMA_NODE" -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true -e HF_TOKEN -e VLLM_CPU_KVCACHE_SPACE=16 -e VLLM_CPU_CI_ENV=1 -e VLLM_CPU_SIM_MULTI_NUMA=1 -e VLLM_CPU_ATTN_SPLIT_KV=0 --shm-size=4g "$IMAGE_NAME" \
+docker run --rm --cpuset-cpus="$CORE_RANGE" --cpuset-mems="$NUMA_NODE" -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true -e HF_TOKEN -e VLLM_CPU_KVCACHE_SPACE=16 -e VLLM_CPU_CI_ENV=1 -e VLLM_CPU_SIM_MULTI_NUMA=1 --shm-size=4g "$IMAGE_NAME" \
        timeout "$TIMEOUT_VAL" bash -c "set -euox pipefail; echo \"--- Print packages\"; pip list; echo \"--- Running tests\"; ${TEST_COMMAND}"
@@ -25,5 +25,5 @@ remove_docker_container
 # Run the image and test offline inference
 docker run -e HF_TOKEN -e VLLM_WORKER_MULTIPROC_METHOD=spawn -v /root/.cache/huggingface:/root/.cache/huggingface --name gh200-test --gpus=all --entrypoint="" gh200-test bash -c '
-    python3 examples/basic/offline_inference/generate.py --model meta-llama/Llama-3.2-1B
+    python3 examples/offline_inference/basic/generate.py --model meta-llama/Llama-3.2-1B
 '
@@ -42,7 +42,7 @@ WORKDIR /workspace/vllm
 ENV no_proxy=localhost,127.0.0.1
 ENV PT_HPU_ENABLE_LAZY_COLLECTIVES=true
-RUN bash -c 'pip install -r <(sed "/^torch/d" requirements/build/cuda.txt)'
+RUN bash -c 'pip install -r <(sed "/^torch/d" requirements/build.txt)'
 RUN VLLM_TARGET_DEVICE=empty pip install --no-build-isolation -e .
 RUN pip install git+https://github.com/vllm-project/vllm-gaudi.git
@@ -76,7 +76,7 @@ docker run --rm --runtime=habana --name="${container_name}" --network=host \
  -e PT_HPU_LAZY_MODE=1 \
  "${image_name}" \
  /bin/bash -c '
-  cd vllm; timeout 120s python -u examples/basic/offline_inference/generate.py --model facebook/opt-125m
+  cd vllm; timeout 120s python -u examples/offline_inference/basic/generate.py --model facebook/opt-125m
 '
 EXITCODE=$?
@@ -1,53 +0,0 @@
 #!/bin/bash
 set -euo pipefail
 test_suite="${1:-}"
 if [[ -z "${test_suite}" ]]; then
  echo "Usage: $0 <example|v1|server>" >&2
  exit 1
 fi
 case "${test_suite}" in
  example)
    pip install tblib==3.1.0
    python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager
    python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 -O3 -cc.cudagraph_mode=NONE
    python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager -tp 2 --distributed-executor-backend mp
    python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --attention-backend=TRITON_ATTN
    python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --quantization fp8
    python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --kv-cache-dtype fp8
    python3 examples/basic/offline_inference/generate.py --model nvidia/Llama-3.1-8B-Instruct-FP8 --block-size 64 --enforce-eager --quantization modelopt --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --max-model-len 4096
    python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --block-size 64 --enforce-eager --max-model-len 8192
    python3 examples/basic/offline_inference/generate.py --model TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ --block-size 64 --enforce-eager 
    python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2
    python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 --enable-expert-parallel
    python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --max-model-len 8192
    ;;
  v1)
    cd tests
    pytest -v -s v1/core --ignore=v1/core/test_reset_prefix_cache_e2e.py --ignore=v1/core/test_scheduler_e2e.py
    pytest -v -s v1/engine --ignore=v1/engine/test_output_processor.py
    pytest -v -s v1/sample --ignore=v1/sample/test_logprobs.py --ignore=v1/sample/test_logprobs_e2e.py -k "not test_topk_only and not test_topp_only and not test_topk_and_topp"
    pytest -v -s v1/worker --ignore=v1/worker/test_gpu_model_runner.py --ignore=v1/worker/test_worker_memory_snapshot.py
    pytest -v -s v1/structured_output
    pytest -v -s v1/test_serial_utils.py
    pytest -v -s v1/e2e/general/test_correctness_sliding_window.py --deselect="tests/v1/e2e/general/test_correctness_sliding_window.py::test_sliding_window_retrieval[True-1-5-google/gemma-3-1b-it]"
    pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py --ignore=v1/spec_decode/test_speculators_correctness.py
    pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_multi_connector.py --ignore=v1/kv_connector/unit/test_example_connector.py --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py --ignore=v1/kv_connector/unit/test_offloading_connector.py
    ;;
  server)
    pip install av
    cd tests
    pytest -v -s entrypoints/multimodal/openai/chat_completion/test_audio_in_video.py
    pytest -v -s benchmarks/test_serve_cli.py
    ;;
  *)
    echo "Unknown Intel test suite: ${test_suite}" >&2
    exit 1
    ;;
 esac
@@ -1,376 +0,0 @@
 #!/bin/bash
 # This script runs tests inside the Intel XPU docker container.
 # It mirrors the structure of run-amd-test.sh while keeping Intel-specific
 # container setup and allowing commands to be sourced from YAML or env.
 #
 # Command sources (in priority order):
 #   1) VLLM_TEST_COMMANDS env var (preferred, preserves quoting)
 #   2) Positional args (legacy)
 #   3) One or more YAML files with a commands list (test-area style)
 ###############################################################################
 set -o pipefail
 DRY_RUN=${DRY_RUN:-0}
 if [[ "${1:-}" == "--dry-run" ]]; then
  DRY_RUN=1
  shift
 fi
 # Export Python path
 export PYTHONPATH=".."
 ###############################################################################
 # Helper Functions
 ###############################################################################
 cleanup_docker() {
  # Share the same lock with image pull to avoid cleanup/pull races on one node.
  local docker_lock="/tmp/docker-pull.lock"
  exec 9>"$docker_lock"
  flock 9
  docker_root=$(docker info -f '{{.DockerRootDir}}')
  if [ -z "$docker_root" ]; then
    echo "Failed to determine Docker root directory." >&2
    flock -u 9
    return 1
  fi
  echo "Docker root directory: $docker_root"
  disk_usage=$(df "$docker_root" | tail -1 | awk '{print $5}' | sed 's/%//')
  threshold=70
  if [ "$disk_usage" -gt "$threshold" ]; then
    echo "Disk usage is above $threshold%. Running aggressive CI image cleanup..."
    cleanup_old_ci_images "${REGISTRY}/${REPO}" "${image_name}" "${DOCKER_IMAGE_CLEANUP_HOURS:-72}" 1
  else
    echo "Disk usage is below $threshold%. Checking old CI images anyway."
    cleanup_old_ci_images "${REGISTRY}/${REPO}" "${image_name}" "${DOCKER_IMAGE_CLEANUP_HOURS:-72}" 0
  fi
  echo "Old CI image cleanup completed."
  flock -u 9
 }
 cleanup_old_ci_images() {
  local repo_prefix="$1"
  local current_image_ref="$2"
  local ttl_hours="$3"
  local aggressive_cleanup="$4"
  if [[ -z "$repo_prefix" || "$repo_prefix" == "/" ]]; then
    echo "Skip old-image cleanup: invalid repo prefix '${repo_prefix}'"
    return 0
  fi
  if ! [[ "$ttl_hours" =~ ^[0-9]+$ ]]; then
    echo "Invalid DOCKER_IMAGE_CLEANUP_HOURS='${ttl_hours}', fallback to 72"
    ttl_hours=72
  fi
  local now_epoch cutoff_epoch
  now_epoch=$(date +%s)
  cutoff_epoch=$((now_epoch - ttl_hours * 3600))
  local -a used_image_ids
  mapfile -t used_image_ids < <(docker ps -aq | xargs -r docker inspect --format '{{.Image}}' | sort -u)
  local removed_count=0
  local examined_count=0
  declare -A seen_ids=()
  while read -r image_ref image_id; do
    [[ -z "$image_ref" || -z "$image_id" ]] && continue
    ((examined_count++))
    # Keep the image this job is going to use.
    if [[ "$image_ref" == "$current_image_ref" ]]; then
      continue
    fi
    # Avoid duplicate deletes when multiple tags point to same image id.
    if [[ -n "${seen_ids[$image_id]:-}" ]]; then
      continue
    fi
    seen_ids[$image_id]=1
    # Never delete images that are used by any container on this node.
    if printf '%s\n' "${used_image_ids[@]}" | grep -qx "$image_id"; then
      continue
    fi
    local created created_epoch
    created=$(docker image inspect -f '{{.Created}}' "$image_id" 2>/dev/null || true)
    [[ -z "$created" ]] && continue
    created_epoch=$(date -d "$created" +%s 2>/dev/null || true)
    [[ -z "$created_epoch" ]] && continue
    if (( created_epoch < cutoff_epoch )) || [[ "$aggressive_cleanup" == "1" ]]; then
      if docker image rm -f "$image_id" >/dev/null 2>&1; then
        ((removed_count++))
      fi
    fi
  done < <(docker image ls --no-trunc "$repo_prefix" --format '{{.Repository}}:{{.Tag}} {{.ID}}')
  # Also trim old dangling layers; this is safe and does not remove referenced images.
  docker image prune -f --filter "until=${ttl_hours}h" >/dev/null 2>&1 || true
  if [[ "$aggressive_cleanup" == "1" ]]; then
    echo "Examined ${examined_count} images under ${repo_prefix}, removed ${removed_count} unused images under disk pressure."
  else
    echo "Examined ${examined_count} images under ${repo_prefix}, removed ${removed_count} old images (>${ttl_hours}h)."
  fi
 }
 re_quote_pytest_markers() {
  local input="$1"
  local output=""
  local collecting=false
  local marker_buf=""
  local flat="${input//$'\n'/ }"
  local restore_glob
  restore_glob="$(shopt -p -o noglob 2>/dev/null || true)"
  set -o noglob
  local -a words
  read -ra words <<< "$flat"
  eval "$restore_glob"
  for word in "${words[@]}"; do
    if $collecting; then
      if [[ "$word" == *"'"* ]]; then
        if [[ -n "$marker_buf" ]]; then
          output+="${marker_buf} "
          marker_buf=""
        fi
        output+="${word} "
        collecting=false
        continue
      fi
      local is_boundary=false
      case "$word" in
        "&&"|"||"|";"|"|")
          is_boundary=true ;;
        --*)
          is_boundary=true ;;
        -[a-zA-Z])
          is_boundary=true ;;
        */*)
          is_boundary=true ;;
        *.py|*.py::*)
          is_boundary=true ;;
        *=*)
          if [[ "$word" =~ ^[A-Z_][A-Z0-9_]*= ]]; then
            is_boundary=true
          fi
          ;;
      esac
      if $is_boundary; then
        if [[ "$marker_buf" == *" "* || "$marker_buf" == *"("* ]]; then
          output+="'${marker_buf}' "
        else
          output+="${marker_buf} "
        fi
        collecting=false
        marker_buf=""
        if [[ "$word" == "-m" || "$word" == "-k" ]]; then
          output+="${word} "
          collecting=true
        else
          output+="${word} "
        fi
      else
        if [[ -n "$marker_buf" ]]; then
          marker_buf+=" ${word}"
        else
          marker_buf="${word}"
        fi
      fi
    elif [[ "$word" == "-m" || "$word" == "-k" ]]; then
      output+="${word} "
      collecting=true
      marker_buf=""
    else
      output+="${word} "
    fi
  done
  if $collecting && [[ -n "$marker_buf" ]]; then
    if [[ "$marker_buf" == *" "* || "$marker_buf" == *"("* ]]; then
      output+="'${marker_buf}'"
    else
      output+="${marker_buf}"
    fi
  fi
  echo "${output% }"
 }
 apply_intel_test_overrides() {
  local cmds="$1"
  # Placeholder for Intel-specific exclusions/overrides.
  echo "$cmds"
 }
 is_yaml_file() {
  local p="$1"
  [[ -f "$p" && "$p" == *.yaml ]]
 }
 extract_yaml_commands() {
  local yaml_path="$1"
  awk '
    $1 == "commands:" { in_cmds=1; next }
    in_cmds && $0 ~ /^[[:space:]]*-[[:space:]]/ {
      sub(/^[[:space:]]*-[[:space:]]/, "");
      print;
      next
    }
    in_cmds && $0 ~ /^[^[:space:]]/ { exit }
  ' "$yaml_path"
 }
 ###############################################################################
 # Main
 ###############################################################################
 default_image_name="${REGISTRY}/${REPO}:${BUILDKITE_COMMIT}-xpu"
 #default_image_name="public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:${BUILDKITE_COMMIT}-xpu"
 image_name="${IMAGE_TAG_XPU:-${default_image_name}}"
 container_name="xpu_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10; echo)"
 # ---- Command source selection ----
 commands=""
 commands_source=""
 if [[ -n "${VLLM_TEST_COMMANDS:-}" ]]; then
  commands="${VLLM_TEST_COMMANDS}"
  commands_source="env"
  echo "Commands sourced from VLLM_TEST_COMMANDS (quoting preserved)"
 elif [[ $# -gt 0 ]]; then
  all_yaml=true
  for arg in "$@"; do
    if ! is_yaml_file "$arg"; then
      all_yaml=false
      break
    fi
  done
  if $all_yaml; then
    for yaml in "$@"; do
      mapfile -t COMMANDS < <(extract_yaml_commands "$yaml")
      if [[ ${#COMMANDS[@]} -eq 0 ]]; then
        echo "Error: No commands found in ${yaml}" >&2
        exit 1
      fi
      for cmd in "${COMMANDS[@]}"; do
        if [[ -z "$commands" ]]; then
          commands="${cmd}"
        else
          commands+=" && ${cmd}"
        fi
      done
    done
    echo "Commands sourced from YAML files: $*"
  else
    commands="$*"
    echo "Commands sourced from positional args (legacy mode)"
  fi
 else
  SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
  DEFAULT_YAML="${SCRIPT_DIR}/intel-test.yaml"
  if [[ ! -f "${DEFAULT_YAML}" ]]; then
    echo "Error: YAML file not found: ${DEFAULT_YAML}" >&2
    exit 1
  fi
  mapfile -t COMMANDS < <(extract_yaml_commands "${DEFAULT_YAML}")
  if [[ ${#COMMANDS[@]} -eq 0 ]]; then
    echo "Error: No commands found in ${DEFAULT_YAML}" >&2
    exit 1
  fi
  for cmd in "${COMMANDS[@]}"; do
    if [[ -z "$commands" ]]; then
      commands="${cmd}"
    else
      commands+=" && ${cmd}"
    fi
  done
  echo "Commands sourced from default YAML: ${DEFAULT_YAML}"
 fi
 if [[ -z "$commands" ]]; then
  echo "Error: No test commands provided." >&2
  exit 1
 fi
 echo "Raw commands: $commands"
 if [[ "$commands_source" != "env" ]]; then
  commands=$(re_quote_pytest_markers "$commands")
  echo "After re-quoting: $commands"
 else
  echo "Skipping re-quoting for VLLM_TEST_COMMANDS input"
 fi
 commands=$(apply_intel_test_overrides "$commands")
 echo "Final commands: $commands"
 # Dry-run mode prints final commands and exits before Docker.
 if [[ "$DRY_RUN" == "1" ]]; then
  echo "DRY_RUN=1 set, skipping Docker execution."
  exit 0
 fi
 # --- Docker housekeeping ---
 cleanup_docker
 aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY"
 # --- Build or pull test image ---
 IMAGE="${IMAGE_TAG_XPU:-${image_name}}"
 echo "Using image: ${IMAGE}"
 remove_docker_container() {
  docker rm -f "${container_name}" || true
 }
 trap remove_docker_container EXIT
 # --- Single-node job ---
 if [[ -z "${ZE_AFFINITY_MASK:-}" ]]; then
  echo "Warning: ZE_AFFINITY_MASK is not set. Proceeding without device affinity." >&2
 fi
 export CMDS="${commands}"
 export HF_TOKEN ZE_AFFINITY_MASK
 {
  flock 9
  if docker image inspect "${IMAGE}" >/dev/null 2>&1; then
    echo "Image already exists locally, skipping pull"
  else
    echo "Image not found locally, pulling image..."
    timeout 900 docker pull "${IMAGE}"
    echo "Pull step completed"
  fi
  docker create \
    --device /dev/dri:/dev/dri \
    --net=host \
    --ipc=host \
    --privileged \
    -v /dev/dri/by-path:/dev/dri/by-path \
    -v "/data/huggingface:/root/.cache/huggingface" \
    --entrypoint='' \
    -e HF_TOKEN \
    -e ZE_AFFINITY_MASK \
    -e BUILDKITE_PARALLEL_JOB \
    -e BUILDKITE_PARALLEL_JOB_COUNT \
    -e CMDS \
    --name "${container_name}" \
    "${IMAGE}" \
    bash -c 'set -e; source /opt/intel/oneapi/setvars.sh --force; source /opt/intel/oneapi/ccl/2021.15/env/vars.sh --force; echo "ZE_AFFINITY_MASK is ${ZE_AFFINITY_MASK:-}"; eval "$CMDS"' \
    >/dev/null
 } 9>/tmp/docker-pull.lock
 docker start -a "${container_name}"
@@ -85,7 +85,7 @@ RUN pip config set global.index-url http://cache-service-vllm.nginx-pypi-cache.s
 # Install for pytest to make the docker build cache layer always valid
 RUN --mount=type=cache,target=/root/.cache/pip \
-    pip install pytest>=6.0  'modelscope<1.38'
+    pip install pytest>=6.0  modelscope
 WORKDIR /workspace/vllm
@@ -61,7 +61,7 @@ echo "Results will be stored in: $RESULTS_DIR"
 echo "--- Installing Python dependencies ---"
 python3 -m pip install --progress-bar off git+https://github.com/thuml/depyf.git \
    && python3 -m pip install --progress-bar off pytest pytest-asyncio tpu-info \
-    && python3 -m pip install --progress-bar off "lm-eval[api]>=0.4.12" \
+    && python3 -m pip install --progress-bar off "lm-eval[api]>=0.4.11" \
    && python3 -m pip install --progress-bar off hf-transfer tblib==3.1.0
 echo "--- Python dependencies installed ---"
@@ -127,7 +127,7 @@ run_and_track_test() {
 # --- Actual Test Execution ---
 run_and_track_test 1 "test_struct_output_generate.py" \
-    "python3 -m pytest -s -v /workspace/vllm/tests/entrypoints/llm/test_struct_output_generate.py -k \"not test_structured_output_with_reasoning_matrices\""
+    "python3 -m pytest -s -v /workspace/vllm/tests/v1/entrypoints/llm/test_struct_output_generate.py -k \"not test_structured_output_with_reasoning_matrices\""
 run_and_track_test 2 "test_moe_pallas.py" \
    "python3 -m pytest -s -v /workspace/vllm/tests/tpu/test_moe_pallas.py"
 run_and_track_test 3 "test_lora.py" \
@@ -61,7 +61,7 @@ echo "Results will be stored in: $RESULTS_DIR"
 echo "--- Installing Python dependencies ---"
 python3 -m pip install --progress-bar off git+https://github.com/thuml/depyf.git \
    && python3 -m pip install --progress-bar off pytest pytest-asyncio tpu-info \
-    && python3 -m pip install --progress-bar off "lm-eval[api]>=0.4.12" \
+    && python3 -m pip install --progress-bar off "lm-eval[api]>=0.4.11" \
    && python3 -m pip install --progress-bar off hf-transfer tblib==3.1.0
 echo "--- Python dependencies installed ---"
@@ -136,6 +136,8 @@ run_and_track_test 3 "test_accuracy.py::test_lm_eval_accuracy_v1_engine" \
    "python3 -m pytest -s -v /workspace/vllm/tests/entrypoints/llm/test_accuracy.py::test_lm_eval_accuracy_v1_engine"
 run_and_track_test 4 "test_quantization_accuracy.py" \
    "python3 -m pytest -s -v /workspace/vllm/tests/tpu/test_quantization_accuracy.py"
 run_and_track_test 5 "examples/offline_inference/tpu.py" \
    "python3 /workspace/vllm/examples/offline_inference/tpu.py"
 run_and_track_test 6 "test_tpu_model_runner.py" \
    "python3 -m pytest -s -v /workspace/vllm/tests/v1/tpu/worker/test_tpu_model_runner.py"
 run_and_track_test 7 "test_sampler.py" \
@@ -0,0 +1,55 @@
 #!/bin/bash
 # This script build the CPU docker image and run the offline inference inside the container.
 # It serves a sanity check for compilation and basic model usage.
 set -ex
 image_name="xpu/vllm-ci:${BUILDKITE_COMMIT}"
 container_name="xpu_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10; echo)"
 # Try building the docker image
 docker build -t "${image_name}" -f docker/Dockerfile.xpu .
 # Setup cleanup
 remove_docker_container() {
  docker rm -f "${container_name}" || true;
  docker image rm -f "${image_name}" || true;
  docker system prune -f || true;
 }
 trap remove_docker_container EXIT
 # Run the image and test offline inference/tensor parallel
 docker run \
    --device /dev/dri:/dev/dri \
    --net=host \
    --ipc=host \
    --privileged \
    -v /dev/dri/by-path:/dev/dri/by-path \
    --entrypoint="" \
    -e "HF_TOKEN=${HF_TOKEN}" \
    -e "ZE_AFFINITY_MASK=${ZE_AFFINITY_MASK}" \
    --name "${container_name}" \
    "${image_name}" \
    bash -c '
    set -e
    echo $ZE_AFFINITY_MASK
    pip install tblib==3.1.0
    python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager
    python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m --block-size 64 -O3 -cc.cudagraph_mode=NONE
    python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager -tp 2 --distributed-executor-backend ray
    python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager -tp 2 --distributed-executor-backend mp
    python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --attention-backend=TRITON_ATTN
    python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --quantization fp8
    python3 examples/offline_inference/basic/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4  --block-size 64 --enforce-eager
    python3 examples/offline_inference/basic/generate.py --model ibm-research/PowerMoE-3b  --block-size 64 --enforce-eager -tp 2
    python3 examples/offline_inference/basic/generate.py --model ibm-research/PowerMoE-3b  --block-size 64 --enforce-eager -tp 2 --enable-expert-parallel
    cd tests
    pytest -v -s v1/core --ignore=v1/core/test_reset_prefix_cache_e2e.py --ignore=v1/core/test_scheduler_e2e.py
    pytest -v -s v1/engine
    pytest -v -s v1/sample --ignore=v1/sample/test_logprobs.py --ignore=v1/sample/test_logprobs_e2e.py
    pytest -v -s v1/worker --ignore=v1/worker/test_gpu_model_runner.py
    pytest -v -s v1/structured_output
    pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_tree_attention.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py
    pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_multi_connector.py --ignore=v1/kv_connector/unit/test_nixl_connector.py --ignore=v1/kv_connector/unit/test_example_connector.py --ignore=v1/kv_connector/unit/test_lmcache_integration.py
    pytest -v -s v1/test_serial_utils.py
 '
@@ -1,44 +0,0 @@
 #!/bin/bash
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 set -euo pipefail
 if python3 -c "import torch; raise SystemExit(0 if torch.version.hip is not None else 1)"; then
    uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
    exit 0
 fi
 REQUIREMENTS_FILE="${KV_CONNECTORS_REQUIREMENTS:-/vllm-workspace/requirements/kv_connectors.txt}"
 uv pip install --system -r "${REQUIREMENTS_FILE}"
 NIXL_METADATA=$(python3 - <<'PY'
 import importlib.metadata as metadata
 import torch
 cuda_version = torch.version.cuda
 if cuda_version is None:
    raise SystemExit("torch.version.cuda is not set")
 print(cuda_version.split(".", 1)[0], metadata.version("nixl"))
 PY
 )
 read -r CUDA_MAJOR NIXL_VERSION <<<"${NIXL_METADATA}"
 # nixl>=1.1.0 can install multiple CUDA wheel variants. Keep only the variant
 # matching this CI image so nixl_ep_cpp links against the available libcudart.
 uv pip uninstall --system nixl-cu12 nixl-cu13 2>/dev/null || true
 uv pip install --system --no-deps "nixl-cu${CUDA_MAJOR}==${NIXL_VERSION}"
 python3 - <<'PY'
 import importlib.metadata as metadata
 for package_name in ("nixl", "nixl-cu12", "nixl-cu13"):
    try:
        version = metadata.version(package_name)
    except metadata.PackageNotFoundError:
        version = "not installed"
    print(f"{package_name}: {version}")
 PY
@@ -1,127 +0,0 @@
 #!/usr/bin/env bash
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 #
 # Shared helper for rewriting a wheel's platform tag from the generic
 # ``linux_<arch>`` to the correct ``manylinux_<major>_<minor>_<arch>``.
 # After sourcing, call ``apply_manylinux_tag <wheel>`` on each wheel
 # that still carries the generic tag; the renamed path is printed on
 # stdout (logs go to stderr).
 #
 # Why a pinned Docker container instead of using whatever Python
 # happens to be on the agent:
 #   - vLLM's release agents are heterogeneous -- they don't agree on
 #     a Python minor version, and we can't rely on a particular
 #     ``auditwheel`` being installed.
 #   - ``detect-manylinux-tag.py`` reads ``auditwheel.wheel_abi`` and
 #     ``Policy.sym_policy``, which are *internal* APIs without a
 #     stability promise. Pinning both Python and auditwheel makes the
 #     detected tag a function of the inputs alone, and shifts version
 #     bumps from "implicit drift" to "deliberate, retested change".
 #   - Other release scripts (``generate-and-upload-nightly-index.sh``,
 #     ``upload-rocm-wheels.sh``) already use the python:3-slim image
 #     when the agent's interpreter is too old; this is the same idea
 #     made stricter.
 #
 # To keep the per-wheel cost down (the ROCm upload retags ~10 wheels
 # each run), we install auditwheel into a long-lived helper container
 # once on source, then ``docker exec`` into it for each call.
 #
 # Trap behaviour:
 # - Sourcing installs an EXIT trap that calls ``manylinux_cleanup`` to
 #   tear down the helper container. Any EXIT trap that was already in
 #   place when this file was sourced is captured and run AFTER our
 #   cleanup, so we don't silently clobber it.
 # - If a caller sets a new EXIT trap *after* sourcing, that trap will
 #   replace ours; in that case the caller should call
 #   ``manylinux_cleanup`` from their own handler.
 if [[ -n "${_MANYLINUX_LIB_SOURCED:-}" ]]; then
    return 0
 fi
 _MANYLINUX_LIB_SOURCED=1
 # Pin both sides. Bump these deliberately and re-run a representative
 # wheel from each build target through the detection.
 _MANYLINUX_PYTHON_IMAGE="python:3.12-slim"
 _MANYLINUX_AUDITWHEEL_VERSION="6.6.0"
 # Resolve our own directory (and the sibling detect script) using the
 # canonical, symlink-resolved path. The container mounts cwd at the
 # same absolute path on both sides, so all paths we hand to it -- the
 # script, the wheel -- must canonicalise to a location under cwd.
 _MANYLINUX_LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
 _MANYLINUX_DETECT_SCRIPT="$(cd "${_MANYLINUX_LIB_DIR}/.." && pwd -P)/detect-manylinux-tag.py"
 _MANYLINUX_CWD="$(pwd -P)"
 docker pull --quiet "$_MANYLINUX_PYTHON_IMAGE" >/dev/null
 # Spin up a long-lived helper container so we install auditwheel once
 # and then ``docker exec`` into it for each wheel.
 #
 # The container runs as root so ``pip install`` can write into the
 # system site-packages; individual ``docker exec`` calls below pin
 # themselves to the host UID so any file rename happens with host
 # ownership, not root.
 _MANYLINUX_CONTAINER="$(docker run -d --rm \
    -v "$_MANYLINUX_CWD:$_MANYLINUX_CWD" \
    -w "$_MANYLINUX_CWD" \
    "$_MANYLINUX_PYTHON_IMAGE" \
    sleep infinity)"
 docker exec "$_MANYLINUX_CONTAINER" \
    pip install --quiet --disable-pip-version-check \
    --root-user-action=ignore \
    "auditwheel==${_MANYLINUX_AUDITWHEEL_VERSION}"
 # Public cleanup -- safe to call multiple times.
 manylinux_cleanup() {
    if [[ -n "${_MANYLINUX_CONTAINER:-}" ]]; then
        docker rm -f "$_MANYLINUX_CONTAINER" >/dev/null 2>&1 || true
        _MANYLINUX_CONTAINER=""
    fi
 }
 # Capture any EXIT trap that was already in place so we can chain to
 # it rather than overwrite it. ``trap -p EXIT`` prints the handler in
 # eval-able form (``trap -- 'CMD' EXIT``) or nothing if unset; we
 # strip the wrapper to recover ``CMD``. Handles the common case --
 # CMDs without embedded single quotes -- and degrades gracefully (we
 # still run our own cleanup) for the pathological case.
 _manylinux_prev_exit_trap_cmd=""
 _manylinux_existing_exit_trap="$(trap -p EXIT)"
 if [[ -n "$_manylinux_existing_exit_trap" ]]; then
    _tmp="${_manylinux_existing_exit_trap#trap -- \'}"
    _manylinux_prev_exit_trap_cmd="${_tmp%\' EXIT}"
    unset _tmp
 fi
 unset _manylinux_existing_exit_trap
 _manylinux_run_exit_chain() {
    manylinux_cleanup
    if [[ -n "$_manylinux_prev_exit_trap_cmd" ]]; then
        eval "$_manylinux_prev_exit_trap_cmd"
    fi
 }
 trap _manylinux_run_exit_chain EXIT
 # Detect the manylinux platform tag for a single wheel and rename it
 # in place, printing the renamed wheel path on stdout. Returns
 # non-zero on failure (which under ``set -e`` propagates to caller).
 #
 # The wheel must be reachable via a path under the host cwd so it's
 # visible inside the helper container; in CI the wheels always live
 # under ``artifacts/`` so this is fine.
 apply_manylinux_tag() {
    local wheel="$1"
    local abs_wheel
    abs_wheel="$(realpath "$wheel")"
    local new_wheel
    new_wheel="$(docker exec -u "$(id -u):$(id -g)" \
        "$_MANYLINUX_CONTAINER" \
        python "$_MANYLINUX_DETECT_SCRIPT" "$abs_wheel")"
    if [[ -z "$new_wheel" || ! -f "$new_wheel" ]]; then
        echo "apply_manylinux_tag: detect-manylinux-tag.py did not produce a valid wheel path for $wheel" >&2
        return 1
    fi
    printf '%s\n' "$new_wheel"
 }
@@ -1,41 +0,0 @@
 #!/usr/bin/env bash
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 #
 # Pick a Python interpreter for buildkite scripts: prefer a local
 # ``python3`` if it is recent enough (>= 3.12), otherwise fall back to
 # a one-shot Docker container running ``python:3-slim``. After
 # ``select_python`` returns, ``$PYTHON`` is set in the caller's shell
 # and is safe to use as a command (e.g. ``$PYTHON some_script.py``).
 #
 # The 3.12 threshold matches what the existing nightly-index work
 # expects -- typing features used by ``generate-nightly-index.py``.
 # This helper does not pin the *minor* version; if you need stricter
 # reproducibility (e.g. relying on auditwheel internals), invoke
 # Docker yourself with a pinned tag rather than calling this.
 if [[ -n "${_SELECT_PYTHON_LIB_SOURCED:-}" ]]; then
    return 0
 fi
 _SELECT_PYTHON_LIB_SOURCED=1
 # Sets ``PYTHON`` in the caller's shell and exports it. Idempotent --
 # calling twice is safe and the second call simply re-runs the probe.
 select_python() {
    local py="${PYTHON_PROG:-python3}"
    local has_new_python
    has_new_python=$("$py" -c \
        "print(1 if __import__('sys').version_info >= (3,12) else 0)" \
        2>/dev/null || echo 0)
    if [[ "$has_new_python" -eq 0 ]]; then
        # ``-u $(id -u):$(id -g)`` so files created via the container
        # end up owned by the host user, not root.
        docker pull python:3-slim
        PYTHON="docker run --rm -u $(id -u):$(id -g) -v $(pwd):/app -w /app python:3-slim python3"
    else
        PYTHON="$py"
    fi
    export PYTHON
    echo "Using python interpreter: $PYTHON"
    echo "Python version: $($PYTHON --version)"
 }
@@ -1,196 +0,0 @@
 #!/bin/bash
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 #
 # Publish release Docker images from ECR to DockerHub.
 # Pulls per-arch images, tags with latest and versioned tags, pushes them,
 # then creates and pushes multi-arch manifests.
 set -euo pipefail
 RELEASE_VERSION=$(buildkite-agent meta-data get release-version --default "" | sed 's/^v//')
 if [ -z "${RELEASE_VERSION}" ]; then
  echo "ERROR: release-version metadata not set"
  exit 1
 fi
 COMMIT="$BUILDKITE_COMMIT"
 ROCM_BASE_CACHE_KEY=$(.buildkite/scripts/cache-rocm-base-wheels.sh key)
 echo "========================================"
 echo "Publishing release images v${RELEASE_VERSION}"
 echo "  Commit: ${COMMIT}"
 echo "  ROCm base cache key: ${ROCM_BASE_CACHE_KEY}"
 echo "========================================"
 # Login to ECR to pull staging images
 aws ecr-public get-login-password --region us-east-1 | \
  docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
 # ---- CUDA (default: 13.0) ----
 docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64
 docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64 vllm/vllm-openai:latest-x86_64
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64
 docker push vllm/vllm-openai:latest-x86_64
 docker push vllm/vllm-openai:v${RELEASE_VERSION}-x86_64
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64 vllm/vllm-openai:latest-aarch64
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64
 docker push vllm/vllm-openai:latest-aarch64
 docker push vllm/vllm-openai:v${RELEASE_VERSION}-aarch64
 docker manifest rm vllm/vllm-openai:latest || true
 docker manifest rm vllm/vllm-openai:v${RELEASE_VERSION} || true
 docker manifest create vllm/vllm-openai:latest vllm/vllm-openai:latest-x86_64 vllm/vllm-openai:latest-aarch64
 docker manifest create vllm/vllm-openai:v${RELEASE_VERSION} vllm/vllm-openai:v${RELEASE_VERSION}-x86_64 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64
 docker manifest push vllm/vllm-openai:latest
 docker manifest push vllm/vllm-openai:v${RELEASE_VERSION}
 # ---- CUDA 12.9 ----
 docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-cu129
 docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64-cu129
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-cu129 vllm/vllm-openai:latest-x86_64-cu129
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-cu129 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu129
 docker push vllm/vllm-openai:latest-x86_64-cu129
 docker push vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu129
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64-cu129 vllm/vllm-openai:latest-aarch64-cu129
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64-cu129 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu129
 docker push vllm/vllm-openai:latest-aarch64-cu129
 docker push vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu129
 docker manifest rm vllm/vllm-openai:latest-cu129 || true
 docker manifest rm vllm/vllm-openai:v${RELEASE_VERSION}-cu129 || true
 docker manifest create vllm/vllm-openai:latest-cu129 vllm/vllm-openai:latest-x86_64-cu129 vllm/vllm-openai:latest-aarch64-cu129
 docker manifest create vllm/vllm-openai:v${RELEASE_VERSION}-cu129 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu129 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu129
 docker manifest push vllm/vllm-openai:latest-cu129
 docker manifest push vllm/vllm-openai:v${RELEASE_VERSION}-cu129
 # ---- Ubuntu 24.04 (CUDA 13.0) ----
 docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-ubuntu2404
 docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64-ubuntu2404
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-ubuntu2404 vllm/vllm-openai:latest-x86_64-ubuntu2404
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-ubuntu2404 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-ubuntu2404
 docker push vllm/vllm-openai:latest-x86_64-ubuntu2404
 docker push vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-ubuntu2404
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64-ubuntu2404 vllm/vllm-openai:latest-aarch64-ubuntu2404
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64-ubuntu2404 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-ubuntu2404
 docker push vllm/vllm-openai:latest-aarch64-ubuntu2404
 docker push vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-ubuntu2404
 docker manifest rm vllm/vllm-openai:latest-ubuntu2404 || true
 docker manifest rm vllm/vllm-openai:v${RELEASE_VERSION}-ubuntu2404 || true
 docker manifest create vllm/vllm-openai:latest-ubuntu2404 vllm/vllm-openai:latest-x86_64-ubuntu2404 vllm/vllm-openai:latest-aarch64-ubuntu2404
 docker manifest create vllm/vllm-openai:v${RELEASE_VERSION}-ubuntu2404 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-ubuntu2404 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-ubuntu2404
 docker manifest push vllm/vllm-openai:latest-ubuntu2404
 docker manifest push vllm/vllm-openai:v${RELEASE_VERSION}-ubuntu2404
 # ---- Ubuntu 24.04 (CUDA 12.9) ----
 docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-cu129-ubuntu2404
 docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64-cu129-ubuntu2404
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-cu129-ubuntu2404 vllm/vllm-openai:latest-x86_64-cu129-ubuntu2404
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-cu129-ubuntu2404 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu129-ubuntu2404
 docker push vllm/vllm-openai:latest-x86_64-cu129-ubuntu2404
 docker push vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu129-ubuntu2404
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64-cu129-ubuntu2404 vllm/vllm-openai:latest-aarch64-cu129-ubuntu2404
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64-cu129-ubuntu2404 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu129-ubuntu2404
 docker push vllm/vllm-openai:latest-aarch64-cu129-ubuntu2404
 docker push vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu129-ubuntu2404
 docker manifest rm vllm/vllm-openai:latest-cu129-ubuntu2404 || true
 docker manifest rm vllm/vllm-openai:v${RELEASE_VERSION}-cu129-ubuntu2404 || true
 docker manifest create vllm/vllm-openai:latest-cu129-ubuntu2404 vllm/vllm-openai:latest-x86_64-cu129-ubuntu2404 vllm/vllm-openai:latest-aarch64-cu129-ubuntu2404
 docker manifest create vllm/vllm-openai:v${RELEASE_VERSION}-cu129-ubuntu2404 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu129-ubuntu2404 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu129-ubuntu2404
 docker manifest push vllm/vllm-openai:latest-cu129-ubuntu2404
 docker manifest push vllm/vllm-openai:v${RELEASE_VERSION}-cu129-ubuntu2404
 # ---- ROCm ----
 docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-rocm
 docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${ROCM_BASE_CACHE_KEY}-rocm-base
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-rocm vllm/vllm-openai-rocm:latest
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-rocm vllm/vllm-openai-rocm:v${RELEASE_VERSION}
 docker push vllm/vllm-openai-rocm:latest
 docker push vllm/vllm-openai-rocm:v${RELEASE_VERSION}
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${ROCM_BASE_CACHE_KEY}-rocm-base vllm/vllm-openai-rocm:latest-base
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${ROCM_BASE_CACHE_KEY}-rocm-base vllm/vllm-openai-rocm:v${RELEASE_VERSION}-base
 docker push vllm/vllm-openai-rocm:latest-base
 docker push vllm/vllm-openai-rocm:v${RELEASE_VERSION}-base
 # ---- XPU ----
 docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-xpu
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-xpu vllm/vllm-openai-xpu:latest-x86_64
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-xpu vllm/vllm-openai-xpu:v${RELEASE_VERSION}-x86_64
 docker push vllm/vllm-openai-xpu:latest-x86_64
 docker push vllm/vllm-openai-xpu:v${RELEASE_VERSION}-x86_64
 docker manifest rm vllm/vllm-openai-xpu:latest || true
 docker manifest rm vllm/vllm-openai-xpu:v${RELEASE_VERSION} || true
 docker manifest create vllm/vllm-openai-xpu:latest vllm/vllm-openai-xpu:latest-x86_64 --amend
 docker manifest create vllm/vllm-openai-xpu:v${RELEASE_VERSION} vllm/vllm-openai-xpu:v${RELEASE_VERSION}-x86_64 --amend
 docker manifest push vllm/vllm-openai-xpu:latest
 docker manifest push vllm/vllm-openai-xpu:v${RELEASE_VERSION}
 # ---- CPU ----
 # CPU images are behind separate block steps and may not have been built.
 # All-or-nothing: inspect both arches first, then either publish everything
 # (per-arch + multi-arch manifest) or skip everything. Publishing only one
 # arch would leave `:latest-x86_64` pointing at the new release while the
 # `:latest` multi-arch manifest still resolves to the previous release.
 CPU_X86_TAG=public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:v${RELEASE_VERSION}
 CPU_ARM_TAG=public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:v${RELEASE_VERSION}
 CPU_X86_AVAILABLE=false
 CPU_ARM_AVAILABLE=false
 docker manifest inspect "${CPU_X86_TAG}" >/dev/null 2>&1 && CPU_X86_AVAILABLE=true
 docker manifest inspect "${CPU_ARM_TAG}" >/dev/null 2>&1 && CPU_ARM_AVAILABLE=true
 if [ "$CPU_X86_AVAILABLE" = "true" ] && [ "$CPU_ARM_AVAILABLE" = "true" ]; then
  docker pull "${CPU_X86_TAG}"
  docker tag "${CPU_X86_TAG}" vllm/vllm-openai-cpu:latest-x86_64
  docker tag "${CPU_X86_TAG}" vllm/vllm-openai-cpu:v${RELEASE_VERSION}-x86_64
  docker push vllm/vllm-openai-cpu:latest-x86_64
  docker push vllm/vllm-openai-cpu:v${RELEASE_VERSION}-x86_64
  docker pull "${CPU_ARM_TAG}"
  docker tag "${CPU_ARM_TAG}" vllm/vllm-openai-cpu:latest-arm64
  docker tag "${CPU_ARM_TAG}" vllm/vllm-openai-cpu:v${RELEASE_VERSION}-arm64
  docker push vllm/vllm-openai-cpu:latest-arm64
  docker push vllm/vllm-openai-cpu:v${RELEASE_VERSION}-arm64
  docker manifest rm vllm/vllm-openai-cpu:latest || true
  docker manifest rm vllm/vllm-openai-cpu:v${RELEASE_VERSION} || true
  docker manifest create vllm/vllm-openai-cpu:latest vllm/vllm-openai-cpu:latest-x86_64 vllm/vllm-openai-cpu:latest-arm64
  docker manifest create vllm/vllm-openai-cpu:v${RELEASE_VERSION} vllm/vllm-openai-cpu:v${RELEASE_VERSION}-x86_64 vllm/vllm-openai-cpu:v${RELEASE_VERSION}-arm64
  docker manifest push vllm/vllm-openai-cpu:latest
  docker manifest push vllm/vllm-openai-cpu:v${RELEASE_VERSION}
 elif [ "$CPU_X86_AVAILABLE" = "false" ] && [ "$CPU_ARM_AVAILABLE" = "false" ]; then
  echo "WARNING: Neither CPU image found in ECR, skipping CPU publish (ensure block-cpu-release-image-build and block-arm64-cpu-release-image-build were unblocked and the builds finished pushing)"
 else
  # Partial state: one arch built, the other did not. Fail loudly rather than
  # ship a Docker Hub state where `:latest-${arch}` and `:latest` (multi-arch)
  # disagree on which release they point at.
  echo "ERROR: Partial CPU build detected (x86_64=${CPU_X86_AVAILABLE}, arm64=${CPU_ARM_AVAILABLE})."
  echo "       Refusing to publish to avoid split-tag drift between per-arch and multi-arch tags."
  echo "       Re-run the missing CPU build and retry, or manually publish if a single-arch release is intended."
  exit 1
 fi
 echo ""
 echo "Successfully published release images for v${RELEASE_VERSION}"
@@ -1,62 +0,0 @@
 #!/bin/bash
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 #
 # Push ROCm nightly base image and nightly image from ECR 
 # to Docker Hub as vllm/vllm-openai-rocm:base-nightly and vllm/vllm-openai-rocm:nightly
 # and vllm/vllm-openai-rocm:base-nightly-<commit> and vllm/vllm-openai-rocm:nightly-<commit>.
 # Run when NIGHTLY=1 after build-rocm-release-image has pushed to ECR.
 #
 # Local testing (no push to Docker Hub):
 #   BUILDKITE_COMMIT=<commit-with-rocm-image-in-ecr> DRY_RUN=1 bash .buildkite/scripts/push-nightly-builds-rocm.sh
 # Requires: AWS CLI configured (for ECR public login), Docker. For full run: Docker Hub login.
 set -ex
 # Use BUILDKITE_COMMIT from env (required; set to a commit that has ROCm image in ECR for local test)
 BUILDKITE_COMMIT="${BUILDKITE_COMMIT:?Set BUILDKITE_COMMIT to the commit SHA that has the ROCm image in ECR (e.g. from a previous release pipeline run)}"
 DRY_RUN="${DRY_RUN:-0}"
 # Get the base image ECR tag (set by build-rocm-release-image pipeline step)
 BASE_ORIG_TAG="$(buildkite-agent meta-data get rocm-base-ecr-tag 2>/dev/null || echo "")"
 if [ -z "$BASE_ORIG_TAG" ]; then
  echo "WARNING: rocm-base-ecr-tag metadata not found, falling back to commit-based tag"
  BASE_ORIG_TAG="public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-rocm-base"
 fi
 ORIG_TAG="${BUILDKITE_COMMIT}-rocm"
 BASE_TAG_NAME="base-nightly"
 TAG_NAME="nightly"
 BASE_TAG_NAME_COMMIT="base-nightly-${BUILDKITE_COMMIT}"
 TAG_NAME_COMMIT="nightly-${BUILDKITE_COMMIT}"
 echo "Pushing ROCm base image from ECR: $BASE_ORIG_TAG"
 echo "Pushing ROCm release image from ECR tag: $ORIG_TAG to Docker Hub as $TAG_NAME and $TAG_NAME_COMMIT"
 [[ "$DRY_RUN" == "1" ]] && echo "[DRY_RUN] Skipping push to Docker Hub"
 # Login to ECR and pull the image built by build-rocm-release-image
 aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
 docker pull "$BASE_ORIG_TAG"
 docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:"$ORIG_TAG"
 # Tag for Docker Hub (base-nightly and base-nightly-<commit>, nightly and nightly-<commit>)
 docker tag "$BASE_ORIG_TAG" vllm/vllm-openai-rocm:"$BASE_TAG_NAME"
 docker tag "$BASE_ORIG_TAG" vllm/vllm-openai-rocm:"$BASE_TAG_NAME_COMMIT"
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:"$ORIG_TAG" vllm/vllm-openai-rocm:"$TAG_NAME"
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:"$ORIG_TAG" vllm/vllm-openai-rocm:"$TAG_NAME_COMMIT"
 if [[ "$DRY_RUN" == "1" ]]; then
  echo "[DRY_RUN] Would push vllm/vllm-openai-rocm:$BASE_TAG_NAME and vllm/vllm-openai-rocm:$BASE_TAG_NAME_COMMIT"
  echo "[DRY_RUN] Would push vllm/vllm-openai-rocm:$TAG_NAME and vllm/vllm-openai-rocm:$TAG_NAME_COMMIT"
  echo "[DRY_RUN] Local tags created. Exiting without push."
  exit 0
 fi
 # Push to Docker Hub (docker-login plugin runs before this step in CI)
 docker push vllm/vllm-openai-rocm:"$BASE_TAG_NAME"
 docker push vllm/vllm-openai-rocm:"$BASE_TAG_NAME_COMMIT"
 docker push vllm/vllm-openai-rocm:"$TAG_NAME"
 docker push vllm/vllm-openai-rocm:"$TAG_NAME_COMMIT"
 echo "Pushed vllm/vllm-openai-rocm:$BASE_TAG_NAME and vllm/vllm-openai-rocm:$BASE_TAG_NAME_COMMIT"
 echo "Pushed vllm/vllm-openai-rocm:$TAG_NAME and vllm/vllm-openai-rocm:$TAG_NAME_COMMIT"
@@ -1,32 +0,0 @@
 #!/usr/bin/env bash
 # Build the ROCm ci_base image, optionally from a freshly rebuilt ROCm base.
 set -euo pipefail
 metadata_get() {
    local key="$1"
    if command -v buildkite-agent >/dev/null 2>&1; then
        buildkite-agent meta-data get "${key}" 2>/dev/null || true
    fi
 }
 main() {
    local base_refreshed=""
    base_refreshed="$(metadata_get rocm-base-refresh)"
    if [[ "${base_refreshed}" == "1" ]]; then
        export BASE_IMAGE
        export CI_BASE_PUSH_STABLE_TAG
        BASE_IMAGE="$(metadata_get rocm-base-image)"
        CI_BASE_PUSH_STABLE_TAG="$(metadata_get rocm-base-push-stable-tag)"
        CI_BASE_PUSH_STABLE_TAG="${CI_BASE_PUSH_STABLE_TAG:-0}"
        echo "Using refreshed ROCm base image for ci_base: ${BASE_IMAGE}"
        echo "Push stable ci_base tag: ${CI_BASE_PUSH_STABLE_TAG}"
    fi
    bash .buildkite/scripts/ci-bake-rocm.sh ci-base-rocm-ci-with-deps
 }
 main "$@"
@@ -1,68 +0,0 @@
 #!/usr/bin/env bash
 # Build the ROCm CI test image or wheel artifact.
 #
 # When Dockerfile.rocm_base changes, always build the full image so downstream
 # ROCm tests can validate the freshly rebuilt base -> ci_base -> ci image chain.
 set -euo pipefail
 metadata_get() {
    local key="$1"
    if command -v buildkite-agent >/dev/null 2>&1; then
        buildkite-agent meta-data get "${key}" 2>/dev/null || true
    fi
 }
 use_ci_base_if_present() {
    local ci_base_image=""
    ci_base_image="$(metadata_get rocm-ci-base-image)"
    if [[ -z "${ci_base_image}" ]]; then
        return 1
    fi
    export CI_BASE_IMAGE="${ci_base_image}"
    echo "Using ROCm ci_base image selected by the preceding build step: ${CI_BASE_IMAGE}"
 }
 use_refreshed_base_if_present() {
    local base_refreshed=""
    base_refreshed="$(metadata_get rocm-base-refresh)"
    if [[ "${base_refreshed}" != "1" ]]; then
        return 1
    fi
    export BASE_IMAGE
    export IMAGE_TAG_LATEST
    BASE_IMAGE="$(metadata_get rocm-base-image)"
    IMAGE_TAG_LATEST="$(metadata_get rocm-ci-image-descriptive)"
    echo "Using refreshed ROCm base image for test image: ${BASE_IMAGE}"
    if [[ -n "${IMAGE_TAG_LATEST}" ]]; then
        echo "Also tagging full ROCm CI image as: ${IMAGE_TAG_LATEST}"
    fi
    return 0
 }
 main() {
    local base_refreshed=0
    use_ci_base_if_present || true
    if use_refreshed_base_if_present; then
        base_refreshed=1
    fi
    if [[ "${ROCM_CI_ARTIFACT_ONLY:-0}" == "1" && "${base_refreshed}" != "1" ]]; then
        echo "ROCM_CI_ARTIFACT_ONLY=1; building ROCm wheel artifact only"
        IMAGE_TAG="" bash .buildkite/scripts/ci-bake-rocm.sh test-rocm-ci-with-artifacts
        return
    fi
    bash .buildkite/scripts/ci-bake-rocm.sh test-rocm-ci-with-wheel
 }
 main "$@"
@@ -1,513 +0,0 @@
 #!/usr/bin/env bash
 # Build and publish a fresh ROCm base image when Dockerfile.rocm_base changes.
 #
 # Normal AMD CI builds should not pay for this path. The script no-ops unless
 # docker/Dockerfile.rocm_base changed relative to the branch base, the previous
 # main commit, or ROCM_BASE_REFRESH_FORCE=1 is set.
 set -euo pipefail
 DOCKERFILE="${ROCM_BASE_DOCKERFILE:-docker/Dockerfile.rocm_base}"
 BASE_REPO="${ROCM_BASE_IMAGE_REPO:-rocm/vllm-dev}"
 CI_IMAGE_REPO="${ROCM_CI_IMAGE_REPO:-rocm/vllm-ci}"
 BUILDER_NAME="${ROCM_BASE_BUILDER_NAME:-vllm-rocm-base-builder}"
 DEFAULT_ROCM_BASE_METADATA_VERSION="1"
 DEFAULT_ROCM_BASE_CONTENT_FILES="${DOCKERFILE}"
 DEFAULT_ROCM_BASE_CONTENT_ARGS="BASE_IMAGE TRITON_BRANCH TRITON_REPO PYTORCH_BRANCH PYTORCH_REPO PYTORCH_VISION_BRANCH PYTORCH_VISION_REPO PYTORCH_AUDIO_BRANCH PYTORCH_AUDIO_REPO FA_BRANCH FA_REPO AITER_BRANCH AITER_REPO MORI_BRANCH MORI_REPO PYTORCH_ROCM_ARCH PYTHON_VERSION USE_SCCACHE"
 metadata_set() {
    local key="$1"
    local value="$2"
    [[ -n "${value}" ]] || return 0
    if command -v buildkite-agent >/dev/null 2>&1; then
        buildkite-agent meta-data set "${key}" "${value}" || true
    fi
 }
 compute_content_hash() {
    local path=""
    local file=""
    for path in "$@"; do
        if [[ -d "${path}" ]]; then
            while IFS= read -r -d '' file; do
                printf 'file:%s\n' "${file}"
                sha256sum "${file}"
            done < <(find "${path}" -type f -print0 | sort -z)
        elif [[ -f "${path}" ]]; then
            printf 'file:%s\n' "${path}"
            sha256sum "${path}"
        else
            printf 'missing:%s\n' "${path}"
        fi
    done | sha256sum | cut -d' ' -f1
 }
 clean_docker_tag() {
    local input="$1"
    echo "${input}" | sed 's/[^a-zA-Z0-9._-]/_/g' | cut -c1-128
 }
 tag_component() {
    local input="$1"
    local max_chars="${2:-24}"
    clean_docker_tag "${input:-unknown}" | cut -c1-"${max_chars}"
 }
 extract_arg_default() {
    local arg_name="$1"
    sed -n -E "s/^[[:space:]]*ARG[[:space:]]+${arg_name}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \
        "${DOCKERFILE}" | head -1
 }
 resolve_image_digest() {
    local image_ref="$1"
    docker buildx imagetools inspect "${image_ref}" 2>/dev/null \
        | sed -n -E 's/^Digest:[[:space:]]+//p' \
        | head -1 || true
 }
 resolve_rocm_base_arg_value() {
    local arg_name="$1"
    local use_sccache="$2"
    case "${arg_name}" in
        USE_SCCACHE)
            printf '%s\n' "${use_sccache}"
            ;;
        *)
            extract_arg_default "${arg_name}"
            ;;
    esac
 }
 hash_rocm_base_arg_values() {
    local use_sccache="$1"
    local base_image_digest="$2"
    local arg_name=""
    local arg_value=""
    shift 2 || true
    for arg_name in "$@"; do
        [[ -n "${arg_name}" ]] || continue
        arg_value=$(resolve_rocm_base_arg_value "${arg_name}" "${use_sccache}")
        printf 'arg:%s=%s\n' "${arg_name}" "${arg_value:-<empty>}"
        if [[ "${arg_name}" == "BASE_IMAGE" && -n "${arg_value}" ]]; then
            printf 'arg:%s.digest=%s\n' "${arg_name}" "${base_image_digest:-unknown}"
        fi
    done
 }
 rocm_version_from_base_image() {
    local base_image="$1"
    local version=""
    version="$(sed -n -E 's/.*:([0-9]+\.[0-9]+(\.[0-9]+)?)-.*/\1/p' <<<"${base_image}")"
    tag_component "${version:-${base_image}}" 16
 }
 git_diff_changed_base() {
    local range="$1"
    [[ -n "$(git diff --name-only "${range}" -- "${DOCKERFILE}" 2>/dev/null)" ]]
 }
 short_git_ref() {
    local ref="$1"
    git rev-parse --short "${ref}" 2>/dev/null || printf '%s\n' "${ref}"
 }
 extract_arg_default_from_ref() {
    local ref="$1"
    local arg_name="$2"
    local content=""
    content="$(git show "${ref}:${DOCKERFILE}" 2>/dev/null || true)"
    sed -n -E "s/^[[:space:]]*ARG[[:space:]]+${arg_name}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \
        <<<"${content}" | head -1
 }
 log_arg_default_changes() {
    local old_ref="$1"
    local new_ref="$2"
    local content_args="${ROCM_BASE_CONTENT_ARGS:-${DEFAULT_ROCM_BASE_CONTENT_ARGS}}"
    local arg_name=""
    local old_value=""
    local new_value=""
    local changed=0
    echo "Changed ROCm base ARG defaults:"
    for arg_name in ${content_args}; do
        old_value="$(extract_arg_default_from_ref "${old_ref}" "${arg_name}")"
        new_value="$(extract_arg_default_from_ref "${new_ref}" "${arg_name}")"
        if [[ "${old_value}" != "${new_value}" ]]; then
            echo "  - ${arg_name}: ${old_value:-<unset>} -> ${new_value:-<unset>}"
            changed=1
        fi
    done
    if [[ "${changed}" == "0" ]]; then
        echo "  - none detected; Dockerfile instructions changed outside tracked ARG defaults"
    fi
 }
 log_arg_line_diff() {
    local range="$1"
    local arg_diff=""
    arg_diff="$(
        git diff --unified=0 "${range}" -- "${DOCKERFILE}" 2>/dev/null \
            | awk '/^[+-][[:space:]]*ARG[[:space:]]/ && $0 !~ /^(---|\+\+\+)/ { print "  " $0 }' \
            || true
    )"
    if [[ -n "${arg_diff}" ]]; then
        echo "Changed Dockerfile ARG lines:"
        printf '%s\n' "${arg_diff}"
    fi
 }
 log_rocm_base_change_check() {
    local context="$1"
    local range="$2"
    local old_ref="$3"
    local old_short=""
    local head_short=""
    old_short="$(short_git_ref "${old_ref}")"
    head_short="$(short_git_ref HEAD)"
    echo "--- :mag: ROCm base refresh check"
    echo "Context: ${context}"
    echo "Dockerfile: ${DOCKERFILE}"
    echo "Base revision: ${old_short}"
    echo "Head revision: ${head_short}"
    echo "Git diff range: ${range}"
 }
 log_rocm_base_rebuild_reason() {
    local context="$1"
    local range="$2"
    local old_ref="$3"
    local changed_files=""
    log_rocm_base_change_check "${context}" "${range}" "${old_ref}"
    changed_files="$(git diff --name-only "${range}" -- "${DOCKERFILE}" 2>/dev/null || true)"
    echo "Changed files:"
    if [[ -n "${changed_files}" ]]; then
        sed 's/^/  - /' <<<"${changed_files}"
    else
        echo "  - ${DOCKERFILE}"
    fi
    log_arg_default_changes "${old_ref}" HEAD
    log_arg_line_diff "${range}"
    echo "Decision: rebuilding ROCm base image because ${DOCKERFILE} changed."
 }
 rocm_base_changed_in_range() {
    local context="$1"
    local range="$2"
    local old_ref="$3"
    if git_diff_changed_base "${range}"; then
        log_rocm_base_rebuild_reason "${context}" "${range}" "${old_ref}"
        return 0
    fi
    log_rocm_base_change_check "${context}" "${range}" "${old_ref}"
    echo "Decision: ROCm base refresh not required; ${DOCKERFILE} is unchanged."
    return 1
 }
 rocm_base_changed() {
    local base_branch="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}"
    local base_ref="refs/remotes/origin/${base_branch}"
    local merge_base=""
    if [[ "${ROCM_BASE_REFRESH_SKIP:-0}" == "1" ]]; then
        echo "ROCM_BASE_REFRESH_SKIP=1 set; skipping ROCm base refresh"
        return 1
    fi
    if [[ "${ROCM_BASE_REFRESH_FORCE:-0}" == "1" ]]; then
        echo "ROCM_BASE_REFRESH_FORCE=1 set; refreshing ROCm base image"
        return 0
    fi
    if ! git rev-parse --is-inside-work-tree >/dev/null 2>&1; then
        echo "Not in a git checkout; skipping ROCm base refresh unless forced"
        return 1
    fi
    if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" ]]; then
        git fetch --no-tags --depth=200 origin \
            "+refs/heads/${base_branch}:${base_ref}" >/dev/null 2>&1 || true
        merge_base=$(git merge-base HEAD "${base_ref}" 2>/dev/null || true)
        if [[ -z "${merge_base}" ]]; then
            echo "Unable to determine merge base with PR base ${base_ref}; skipping ROCm base refresh unless forced"
            return 1
        fi
        if rocm_base_changed_in_range \
            "pull request build against ${base_ref}" \
            "${merge_base}...HEAD" \
            "${merge_base}"; then
            return 0
        fi
    elif [[ "${BUILDKITE_BRANCH:-}" == "${ROCM_BASE_STABLE_BRANCH:-main}" ]] \
        && git rev-parse --verify HEAD~1 >/dev/null 2>&1; then
        if rocm_base_changed_in_range \
            "stable branch build; comparing against previous ${ROCM_BASE_STABLE_BRANCH:-main} commit" \
            "HEAD~1..HEAD" \
            "HEAD~1"; then
            return 0
        fi
    else
        git fetch --no-tags --depth=200 origin \
            "+refs/heads/${base_branch}:${base_ref}" >/dev/null 2>&1 || true
        merge_base=$(git merge-base HEAD "${base_ref}" 2>/dev/null || true)
        if [[ -z "${merge_base}" ]]; then
            echo "Unable to determine merge base with branch base ${base_ref}; skipping ROCm base refresh unless forced"
            return 1
        fi
        if rocm_base_changed_in_range \
            "branch build against ${base_ref}" \
            "${merge_base}...HEAD" \
            "${merge_base}"; then
            return 0
        fi
    fi
    return 1
 }
 should_push_stable_tag() {
    if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" ]]; then
        return 1
    fi
    if [[ "${ROCM_BASE_PUSH_STABLE_TAG:-}" == "1" ]]; then
        return 0
    fi
    if [[ "${ROCM_BASE_PUSH_STABLE_TAG:-}" == "0" ]]; then
        return 1
    fi
    [[ "${BUILDKITE_PULL_REQUEST:-false}" == "false" \
        && "${BUILDKITE_BRANCH:-}" == "${ROCM_BASE_STABLE_BRANCH:-main}" ]]
 }
 setup_builder() {
    echo "--- :buildkite: Setting up buildx builder for ROCm base"
    if docker buildx inspect "${BUILDER_NAME}" >/dev/null 2>&1; then
        docker buildx use "${BUILDER_NAME}"
    else
        docker buildx create --name "${BUILDER_NAME}" --driver docker-container --use
    fi
    docker buildx inspect --bootstrap
 }
 compute_base_content_hash() {
    local use_sccache="$1"
    local base_image_digest="$2"
    local content_files="${ROCM_BASE_CONTENT_FILES:-${DEFAULT_ROCM_BASE_CONTENT_FILES}}"
    local content_args="${ROCM_BASE_CONTENT_ARGS:-${DEFAULT_ROCM_BASE_CONTENT_ARGS}}"
    local -a content_paths=()
    local -a content_arg_names=()
    read -r -a content_paths <<< "${content_files}"
    read -r -a content_arg_names <<< "${content_args}"
    {
        printf 'content-files-hash:%s\n' "$(compute_content_hash "${content_paths[@]}")"
        printf 'dockerfile:%s\n' "${DOCKERFILE}"
        printf 'resolved-build-args:\n'
        hash_rocm_base_arg_values \
            "${use_sccache}" "${base_image_digest}" "${content_arg_names[@]}"
    } | sha256sum | cut -d' ' -f1
 }
 build_base_image() {
    local use_sccache="${ROCM_BASE_USE_SCCACHE:-${USE_SCCACHE:-0}}"
    local base_hash=""
    local build_date=""
    local build_suffix=""
    local base_image_arg=""
    local base_image_digest=""
    local rocm_version=""
    local triton_arg=""
    local pytorch_arg=""
    local pytorch_vision_arg=""
    local pytorch_audio_arg=""
    local fa_arg=""
    local aiter_arg=""
    local mori_arg=""
    local python_version_arg=""
    local pytorch_rocm_arch_arg=""
    local pytorch_branch=""
    local aiter_branch=""
    local dependency_summary=""
    local descriptor=""
    local ci_descriptor=""
    local descriptive_tag=""
    local stable_tag="${BASE_REPO}:base"
    local ci_descriptive_tag=""
    local content_files="${ROCM_BASE_CONTENT_FILES:-${DEFAULT_ROCM_BASE_CONTENT_FILES}}"
    local content_args="${ROCM_BASE_CONTENT_ARGS:-${DEFAULT_ROCM_BASE_CONTENT_ARGS}}"
    local content_files_hash=""
    local metadata_version="${ROCM_BASE_METADATA_VERSION:-${DEFAULT_ROCM_BASE_METADATA_VERSION}}"
    local -a tags=()
    local -a no_cache_args=()
    local -a sccache_args=()
    local -a content_paths=()
    if [[ ! -f "${DOCKERFILE}" ]]; then
        echo "Error: ROCm base Dockerfile not found: ${DOCKERFILE}" >&2
        exit 1
    fi
    build_date="${ROCM_BASE_TAG_DATE:-$(date -u +%Y%m%d)}"
    if [[ -n "${BUILDKITE_BUILD_NUMBER:-}" ]]; then
        build_suffix="_bk_${BUILDKITE_BUILD_NUMBER}"
    fi
    base_image_arg="$(extract_arg_default BASE_IMAGE)"
    base_image_digest="$(resolve_image_digest "${base_image_arg}")"
    read -r -a content_paths <<< "${content_files}"
    content_files_hash="$(compute_content_hash "${content_paths[@]}")"
    base_hash=$(compute_base_content_hash "${use_sccache}" "${base_image_digest}")
    rocm_version="$(rocm_version_from_base_image "${base_image_arg}")"
    triton_arg="$(extract_arg_default TRITON_BRANCH)"
    pytorch_arg="$(extract_arg_default PYTORCH_BRANCH)"
    pytorch_vision_arg="$(extract_arg_default PYTORCH_VISION_BRANCH)"
    pytorch_audio_arg="$(extract_arg_default PYTORCH_AUDIO_BRANCH)"
    fa_arg="$(extract_arg_default FA_BRANCH)"
    aiter_arg="$(extract_arg_default AITER_BRANCH)"
    mori_arg="$(extract_arg_default MORI_BRANCH)"
    python_version_arg="$(extract_arg_default PYTHON_VERSION)"
    pytorch_rocm_arch_arg="$(extract_arg_default PYTORCH_ROCM_ARCH)"
    pytorch_branch="$(tag_component "${pytorch_arg}" 16)"
    aiter_branch="$(tag_component "${aiter_arg}" 24)"
    dependency_summary="base=${base_image_arg},rocm=${rocm_version},python=${python_version_arg},pytorch=${pytorch_arg},torchvision=${pytorch_vision_arg},torchaudio=${pytorch_audio_arg},triton=${triton_arg},flash-attn=${fa_arg},aiter=${aiter_arg},mori=${mori_arg},pytorch-rocm-arch=${pytorch_rocm_arch_arg}"
    descriptor="$(clean_docker_tag "base_custom_aiter_${aiter_branch}_torch_${pytorch_branch}_${build_date}${build_suffix}")"
    ci_descriptor="$(clean_docker_tag "ci_custom_aiter_${aiter_branch}_torch_${pytorch_branch}_${build_date}${build_suffix}")"
    descriptive_tag="${BASE_REPO}:${descriptor}"
    ci_descriptive_tag="${CI_IMAGE_REPO}:${ci_descriptor}"
    tags=(-t "${descriptive_tag}")
    if should_push_stable_tag; then
        tags+=(-t "${stable_tag}")
        metadata_set "rocm-base-push-stable-tag" "1"
    else
        metadata_set "rocm-base-push-stable-tag" "0"
    fi
    if [[ "${ROCM_BASE_NO_CACHE:-1}" == "1" ]]; then
        no_cache_args=(--no-cache)
    fi
    for env_name in \
        SCCACHE_DOWNLOAD_URL \
        SCCACHE_ENDPOINT \
        SCCACHE_BUCKET_NAME \
        SCCACHE_REGION_NAME \
        SCCACHE_S3_NO_CREDENTIALS; do
        if [[ -n "${!env_name:-}" ]]; then
            sccache_args+=(--build-arg "${env_name}=${!env_name}")
        fi
    done
    echo "--- :docker: Building ROCm base image"
    echo "Dockerfile: ${DOCKERFILE}"
    echo "Descriptive tag: ${descriptive_tag}"
    echo "Stable tag: ${stable_tag} ($(should_push_stable_tag && echo enabled || echo disabled))"
    echo "Content hash: ${base_hash}"
    echo "Dependency summary: ${dependency_summary}"
    echo "USE_SCCACHE: ${use_sccache}"
    docker buildx build \
        "${no_cache_args[@]}" \
        --pull \
        --progress "${BUILDKIT_PROGRESS:-plain}" \
        --file "${DOCKERFILE}" \
        --build-arg "USE_SCCACHE=${use_sccache}" \
        "${sccache_args[@]}" \
        --label "org.opencontainers.image.source=https://github.com/vllm-project/vllm" \
        --label "org.opencontainers.image.vendor=vLLM" \
        --label "org.opencontainers.image.title=vLLM ROCm base" \
        --label "org.opencontainers.image.revision=${BUILDKITE_COMMIT:-}" \
        --label "vllm.rocm_base.metadata_version=${metadata_version}" \
        --label "vllm.rocm_base.content_hash=${base_hash}" \
        --label "vllm.rocm_base.content_files_hash=${content_files_hash}" \
        --label "vllm.rocm_base.dockerfile=${DOCKERFILE}" \
        --label "vllm.rocm_base.image.descriptive=${descriptive_tag}" \
        --label "vllm.rocm_base.image.stable=${stable_tag}" \
        --label "vllm.rocm_base.git_commit=${BUILDKITE_COMMIT:-}" \
        --label "vllm.rocm_base.stable_branch=${ROCM_BASE_STABLE_BRANCH:-main}" \
        --label "vllm.rocm_base.descriptor=${descriptor}" \
        --label "vllm.rocm_base.dependency_summary=${dependency_summary}" \
        --label "vllm.rocm_base.base_image=${base_image_arg}" \
        --label "vllm.rocm_base.base_image_digest=${base_image_digest}" \
        --label "vllm.rocm_base.dependency.rocm=${rocm_version}" \
        --label "vllm.rocm_base.dependency.python=${python_version_arg}" \
        --label "vllm.rocm_base.dependency.pytorch=${pytorch_arg}" \
        --label "vllm.rocm_base.dependency.torchvision=${pytorch_vision_arg}" \
        --label "vllm.rocm_base.dependency.torchaudio=${pytorch_audio_arg}" \
        --label "vllm.rocm_base.dependency.triton=${triton_arg}" \
        --label "vllm.rocm_base.dependency.flash_attention=${fa_arg}" \
        --label "vllm.rocm_base.dependency.aiter=${aiter_arg}" \
        --label "vllm.rocm_base.dependency.mori=${mori_arg}" \
        --label "vllm.rocm_base.pytorch_rocm_arch=${pytorch_rocm_arch_arg}" \
        "${tags[@]}" \
        --push \
        .
    docker buildx imagetools inspect "${descriptive_tag}" >/dev/null
    metadata_set "rocm-base-refresh" "1"
    metadata_set "rocm-base-image" "${descriptive_tag}"
    metadata_set "rocm-base-image-descriptive" "${descriptive_tag}"
    metadata_set "rocm-base-image-stable" "${stable_tag}"
    metadata_set "rocm-base-image-ci-descriptive" "${ci_descriptive_tag}"
    metadata_set "rocm-base-metadata-version" "${metadata_version}"
    metadata_set "rocm-base-content-hash" "${base_hash}"
    metadata_set "rocm-base-content-files-hash" "${content_files_hash}"
    metadata_set "rocm-base-content-files" "${content_files}"
    metadata_set "rocm-base-content-args" "${content_args}"
    metadata_set "rocm-base-base-image-digest" "${base_image_digest}"
    metadata_set "rocm-base-dockerfile" "${DOCKERFILE}"
    metadata_set "rocm-base-descriptor" "${descriptor}"
    metadata_set "rocm-base-dependency-summary" "${dependency_summary}"
    metadata_set "rocm-base-dependency-rocm" "${rocm_version}"
    metadata_set "rocm-base-dependency-python" "${python_version_arg}"
    metadata_set "rocm-base-dependency-pytorch" "${pytorch_arg}"
    metadata_set "rocm-base-dependency-torchvision" "${pytorch_vision_arg}"
    metadata_set "rocm-base-dependency-torchaudio" "${pytorch_audio_arg}"
    metadata_set "rocm-base-dependency-triton" "${triton_arg}"
    metadata_set "rocm-base-dependency-flash-attention" "${fa_arg}"
    metadata_set "rocm-base-dependency-aiter" "${aiter_arg}"
    metadata_set "rocm-base-dependency-mori" "${mori_arg}"
    metadata_set "rocm-base-pytorch-rocm-arch" "${pytorch_rocm_arch_arg}"
    metadata_set "rocm-ci-image-descriptive" "${ci_descriptive_tag}"
    echo "--- :white_check_mark: ROCm base image published"
    echo "Use BASE_IMAGE=${descriptive_tag} for downstream ROCm CI builds"
 }
 main() {
    metadata_set "rocm-base-refresh" "0"
    if ! rocm_base_changed; then
        echo "ROCm base Dockerfile did not change; skipping base image refresh"
        return 0
    fi
    setup_builder
    build_base_image
 }
 main "$@"
@@ -1,32 +0,0 @@
 #!/usr/bin/env bash
 # Fast structural smoke test for the full ROCm CI image.
 set -euo pipefail
 image_ref="${VLLM_CI_SMOKE_IMAGE:-rocm/vllm-ci:${BUILDKITE_COMMIT:?BUILDKITE_COMMIT is required}}"
 docker run --rm --network=none --entrypoint /bin/bash "${image_ref}" -ec '
  if [ ! -d /vllm-workspace ]; then echo Missing directory: /vllm-workspace >&2; exit 1; fi
  if [ ! -d /vllm-workspace/tests ]; then echo Missing directory: /vllm-workspace/tests >&2; exit 1; fi
  if [ ! -d /vllm-workspace/src/vllm ]; then echo Missing directory: /vllm-workspace/src/vllm >&2; exit 1; fi
  if [ ! -x /vllm-workspace/src/vllm/vllm-rs ]; then echo Missing executable: /vllm-workspace/src/vllm/vllm-rs >&2; exit 1; fi
  command -v python3
  command -v uv
  command -v pytest
  if ! command -v amd-smi >/dev/null 2>&1 && ! command -v rocminfo >/dev/null 2>&1; then
    echo No ROCm CLI found in image >&2
    exit 1
  fi
  python3 - <<PY
 import torch
 import vllm
 print(torch.__version__)
 print(vllm.__version__)
 PY
  echo AMD image smoke OK
 '
@@ -109,9 +109,7 @@ run_nodes() {
        if [ "$node" -ne 0 ]; then
            docker exec -d "node$node" /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}"
        else
-            # Allocate a TTY (-t -i) for the foreground head node so its output
+            docker exec "node$node" /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}"
            # keeps ANSI color in the Buildkite log (see run-amd-test.sh).
            docker exec -t -i "node$node" /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}"
        fi
    done
 }
@@ -1,239 +0,0 @@
 #!/usr/bin/env bash
 set -euo pipefail
 MODE="${1:-}"
 if [[ "$MODE" != "style-clippy" && "$MODE" != "test" ]]; then
  echo "Usage: $0 {style-clippy|test}" >&2
  exit 2
 fi
 if ROOT_DIR="$(git rev-parse --show-toplevel 2>/dev/null)"; then
  :
 else
  SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
  ROOT_DIR="$(cd -- "${SCRIPT_DIR}/../.." && pwd -P)"
 fi
 cd "$ROOT_DIR"
 export CARGO_TERM_COLOR="${CARGO_TERM_COLOR:-always}"
 export CARGO_HOME="${CARGO_HOME:-$HOME/.cargo}"
 export RUSTUP_HOME="${RUSTUP_HOME:-$HOME/.rustup}"
 export PATH="$CARGO_HOME/bin:$PATH"
 PROTOC_VERSION="${PROTOC_VERSION:-31.1}"
 CARGO_BINSTALL_VERSION="${CARGO_BINSTALL_VERSION:-1.20.1}"
 UV_VERSION="${UV_VERSION:-0.11.28}"
 PYO3_PYTHON_VERSION="${PYO3_PYTHON_VERSION:-3.12}"
 CARGO_SORT_VERSION_REQ="${CARGO_SORT_VERSION_REQ:-2}"
 CARGO_DENY_VERSION_REQ="${CARGO_DENY_VERSION_REQ:-0.20}"
 CARGO_NEXTEST_VERSION_REQ="${CARGO_NEXTEST_VERSION_REQ:-0.9}"
 CARGO_LLVM_COV_VERSION="${CARGO_LLVM_COV_VERSION:-0.8.7}"
 log_section() {
  echo "--- $*"
 }
 install_protoc() {
  local arch
  case "$(uname -m)" in
    x86_64)
      arch="x86_64"
      ;;
    aarch64|arm64)
      arch="aarch_64"
      ;;
    *)
      echo "Unsupported protoc architecture: $(uname -m)" >&2
      return 1
      ;;
  esac
  local url="https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-${arch}.zip"
  local tmp_dir
  tmp_dir="$(mktemp -d)"
  log_section "Installing protoc ${PROTOC_VERSION}"
  curl -L --proto '=https' --tlsv1.2 -sSf "$url" -o "$tmp_dir/protoc.zip"
  mkdir -p "$CARGO_HOME/bin"
  unzip -q "$tmp_dir/protoc.zip" bin/protoc 'include/*' -d "$CARGO_HOME"
  chmod +x "$CARGO_HOME/bin/protoc"
  rm -rf "$tmp_dir"
  protoc --version
 }
 rust_toolchain() {
  awk -F '"' '/channel[[:space:]]*=/ { print $2; exit }' rust-toolchain.toml
 }
 install_rust_toolchain() {
  log_section "Installing Rust toolchain"
  if ! command -v rustup >/dev/null 2>&1; then
    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \
      | sh -s -- -y --profile minimal --default-toolchain none
  fi
  local toolchain
  toolchain="$(rust_toolchain)"
  rustup toolchain install "$toolchain" --profile minimal --component rustfmt,clippy
  rustup component add --toolchain "$toolchain" rustfmt clippy
 }
 install_cargo_binstall() {
  log_section "Installing cargo-binstall ${CARGO_BINSTALL_VERSION}"
  curl -L --proto '=https' --tlsv1.2 -sSf \
    "https://raw.githubusercontent.com/cargo-bins/cargo-binstall/v${CARGO_BINSTALL_VERSION}/install-from-binstall-release.sh" \
    | env BINSTALL_VERSION="$CARGO_BINSTALL_VERSION" bash
  cargo-binstall -V
 }
 install_cargo_sort() {
  log_section "Installing cargo-sort ${CARGO_SORT_VERSION_REQ}"
  cargo binstall --no-confirm --force "cargo-sort@${CARGO_SORT_VERSION_REQ}"
 }
 install_cargo_deny() {
  log_section "Installing cargo-deny ${CARGO_DENY_VERSION_REQ}"
  cargo binstall --no-confirm --force "cargo-deny@${CARGO_DENY_VERSION_REQ}"
 }
 install_cargo_nextest() {
  log_section "Installing cargo-nextest ${CARGO_NEXTEST_VERSION_REQ}"
  cargo binstall \
    --no-confirm \
    --force \
    --secure \
    "cargo-nextest@${CARGO_NEXTEST_VERSION_REQ}"
 }
 install_cargo_llvm_cov() {
  log_section "Installing cargo-llvm-cov ${CARGO_LLVM_COV_VERSION}"
  local toolchain
  toolchain="$(rust_toolchain)"
  rustup component add --toolchain "$toolchain" llvm-tools-preview
  cargo binstall \
    --no-confirm \
    --force \
    --secure \
    "cargo-llvm-cov@${CARGO_LLVM_COV_VERSION}"
 }
 install_uv() {
  log_section "Installing uv ${UV_VERSION}"
  curl -L --proto '=https' --tlsv1.2 -sSf \
    "https://github.com/astral-sh/uv/releases/download/${UV_VERSION}/uv-installer.sh" \
    | env UV_INSTALL_DIR="$CARGO_HOME/bin" sh
  uv --version
 }
 setup_pyo3_python() {
  log_section "Installing Python ${PYO3_PYTHON_VERSION} for PyO3 tests"
  uv python install "$PYO3_PYTHON_VERSION"
  PYO3_PYTHON="$(uv python find \
    --managed-python \
    --no-project \
    --resolve-links \
    "$PYO3_PYTHON_VERSION")"
  export PYO3_PYTHON
  local python_libdir
  python_libdir="$("$PYO3_PYTHON" - <<'PY'
 import pathlib
 import sysconfig
 libdir = pathlib.Path(sysconfig.get_config_var("LIBDIR"))
 ldlibrary = sysconfig.get_config_var("LDLIBRARY")
 assert sysconfig.get_config_var("Py_ENABLE_SHARED") == 1
 assert ldlibrary
 assert (libdir / ldlibrary).exists(), libdir / ldlibrary
 print(libdir)
 PY
 )"
  export LD_LIBRARY_PATH="${python_libdir}:${LD_LIBRARY_PATH:-}"
  export LIBRARY_PATH="${python_libdir}:${LIBRARY_PATH:-}"
 }
 run_style_clippy() {
  install_cargo_binstall
  install_cargo_sort
  install_cargo_deny
  log_section "Checking Rust formatting"
  cargo fmt --manifest-path rust/Cargo.toml --all -- --check
  log_section "Checking Cargo.toml ordering"
  cargo sort --workspace --check rust
  log_section "Checking Rust dependency bans"
  cargo deny \
    --manifest-path rust/Cargo.toml \
    --config rust/deny.toml \
    check \
    bans
  log_section "Running clippy"
  cargo clippy \
    --manifest-path rust/Cargo.toml \
    --workspace \
    --all-targets \
    --all-features \
    --locked \
    -- \
    -D warnings
 }
 run_tests() {
  install_uv
  setup_pyo3_python
  install_cargo_binstall
  install_cargo_nextest
  install_cargo_llvm_cov
  log_section "Running cargo nextest with Rust coverage"
  mkdir -p artifacts
  export LLVM_PROFILE_FILE_NAME="vllm-rust-unit-%4m.profraw"
  cargo llvm-cov clean \
    --manifest-path rust/Cargo.toml \
    --profraw-only
  set +e
  cargo llvm-cov nextest \
    --manifest-path rust/Cargo.toml \
    --workspace \
    --all-features \
    --locked \
    --no-fail-fast \
    --no-clean \
    --lcov \
    --output-path artifacts/rust-unit.lcov \
    --ignore-filename-regex='/\.cargo/(registry|git)/|/rustc/|/target/'
  local coverage_rc=$?
  local upload_rc=0
  if [[ $coverage_rc -eq 0 ]]; then
    # shellcheck source=.buildkite/scripts/rust-coverage.sh
    source .buildkite/scripts/rust-coverage.sh
    rust_coverage_upload artifacts/rust-unit.lcov rust-unit
    upload_rc=$?
  fi
  set -e
  if [[ $coverage_rc -ne 0 ]]; then
    return "$coverage_rc"
  fi
  return "$upload_rc"
 }
 install_protoc
 install_rust_toolchain
 case "$MODE" in
  style-clippy)
    run_style_clippy
    ;;
  test)
    run_tests
    ;;
 esac
@@ -1,182 +0,0 @@
 #!/bin/sh
 RUST_CODECOV_VERSION="v11.3.1"
 RUST_CODECOV_SHA256="ca1d64196d2d34771084afe76ea657d581bf628e31d993ff8e52ea09cc88a56d"
 rust_coverage_repo_root() {
    if [ -f /vllm-workspace/.buildkite/scripts/rust-coverage.sh ]; then
        printf '%s\n' /vllm-workspace
    elif [ -n "${BUILDKITE_BUILD_CHECKOUT_PATH:-}" ] \
        && [ -d "$BUILDKITE_BUILD_CHECKOUT_PATH" ]; then
        printf '%s\n' "$BUILDKITE_BUILD_CHECKOUT_PATH"
    else
        git rev-parse --show-toplevel
    fi
 }
 rust_coverage_start() {
    RUST_COVERAGE_FLAG=${1:?coverage flag is required}
    RUST_COVERAGE_DIR="/tmp/vllm-rust-coverage/${BUILDKITE_JOB_ID:-local}"
    export RUST_COVERAGE_FLAG RUST_COVERAGE_DIR
    mkdir -p "$RUST_COVERAGE_DIR"
    LLVM_PROFILE_FILE="$RUST_COVERAGE_DIR/rust-%4m.profraw"
    export LLVM_PROFILE_FILE
    trap rust_coverage_finalize 0
 }
 rust_coverage_objects() {
    rust_cov_objects_manifest="$(dirname "$(command -v llvm-cov)")/../objects"
    python3 - "$rust_cov_objects_manifest" <<'PY'
 from pathlib import Path
 import sys
 for relative in Path(sys.argv[1]).read_text().splitlines():
    for entry in sys.path:
        path = Path(entry or ".").resolve() / relative
        if path.is_file():
            print(path)
            break
    else:
        raise RuntimeError(f"installed Rust coverage object was not found: {relative}")
 PY
 }
 rust_coverage_collect() {
    rust_cov_collect_flag=${1:?coverage flag is required}
    rust_cov_collect_lcov="$RUST_COVERAGE_DIR/$rust_cov_collect_flag.lcov"
    rust_cov_collect_objects=$(rust_coverage_objects) || return 1
    rust_cov_collect_primary=
    set --
    while IFS= read -r rust_cov_collect_object; do
        if [ -z "$rust_cov_collect_primary" ]; then
            rust_cov_collect_primary=$rust_cov_collect_object
        else
            set -- "$@" "--object=$rust_cov_collect_object"
        fi
    done <<EOF
 $rust_cov_collect_objects
 EOF
    llvm-profdata merge \
        -sparse \
        "$RUST_COVERAGE_DIR"/*.profraw \
        -o "$RUST_COVERAGE_DIR/merged.profdata" || return 1
    llvm-cov export \
        "$rust_cov_collect_primary" \
        "$@" \
        --format=lcov \
        --instr-profile="$RUST_COVERAGE_DIR/merged.profdata" \
        --ignore-filename-regex='/\.cargo/(registry|git)/|/rustc/|/target/' \
        > "$rust_cov_collect_lcov" || return 1
    RUST_COVERAGE_LCOV=$rust_cov_collect_lcov
    export RUST_COVERAGE_LCOV
 }
 rust_coverage_upload() {
    rust_cov_upload_lcov=${1:?LCOV path is required}
    rust_cov_upload_flag=${2:?coverage flag is required}
    rust_cov_upload_repo_root=$(rust_coverage_repo_root) || return 1
    if [ "$(uname -m)" != "x86_64" ]; then
        echo "Rust coverage upload currently supports x86_64 CI agents" >&2
        return 1
    fi
    rust_cov_upload_codecov_dir=$(mktemp -d /tmp/codecov-bin.XXXXXX) \
        || return 1
    curl -fsSL \
        "https://github.com/codecov/codecov-cli/releases/download/${RUST_CODECOV_VERSION}/codecovcli_linux" \
        -o "$rust_cov_upload_codecov_dir/codecov" || return 1
    echo "$RUST_CODECOV_SHA256  $rust_cov_upload_codecov_dir/codecov" \
        | sha256sum -c - || return 1
    chmod +x "$rust_cov_upload_codecov_dir/codecov" || return 1
    rust_cov_upload_slug="vllm-project/vllm"
    if [ -n "${BUILDKITE_PULL_REQUEST:-}" ] \
        && [ "${BUILDKITE_PULL_REQUEST}" != "false" ] \
        && [ -n "${BUILDKITE_PULL_REQUEST_REPO:-}" ]; then
        rust_cov_upload_slug=$(echo "$BUILDKITE_PULL_REQUEST_REPO" \
            | sed -E 's#(git@|https?://)([^/:]+)[:/]([^/]+/[^/.]+)(\.git)?$#\3#')
        case "$rust_cov_upload_slug" in
            */*) ;;
            *) rust_cov_upload_slug="vllm-project/vllm" ;;
        esac
    fi
    rust_cov_upload_branch=${BUILDKITE_BRANCH:?BUILDKITE_BRANCH is required}
    if [ -z "${CODECOV_TOKEN:-}" ]; then
        # Codecov accepts tokenless public uploads on unprotected branch names.
        # A colon-separated prefix keeps feature-branch and fork uploads from
        # requiring a repository secret.
        if [ -n "${BUILDKITE_PULL_REQUEST:-}" ] \
            && [ "${BUILDKITE_PULL_REQUEST}" != "false" ]; then
            rust_cov_upload_branch="pr${BUILDKITE_PULL_REQUEST}:$rust_cov_upload_branch"
        else
            rust_cov_upload_branch="buildkite:$rust_cov_upload_branch"
        fi
    fi
    set --
    set -- "$@" upload-process
    set -- "$@" --file "$rust_cov_upload_lcov"
    # LCOV paths are mapped server-side by codecov.yml. Skip the CLI's local
    # source-line fix scanning, which is unrelated to path mapping.
    set -- "$@" --disable-search --disable-file-fixes
    set -- "$@" --fail-on-error --git-service github
    set -- "$@" --build "${BUILDKITE_BUILD_NUMBER:?BUILDKITE_BUILD_NUMBER is required}"
    set -- "$@" --branch "$rust_cov_upload_branch"
    set -- "$@" --sha "${BUILDKITE_COMMIT:?BUILDKITE_COMMIT is required}"
    set -- "$@" --slug "$rust_cov_upload_slug"
    set -- "$@" --flag "$rust_cov_upload_flag"
    set -- "$@" --name "${rust_cov_upload_flag}-${BUILDKITE_JOB_ID:?BUILDKITE_JOB_ID is required}"
    set -- "$@" --dir "$rust_cov_upload_repo_root"
    set -- "$@" --network-root-folder "$rust_cov_upload_repo_root"
    if [ -n "${BUILDKITE_PULL_REQUEST:-}" ] \
        && [ "${BUILDKITE_PULL_REQUEST}" != "false" ]; then
        set -- "$@" --pr "$BUILDKITE_PULL_REQUEST"
    fi
    rust_cov_upload_log="$rust_cov_upload_codecov_dir/codecov.log"
    # E2E steps run from tests/, so execute from the repository root to resolve
    # codecov.yml and repository paths consistently.
    (
        cd "$rust_cov_upload_repo_root" || exit 1
        "$rust_cov_upload_codecov_dir/codecov" "$@"
    ) >"$rust_cov_upload_log" 2>&1
    rust_cov_upload_rc=$?
    cat "$rust_cov_upload_log"
    # v11.3.1 can log API failures while returning zero even with
    # --fail-on-error. Preserve the strict CI contract explicitly.
    if grep -aEq 'error.* -- ' "$rust_cov_upload_log"; then
        echo "Codecov CLI reported an upload error" >&2
        rust_cov_upload_rc=1
    fi
    rm -rf "$rust_cov_upload_codecov_dir"
    return "$rust_cov_upload_rc"
 }
 rust_coverage_finalize() {
    rust_cov_finalize_test_rc=$?
    trap - 0
    set +e
    rust_coverage_collect "$RUST_COVERAGE_FLAG"
    rust_cov_finalize_collect_rc=$?
    rust_cov_finalize_upload_rc=0
    if [ "$rust_cov_finalize_collect_rc" -eq 0 ]; then
        rust_coverage_upload "$RUST_COVERAGE_LCOV" "$RUST_COVERAGE_FLAG"
        rust_cov_finalize_upload_rc=$?
    fi
    find "$RUST_COVERAGE_DIR" -type f -name '*.profraw' -delete
    if [ "$rust_cov_finalize_test_rc" -ne 0 ]; then
        exit "$rust_cov_finalize_test_rc"
    fi
    if [ "$rust_cov_finalize_collect_rc" -ne 0 ]; then
        exit "$rust_cov_finalize_collect_rc"
    fi
    exit "$rust_cov_finalize_upload_rc"
 }
@@ -49,7 +49,6 @@ for BACK in "${BACKENDS[@]}"; do
    --data-parallel-size 2 \
    --enable-expert-parallel \
    --enable-eplb \
    --eplb-config '{"use_async": false}' \
    --trust-remote-code \
    --max-model-len 2048 \
    --all2all-backend "$BACK" \
@@ -1,14 +1,11 @@
 #!/usr/bin/env bash
 set -euxo pipefail
 # Nightly e2e test for prefetch offloading with a MoE model.
 # Runs DeepSeek-V2-Lite with prefetch offloading of MoE expert weights
 # and validates GSM8K accuracy matches baseline (no offloading).
 #
 # args: [THRESHOLD] [NUM_QUESTIONS] [START_PORT]
 #
 # Environment variables:
 #   ATTENTION_BACKEND   - attention backend to use (e.g., FLASH_ATTN,
 #                         ROCM_ATTN, FLASHINFER). If unset, uses vllm default.
 THRESHOLD=${1:-0.25}
 NUM_Q=${2:-1319}
 PORT=${3:-8030}
@@ -25,22 +22,6 @@ wait_for_server() {
 MODEL="deepseek-ai/DeepSeek-V2-Lite"
 # ── Build optional vllm serve flags ─────────────────────────────────────
 EXTRA_ARGS=()
 if [[ -n "${ATTENTION_BACKEND:-}" ]]; then
  echo "Using attention backend: ${ATTENTION_BACKEND}"
  EXTRA_ARGS+=(--attention-backend "${ATTENTION_BACKEND}")
 fi
 # ROCm: run eager to avoid intermittent HIP-graph decode corruption.
 # See https://github.com/ROCm/clr/issues/279
 # TODO(aarushjain29): Revert after TheRock 7.14
 if command -v rocm-smi &> /dev/null || command -v amd-smi &> /dev/null || [[ -d /opt/rocm ]] || [[ -n "${ROCM_PATH:-}" ]]; then
  echo "ROCm platform detected: adding --enforce-eager to avoid HIP-graph decode corruption"
  EXTRA_ARGS+=(--enforce-eager)
 fi
 cleanup() {
  if [[ -n "${SERVER_PID:-}" ]] && kill -0 "${SERVER_PID}" 2>/dev/null; then
    kill "${SERVER_PID}" 2>/dev/null || true
@@ -59,9 +40,7 @@ vllm serve "$MODEL" \
  --offload-num-in-group 2 \
  --offload-prefetch-step 1 \
  --offload-params w13_weight w2_weight \
-  --generation-config vllm \
+  --port "$PORT" &
  --port "$PORT" \
  ${EXTRA_ARGS+"${EXTRA_ARGS[@]}"} &
 SERVER_PID=$!
 wait_for_server "$PORT"
@@ -48,7 +48,7 @@ for BACK in "${BACKENDS[@]}"; do
    --enforce-eager \
    --enable-eplb \
    --all2all-backend "$BACK" \
-    --eplb-config '{"window_size":10, "step_interval":100, "num_redundant_experts":0, "log_balancedness":true, "use_async":false}' \
+    --eplb-config '{"window_size":10, "step_interval":100, "num_redundant_experts":0, "log_balancedness":true}' \
    --tensor-parallel-size "${TENSOR_PARALLEL_SIZE}" \
    --data-parallel-size "${DATA_PARALLEL_SIZE}" \
    --enable-expert-parallel \
@@ -1,59 +0,0 @@
 #!/usr/bin/env bash
 set -euxo pipefail
 # args: [THRESHOLD] [NUM_QUESTIONS] [START_PORT]
 THRESHOLD=${1:-0.8}
 NUM_Q=${2:-1319}
 PORT=${3:-8050}
 OUT_DIR=${OUT_DIR:-/tmp/vllm-scheduled}
 mkdir -p "${OUT_DIR}"
 wait_for_server() {
  local port=$1
  timeout 600 bash -c '
    until curl -sf "http://127.0.0.1:'"$port"'/health" > /dev/null; do
      sleep 1
    done'
 }
 MODEL="Qwen/Qwen3-30B-A3B-FP8"
 BACK="allgather_reducescatter"
 if command -v rocm-smi &> /dev/null || [[ -d /opt/rocm ]] || [[ -n "${ROCM_PATH:-}" ]]; then
  # Disable MOE padding for ROCm since it is causing eplb to fail.
  export VLLM_ROCM_MOE_PADDING=0
 fi
 cleanup() {
  if [[ -n "${SERVER_PID:-}" ]] && kill -0 "${SERVER_PID}" 2>/dev/null; then
    kill "${SERVER_PID}" 2>/dev/null || true
    for _ in {1..20}; do
      kill -0 "${SERVER_PID}" 2>/dev/null || break
      sleep 0.5
    done
    kill -9 "${SERVER_PID}" 2>/dev/null || true
  fi
 }
 trap cleanup EXIT
 VLLM_DEEP_GEMM_WARMUP=skip \
 vllm serve "$MODEL" \
 --enforce-eager \
 --data-parallel-size 4 \
 --enable-expert-parallel \
 --enable-eplb \
 --all2all-backend "$BACK" \
 --eplb-config '{"window_size":20, "step_interval":100, "use_async":true}' \
 --trust-remote-code \
 --max-model-len 2048 \
 --port "$PORT" &
 SERVER_PID=$!
 wait_for_server "$PORT"
 TAG=$(echo "$MODEL" | tr '/: \\n' '_____')
 OUT="${OUT_DIR}/${TAG}_${BACK}.json"
 python3 tests/evals/gsm8k/gsm8k_eval.py --host http://127.0.0.1 --port "$PORT" --num-questions "${NUM_Q}" --save-results "${OUT}"
 python3 - <<PY
 import json; acc=json.load(open('${OUT}'))['accuracy']
 print(f"${MODEL} ${BACK}: accuracy {acc:.3f}")
 assert acc >= ${THRESHOLD}, f"${MODEL} ${BACK} accuracy {acc}"
 PY
@@ -24,7 +24,7 @@ if command -v rocm-smi &> /dev/null || [[ -d /opt/rocm ]] || [[ -n "${ROCM_PATH:
  BACKENDS=("allgather_reducescatter")
  # Disable MOE padding for ROCm since it is causing eplb to fail
  export VLLM_ROCM_MOE_PADDING=0
-  PLATFORM_ARGS=("--no-async-scheduling" "--attention-backend=TRITON_ATTN")
+  PLATFORM_ARGS=("--no-async-scheduling")
  echo "Disabled async scheduling for ROCm platform due to issues with spec decode."
 else
  # Non-ROCm platform (CUDA/other)
@@ -1,252 +0,0 @@
 #!/bin/bash
 # Run BFCL (Berkeley Function Call Leaderboard) tool-calling correctness
 # evaluation against a local vLLM server.
 #
 # Usage:
 #   # Run with defaults (gpt-oss-20b, multi_turn)
 #   bash .buildkite/scripts/tool_call/run-bfcl-eval.sh
 #
 #   # Run with gpt-oss-120b and multiple test categories
 #   BFCL_MODEL="openai/gpt-oss-120b" BFCL_TP_SIZE=4 \
 #     BFCL_TEST_CATEGORY="live_simple, multiple, parallel_multiple" \
 #     bash .buildkite/scripts/tool_call/run-bfcl-eval.sh
 #
 #   # Chain both API types (use BFCL_OUTPUT_DIR to avoid overwriting results)
 #   BFCL_OUTPUT_DIR=./bfcl-chat-completions BFCL_API_TYPE=chat_completions \
 #     bash .buildkite/scripts/tool_call/run-bfcl-eval.sh && \
 #   BFCL_OUTPUT_DIR=./bfcl-responses BFCL_API_TYPE=responses \
 #     bash .buildkite/scripts/tool_call/run-bfcl-eval.sh
 #
 # Environment variables (all optional, with defaults):
 #   BFCL_MODEL          - HF model name (default: openai/gpt-oss-20b)
 #   BFCL_API_TYPE       - API type: "chat_completions" or "responses" (default: chat_completions)
 #   BFCL_OUTPUT_DIR     - Directory for BFCL results (default: current working directory)
 #   BFCL_TEST_CATEGORY  - BFCL test categories (default: multi_turn)
 #   BFCL_TOOL_CALL_PARSER - Tool call parser name (default: openai)
 #   BFCL_NUM_THREADS    - Threads for BFCL generate (default: 8)
 #   BFCL_TP_SIZE        - Tensor parallel size (default: 1)
 #   BFCL_MAX_MODEL_LEN  - Max model length (default: 4096)
 #   BFCL_PORT           - Server port (default: 8000)
 #   BFCL_REASONING_PARSER - Reasoning parser name (default: disabled)
 #   BFCL_TEMPERATURE    - Temperature (default: 0.0)
 #   BFCL_EXTRA_ARGS     - Additional vLLM server args
 set -euo pipefail
 # ---- Configuration ----
 MODEL="${BFCL_MODEL:-openai/gpt-oss-20b}"
 API_TYPE="${BFCL_API_TYPE:-chat_completions}"
 OUTPUT_DIR="${BFCL_OUTPUT_DIR:-}"
 TEST_CATEGORY="${BFCL_TEST_CATEGORY:-multi_turn}"
 TOOL_CALL_PARSER="${BFCL_TOOL_CALL_PARSER:-openai}"
 NUM_THREADS="${BFCL_NUM_THREADS:-8}"
 TP_SIZE="${BFCL_TP_SIZE:-1}"
 MAX_MODEL_LEN="${BFCL_MAX_MODEL_LEN:-4096}"
 PORT="${BFCL_PORT:-8000}"
 REASONING_PARSER="${BFCL_REASONING_PARSER:-}"
 TEMPERATURE="${BFCL_TEMPERATURE:-0.0}"
 EXTRA_ARGS="${BFCL_EXTRA_ARGS:-}"
 # Set up output directory
 if [ -n "$OUTPUT_DIR" ]; then
    mkdir -p "$OUTPUT_DIR"
    OUTPUT_DIR="$(cd "$OUTPUT_DIR" && pwd)"
 fi
 echo "============================================"
 echo "BFCL Tool Call Correctness Evaluation"
 echo "============================================"
 echo "Model:          $MODEL"
 echo "Tool parser:    $TOOL_CALL_PARSER"
 echo "API type:       $API_TYPE"
 echo "Output dir:     ${OUTPUT_DIR:-<cwd>}"
 echo "Test category:  $TEST_CATEGORY"
 echo "TP size:        $TP_SIZE"
 echo "Max model len:  $MAX_MODEL_LEN"
 echo "Port:           $PORT"
 echo "Num threads:    $NUM_THREADS"
 echo "============================================"
 # ---- Install bfcl-eval if missing ----
 if ! python3 -c "import bfcl_eval" 2>/dev/null; then
    echo "Installing bfcl-eval..."
    uv pip install "bfcl-eval>=2025.10.20.1,<2026"
 fi
 # ---- Cleanup handler ----
 SERVER_PID=""
 cleanup() {
    if [ -n "$SERVER_PID" ]; then
        echo "Stopping vLLM server (pid=$SERVER_PID)..."
        kill "$SERVER_PID" 2>/dev/null || true
        wait "$SERVER_PID" 2>/dev/null || true
    fi
    # Remove BFCL lock files (created by filelock for thread-safe writes)
    rm -rf .file_locks/
    if [ -n "${OUTPUT_DIR:-}" ]; then
        rm -rf "$OUTPUT_DIR/.file_locks/"
    fi
 }
 trap cleanup EXIT
 # ---- Start vLLM server ----
 echo "Starting vLLM server..."
 SERVE_ARGS=(
    "$MODEL"
    --port "$PORT"
    --enable-auto-tool-choice
    --tool-call-parser "$TOOL_CALL_PARSER"
    --tensor-parallel-size "$TP_SIZE"
    --max-model-len "$MAX_MODEL_LEN"
    --enforce-eager
    --enable-prefix-caching
 )
 # Append reasoning parser if specified
 if [ -n "$REASONING_PARSER" ]; then
    SERVE_ARGS+=(--reasoning-parser "$REASONING_PARSER")
 fi
 # Append any extra args
 if [ -n "$EXTRA_ARGS" ]; then
    read -ra EXTRA_ARGS_ARRAY <<< "$EXTRA_ARGS"
    SERVE_ARGS+=("${EXTRA_ARGS_ARRAY[@]}")
 fi
 echo "Command: vllm serve ${SERVE_ARGS[*]}"
 vllm serve "${SERVE_ARGS[@]}" &
 SERVER_PID=$!
 # ---- Wait for server to be ready ----
 echo "Waiting for vLLM server to start (timeout: 600s)..."
 SECONDS_WAITED=0
 until curl -sf "http://localhost:${PORT}/health" > /dev/null 2>&1; do
    if [ $SECONDS_WAITED -ge 600 ]; then
        echo ""
        echo "ERROR: vLLM server failed to start within 600s"
        exit 1
    fi
    if (( SECONDS_WAITED % 30 == 0 && SECONDS_WAITED > 0 )); then
        echo "  Still waiting... (${SECONDS_WAITED}s elapsed)"
    fi
    sleep 2
    SECONDS_WAITED=$((SECONDS_WAITED + 2))
 done
 echo "vLLM server is ready. (started in ${SECONDS_WAITED}s)"
 # ---- Run BFCL evaluation ----
 # bfcl-eval has no CLI entry point; generate() and evaluate() are Typer
 # functions that must be called from Python. The MODEL_CONFIG_MAPPING must
 # be patched in-process so BFCL knows to use the OpenAI-compatible handler
 # against our local vLLM server.
 bfcl_exit_code=0
 python3 - "$MODEL" "$TEST_CATEGORY" "$NUM_THREADS" "$PORT" "$API_TYPE" "$TEMPERATURE" "$OUTPUT_DIR" << 'PYEOF' || bfcl_exit_code=$?
 import os
 import sys
 model = sys.argv[1]
 test_category = sys.argv[2]
 num_threads = int(sys.argv[3])
 port = sys.argv[4]
 api_type = sys.argv[5]
 temperature = float(sys.argv[6])
 output_dir = sys.argv[7] if len(sys.argv) > 7 and sys.argv[7] else os.getcwd()
 os.environ["OPENAI_BASE_URL"] = f"http://localhost:{port}/v1"
 os.environ["OPENAI_API_KEY"] = "dummy"
 os.environ["BFCL_PROJECT_ROOT"] = output_dir
 import bfcl_eval.constants.model_config as bfcl_model_config
 from bfcl_eval.constants.model_config import ModelConfig
 from bfcl_eval.model_handler.api_inference.openai_completion import (
    OpenAICompletionsHandler,
 )
 from bfcl_eval.model_handler.api_inference.openai_response import (
    OpenAIResponsesHandler,
 )
 if api_type == "responses":
    handler = OpenAIResponsesHandler
 else:
    handler = OpenAICompletionsHandler
 bfcl_model_config.MODEL_CONFIG_MAPPING[model] = ModelConfig(
    model_name=model,
    display_name=f"{model} (FC) (vLLM)",
    url=f"https://huggingface.co/{model}",
    org="",
    license="apache-2.0",
    model_handler=handler,
    input_price=None,
    output_price=None,
    is_fc_model=True,
    underscore_to_dot=True,
 )
 from bfcl_eval.__main__ import evaluate, generate
 import inspect
 import typer
 def _get_default_kwargs(function):
    kwargs = {}
    for k, v in inspect.signature(function).parameters.items():
        if v.default is not inspect.Parameter.empty:
            default = v.default
            if isinstance(default, typer.models.OptionInfo):
                default = default.default
            kwargs[k] = default
    return kwargs
 # ---- generate ----
 print(f"=== BFCL generate: model={model} test_category={test_category} ===")
 gen_kwargs = _get_default_kwargs(generate)
 gen_kwargs["model"] = [model]
 gen_kwargs["test_category"] = [c.strip() for c in test_category.split(",")]
 gen_kwargs["skip_server_setup"] = True
 gen_kwargs["num_threads"] = num_threads
 gen_kwargs["temperature"] = temperature
 generate(**gen_kwargs)
 # ---- evaluate ----
 print(f"=== BFCL evaluate: model={model} test_category={test_category} ===")
 eval_kwargs = _get_default_kwargs(evaluate)
 eval_kwargs["model"] = [model]
 eval_kwargs["test_category"] = [c.strip() for c in test_category.split(",")]
 evaluate(**eval_kwargs)
 print("=== BFCL evaluation completed successfully ===")
 PYEOF
 # ---- Upload results to buildkite ----
 if command -v buildkite-agent &>/dev/null; then
    if [ $bfcl_exit_code -eq 0 ]; then
        STYLE="success"
        STATUS="PASSED"
    else
        STYLE="error"
        STATUS="FAILED"
    fi
    buildkite-agent annotate --style "$STYLE" --context "bfcl-results" <<EOF
 ### BFCL Tool Call Correctness - ${STATUS}
 - **Model:** \`${MODEL}\`
 - **Parser:** \`${TOOL_CALL_PARSER}\`
 - **API type:** \`${API_TYPE}\`
 - **Test category:** \`${TEST_CATEGORY}\`
 EOF
    # BFCL writes results to $BFCL_PROJECT_ROOT/result/ and scores to
    # $BFCL_PROJECT_ROOT/score/
    RESULTS_ROOT="${OUTPUT_DIR:-.}"
    if [ -d "$RESULTS_ROOT/result" ]; then
        buildkite-agent artifact upload "$RESULTS_ROOT/result/**/*"
    fi
    if [ -d "$RESULTS_ROOT/score" ]; then
        buildkite-agent artifact upload "$RESULTS_ROOT/score/**/*"
    fi
 fi
 exit $bfcl_exit_code
@@ -23,7 +23,6 @@ NC='\033[0m' # No Color
 # Default configuration
 PIPELINE="ci"
 DRY_RUN=true
 TORCH_NIGHTLY=false
 usage() {
    cat <<EOF
@@ -35,14 +34,12 @@ Sets RUN_ALL=1 and NIGHTLY=1 environment variables.
 SAFETY: Dry-run by default. Use --execute to actually trigger a build.
 Options:
-    --execute        Actually trigger the build (default: dry-run)
+    --execute       Actually trigger the build (default: dry-run)
-    --pipeline       Buildkite pipeline slug (default: ${PIPELINE})
+    --pipeline      Buildkite pipeline slug (default: ${PIPELINE})
-    --commit         Override commit SHA (default: current HEAD)
+    --commit        Override commit SHA (default: current HEAD)
-    --branch         Override branch name (default: current branch)
+    --branch        Override branch name (default: current branch)
-    --message        Custom build message (default: auto-generated)
+    --message       Custom build message (default: auto-generated)
-    --torch-nightly  Also build and run the full suite against torch nightly
+    --help          Show this help message
                     (sets TORCH_NIGHTLY=1)
    --help           Show this help message
 Prerequisites:
    - bk CLI installed: brew tap buildkite/buildkite && brew install buildkite/buildkite/bk
@@ -52,7 +49,6 @@ Examples:
    $(basename "$0")                        # Dry-run, show what would happen
    $(basename "$0") --execute              # Actually trigger the build
    $(basename "$0") --pipeline ci-shadow   # Dry-run with different pipeline
    $(basename "$0") --torch-nightly        # Dry-run a full torch-nightly run
 EOF
    exit 1
 }
@@ -100,10 +96,6 @@ while [[ $# -gt 0 ]]; do
            MESSAGE="$2"
            shift 2
            ;;
        --torch-nightly)
            TORCH_NIGHTLY=true
            shift
            ;;
        --help|-h)
            usage
            ;;
@@ -179,17 +171,11 @@ if [[ $(echo "$REMOTE_BRANCHES" | wc -l) -gt 5 ]]; then
 fi
 echo ""
 # Environment variables passed to the build.
 BUILD_ENV=("RUN_ALL=1" "NIGHTLY=1")
 if [[ "$TORCH_NIGHTLY" == true ]]; then
    BUILD_ENV+=("TORCH_NIGHTLY=1")
 fi
 log_info "Pipeline: ${PIPELINE}"
 log_info "Branch: ${BRANCH}"
 log_info "Commit: ${COMMIT}"
 log_info "Message: ${MESSAGE}"
-log_info "Environment: ${BUILD_ENV[*]}"
+log_info "Environment: RUN_ALL=1, NIGHTLY=1"
 echo ""
 # Build the command
@@ -201,10 +187,9 @@ CMD=(bk build create
    --commit "${COMMIT}"
    --branch "${BRANCH}"
    --message "${MESSAGE}"
    --env "RUN_ALL=1"
    --env "NIGHTLY=1"
 )
 for env_var in "${BUILD_ENV[@]}"; do
    CMD+=(--env "${env_var}")
 done
 if [[ "$DRY_RUN" == true ]]; then
    echo "=========================================="
@@ -225,14 +210,8 @@ if [[ "$DRY_RUN" == true ]]; then
    echo "    --commit '$(escape_for_shell "${COMMIT}")' \\"
    echo "    --branch '$(escape_for_shell "${BRANCH}")' \\"
    echo "    --message '$(escape_for_shell "${MESSAGE}")' \\"
-    last_idx=$(( ${#BUILD_ENV[@]} - 1 ))
+    echo "    --env 'RUN_ALL=1' \\"
-    for i in "${!BUILD_ENV[@]}"; do
+    echo "    --env 'NIGHTLY=1'"
        if [[ $i -eq $last_idx ]]; then
            echo "    --env '$(escape_for_shell "${BUILD_ENV[$i]}")'"
        else
            echo "    --env '$(escape_for_shell "${BUILD_ENV[$i]}")' \\"
        fi
    done
    echo ""
    echo "=========================================="
    echo -e "${YELLOW}To actually trigger this build, run:${NC}"
--- a/Show More
+++ b/Show More