fix python install ci error

Signed-off-by: yewentao256 <zhyanwentao@126.com>
2025-12-01 14:19:43 -08:00
5645 changed files with 280674 additions and 1166632 deletions
@@ -1,26 +0,0 @@
 name: vllm_ci
 job_dirs:
  - ".buildkite/image_build"
  - ".buildkite/test_areas"
  - ".buildkite/hardware_tests"
 run_all_patterns:
  - "docker/Dockerfile"
  - "CMakeLists.txt"
  - "requirements/common.txt"
  - "requirements/cuda.txt"
  - "requirements/kv_connectors.txt"
  - "requirements/build/cuda.txt"
  - "requirements/test/cuda.txt"
  - "setup.py"
  - "csrc/"
  - "cmake/"
 run_all_exclude_patterns:
  - "docker/Dockerfile."
  - "csrc/cpu/"
  - "csrc/rocm/"
  - "cmake/hipify.py"
  - "cmake/cpu_extension.cmake"
 registries: public.ecr.aws/q9t5s3a7
 repositories:
  main: "vllm-ci-postmerge-repo"
  premerge: "vllm-ci-test-repo"
@@ -1,23 +0,0 @@
 name: vllm_intel_ci
 job_dirs:
  - ".buildkite/intel_jobs"
 run_all_patterns:
  - ".buildkite/ci_config_intel.yaml"
  - ".buildkite/scripts/hardware_ci/run-intel-test.sh"
  - "docker/Dockerfile"
  - "docker/Dockerfile.xpu"
  - "CMakeLists.txt"
  - "requirements/common.txt"
  - "requirements/xpu.txt"
  - "setup.py"
  - "csrc/"
  - "cmake/"
 run_all_exclude_patterns:
  - "csrc/cpu/"
  - "csrc/rocm/"
  - "cmake/hipify.py"
  - "cmake/cpu_extension.cmake"
 registries: public.ecr.aws/q9t5s3a7
 repositories:
  main: "vllm-ci-test-repo"
  premerge: "vllm-ci-test-repo"
@@ -1,24 +0,0 @@
 name: vllm_rocm_ci
 job_dirs:
  - ".buildkite/hardware_tests"
 run_all_patterns:
  - "docker/Dockerfile.rocm"
  - "docker/Dockerfile.rocm_base"
  - "docker/ci-rocm.hcl"
  - "docker/docker-bake-rocm.hcl"
  - ".buildkite/hardware_tests/amd.yaml"
  - ".buildkite/scripts/ci-bake-rocm.sh"
  - ".buildkite/scripts/rocm/"
  - ".buildkite/scripts/hardware_ci/run-amd-test.py"
  - ".buildkite/scripts/hardware_ci/run-amd-test.sh"
  - "CMakeLists.txt"
  - "requirements/common.txt"
  - "requirements/rocm.txt"
  - "requirements/build/rocm.txt"
  - "requirements/test/rocm.txt"
  - "setup.py"
  - "csrc/"
  - "cmake/"
 run_all_exclude_patterns:
  - "csrc/cpu/"
  - "cmake/cpu_extension.cmake"
@@ -0,0 +1,46 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 import argparse
 import os
 template = """<!DOCTYPE html>
 <html>
    <body>
    <h1>Links for vLLM</h1/>
        <a href="../{x86_wheel_html_escaped}">{x86_wheel}</a><br/>
        <a href="../{arm_wheel_html_escaped}">{arm_wheel}</a><br/>
    </body>
 </html>
 """
 parser = argparse.ArgumentParser()
 parser.add_argument("--wheel", help="The wheel path.", required=True)
 args = parser.parse_args()
 filename = os.path.basename(args.wheel)
 with open("index.html", "w") as f:
    print(f"Generated index.html for {args.wheel}")
    # sync the abi tag with .buildkite/scripts/upload-wheels.sh
    if "x86_64" in filename:
        x86_wheel = filename
        arm_wheel = filename.replace("x86_64", "aarch64").replace(
            "manylinux1", "manylinux2014"
        )
    elif "aarch64" in filename:
        x86_wheel = filename.replace("aarch64", "x86_64").replace(
            "manylinux2014", "manylinux1"
        )
        arm_wheel = filename
    else:
        raise ValueError(f"Unsupported wheel: {filename}")
    # cloudfront requires escaping the '+' character
    f.write(
        template.format(
            x86_wheel=x86_wheel,
            x86_wheel_html_escaped=x86_wheel.replace("+", "%2B"),
            arm_wheel=arm_wheel,
            arm_wheel_html_escaped=arm_wheel.replace("+", "%2B"),
        )
    )
@@ -1,84 +0,0 @@
 group: Hardware - AMD Build
 # ROCm image flow:
 # 1. Refresh the long-lived ROCm base image only when Dockerfile.rocm_base changes.
 # 2. Build ci_base from either the stable base or the freshly refreshed base.
 # 3. Build the per-commit ROCm CI image and smoke-test it before GPU jobs run.
 steps:
  - label: "AMD: :docker: refresh ROCm base"
    key: refresh-rocm-base-amd
    depends_on: []
    device: amd_cpu
    no_plugin: true
    commands:
      - bash .buildkite/scripts/rocm/refresh-base-image.sh
    env:
      DOCKER_BUILDKIT: "1"
      BUILDKIT_PROGRESS: "tty"
      TERM: "xterm-256color"
    retry:
      automatic:
        - exit_status: 1  # Transient Docker/BuildKit failure
          limit: 1
        - exit_status: -1  # Agent was lost
          limit: 1
        - exit_status: -10  # Agent was lost
          limit: 1
  # Ensure ci_base is up-to-date before building the test image.
  # Compares a content hash of ci_base-affecting files against the remote
  # image label. If hashes match the build is skipped (< 30 s); if they
  # differ ci_base is rebuilt and pushed automatically.
  - label: "AMD: :docker: ensure ci_base"
    key: ensure-ci-base-amd
    soft_fail: false
    depends_on:
      - refresh-rocm-base-amd
    device: amd_cpu
    no_plugin: true
    commands:
      - bash .buildkite/scripts/rocm/build-ci-base.sh
    env:
      DOCKER_BUILDKIT: "1"
      BUILDKIT_PROGRESS: "tty"
      TERM: "xterm-256color"
      VLLM_BAKE_FILE: "docker/docker-bake-rocm.hcl"
      PYTORCH_ROCM_ARCH: "gfx90a;gfx942;gfx950"
      REMOTE_VLLM: "1"
      VLLM_BRANCH: "$BUILDKITE_COMMIT"
    retry:
      automatic:
        - exit_status: 1  # Transient Docker/BuildKit failure
          limit: 1
        - exit_status: -1  # Agent was lost
          limit: 1
        - exit_status: -10  # Agent was lost
          limit: 1
  - label: "AMD: :docker: build test image and artifacts"
    key: image-build-amd
    soft_fail: false
    depends_on:
      - ensure-ci-base-amd
    device: amd_cpu
    no_plugin: true
    commands:
      - bash .buildkite/scripts/rocm/build-test-image.sh
      - bash .buildkite/scripts/rocm/smoke-test-image.sh
    env:
      DOCKER_BUILDKIT: "1"
      BUILDKIT_PROGRESS: "tty"
      TERM: "xterm-256color"
      VLLM_BAKE_FILE: "docker/docker-bake-rocm.hcl"
      PYTORCH_ROCM_ARCH: "gfx90a;gfx942;gfx950"
      IMAGE_TAG: "rocm/vllm-ci:$BUILDKITE_COMMIT"
      REMOTE_VLLM: "1"
      VLLM_BRANCH: "$BUILDKITE_COMMIT"
    retry:
      automatic:
        - exit_status: 1  # Transient Docker/BuildKit failure
          limit: 1
        - exit_status: -1  # Agent was lost
          limit: 1
        - exit_status: -10  # Agent was lost
          limit: 1
@@ -1,10 +0,0 @@
 group: Hardware
 depends_on: ~
 steps:
  - label: "Ascend NPU Test"
    soft_fail: true
    timeout_in_minutes: 20
    no_plugin: true
    device: ascend_npu
    commands: 
    - bash .buildkite/scripts/hardware_ci/run-npu-test.sh
@@ -1,170 +0,0 @@
 group: CPU
 depends_on: []
 steps:
 - label: CPU-Kernel Tests
  depends_on: []
  device: intel_cpu
  no_plugin: true
  source_file_dependencies:
  - csrc/cpu/
  - cmake/cpu_extension.cmake
  - CMakeLists.txt
  - vllm/_custom_ops.py
  - tests/kernels/attention/test_cpu_attn.py
  - tests/kernels/moe/test_cpu_fused_moe.py
  - tests/kernels/moe/test_cpu_quant_fused_moe.py
  - tests/kernels/test_onednn.py
  - tests/kernels/test_awq_int4_to_int8.py
  - tests/kernels/quantization/test_cpu_fp8_scaled_mm.py
  - tests/kernels/mamba/cpu/test_cpu_gdn_ops.py
  - tests/kernels/mamba/test_cpu_short_conv.py
  - tests/kernels/mamba/test_causal_conv1d.py
  - tests/kernels/mamba/test_mamba_ssm.py
  commands:
    - |
      bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 30m "
      pytest -x -v -s tests/kernels/attention/test_cpu_attn.py
      pytest -x -v -s tests/kernels/moe/test_cpu_fused_moe.py
      pytest -x -v -s tests/kernels/moe/test_cpu_quant_fused_moe.py
      pytest -x -v -s tests/kernels/mamba/test_cpu_short_conv.py
      pytest -x -v -s tests/kernels/test_onednn.py
      pytest -x -v -s tests/kernels/test_awq_int4_to_int8.py
      pytest -x -v -s tests/kernels/quantization/test_cpu_fp8_scaled_mm.py
      pytest -x -v -s tests/kernels/mamba/cpu/test_cpu_gdn_ops.py
      pytest -x -v -s tests/kernels/mamba/test_causal_conv1d.py
      pytest -x -v -s tests/kernels/mamba/test_mamba_ssm.py"
 # Note: SDE can't be downloaded from CI host because of AWS WAF
 # - label: CPU-Compatibility Tests
 #   depends_on: []
 #   device: intel_cpu
 #   no_plugin: true
 #   source_file_dependencies:
 #   - cmake/cpu_extension.cmake
 #   - setup.py
 #   - vllm/platforms/cpu.py
 #   commands:
 #     - |
 #       bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 20m "
 #       bash .buildkite/scripts/hardware_ci/run-cpu-compatibility-test.sh"
 - label: CPU-Language Generation and Pooling Model Tests
  depends_on: []
  device: intel_cpu
  no_plugin: true
  source_file_dependencies:
  - csrc/cpu/
  - vllm/
  - tests/models/language/generation/
  - tests/models/language/pooling/
  commands:
    - |
      bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 50m "
      pytest -x -v -s tests/models/language/generation -m cpu_model
      pytest -x -v -s tests/models/language/pooling -m cpu_model"
 - label: CPU-ModelRunnerV2 Tests
  depends_on: []
  device: intel_cpu
  no_plugin: true
  soft_fail: true
  source_file_dependencies:
  - vllm/v1/worker/cpu/
  - vllm/v1/worker/gpu/
  - vllm/v1/sample/ops/topk_topp_triton.py
  - vllm/v1/sample/ops/topk_topp_sampler.py
  - tests/v1/sample/test_topk_topp_sampler.py
  - tests/v1/e2e/test_cpu_linear_attn_chunked_prefix.py
  commands:
    - |
      bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 45m "
      uv pip install git+https://github.com/triton-lang/triton-cpu.git@270e696d
      VLLM_USE_V2_MODEL_RUNNER=1 pytest -x -v -s tests/models/language/generation/test_granite.py -m cpu_model
      # TODO: move to CPU-Kernel Tests once triton-cpu has a pre-built wheel
      pytest -x -v -s tests/v1/sample/test_topk_topp_sampler.py::TestTritonTopkTopp
      pytest -x -v -s tests/v1/e2e/test_cpu_linear_attn_chunked_prefix.py"
 - label: CPU-Quantization Model Tests
  depends_on: []
  device: intel_cpu
  no_plugin: true
  source_file_dependencies:
  - csrc/cpu/
  - vllm/model_executor/layers/quantization/auto_gptq.py
  - vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py
  - vllm/model_executor/kernels/linear/mixed_precision/cpu.py
  - vllm/model_executor/kernels/linear/scaled_mm/cpu.py
  - vllm/model_executor/layers/fused_moe/experts/cpu_moe.py
  - tests/quantization/test_compressed_tensors.py
  - tests/quantization/test_cpu_wna16.py
  - tests/quantization/test_cpu_w8a8.py
  commands:
    - |
      bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 45m "
      pytest -x -v -s tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_logprobs
      pytest -x -v -s tests/quantization/test_cpu_wna16.py
      pytest -x -v -s tests/quantization/test_cpu_w8a8.py"
 - label: CPU-Distributed Tests (PP+TP)
  depends_on: []
  device: intel_cpu
  no_plugin: true
  source_file_dependencies: &cpu_distributed_deps
  - csrc/cpu/shm.cpp
  - vllm/v1/worker/cpu_worker.py
  - vllm/v1/worker/gpu_worker.py
  - vllm/v1/worker/cpu_model_runner.py
  - vllm/v1/worker/gpu_model_runner.py
  - vllm/platforms/cpu.py
  - vllm/distributed/parallel_state.py
  - vllm/distributed/device_communicators/cpu_communicator.py
  - .buildkite/scripts/hardware_ci/run-cpu-distributed-smoke-test.sh
  commands:
    - |
      bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 10m "
      bash .buildkite/scripts/hardware_ci/run-cpu-distributed-smoke-test.sh tp_pp"
 - label: CPU-Distributed Tests (DP+TP)
  depends_on: []
  device: intel_cpu
  no_plugin: true
  source_file_dependencies: *cpu_distributed_deps
  commands:
    - |
      bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 10m "
      bash .buildkite/scripts/hardware_ci/run-cpu-distributed-smoke-test.sh dp_tp"
 - label: CPU-Multi-Modal Model Tests %N
  depends_on: []
  device: intel_cpu
  no_plugin: true
  source_file_dependencies:
  # - vllm/
  - vllm/model_executor/layers/rotary_embedding
  - tests/models/multimodal/generation/
  commands:
    - |
      bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 45m "
      pytest -x -v -s tests/models/multimodal/generation --ignore=tests/models/multimodal/generation/test_pixtral.py --ignore=tests/models/multimodal/generation/test_qwen2_5_vl.py -m cpu_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
  parallelism: 4
 - label: CPU-Qwen2.5-VL Multimodal Tests
  depends_on: []
  device: intel_cpu
  no_plugin: true
  source_file_dependencies:
  # - vllm/
  - vllm/model_executor/layers/rotary_embedding
  - tests/models/multimodal/generation/
  commands:
    - |
      bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 40m "
      VLLM_CI_ENV=0 pytest -x -v -s tests/models/multimodal/generation/test_qwen2_5_vl.py"
 - label: "Arm CPU Test"
  depends_on: []
  soft_fail: false
  device: arm_cpu
  no_plugin: true
  commands: 
  - bash .buildkite/scripts/hardware_ci/run-cpu-test-arm.sh
@@ -1,10 +0,0 @@
 group: Hardware
 steps:
  - label: "GH200 Test"
    soft_fail: true
    device: gh200
    no_plugin: true
    optional: true
    commands: 
    - nvidia-smi 
    - bash .buildkite/scripts/hardware_ci/run-gh200-test.sh
@@ -1,10 +0,0 @@
 group: Hardware
 depends_on: ~
 steps:
  - label: "Intel HPU Test"
    soft_fail: true
    device: intel_hpu
    no_plugin: true
    commands: 
    - bash .buildkite/scripts/hardware_ci/run-hpu-test.sh
@@ -1,80 +0,0 @@
 group: Intel
 steps:
  - label: ":docker: Build XPU image"
    soft_fail: true
    optional: true
    depends_on: []
    key: image-build-xpu
    commands:
      - bash -lc '.buildkite/image_build/image_build_xpu.sh "public.ecr.aws/q9t5s3a7" "vllm-ci-test-repo" "$BUILDKITE_COMMIT"'
    env:
      DOCKER_BUILDKIT: "1"
    retry:
      automatic:
        - exit_status: -1  # Agent was lost
          limit: 2
        - exit_status: -10  # Agent was lost
          limit: 2
  - label: "XPU example Test"
    depends_on:
      - image-build-xpu
    timeout_in_minutes: 50
    optional: true
    device: intel_gpu
    agent_tags:
      label: production
      gpu: 2+
      mem: 24+
    no_plugin: true
    env:
      REGISTRY: "public.ecr.aws/q9t5s3a7"
      REPO: "vllm-ci-test-repo"
    source_file_dependencies:
      - .buildkite/hardware_tests/intel_xpu_ci/test-intel.yaml
      - .buildkite/scripts/hardware_ci/run-intel-ci-test.sh
    commands:
      - >-
        bash .buildkite/scripts/hardware_ci/run-intel-test.sh
        'bash .buildkite/scripts/hardware_ci/run-intel-ci-test.sh example'
  - label: "XPU V1 test"
    depends_on:
      - image-build-xpu
    timeout_in_minutes: 70
    optional: true
    device: intel_gpu
    agent_tags:
      label: production
      gpu: 1+
      mem: 24+
    no_plugin: true
    env:
      REGISTRY: "public.ecr.aws/q9t5s3a7"
      REPO: "vllm-ci-test-repo"
    source_file_dependencies:
      - .buildkite/hardware_tests/intel_xpu_ci/test-intel.yaml
      - .buildkite/scripts/hardware_ci/run-intel-ci-test.sh
    commands:
      - >-
        bash .buildkite/scripts/hardware_ci/run-intel-test.sh
        'bash .buildkite/scripts/hardware_ci/run-intel-ci-test.sh v1'
  - label: "XPU server test"
    depends_on:
      - image-build-xpu
    timeout_in_minutes: 45
    optional: true
    device: intel_gpu
    agent_tags:
      label: production
      gpu: 1+
      mem: 16+
    no_plugin: true
    env:
      REGISTRY: "public.ecr.aws/q9t5s3a7"
      REPO: "vllm-ci-test-repo"
    source_file_dependencies:
      - .buildkite/hardware_tests/intel_xpu_ci/test-intel.yaml
      - .buildkite/scripts/hardware_ci/run-intel-ci-test.sh
    commands:
      - >-
        bash .buildkite/scripts/hardware_ci/run-intel-test.sh
        'bash .buildkite/scripts/hardware_ci/run-intel-ci-test.sh server'
@@ -1,276 +0,0 @@
 #!/bin/bash
 set -euo pipefail
 # replace invalid characters in Docker image tags and truncate to 128 chars
 clean_docker_tag() {
    local input="$1"
    echo "$input" | sed 's/[^a-zA-Z0-9._-]/_/g' | cut -c1-128
 }
 print_usage_and_exit() {
    echo "Usage: $0 <registry> <repo> <commit> <branch> <image_tag> [<image_tag_latest>]"
    exit 1
 }
 print_instance_info() {
    echo ""
    echo "=== Debug: Instance Information ==="
    # Get IMDSv2 token
    if TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" \
            -H "X-aws-ec2-metadata-token-ttl-seconds: 21600" 2>/dev/null); then
        AMI_ID=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" \
            http://169.254.169.254/latest/meta-data/ami-id 2>/dev/null || echo "unknown")
        INSTANCE_TYPE=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" \
            http://169.254.169.254/latest/meta-data/instance-type 2>/dev/null || echo "unknown")
        INSTANCE_ID=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" \
            http://169.254.169.254/latest/meta-data/instance-id 2>/dev/null || echo "unknown")
        AZ=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" \
            http://169.254.169.254/latest/meta-data/placement/availability-zone 2>/dev/null || echo "unknown")
        echo "AMI ID:        ${AMI_ID}"
        echo "Instance Type: ${INSTANCE_TYPE}"
        echo "Instance ID:   ${INSTANCE_ID}"
        echo "AZ:            ${AZ}"
    else
        echo "Not running on EC2 or IMDS not available"
    fi
    # Check for warm cache AMI (marker file baked into custom AMI)
    if [[ -f /etc/vllm-ami-info ]]; then
        echo "Cache:         warm (custom vLLM AMI)"
        cat /etc/vllm-ami-info
    else
        echo "Cache:         cold (standard AMI)"
    fi
    echo "==================================="
    echo ""
 }
 setup_buildx_builder() {
    echo "--- :buildkite: Setting up buildx builder"
    if [[ -S "${BUILDKIT_SOCKET}" ]]; then
        # Custom AMI with standalone buildkitd - use remote driver for warm cache
        echo "✅ Found local buildkitd socket at ${BUILDKIT_SOCKET}"
        echo "Using remote driver to connect to buildkitd (warm cache available)"
        if docker buildx inspect baked-vllm-builder >/dev/null 2>&1; then
            echo "Using existing baked-vllm-builder"
            docker buildx use baked-vllm-builder
        else
            echo "Creating baked-vllm-builder with remote driver"
            docker buildx create \
                --name baked-vllm-builder \
                --driver remote \
                --use \
                "unix://${BUILDKIT_SOCKET}"
        fi
        docker buildx inspect --bootstrap
    elif docker buildx inspect "${BUILDER_NAME}" >/dev/null 2>&1; then
        # Existing builder available
        echo "Using existing builder: ${BUILDER_NAME}"
        docker buildx use "${BUILDER_NAME}"
        docker buildx inspect --bootstrap
    else
        # No local buildkitd, no existing builder - create new docker-container builder
        echo "No local buildkitd found, using docker-container driver"
        docker buildx create --name "${BUILDER_NAME}" --driver docker-container --use
        docker buildx inspect --bootstrap
    fi
    # builder info
    echo "Active builder:"
    docker buildx ls | grep -E '^\*|^NAME' || docker buildx ls
 }
 annotate_image_tags() {
    .buildkite/scripts/annotate-image-build.sh \
        "${IMAGE_TAG:-}" "${IMAGE_TAG_LATEST:-}"
 }
 check_and_skip_if_image_exists() {
    if [[ -n "${IMAGE_TAG:-}" ]]; then
        echo "--- :mag: Checking if image exists"
        if docker manifest inspect "${IMAGE_TAG}" >/dev/null 2>&1; then
            echo "Image already exists: ${IMAGE_TAG}"
            echo "Skipping build"
            annotate_image_tags
            exit 0
        fi
        echo "Image not found, proceeding with build"
    fi
 }
 ecr_login() {
    aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true
    aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 936637512419.dkr.ecr.us-east-1.amazonaws.com || true
 }
 prepare_cache_tags() {
    # resolve and set: CACHE_TO, CACHE_FROM, CACHE_FROM_BASE_BRANCH, CACHE_FROM_MAIN
    TEST_CACHE_ECR="936637512419.dkr.ecr.us-east-1.amazonaws.com/vllm-ci-test-cache"
    MAIN_CACHE_ECR="936637512419.dkr.ecr.us-east-1.amazonaws.com/vllm-ci-postmerge-cache"
    if [[ "$BUILDKITE_PULL_REQUEST" == "false" ]]; then
        if [[ "$BUILDKITE_BRANCH" == "main" ]]; then
            cache="${MAIN_CACHE_ECR}:latest"
        else
            clean_branch=$(clean_docker_tag "$BUILDKITE_BRANCH")
            cache="${TEST_CACHE_ECR}:${clean_branch}"
        fi
        CACHE_TO="$cache"
        CACHE_FROM="$cache"
        CACHE_FROM_BASE_BRANCH="$cache"
    else
        CACHE_TO="${TEST_CACHE_ECR}:pr-${BUILDKITE_PULL_REQUEST}"
        CACHE_FROM="${TEST_CACHE_ECR}:pr-${BUILDKITE_PULL_REQUEST}"
        if [[ "$BUILDKITE_PULL_REQUEST_BASE_BRANCH" == "main" ]]; then
            CACHE_FROM_BASE_BRANCH="${MAIN_CACHE_ECR}:latest"
        else
            clean_base=$(clean_docker_tag "$BUILDKITE_PULL_REQUEST_BASE_BRANCH")
            CACHE_FROM_BASE_BRANCH="${TEST_CACHE_ECR}:${clean_base}"
        fi
    fi
    CACHE_FROM_MAIN="${MAIN_CACHE_ECR}:latest"
    export CACHE_TO CACHE_FROM CACHE_FROM_BASE_BRANCH CACHE_FROM_MAIN
 }
 resolve_parent_commit() {
    if [[ -z "${PARENT_COMMIT:-}" ]]; then
        PARENT_COMMIT=$(git rev-parse HEAD~1 2>/dev/null || echo "")
        if [[ -n "${PARENT_COMMIT}" ]]; then
            echo "Computed parent commit for cache fallback: ${PARENT_COMMIT}"
            export PARENT_COMMIT
        else
            echo "Could not determine parent commit (may be first commit in repo)"
        fi
    else
        echo "Using provided PARENT_COMMIT: ${PARENT_COMMIT}"
    fi
 }
 print_bake_config() {
    echo "--- :page_facing_up: Resolved bake configuration"
    # Write to a temp directory to avoid polluting the repo root (which is the
    # Docker build context). Files left in the repo root get COPY'd into the
    # image and can cause duplicate artifact uploads from downstream steps.
    local bake_tmp
    bake_tmp="$(mktemp -d)"
    BAKE_CONFIG_FILE="${bake_tmp}/bake-config-build-${BUILDKITE_BUILD_NUMBER:-local}.json"
    docker buildx bake -f "${VLLM_BAKE_FILE_PATH}" -f "${CI_HCL_PATH}" --print "${TARGET}" | tee "${BAKE_CONFIG_FILE}" || true
    echo "Saved bake config to ${BAKE_CONFIG_FILE}"
    echo "--- :arrow_down: Uploading bake config to Buildkite"
    (cd "$(dirname "${BAKE_CONFIG_FILE}")" && buildkite-agent artifact upload "$(basename "${BAKE_CONFIG_FILE}")")
 }
 #################################
 #         Main Script           #
 #################################
 print_instance_info
 if [[ $# -lt 5 ]]; then
    print_usage_and_exit
 fi
 # input args
 REGISTRY=$1
 REPO=$2
 BUILDKITE_COMMIT=$3
 BRANCH=$4
 IMAGE_TAG=$5
 IMAGE_TAG_LATEST=${6:-} # only used for main branch, optional
 # When TORCH_NIGHTLY=1, build the base CI image against PyTorch nightly so the
 # entire existing pipeline runs on nightly torch (CUDA/GPU lane only). Delegate
 # to the dedicated nightly build (PYTORCH_NIGHTLY=1, CUDA 13.0) and tag it at the
 # normal IMAGE_TAG that every test step already pulls -- no separate image tag,
 # no duplicate "vLLM Against PyTorch Nightly" pipeline section.
 if [[ "${TORCH_NIGHTLY:-0}" == "1" ]]; then
    echo "--- :warning: TORCH_NIGHTLY=1 -- building base image on PyTorch nightly"
    SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
    exec "${SCRIPT_DIR}/image_build_torch_nightly.sh" \
        "${REGISTRY}" "${REPO}" "${BUILDKITE_COMMIT}" "${BRANCH}" "${IMAGE_TAG}"
 fi
 # build config
 TARGET="test-ci"
 VLLM_BAKE_FILE_PATH="${VLLM_BAKE_FILE_PATH:-docker/docker-bake.hcl}"
 BUILDER_NAME="${BUILDER_NAME:-vllm-builder}"
 CI_HCL_URL="${CI_HCL_URL:-https://raw.githubusercontent.com/vllm-project/ci-infra/main/docker/ci.hcl}"
 CI_HCL_PATH="/tmp/ci.hcl"
 BUILDKIT_SOCKET="/run/buildkit/buildkitd.sock"
 prepare_cache_tags
 ecr_login
 # Environment info (for docs and human readers)
 #   VLLM_CI_BRANCH      - ci-infra branch to use (default: main)
 #   VLLM_BAKE_FILE_PATH      - Path to vLLM's bake file (default: docker/docker-bake.hcl)
 #   BUILDER_NAME        - Name for buildx builder (default: vllm-builder)
 #
 # Build configuration (exported as environment variables for bake):
 export BUILDKITE_COMMIT
 export PARENT_COMMIT
 export IMAGE_TAG
 export IMAGE_TAG_LATEST
 export COMMIT="${COMMIT:-${BUILDKITE_COMMIT}}"
 export CACHE_FROM
 export CACHE_FROM_BASE_BRANCH
 export CACHE_FROM_MAIN
 export CACHE_TO
 # print args
 echo "--- :mag: Arguments"
 echo "REGISTRY: ${REGISTRY}"
 echo "REPO: ${REPO}"
 echo "BUILDKITE_COMMIT: ${BUILDKITE_COMMIT}"
 echo "BRANCH: ${BRANCH}"
 echo "IMAGE_TAG: ${IMAGE_TAG}"
 echo "IMAGE_TAG_LATEST: ${IMAGE_TAG_LATEST}"
 # print build configuration
 echo "--- :mag: Build configuration"
 echo "TARGET: ${TARGET}"
 echo "vLLM bake file: ${VLLM_BAKE_FILE_PATH}"
 echo "BUILDER_NAME: ${BUILDER_NAME}"
 echo "CI_HCL_URL: ${CI_HCL_URL}"
 echo "BUILDKIT_SOCKET: ${BUILDKIT_SOCKET}"
 echo "--- :mag: Cache tags"
 echo "CACHE_TO: ${CACHE_TO}"
 echo "CACHE_FROM: ${CACHE_FROM}"
 echo "CACHE_FROM_BASE_BRANCH: ${CACHE_FROM_BASE_BRANCH}"
 echo "CACHE_FROM_MAIN: ${CACHE_FROM_MAIN}"
 check_and_skip_if_image_exists
 echo "--- :docker: Setting up Docker buildx bake"
 echo "Target: ${TARGET}"
 echo "vLLM bake file: ${VLLM_BAKE_FILE_PATH}"
 echo "CI HCL path: ${CI_HCL_PATH}"
 if [[ ! -f "${VLLM_BAKE_FILE_PATH}" ]]; then
    echo "Error: vLLM bake file not found at ${VLLM_BAKE_FILE_PATH}"
    echo "Make sure you're running from the vLLM repository root"
    exit 1
 fi
 echo "--- :arrow_down: Downloading ci.hcl"
 curl -sSfL -o "${CI_HCL_PATH}" "${CI_HCL_URL}"
 echo "Downloaded to ${CI_HCL_PATH}"
 if [[ ! -f "${CI_HCL_PATH}" ]]; then
    echo "Error: ci.hcl not found at ${CI_HCL_PATH}"
    exit 1
 fi
 setup_buildx_builder
 resolve_parent_commit
 export PARENT_COMMIT
 print_bake_config
 echo "--- :docker: Building ${TARGET}"
 docker --debug buildx bake -f "${VLLM_BAKE_FILE_PATH}" -f "${CI_HCL_PATH}" --progress plain "${TARGET}"
 echo "--- :white_check_mark: Build complete"
 annotate_image_tags
@@ -1,130 +0,0 @@
 group: Abuild
 steps:
  - label: ":docker: Build image"
    key: image-build
    depends_on: []
    timeout_in_minutes: 600
    commands:
    - if [[ "$BUILDKITE_BRANCH" == "main" ]]; then .buildkite/image_build/image_build.sh $REGISTRY $REPO $BUILDKITE_COMMIT $BRANCH $IMAGE_TAG $IMAGE_TAG_LATEST; else .buildkite/image_build/image_build.sh $REGISTRY $REPO $BUILDKITE_COMMIT $BRANCH $IMAGE_TAG; fi
    retry:
      automatic:
        - exit_status: -1  # Agent was lost
          limit: 2
        - exit_status: -10  # Agent was lost
          limit: 2
  - label: ":docker: :smoking: Non-root smoke tests"
    key: image-build-smoke-test
    depends_on:
      - image-build
    commands:
    # Smoke 1: the default (root) image must still be importable
    # under a non-root UID via `--user 2000:0`. Validates the `vllm` passwd
    # entry + group-0-writable /home/vllm + uv path cleanup from #31959.
    # Uses `import vllm` rather than `vllm serve --help` because the latter
    # instantiates `VllmConfig` which requires a GPU attached to the
    # container.
    - docker run --rm --user 2000:0 --entrypoint python3 "$IMAGE_TAG" -c "import vllm; print(vllm.__version__)"
    # Smoke 2: assert the non-root enabling invariants are baked
    # into the image. Runs as UID 2000:0 via a shell so we can verify
    # filesystem perms + passwd/group file state + wrapper presence without
    # triggering vLLM's GPU-requiring config-init path. The opt-in
    # `vllm-openai-nonroot` target adds only `USER vllm`, `WORKDIR
    # /home/vllm`, and an `ENTRYPOINT` override on top of these invariants;
    # its build correctness is reviewed at the Dockerfile level. Wrapper
    # logic is covered separately by the pre-commit hook
    # `test-nonroot-entrypoint` (see .pre-commit-config.yaml).
    - |
      docker run --rm --user 2000:0 --entrypoint /bin/sh "$IMAGE_TAG" -ec '
        if ! getent passwd 2000 | grep -q ^vllm:; then
          echo FAIL: UID 2000 != vllm
          exit 1
        fi
        if ! id -gn 2>/dev/null | grep -qx root; then
          echo FAIL: GID 0 not root group
          exit 1
        fi
        touch /home/vllm/.smoke && rm /home/vllm/.smoke
        touch /opt/uv/cache/.smoke && rm /opt/uv/cache/.smoke
        if ! test -x /usr/local/bin/vllm-nonroot-entrypoint.sh; then
          echo FAIL: wrapper missing
          exit 1
        fi
        if ! test -w /etc/passwd; then
          echo FAIL: /etc/passwd not group-writable
          exit 1
        fi
        if ! test -w /etc/group; then
          echo FAIL: /etc/group not group-writable
          exit 1
        fi
        echo non-root invariants OK
      '
    retry:
      automatic:
        - exit_status: -1  # Agent was lost
          limit: 2
        - exit_status: -10  # Agent was lost
          limit: 2
  - label: ":docker: Build CPU image"
    key: image-build-cpu
    depends_on: []
    commands:
    - .buildkite/image_build/image_build_cpu.sh $REGISTRY $REPO $BUILDKITE_COMMIT
    env:
      DOCKER_BUILDKIT: "1"
    retry:
      automatic:
        - exit_status: -1  # Agent was lost
          limit: 2
        - exit_status: -10  # Agent was lost
          limit: 2
  - label: ":docker: Build HPU image"
    soft_fail: true
    depends_on: []
    key: image-build-hpu
    commands:
    - .buildkite/image_build/image_build_hpu.sh $REGISTRY $REPO $BUILDKITE_COMMIT
    env:
      DOCKER_BUILDKIT: "1"
    retry:
      automatic:
        - exit_status: -1  # Agent was lost
          limit: 2
        - exit_status: -10  # Agent was lost
          limit: 2
  - label: ":docker: Build CPU arm64 image"
    key: cpu-arm64-image-build
    depends_on: []
    optional: true
    commands:
    - .buildkite/image_build/image_build_cpu_arm64.sh $REGISTRY $REPO $BUILDKITE_COMMIT
    env:
      DOCKER_BUILDKIT: "1"
    retry:
      automatic:
        - exit_status: -1  # Agent was lost
          limit: 2
        - exit_status: -10  # Agent was lost
          limit: 2
  - label: ":docker: Build arm64 image"
    key: arm64-image-build
    depends_on: []
    source_file_dependencies:
      - ".buildkite/image_build/image_build.yaml"
      - ".buildkite/image_build/image_build_arm64.sh"
      - "docker/Dockerfile"
    commands:
    - .buildkite/image_build/image_build_arm64.sh $REGISTRY $REPO $BUILDKITE_COMMIT
    env:
      DOCKER_BUILDKIT: "1"
    retry:
      automatic:
        - exit_status: -1  # Agent was lost
          limit: 2
        - exit_status: -10  # Agent was lost
          limit: 2
@@ -1,39 +0,0 @@
 #!/bin/bash
 set -e
 if [[ $# -lt 3 ]]; then
  echo "Usage: $0 <registry> <repo> <commit>"
  exit 1
 fi
 REGISTRY=$1
 REPO=$2
 BUILDKITE_COMMIT=$3
 IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-arm64"
 # authenticate with AWS ECR
 aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true
 # skip build if image already exists
 if docker manifest inspect "$IMAGE" >/dev/null 2>&1; then
  echo "Image found"
 else
  echo "Image not found, proceeding with build..."
  # build for arm64 GPU targets: Grace/GH200 (sm_90),
  # Blackwell/Thor (sm_100/sm_103/sm_110), and DGX Spark/GB10
  # (sm_121, family-covered by 12.0 under CUDA 13)
  docker build --file docker/Dockerfile \
    --platform linux/arm64 \
    --build-arg max_jobs=16 \
    --build-arg nvcc_threads=4 \
    --build-arg torch_cuda_arch_list="9.0 10.0 11.0 12.0" \
    --build-arg USE_SCCACHE=1 \
    --build-arg buildkite_commit="$BUILDKITE_COMMIT" \
    --tag "$IMAGE" \
    --target test \
    --progress plain .
  # push
  docker push "$IMAGE"
 fi
 .buildkite/scripts/annotate-image-build.sh "$IMAGE"
@@ -1,34 +0,0 @@
 #!/bin/bash
 set -e
 if [[ $# -lt 3 ]]; then
  echo "Usage: $0 <registry> <repo> <commit>"
  exit 1
 fi
 REGISTRY=$1
 REPO=$2
 BUILDKITE_COMMIT=$3
 IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-cpu"
 # authenticate with AWS ECR
 aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true
 # skip build if image already exists
 if docker manifest inspect "$IMAGE" >/dev/null 2>&1; then
  echo "Image found"
 else
  echo "Image not found, proceeding with build..."
  # build
  docker build --file docker/Dockerfile.cpu \
    --build-arg max_jobs=16 \
    --build-arg buildkite_commit="$BUILDKITE_COMMIT" \
    --build-arg VLLM_CPU_X86=true \
    --tag "$IMAGE" \
    --target vllm-test \
    --progress plain .
  # push
  docker push "$IMAGE"
 fi
 .buildkite/scripts/annotate-image-build.sh "$IMAGE"
@@ -1,33 +0,0 @@
 #!/bin/bash
 set -e
 if [[ $# -lt 3 ]]; then
  echo "Usage: $0 <registry> <repo> <commit>"
  exit 1
 fi
 REGISTRY=$1
 REPO=$2
 BUILDKITE_COMMIT=$3
 IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-arm64-cpu"
 # authenticate with AWS ECR
 aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true
 # skip build if image already exists
 if docker manifest inspect "$IMAGE" >/dev/null 2>&1; then
  echo "Image found"
 else
  echo "Image not found, proceeding with build..."
  # build
  docker build --file docker/Dockerfile.cpu \
    --build-arg max_jobs=16 \
    --build-arg buildkite_commit="$BUILDKITE_COMMIT" \
    --tag "$IMAGE" \
    --target vllm-test \
    --progress plain .
  # push
  docker push "$IMAGE"
 fi
 .buildkite/scripts/annotate-image-build.sh "$IMAGE"
@@ -1,34 +0,0 @@
 #!/bin/bash
 set -e
 if [[ $# -lt 3 ]]; then
  echo "Usage: $0 <registry> <repo> <commit>"
  exit 1
 fi
 REGISTRY=$1
 REPO=$2
 BUILDKITE_COMMIT=$3
 IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-hpu"
 # authenticate with AWS ECR
 aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true
 # skip build if image already exists
 if docker manifest inspect "$IMAGE" >/dev/null 2>&1; then
  echo "Image found"
 else
  echo "Image not found, proceeding with build..."
  # build
  docker build \
    --file tests/pytorch_ci_hud_benchmark/Dockerfile.hpu \
    --build-arg max_jobs=16 \
    --build-arg buildkite_commit="$BUILDKITE_COMMIT" \
    --tag "$IMAGE" \
    --progress plain \
    https://github.com/vllm-project/vllm-gaudi.git
  # push
  docker push "$IMAGE"
 fi
 .buildkite/scripts/annotate-image-build.sh "$IMAGE"
@@ -1,71 +0,0 @@
 #!/bin/bash
 set -euo pipefail
 # Build a vLLM test image with PyTorch nightly installed.
 # Called by the pipeline generator's "vLLM Against PyTorch Nightly" group.
 if [[ $# -lt 5 ]]; then
  echo "Usage: $0 <registry> <repo> <commit> <branch> <image_tag>"
  exit 1
 fi
 REGISTRY=$1
 REPO=$2
 BUILDKITE_COMMIT=$3
 BRANCH=$4
 IMAGE_TAG=$5
 # --- Arguments ---
 echo "--- :mag: Arguments"
 echo "REGISTRY: ${REGISTRY}"
 echo "REPO: ${REPO}"
 echo "BUILDKITE_COMMIT: ${BUILDKITE_COMMIT}"
 echo "BRANCH: ${BRANCH}"
 echo "IMAGE_TAG: ${IMAGE_TAG}"
 # --- ECR login ---
 echo "--- :key: ECR login"
 aws ecr-public get-login-password --region us-east-1 \
  | docker login --username AWS --password-stdin "$REGISTRY"
 aws ecr get-login-password --region us-east-1 \
  | docker login --username AWS --password-stdin 936637512419.dkr.ecr.us-east-1.amazonaws.com
 # --- Set up buildx ---
 echo "--- :docker: Setting up buildx"
 docker buildx create --name vllm-builder --driver docker-container --use || true
 docker buildx inspect --bootstrap
 docker buildx ls
 # --- Skip if image already exists ---
 echo "--- :mag: Checking if image already exists"
 if docker manifest inspect "$IMAGE_TAG" >/dev/null 2>&1; then
  echo "Image found: $IMAGE_TAG — skipping build"
  .buildkite/scripts/annotate-image-build.sh "$IMAGE_TAG"
  exit 0
 fi
 echo "Image not found, proceeding with build..."
 # --- CUDA 13.0 for nightly builds ---
 # Nightly CI uses CUDA 13.0 while regular CI stays on CUDA 12.9
 NIGHTLY_CUDA_VERSION="13.0.2"
 NIGHTLY_BUILD_BASE_IMAGE="nvidia/cuda:${NIGHTLY_CUDA_VERSION}-devel-ubuntu22.04"
 NIGHTLY_FINAL_BASE_IMAGE="nvidia/cuda:${NIGHTLY_CUDA_VERSION}-base-ubuntu22.04"
 echo "--- :docker: Building torch nightly image (CUDA ${NIGHTLY_CUDA_VERSION})"
 docker buildx build --file docker/Dockerfile \
  --build-arg max_jobs=16 \
  --build-arg buildkite_commit="$BUILDKITE_COMMIT" \
  --build-arg USE_SCCACHE=1 \
  --build-arg PYTORCH_NIGHTLY=1 \
  --build-arg CUDA_VERSION="${NIGHTLY_CUDA_VERSION}" \
  --build-arg BUILD_BASE_IMAGE="${NIGHTLY_BUILD_BASE_IMAGE}" \
  --build-arg FINAL_BASE_IMAGE="${NIGHTLY_FINAL_BASE_IMAGE}" \
  --build-arg torch_cuda_arch_list="8.0 8.9 9.0 10.0 12.0" \
  --tag "$IMAGE_TAG" \
  --push \
  --target test \
  --progress plain .
 echo "--- :white_check_mark: Torch nightly image build complete: $IMAGE_TAG"
 .buildkite/scripts/annotate-image-build.sh "$IMAGE_TAG"
@@ -1,34 +0,0 @@
 #!/bin/bash
 set -e
 if [[ $# -lt 3 ]]; then
  echo "Usage: $0 <registry> <repo> <commit>"
  exit 1
 fi
 REGISTRY=$1
 REPO=$2
 BUILDKITE_COMMIT=$3
 IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-xpu"
 # authenticate with AWS ECR
 aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true
 aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 936637512419.dkr.ecr.us-east-1.amazonaws.com || true
 # skip build if image already exists
 if docker manifest inspect "$IMAGE" &> /dev/null; then
  echo "Image found"
 else
  echo "Image not found, proceeding with build..."
  # build
  docker build \
    --file docker/Dockerfile.xpu \
    --build-arg max_jobs=16 \
    --build-arg buildkite_commit="$BUILDKITE_COMMIT" \
    --tag "$IMAGE" \
    --progress plain .
  # push
  docker push "$IMAGE"
 fi
 .buildkite/scripts/annotate-image-build.sh "$IMAGE"
@@ -1,27 +0,0 @@
 group: Basic Correctness
 depends_on:
  - image-build-xpu
 steps:
 - label: XPU Sleep Mode
  timeout_in_minutes: 45
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
  - vllm/
  - tests/basic_correctness/test_cumem.py
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'cd tests &&
      export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
      pytest -v -s basic_correctness/test_cpu_offload.py &&
      pytest -v -s basic_correctness/test_mem.py::test_end_to_end'
@@ -1,25 +0,0 @@
 group: Engine Intel
 depends_on:
  - image-build-xpu
 steps:
 - label: Engine (1 GPU)
  timeout_in_minutes: 30
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
    - vllm/v1/engine/
    - tests/v1/engine/
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'cd tests &&
      pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py'
@@ -1,27 +0,0 @@
 group: Expert Parallelism
 depends_on:
  - image-build-xpu
 steps:
 - label: EPLB Algorithm
  key: eplb-algorithm
  timeout_in_minutes: 45
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
  - vllm/distributed/eplb
  - tests/distributed/test_eplb_algo.py
  - tests/distributed/test_eplb_utils.py
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'cd tests &&
      pytest -v -s distributed/test_eplb_algo.py'
@@ -1,25 +0,0 @@
 group: Kernels Intel
 depends_on: 
  - image-build-xpu
 steps:
 - label: vLLM IR Tests
  timeout_in_minutes: 30
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
    - vllm/ir
    - vllm/kernels
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'cd tests &&
      pytest -v -s kernels/ir'
@@ -1,161 +0,0 @@
 group: LoRA Intel
 depends_on:
  - image-build-xpu
 steps:
 - label: LoRA Runtime + Utils
  timeout_in_minutes: 45
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 24+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
    - vllm/lora
    - tests/lora
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'cd tests &&
      export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
      pytest -v -s lora/test_layers.py &&
      pytest -v -s lora/test_lora_checkpoints.py &&
      pytest -v -s lora/test_lora_functions.py &&
      pytest -v -s lora/test_lora_huggingface.py &&
      pytest -v -s lora/test_lora_manager.py &&
      pytest -v -s lora/test_lora_utils.py &&
      pytest -v -s lora/test_peft_helper.py &&
      pytest -v -s lora/test_resolver.py &&
      pytest -v -s lora/test_utils.py &&
      pytest -v -s lora/test_add_lora.py  &&
      pytest -v -s lora/test_worker.py'
 - label: LoRA Fused/MoE Kernels
  timeout_in_minutes: 45
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
    - vllm/lora
    - tests/lora
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'cd tests &&
      export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
      pytest -v -s lora/test_fused_moe_lora_kernel.py && 
      pytest -v -s lora/test_moe_lora_align_sum.py --deselect="tests/lora/test_moe_lora_align_sum.py::test_moe_lora_align_block_size_mixed_base_and_lora[1]"'
 - label: LoRA Punica Kernels
  timeout_in_minutes: 45
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
    - vllm/lora
    - tests/lora
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'cd tests &&
      export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
      set -o pipefail &&
      pytest -v -s lora/test_punica_ops.py::test_kernels &&
      pytest -v -s lora/test_punica_ops.py::test_kernels_hidden_size &&
      pytest -v -s lora/test_punica_ops.py::test_add_lora_fused_moe_early_exit'
 - label: LoRA Punica FP8/XPU Ops
  timeout_in_minutes: 60
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
    - vllm/lora
    - tests/lora
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'cd tests &&
      export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
      pytest -v -s lora/test_punica_ops_fp8.py &&
      pytest -v -s lora/test_punica_xpu_ops.py'
 - label: LoRA Models
  timeout_in_minutes: 45
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 2+
    mem: 24+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
    - vllm/lora
    - tests/lora
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'cd tests &&
      export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
      pytest -v -s lora/test_quant_model.py --deselect="tests/lora/test_quant_model.py::test_quant_model_lora[model0]" --deselect="tests/lora/test_quant_model.py::test_quant_model_lora[model1]" --deselect="tests/lora/test_quant_model.py::test_quant_model_tp_equality[model0]" &&
      pytest -v -s lora/test_transformers_model.py &&
      pytest -v -s lora/test_chatglm3_tp.py &&
      pytest -v -s lora/test_llama_tp.py::test_llama_lora &&
      pytest -s -v lora/test_minicpmv_tp.py'
 - label: LoRA Multimodal
  timeout_in_minutes: 45
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
    - vllm/lora
    - tests/lora
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'cd tests &&
      export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
      pytest -v -s lora/test_default_mm_loras.py && 
      pytest -v -s lora/test_whisper.py'
@@ -1,261 +0,0 @@
 group: Miscellaneous Intel
 depends_on:
  - image-build-xpu
 steps:
 - label: V1 Core + KV + Metrics
  timeout_in_minutes: 45
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
    - vllm/
    - tests/v1/core
    - tests/v1/executor
    - tests/v1/kv_offload
    - tests/v1/worker
    - tests/v1/kv_connector/unit
    - tests/v1/metrics
    - tests/entrypoints/openai/correctness/test_lmeval.py
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'pip install -r requirements/kv_connectors.txt &&
      export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
      cd tests &&
      pytest -v -s v1/executor'
 - label: V1 Sample + Logits
  timeout_in_minutes: 90
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 24+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
    - vllm/config/
    - vllm/distributed/
    - vllm/engine/
    - vllm/inputs/
    - vllm/logger.py
    - vllm/model_executor/
    - vllm/platforms/
    - vllm/sampling_params.py
    - vllm/transformers_utils/
    - vllm/utils/
    - vllm/v1/
    - tests/v1/sample
    - tests/v1/logits_processors
    - tests/v1/test_oracle.py
    - tests/v1/test_request.py
    - tests/v1/test_outputs.py
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'pip install lm_eval[api]>=0.4.12 &&
      export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
      cd tests &&
      pytest -v -s v1/logits_processors --ignore=v1/logits_processors/test_custom_online.py --ignore=v1/logits_processors/test_custom_offline.py &&
      pytest -v -s v1/test_oracle.py &&
      pytest -v -s v1/test_request.py &&
      pytest -v -s v1/test_outputs.py &&
      pytest -v -s v1/sample'
 - label: Basic Models Tests (Initialization)
  timeout_in_minutes: 60
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
    - vllm/
    - tests/models/test_initialization.py
    - tests/models/registry.py
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'export VLLM_XPU_FUSED_MOE_USE_REF=1 &&
      cd tests &&
      pytest -v -s models/test_initialization.py::test_can_initialize_large_subset[Eagle3MiniMaxM2ForCausalLM]'
 - label: XPU CPU Offload
  timeout_in_minutes: 60
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
    - vllm/
    - vllm/v1/kv_offload/
    - vllm/v1/kv_connector/
    - tests/v1/kv_offload/
    - tests/v1/kv_connector/unit/test_offloading_connector.py
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
      cd tests &&
      pytest -v -s v1/kv_offload &&
      pytest -v -s v1/kv_connector/unit/test_offloading_connector.py'
 - label: NixlConnector PD accuracy (2 GPUs)
  timeout_in_minutes: 60
  num_devices: 2
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 2+
    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
    - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
    - vllm/v1/worker/kv_connector_model_runner_mixin.py
    - tests/v1/kv_connector/nixl_integration/
    - vllm/platforms/xpu.py
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'cd tests &&
      bash v1/kv_connector/nixl_integration/run_xpu_disagg_accuracy_test.sh'
 - label: Regression
  key: regression
  timeout_in_minutes: 50
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
  - vllm/config/
  - vllm/distributed/
  - vllm/engine/
  - vllm/inputs/
  - vllm/model_executor/
  - vllm/multimodal/
  - vllm/platforms/
  - vllm/sampling_params.py
  - vllm/transformers_utils/
  - vllm/utils/
  - vllm/v1/
  - tests/test_regression
  commands:
  - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'pip install modelscope\<1.38 &&
      cd tests &&
      pytest -v -s test_regression.py'
 - label: Metrics, Tracing (2 GPUs)
  key: metrics-tracing-2-gpus
  timeout_in_minutes: 45
  num_devices: 2
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 2+
    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
  - vllm/config/
  - vllm/distributed/
  - vllm/engine/
  - vllm/inputs/
  - vllm/model_executor/
  - vllm/multimodal/
  - vllm/platforms/
  - vllm/sampling_params.py
  - vllm/tracing/
  - vllm/transformers_utils/
  - vllm/utils/
  - vllm/v1/
  - tests/v1/tracing
  commands:
  - >-
    bash .buildkite/scripts/hardware_ci/run-intel-test.sh
    'pip install opentelemetry-sdk\>=1.26.0 opentelemetry-api\>=1.26.0 opentelemetry-exporter-otlp\>=1.26.0 opentelemetry-semantic-conventions-ai\>=0.4.1 &&
    cd tests &&
    pytest -v -s v1/tracing'
 - label: Async Engine, Inputs, Utils, Worker
  key: async-engine-inputs-utils-worker
  timeout_in_minutes: 55
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 24+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
  - vllm/assets/
  - vllm/config/
  - vllm/distributed/
  - vllm/engine/
  - vllm/inputs/
  - vllm/model_executor/
  - vllm/multimodal/
  - vllm/platforms/
  - vllm/sampling_params.py
  - vllm/tokenizers/
  - vllm/transformers_utils/
  - vllm/utils/
  - vllm/v1/
  - tests/detokenizer
  - tests/multimodal
  - tests/utils_
  commands:
  - >-
    bash .buildkite/scripts/hardware_ci/run-intel-test.sh
    'cd tests &&
    pip install av &&
    pytest -v -s detokenizer &&
    pytest -v -s -m "not cpu_test" ./multimodal &&
    pytest -v -s utils_ --ignore=utils_/test_mem_utils.py'
@@ -1,62 +0,0 @@
 group: Model Runner V2 Intel
 depends_on:
  - image-build-xpu
 steps:
 - label: Model Runner V2 Core Tests (Intel)
  timeout_in_minutes: 45
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 2+
    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
    - vllm/v1/worker/gpu/
    - vllm/v1/worker/gpu_worker.py
    - vllm/v1/core/sched/
    - vllm/v1/attention/
    - tests/v1/engine/test_llm_engine.py
    - tests/v1/e2e/
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'export VLLM_USE_V2_MODEL_RUNNER=1 &&
      cd tests &&
      pytest -v -s v1/engine/test_llm_engine.py -k "not test_engine_metrics" &&
      ENFORCE_EAGER=1 pytest -v -s v1/e2e/general/test_async_scheduling.py -k "not ngram" &&
      pytest -v -s v1/e2e/general/test_min_tokens.py'
 - label: Model Runner V2 Examples (Intel)
  timeout_in_minutes: 45
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 24+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
    - vllm/v1/worker/gpu/
    - vllm/v1/core/sched/
    - vllm/v1/worker/gpu_worker.py
    - examples/basic/offline_inference/
    - examples/generate/multimodal/
    - examples/features/
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'export VLLM_USE_V2_MODEL_RUNNER=1 &&
      cd examples &&
      python3 basic/offline_inference/chat.py &&
      python3 basic/offline_inference/generate.py --model facebook/opt-125m &&
      python3 generate/multimodal/vision_language_offline.py --seed 0 &&
      python3 features/automatic_prefix_caching/prefix_caching_offline.py'
@@ -1,27 +0,0 @@
 group: Models - Distributed
 depends_on: 
  - image-build-xpu
 steps:
 - label: Distributed Model Tests (2 GPUs)
  key: distributed-model-tests-2-gpus
  timeout_in_minutes: 65
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 2+
    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
  - vllm/model_executor/model_loader/sharded_state_loader.py
  - vllm/model_executor/models/
  - tests/model_executor/model_loader/test_sharded_state_loader.py
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'cd tests &&
      pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m "not slow_test"'
@@ -1,127 +0,0 @@
 group: Models - Multimodal
 depends_on:
  - image-build-xpu
 steps:
 - label: "Multi-Modal Models (Standard) 1: qwen2"
  key: multi-modal-models-standard-1-qwen2
  timeout_in_minutes: 70
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 24+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
  - vllm/
  - tests/models/multimodal
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'pip install av &&
      cd tests &&
      pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2" &&
      pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model'
 - label: "Multi-Modal Models (Standard) 2: qwen3 + gemma"
  key: multi-modal-models-standard-2-qwen3-gemma
  timeout_in_minutes: 70
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 16+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
  - vllm/
  - tests/models/multimodal
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'cd tests &&
      pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model'
 - label: "Multi-Modal Models (Standard) 3: llava + qwen2_vl"
  key: multi-modal-models-standard-3-llava-qwen2-vl
  timeout_in_minutes: 65
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 24+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
  - vllm/
  - tests/models/multimodal
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'cd tests &&
      pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma" &&
      pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model'
 - label: "Multi-Modal Models (Standard) 4: other + whisper"
  key: multi-modal-models-standard-4-other-whisper
  timeout_in_minutes: 45
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 24+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
  - vllm/
  - tests/models/multimodal
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'pip install av &&
      cd tests &&
      pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py  --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/processing'
 - label: Multi-Modal Processor # 44min
  key: multi-modal-processor
  timeout_in_minutes: 60
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 24+
  no_plugin: true
  working_dir: "."
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  source_file_dependencies:
  - vllm/
  - tests/models/multimodal
  - tests/models/registry.py
  commands:
    - >-
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'pip install av matplotlib ftfy &&
      pip install open-clip-torch --no-deps &&
      cd tests &&
      pytest -v -s models/multimodal/processing/test_tensor_schema.py
      --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB'
  parallelism: 4
@@ -1,28 +0,0 @@
 group: Quantization
 depends_on:
  - image-build-xpu
 steps:
 - label: Quantization
  key: quantization
  timeout_in_minutes: 30
  env:
    REGISTRY: "public.ecr.aws/q9t5s3a7"
    REPO: "vllm-ci-test-repo"
    VLLM_TEST_DEVICE: "xpu"
  no_plugin: true
  working_dir: "."
  device: intel_gpu
  agent_tags:
    label: production
    gpu: 1+
    mem: 16+
  source_file_dependencies:
  - csrc/
  - vllm/model_executor/layers/quantization
  - tests/quantization
  commands:
  # - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py
  - >-
    bash .buildkite/scripts/hardware_ci/run-intel-test.sh
    'VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s tests/quantization/test_per_token_kv_cache.py --deselect="tests/quantization/test_per_token_kv_cache.py::test_triton_unified_attention_per_token_head_scale[int4-16-128-num_heads0-seq_lens1]"'
@@ -1,171 +0,0 @@
 group: Intel
 steps:
  - label: ":docker: Build XPU image"
    soft_fail: true
    depends_on: []
    key: image-build-xpu
    commands:
      - bash -lc '.buildkite/image_build/image_build_xpu.sh "public.ecr.aws/q9t5s3a7" "vllm-ci-test-repo" "$BUILDKITE_COMMIT"'
    env:
      DOCKER_BUILDKIT: "1"
    retry:
      automatic:
        - exit_status: -1  # Agent was lost
          limit: 2
        - exit_status: -10  # Agent was lost
          limit: 2
  - label: "XPU example Test"
    depends_on:
      - image-build-xpu
    timeout_in_minutes: 50
    device: intel_gpu
    agent_tags:
      label: production
      gpu: 2+
      mem: 24+
    no_plugin: true
    env:
      REGISTRY: "public.ecr.aws/q9t5s3a7"
      REPO: "vllm-ci-test-repo"
    source_file_dependencies:
      - vllm/
      - .buildkite/intel_jobs/test-intel.yaml 
    commands:
      - >-
        bash .buildkite/scripts/hardware_ci/run-intel-test.sh
        'pip install tblib==3.1.0 &&
        python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager &&
        python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 -O3 -cc.cudagraph_mode=NONE &&
        python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager -tp 2 --distributed-executor-backend mp &&
        python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --attention-backend=TRITON_ATTN &&
        python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --quantization fp8 &&
        python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --kv-cache-dtype fp8 &&
        python3 examples/basic/offline_inference/generate.py --model nvidia/Llama-3.1-8B-Instruct-FP8 --block-size 64 --enforce-eager --quantization modelopt --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --max-model-len 4096 &&
        python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --block-size 64 --enforce-eager --max-model-len 8192 &&
        python3 examples/basic/offline_inference/generate.py --model TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ --block-size 64 --enforce-eager &&
        python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 &&
        python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 --enable-expert-parallel &&
        python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --max-model-len 8192 &&
        VLLM_XPU_FUSED_MOE_USE_REF=1 python3 examples/basic/offline_inference/generate.py --model Qwen/Qwen3-30B-A3B-Instruct-2507-FP8 --enforce-eager -tp 2 --max-model-len 8192 &&
        python3 examples/basic/offline_inference/generate.py --model INCModel/Qwen3-30B-A3B-Instruct-2507-MXFP4-LLMC --enforce-eager -tp 2 --max-model-len 8192
        '
  - label: "XPU W8A8 FP8 Linear Examples"
    depends_on:
      - image-build-xpu
    timeout_in_minutes: 60
    device: intel_gpu
    agent_tags:
      label: production
      gpu: 1+
      mem: 24+
    no_plugin: true
    env:
      REGISTRY: "public.ecr.aws/q9t5s3a7"
      REPO: "vllm-ci-test-repo"
      VLLM_TEST_DEVICE: "xpu"
    source_file_dependencies:
      - vllm/
      - .buildkite/intel_jobs/test-intel.yaml
    commands:
      - >-
        bash .buildkite/scripts/hardware_ci/run-intel-test.sh
        'python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8 --enforce-eager --max-model-len 4096 &&
        python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model neuralmagic/Llama-3.2-1B-Instruct-FP8-dynamic --enforce-eager --max-model-len 4096 &&
        python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8 --enforce-eager --max-model-len 4096
        '
  - label: "XPU V1 test"
    depends_on:
      - image-build-xpu
    timeout_in_minutes: 70
    device: intel_gpu
    agent_tags:
      label: production
      gpu: 1+
      mem: 24+
    no_plugin: true
    env:
      REGISTRY: "public.ecr.aws/q9t5s3a7"
      REPO: "vllm-ci-test-repo"
    source_file_dependencies:
      - vllm/
      - .buildkite/intel_jobs/test-intel.yaml 
    commands:
      - >-
        bash .buildkite/scripts/hardware_ci/run-intel-test.sh
        'cd tests &&
        pytest -v -s v1/core --ignore=v1/core/test_reset_prefix_cache_e2e.py --ignore=v1/core/test_scheduler_e2e.py &&
        pytest -v -s v1/engine --ignore=v1/engine/test_output_processor.py &&
        pytest -v -s v1/sample --ignore=v1/sample/test_logprobs.py --ignore=v1/sample/test_logprobs_e2e.py -k "not test_topk_only and not test_topp_only and not test_topk_and_topp" &&
        pytest -v -s v1/worker --ignore=v1/worker/test_gpu_model_runner.py --ignore=v1/worker/test_worker_memory_snapshot.py &&
        pytest -v -s v1/structured_output &&
        pytest -v -s v1/test_serial_utils.py &&
        pytest -v -s v1/e2e/general/test_correctness_sliding_window.py --deselect="tests/v1/e2e/general/test_correctness_sliding_window.py::test_sliding_window_retrieval[True-1-5-google/gemma-3-1b-it]" &&
        pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py --ignore=v1/spec_decode/test_speculators_correctness.py &&
        pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_multi_connector.py --ignore=v1/kv_connector/unit/test_example_connector.py --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py --ignore=v1/kv_connector/unit/test_offloading_connector.py'
  - label: "XPU server test"
    depends_on:
      - image-build-xpu
    timeout_in_minutes: 45
    device: intel_gpu
    agent_tags:
      label: production
      gpu: 1+
      mem: 16+
    no_plugin: true
    env:
      REGISTRY: "public.ecr.aws/q9t5s3a7"
      REPO: "vllm-ci-test-repo"
    source_file_dependencies:
      - vllm/
      - .buildkite/intel_jobs/test-intel.yaml 
    commands:
      - >-
        bash .buildkite/scripts/hardware_ci/run-intel-test.sh
        'pip install av &&
        cd tests &&
        pytest -v -s entrypoints/multimodal/openai/chat_completion/test_audio_in_video.py &&
        pytest -v -s benchmarks/test_serve_cli.py'
  - label: "XPU quantization test"
    depends_on:
      - image-build-xpu
    timeout_in_minutes: 30
    device: intel_gpu
    agent_tags:
      label: production
      gpu: 1+
      mem: 16+
    no_plugin: true
    env:
      REGISTRY: "public.ecr.aws/q9t5s3a7"
      REPO: "vllm-ci-test-repo"
    source_file_dependencies:
      - vllm/
      - .buildkite/intel_jobs/test-intel.yaml 
    commands:
      - >-
        bash .buildkite/scripts/hardware_ci/run-intel-test.sh
        'cd tests &&
        pytest -v -s quantization/test_auto_round.py'
  - label: "XPU compressed tensors FP8 test"
    depends_on:
      - image-build-xpu
    timeout_in_minutes: 60
    device: intel_gpu
    agent_tags:
      label: production
      gpu: 1+
      mem: 16+
    no_plugin: true
    env:
      REGISTRY: "public.ecr.aws/q9t5s3a7"
      REPO: "vllm-ci-test-repo"
      VLLM_TEST_DEVICE: "xpu"
    source_file_dependencies:
      - vllm/
      - tests/quantization/test_compressed_tensors.py
      - .buildkite/intel_jobs/test-intel.yaml
    commands:
      - >-
        bash .buildkite/scripts/hardware_ci/run-intel-test.sh
        'cd tests &&
        pytest -v -s quantization/test_compressed_tensors.py::test_compressed_tensors_fp8'
@@ -1,10 +1,6 @@
 # For hf script, without -t option (tensor parallel size).
 # bash .buildkite/lm-eval-harness/run-lm-eval-mmlupro-vllm-baseline.sh -m meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 -l 250 -t 8 -f 5
 model_name: "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8"
 rocm_safetensors_load_strategy: lazy
 required_gpu_arch:
  - gfx942
  - gfx950
 tasks:
 - name: "mmlu_pro"
  metrics:
@@ -12,4 +8,3 @@ tasks:
    value: 0.80
 limit: 250 # will run on 250 * 14 subjects = 3500 samples
 num_fewshot: 5
 rtol: 0.05
@@ -1,15 +0,0 @@
 model_name: "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16"
 tasks:
 - name: "gsm8k"
  metrics:
  - name: "exact_match,strict-match"
    value: 0.695
  - name: "exact_match,flexible-extract"
    value: 0.447
 limit: 1319
 num_fewshot: 5
 max_model_len: 262144
 enforce_eager: false
 apply_chat_template: true
 fewshot_as_multiturn: true
 trust_remote_code: true
@@ -1,17 +0,0 @@
 model_name: "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8"
 tasks:
 - name: "gsm8k"
  metrics:
  - name: "exact_match,strict-match"
    value: 0.7142
  - name: "exact_match,flexible-extract"
    value: 0.4579
 moe_backend: "flashinfer_cutlass"
 limit: 1319
 num_fewshot: 5
 max_model_len: 262144
 kv_cache_dtype: fp8
 enforce_eager: false
 apply_chat_template: true
 fewshot_as_multiturn: true
 trust_remote_code: true
@@ -1,9 +1,6 @@
 # For vllm script, with -t option (tensor parallel size)
 # bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m RedHatAI/Qwen2.5-VL-3B-Instruct-FP8-Dynamic -l 1319 -t 1
 model_name: "RedHatAI/Qwen2.5-VL-3B-Instruct-FP8-Dynamic"
 required_gpu_arch:
  - gfx942
  - gfx950
 tasks:
 - name: "gsm8k"
  metrics:
@@ -1,7 +1,4 @@
 model_name: "Qwen/Qwen3-235B-A22B-Instruct-2507-FP8"
 required_gpu_arch:
  - gfx942
  - gfx950
 tasks:
  - name: "mmlu_pro"
    metrics:
@@ -0,0 +1,12 @@
 # For vllm script, with -t option (tensor parallel size).
 # bash ./run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/SparseLlama-3.1-8B-gsm8k-pruned.2of4-chnl_wts_per_tok_dyn_act_fp8-BitM -b "auto" -t 2
 model_name: "nm-testing/SparseLlama-3.1-8B-gsm8k-pruned.2of4-chnl_wts_per_tok_dyn_act_fp8-BitM"
 tasks:
 - name: "gsm8k"
  metrics:
  - name: "exact_match,strict-match"
    value: 0.6353
  - name: "exact_match,flexible-extract"
    value: 0.637
 limit: null
 num_fewshot: null 
@@ -1,2 +1 @@
 Qwen3-235B-A22B-Instruct-2507-FP8.yaml
 NVIDIA-Nemotron-3-Nano-30B-A3B-FP8.yaml
@@ -1 +0,0 @@
 Qwen3-235B-A22B-Instruct-2507-FP8.yaml
@@ -1 +0,0 @@
 Meta-Llama-4-Maverick-17B-128E-Instruct-FP8.yaml
@@ -3,4 +3,3 @@ Meta-Llama-3-70B-Instruct.yaml
 Mixtral-8x7B-Instruct-v0.1.yaml
 Qwen2-57B-A14-Instruct.yaml
 DeepSeek-V2-Lite-Chat.yaml
 NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.yaml
@@ -1,6 +0,0 @@
 Qwen2.5-1.5B-Instruct.yaml
 Meta-Llama-3.2-1B-Instruct-INT8-compressed-tensors.yaml
 Meta-Llama-3-8B-Instruct-INT8-compressed-tensors-asym.yaml
 Meta-Llama-3-8B-Instruct-nonuniform-compressed-tensors.yaml
 Qwen2.5-VL-3B-Instruct-FP8-dynamic.yaml
 Qwen1.5-MoE-W4A16-compressed-tensors.yaml
@@ -2,7 +2,7 @@
 # We can use this script to compute baseline accuracy on chartqa for vllm.
 #
 # Make sure you have lm-eval-harness installed:
-#   pip install "lm-eval[api]>=0.4.12"
+#   pip install lm-eval==0.4.9
 usage() {
    echo``
@@ -41,4 +41,4 @@ lm_eval --model vllm-vlm \
  --tasks chartqa \
  --batch_size auto \
  --apply_chat_template \
-  --limit "$LIMIT"
+  --limit $LIMIT
@@ -2,7 +2,7 @@
 # We can use this script to compute baseline accuracy on GSM for transformers.
 #
 # Make sure you have lm-eval-harness installed:
-#   pip install "lm-eval[api]>=0.4.12"
+#   pip install git+https://github.com/EleutherAI/lm-evaluation-harness.git@206b7722158f58c35b7ffcd53b035fdbdda5126d#egg=lm-eval[api]
 usage() {
    echo``
@@ -3,7 +3,7 @@
 # We use this for fp8, which HF does not support.
 #
 # Make sure you have lm-eval-harness installed:
-#   pip install "lm-eval[api]>=0.4.12"
+#   pip install git+https://github.com/EleutherAI/lm-evaluation-harness.git@206b7722158f58c35b7ffcd53b035fdbdda5126d#egg=lm-eval[api]
 usage() {
    echo``
@@ -3,7 +3,7 @@
 # We use this for fp8, which HF does not support.
 #
 # Make sure you have lm-eval-harness installed:
-#   pip install "lm-eval[api]>=0.4.12"
+#   pip install git+https://github.com/EleutherAI/lm-evaluation-harness.git@206b7722158f58c35b7ffcd53b035fdbdda5126d#egg=lm-eval[api]
 usage() {
    echo``
@@ -20,11 +20,14 @@ usage() {
    echo
 }
-while getopts "m:l:f:t:" OPT; do
+while getopts "m:b:l:f:t:" OPT; do
  case ${OPT} in
    m )
        MODEL="$OPTARG"
        ;;
    b )
        BATCH_SIZE="$OPTARG"
        ;;
    l )
        LIMIT="$OPTARG"
        ;;
@@ -9,42 +9,11 @@ pytest -s -v test_lm_eval_correctness.py \
    --tp-size=1
 """
 import os
 from contextlib import contextmanager
 import lm_eval
-import pytest
+import numpy as np
 import yaml
-from vllm.platforms import current_platform
+RTOL = 0.08
 DEFAULT_RTOL = 0.08
@contextmanager
 def scoped_env_vars(new_env: dict[str, str]):
    if not new_env:
        # Fast path: nothing to do
        yield
        return
    old_values = {}
    new_keys = []
    try:
        for key, value in new_env.items():
            if key in os.environ:
                old_values[key] = os.environ[key]
            else:
                new_keys.append(key)
            os.environ[key] = str(value)
        yield
    finally:
        # Restore / clean up
        for key, value in old_values.items():
            os.environ[key] = value
        for key in new_keys:
            os.environ.pop(key, None)
 def launch_lm_eval(eval_config, tp_size):
@@ -62,81 +31,32 @@ def launch_lm_eval(eval_config, tp_size):
        f"add_bos_token=true,"
        f"trust_remote_code={trust_remote_code},"
        f"max_model_len={max_model_len},"
        "allow_deprecated_quantization=True,"
    )
-
+    results = lm_eval.simple_evaluate(
-    if current_platform.is_rocm() and "Nemotron-3" in eval_config["model_name"]:
+        model=backend,
-        model_args += "attention_backend=TRITON_ATTN"
+        model_args=model_args,
-
+        tasks=[task["name"] for task in eval_config["tasks"]],
-    moe_backend = eval_config.get("moe_backend", None)
+        num_fewshot=eval_config["num_fewshot"],
-    if moe_backend is not None:
+        limit=eval_config["limit"],
-        model_args += f"moe_backend={moe_backend},"
+        # TODO(yeq): using chat template w/ fewshot_as_multiturn is supposed help
-
+        # text models. however, this is regressing measured strict-match for
-    if current_platform.is_rocm():
+        # existing text models in CI, so only apply it for mm, or explicitly set
-        rocm_load_strategy = eval_config.get("rocm_safetensors_load_strategy")
+        apply_chat_template=eval_config.get(
-        if rocm_load_strategy is not None:
+            "apply_chat_template", backend == "vllm-vlm"
-            model_args += f"safetensors_load_strategy={rocm_load_strategy},"
+        ),
-
+        fewshot_as_multiturn=eval_config.get("fewshot_as_multiturn", False),
-    env_vars = eval_config.get("env_vars", None)
+        # Forward decoding and early-stop controls (e.g., max_gen_toks, until=...)
-    with scoped_env_vars(env_vars):
+        gen_kwargs=eval_config.get("gen_kwargs"),
-        results = lm_eval.simple_evaluate(
+        batch_size=batch_size,
-            model=backend,
+    )
            model_args=model_args,
            tasks=[task["name"] for task in eval_config["tasks"]],
            num_fewshot=eval_config["num_fewshot"],
            limit=eval_config["limit"],
            # TODO(yeq): using chat template w/ fewshot_as_multiturn is supposed help
            # text models. however, this is regressing measured strict-match for
            # existing text models in CI, so only apply it for mm, or explicitly set
            apply_chat_template=eval_config.get(
                "apply_chat_template", backend == "vllm-vlm"
            ),
            fewshot_as_multiturn=eval_config.get("fewshot_as_multiturn", False),
            # Forward decoding and early-stop controls (e.g., max_gen_toks, until=...)
            gen_kwargs=eval_config.get("gen_kwargs"),
            batch_size=batch_size,
        )
    return results
 def _check_rocm_gpu_arch_requirement(eval_config):
    """Skip the test if the model requires a ROCm GPU arch not present.
    Model YAML configs can specify::
        required_gpu_arch:
          - gfx942
          - gfx950
    The check only applies on ROCm.  On other platforms (e.g. CUDA) the
    field is ignored so that shared config files work for both NVIDIA and
    AMD CI pipelines.
    """
    required_archs = eval_config.get("required_gpu_arch")
    if not required_archs:
        return
    if not current_platform.is_rocm():
        return
    from vllm.platforms.rocm import _GCN_ARCH  # noqa: E402
    if not any(arch in _GCN_ARCH for arch in required_archs):
        pytest.skip(
            f"Model requires GPU arch {required_archs}, "
            f"but detected arch is '{_GCN_ARCH}'"
        )
 def test_lm_eval_correctness_param(config_filename, tp_size):
    eval_config = yaml.safe_load(config_filename.read_text(encoding="utf-8"))
    _check_rocm_gpu_arch_requirement(eval_config)
    results = launch_lm_eval(eval_config, tp_size)
    rtol = eval_config.get("rtol", DEFAULT_RTOL)
    success = True
    for task in eval_config["tasks"]:
        for metric in task["metrics"]:
@@ -144,11 +64,8 @@ def test_lm_eval_correctness_param(config_filename, tp_size):
            measured_value = results["results"][task["name"]][metric["name"]]
            print(
                f"{task['name']} | {metric['name']}: "
-                f"ground_truth={ground_truth:.3f} | "
+                f"ground_truth={ground_truth} | measured={measured_value}"
                f"measured={measured_value:.3f} | rtol={rtol}"
            )
-
+            success = success and np.isclose(ground_truth, measured_value, rtol=RTOL)
            min_acceptable = ground_truth * (1 - rtol)
            success = success and measured_value >= min_acceptable
    assert success
@@ -7,7 +7,7 @@ vLLM also maintains a continuous performance benchmark under [perf.vllm.ai](http
 ## Performance benchmark quick overview
-**Benchmarking Coverage**: latency, throughput and fix-qps serving on B200, A100, H100, Intel® Xeon® Processors, Intel® Gaudi® 3 Accelerators and Arm® Neoverse™ with different models.
+**Benchmarking Coverage**: latency, throughput and fix-qps serving on B200, A100, H100, Intel® Xeon® Processors and Intel® Gaudi® 3 Accelerators with different models.
 **Benchmarking Duration**: about 1hr.
@@ -23,7 +23,7 @@ bash .buildkite/performance-benchmarks/scripts/run-performance-benchmarks.sh
 Runtime environment variables:
- `ON_CPU`: set the value to '1' on Intel® Xeon® and Arm® Neoverse™ Processors. Default value is 0.
+- `ON_CPU`: set the value to '1' on Intel® Xeon® Processors. Default value is 0.
 - `SERVING_JSON`: JSON file to use for the serving tests. Default value is empty string (use default file).
 - `LATENCY_JSON`: JSON file to use for the latency tests. Default value is empty string (use default file).
 - `THROUGHPUT_JSON`: JSON file to use for the throughout tests. Default value is empty string (use default file).
@@ -34,9 +34,8 @@ Runtime environment variables:
 See [performance-benchmarks-descriptions.md](performance-benchmarks-descriptions.md) for detailed descriptions, and use `tests/latency-tests.json`, `tests/throughput-tests.json`, `tests/serving-tests.json` to configure the test cases.
 > NOTE: For Intel® Xeon® Processors, use `tests/latency-tests-cpu.json`, `tests/throughput-tests-cpu.json`, `tests/serving-tests-cpu.json` instead.
-> For Intel® Gaudi® 3 Accelerators, use `tests/latency-tests-hpu.json`, `tests/throughput-tests-hpu.json`, `tests/serving-tests-hpu.json` instead.
+For Intel® Gaudi® 3 Accelerators, use `tests/latency-tests-hpu.json`, `tests/throughput-tests-hpu.json`, `tests/serving-tests-hpu.json` instead.
-> For Arm® Neoverse™, use `tests/latency-tests-arm64-cpu.json`, `tests/throughput-tests-arm64-cpu.json`, `tests/serving-tests-arm64-cpu.json` instead.
+>
 ### Latency test
 Here is an example of one test inside `latency-tests.json`:
@@ -83,6 +82,7 @@ We test the throughput by using `vllm bench serve` with request rate = inf to co
        "server_parameters": {
            "model": "meta-llama/Meta-Llama-3-8B",
            "tensor_parallel_size": 1,
            "swap_space": 16,
            "disable_log_stats": "",
            "load_format": "dummy"
        },
@@ -108,65 +108,6 @@ The number of this test is less stable compared to the delay and latency benchma
 WARNING: The benchmarking script will save json results by itself, so please do not configure `--save-results` or other results-saving-related parameters in `serving-tests.json`.
 #### Default Parameters Field
 We can specify default parameters in a JSON field with key `defaults`. Parameters defined in the field are applied globally to all serving tests, and can be overridden in test case fields. Here is an example:
 <details>
 <summary> An Example of default parameters field </summary>
 ```json
 {
  "defaults": {
    "qps_list": [
      "inf"
    ],
    "server_environment_variables": {
      "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1
    },
    "server_parameters": {
      "tensor_parallel_size": 1,
      "dtype": "bfloat16",
      "block_size": 128,
      "disable_log_stats": "",
      "load_format": "dummy"
    },
    "client_parameters": {
      "backend": "vllm",
      "dataset_name": "random",
      "random-input-len": 128,
      "random-output-len": 128,
      "num_prompts": 200,
      "ignore-eos": ""
    }
  },
  "tests": [
    {
      "test_name": "serving_llama3B_tp2_random_128_128",
      "server_parameters": {
        "model": "meta-llama/Llama-3.2-3B-Instruct",
        "tensor_parallel_size": 2,
      },
      "client_parameters": {
        "model": "meta-llama/Llama-3.2-3B-Instruct",
      }
    },
    {
      "test_name": "serving_qwen3_tp4_random_128_128",
      "server_parameters": {
        "model": "Qwen/Qwen3-14B",
        "tensor_parallel_size": 4,
      },
      "client_parameters": {
        "model": "Qwen/Qwen3-14B",
      }
    },
  ]
 }
 ```
 </details>
 ### Visualizing the results
 The `convert-results-json-to-markdown.py` helps you put the benchmarking results inside a markdown table, by formatting [descriptions.md](performance-benchmarks-descriptions.md) with real benchmarking results.
@@ -175,6 +116,19 @@ If you do not see the table, please wait till the benchmark finish running.
 The json version of the table (together with the json version of the benchmark) will be also attached to the markdown file.
 The raw benchmarking results (in the format of json files) are in the `Artifacts` tab of the benchmarking.
-#### Performance Results Comparison  
+The `compare-json-results.py` helps to compare benchmark results JSON files converted using `convert-results-json-to-markdown.py`.
 When run, benchmark script generates results under `benchmark/results` folder, along with the `benchmark_results.md` and `benchmark_results.json`.
 `compare-json-results.py` compares two `benchmark_results.json` files and provides performance ratio e.g. for Output Tput, Median TTFT and Median TPOT.  
 If only one benchmark_results.json is passed, `compare-json-results.py` compares different TP and PP configurations in the benchmark_results.json instead.
-Follow the instructions in [performance results comparison](https://docs.vllm.ai/en/latest/benchmarking/dashboard/#performance-results-comparison) to analyze performance results and the sizing guide.
+Here is an example using the script to compare result_a and result_b with Model, Dataset name, input/output length, max concurrency and qps.
 `python3 compare-json-results.py -f results_a/benchmark_results.json -f results_b/benchmark_results.json`
 |   | Model | Dataset Name | Input Len | Output Len | # of max concurrency | qps  | results_a/benchmark_results.json | results_b/benchmark_results.json | perf_ratio        |
 |----|---------------------------------------|--------|-----|-----|------|-----|-----------|----------|----------|
 | 0  | meta-llama/Meta-Llama-3.1-8B-Instruct | random | 128 | 128 | 1000 | 1 | 142.633982                             | 156.526018                             | 1.097396 |
 | 1  | meta-llama/Meta-Llama-3.1-8B-Instruct | random | 128 | 128 | 1000 | inf| 241.620334                             | 294.018783                             | 1.216863 |
 A comparison diagram will be generated below the table.
 Here is an example to compare between 96c/results_gnr_96c_091_tp2pp3 and 128c/results_gnr_128c_091_tp2pp3
 <img width="1886" height="828" alt="image" src="https://github.com/user-attachments/assets/c02a43ef-25d0-4fd6-90e5-2169a28682dd" />
@@ -393,7 +393,7 @@ if __name__ == "__main__":
    with open(results_folder / md_file, "w") as f:
        results = read_markdown(
            "../.buildkite/performance-benchmarks/"
-            "performance-benchmarks-descriptions.md"
+            + "performance-benchmarks-descriptions.md"
        )
        results = results.format(
            latency_tests_markdown_table=latency_md_table,
@@ -1,4 +1,6 @@
 #!/bin/bash
 # This script should be run inside the CI process
 # This script assumes that we are already inside the vllm/ directory
 # Benchmarking results will be available inside vllm/benchmarks/results/
@@ -7,26 +9,14 @@
 set -x
 set -o pipefail
 # Environment-driven debug controls (like ON_CPU=1)
 DRY_RUN="${DRY_RUN:-0}"
 MODEL_FILTER="${MODEL_FILTER:-}"
 DTYPE_FILTER="${DTYPE_FILTER:-}"
 # Adaptive search controls
 ENABLE_ADAPTIVE_CONCURRENCY="${ENABLE_ADAPTIVE_CONCURRENCY:-0}"
 SLA_TTFT_MS="${SLA_TTFT_MS:-3000}"
 SLA_TPOT_MS="${SLA_TPOT_MS:-100}"
 ADAPTIVE_MAX_PROBES="${ADAPTIVE_MAX_PROBES:-8}"
 ADAPTIVE_MAX_CONCURRENCY="${ADAPTIVE_MAX_CONCURRENCY:-1024}"
 check_gpus() {
  if command -v nvidia-smi; then
    # check the number of GPUs and GPU type.
-    declare -g gpu_count=$(nvidia-smi --list-gpus | grep -c . || true)
+    declare -g gpu_count=$(nvidia-smi --list-gpus | wc -l)
  elif command -v amd-smi; then
-    declare -g gpu_count=$(amd-smi list | grep -c 'GPU' || true)
+    declare -g gpu_count=$(amd-smi list | grep 'GPU' | wc -l)
  elif command -v hl-smi; then
-    declare -g gpu_count=$(hl-smi --list | grep -ci "Module ID" || true)
+    declare -g gpu_count=$(hl-smi --list | grep -i "Module ID" | wc -l)
  fi
  if [[ $gpu_count -gt 0 ]]; then
@@ -35,9 +25,9 @@ check_gpus() {
    echo "Need at least 1 GPU to run benchmarking."
    exit 1
  fi
-
+  
  declare -g arch_suffix=''
-
+  
  if command -v nvidia-smi; then
    declare -g gpu_type=$(nvidia-smi --query-gpu=name --format=csv,noheader | awk '{print $2}')
  elif command -v amd-smi; then
@@ -54,16 +44,12 @@ check_cpus() {
  declare -g numa_count=$(lscpu | grep "NUMA node(s):" | awk '{print $3}')
  if [[ $numa_count -gt 0 ]]; then
    echo "NUMA found."
-    echo "$numa_count"
+    echo $numa_count
  else
    echo "Need at least 1 NUMA to run benchmarking."
    exit 1
  fi
-  if [[ "$(uname -m)" == "aarch64" ]] || [[ "$(uname -m)" == "arm64" ]]; then
+  declare -g gpu_type="cpu"
    declare -g gpu_type="arm64-cpu"
  else
    declare -g gpu_type="cpu"
  fi
  echo "GPU type is $gpu_type"
 }
@@ -122,12 +108,12 @@ json2envs() {
 }
 wait_for_server() {
-  local timeout_val="1200"
+  # wait for vllm server to start
-  timeout "$timeout_val" bash -c '
+  # return 1 if vllm server crashes
-    until curl -sf http://localhost:8000/v1/models >/dev/null; do
+  timeout 1200 bash -c '
    until curl -X POST localhost:8000/v1/completions; do
      sleep 1
-    done
+    done' && return 0 || return 1
  '
 }
 kill_processes_launched_by_current_bash() {
@@ -190,318 +176,19 @@ upload_to_buildkite() {
  $BUILDKITE_AGENT_COMMAND artifact upload "$RESULTS_FOLDER/*"
 }
-# -------------------------------
+run_latency_tests() {
-# Adaptive concurrency helpers
+  # run latency tests using `vllm bench latency` command
-# -------------------------------
+  # $1: a json file specifying latency test cases
 result_json_path_for_serving() {
  local test_name=$1
  local qps=$2
  local max_concurrency=$3
  echo "$RESULTS_FOLDER/${test_name}_qps_${qps}_concurrency_${max_concurrency}.json"
 }
-extract_metric_ms() {
+  local latency_test_file
-  local metric_name=$1
+  latency_test_file=$1
  local json_file=$2
-  [[ -f "$json_file" ]] || return 0
+  # Iterate over latency tests
-
+  jq -c '.[]' "$latency_test_file" | while read -r params; do
  if [[ "$metric_name" == "ttft" ]]; then
    jq -r '
      [
        .ttft_ms.p99?,
        .metrics.ttft_ms.p99?,
        .ttft.p99?,
        .metrics.ttft.p99?,
        .p99_ttft_ms?,
        .ttft_ms.mean?,
        .metrics.ttft_ms.mean?,
        .ttft.mean?,
        .metrics.ttft.mean?,
        .mean_ttft_ms?
      ] | map(select(. != null)) | .[0] // empty
    ' "$json_file"
  else
    jq -r '
      [
        .tpot_ms.p99?,
        .metrics.tpot_ms.p99?,
        .tpot.p99?,
        .metrics.tpot.p99?,
        .p99_tpot_ms?,
        .itl_ms.p99?,
        .metrics.itl_ms.p99?,
        .inter_token_latency_ms.p99?,
        .tpot_ms.mean?,
        .metrics.tpot_ms.mean?,
        .tpot.mean?,
        .metrics.tpot.mean?,
        .itl_ms.mean?,
        .metrics.itl_ms.mean?,
        .mean_tpot_ms?,
        .mean_itl_ms?
      ] | map(select(. != null)) | .[0] // empty
    ' "$json_file"
  fi
 }
 evaluate_sla_from_json() {
  local json_file=$1
  local ttft
  local tpot
  local pass
  [[ -f "$json_file" ]] || return 2
  ttft=$(extract_metric_ms ttft "$json_file")
  tpot=$(extract_metric_ms tpot "$json_file")
  [[ -n "$ttft" && -n "$tpot" ]] || return 2
  pass=$(jq -n \
    --argjson ttft "$ttft" \
    --argjson tpot "$tpot" \
    --argjson sla_ttft "$SLA_TTFT_MS" \
    --argjson sla_tpot "$SLA_TPOT_MS" \
    '($ttft <= $sla_ttft) and ($tpot <= $sla_tpot)')
  [[ "$pass" == "true" ]]
 }
 write_adaptive_summary_json() {
  local summary_file=$1
  local test_name=$2
  local qps=$3
  local static_last_pass=$4
  local static_first_fail=$5
  local final_last_pass=$6
  local final_first_fail=$7
  jq -n \
    --arg test_name "$test_name" \
    --arg qps "$qps" \
    --argjson sla_ttft "$SLA_TTFT_MS" \
    --argjson sla_tpot "$SLA_TPOT_MS" \
    --arg static_last_pass "${static_last_pass:-}" \
    --arg static_first_fail "${static_first_fail:-}" \
    --arg final_last_pass "${final_last_pass:-}" \
    --arg final_first_fail "${final_first_fail:-}" \
    '{
      test_name: $test_name,
      qps: $qps,
      sla_ttft_ms: $sla_ttft,
      sla_tpot_ms: $sla_tpot,
      static_last_pass: (if $static_last_pass == "" then null else ($static_last_pass | tonumber) end),
      static_first_fail: (if $static_first_fail == "" then null else ($static_first_fail | tonumber) end),
      final_last_pass: (if $final_last_pass == "" then null else ($final_last_pass | tonumber) end),
      final_first_fail: (if $final_first_fail == "" then null else ($final_first_fail | tonumber) end)
    }' > "$summary_file"
 }
 run_single_serving_probe() {
  local test_name=$1
  local qps=$2
  local max_concurrency=$3
  local tp=$4
  local compilation_config_mode=$5
  local optimization_level=$6
  local client_args_effective=$7
  local client_remote_args=$8
  local server_command=$9
  local new_test_name="${test_name}_qps_${qps}_concurrency_${max_concurrency}"
  local result_json
  local num_prompts_arg=""
  local client_command
  result_json=$(result_json_path_for_serving "$test_name" "$qps" "$max_concurrency")
  if [[ -f "$result_json" ]]; then
    evaluate_sla_from_json "$result_json"
    return $?
  fi
  if [[ -n "${PROMPTS_PER_CONCURRENCY}" ]]; then
    num_prompts=$(( max_concurrency * PROMPTS_PER_CONCURRENCY ))
    if (( num_prompts < MIN_NUM_PROMPTS )); then num_prompts=$MIN_NUM_PROMPTS; fi
    if (( num_prompts > MAX_NUM_PROMPTS )); then num_prompts=$MAX_NUM_PROMPTS; fi
    num_prompts_arg="--num-prompts $num_prompts"
  fi
  client_command="vllm bench serve \
    --save-result \
    --result-dir $RESULTS_FOLDER \
    --result-filename ${new_test_name}.json \
    --request-rate $qps \
    --max-concurrency $max_concurrency \
    $num_prompts_arg \
    --metadata tensor_parallel_size=$tp compilation_config.mode=$compilation_config_mode optimization_level=$optimization_level adaptive_search=1 \
    $client_args_effective $client_remote_args "
  echo "Adaptive probe: $client_command"
  if [[ "${DRY_RUN:-0}" != "1" ]]; then
    bash -c "$client_command"
  fi
  jq_output=$(jq -n \
    --arg server "$server_command" \
    --arg client "$client_command" \
    --arg gpu "$gpu_type" \
    '{
      server_command: $server,
      client_command: $client,
      gpu_type: $gpu,
      adaptive_search: true
    }')
  echo "$jq_output" > "$RESULTS_FOLDER/${new_test_name}.commands"
  evaluate_sla_from_json "$result_json"
 }
 adaptive_refine_from_static_results() {
  local test_name=$1
  local qps=$2
  local max_concurrency_list_raw=$3
  local tp=$4
  local compilation_config_mode=$5
  local optimization_level=$6
  local client_args_effective=$7
  local client_remote_args=$8
  local server_command=$9
  local sorted_points
  local point
  local rc
  local static_last_pass=""
  local static_first_fail=""
  local largest_static=""
  local step_hint=1
  local previous_point=""
  local low
  local high
  local mid
  local probes=0
  local summary_file="$RESULTS_FOLDER/${test_name}_qps_${qps}_sla_summary.json"
  [[ "${ENABLE_ADAPTIVE_CONCURRENCY}" == "1" ]] || return 0
  [[ "${DRY_RUN:-0}" != "1" ]] || return 0
  sorted_points=$(for point in $max_concurrency_list_raw; do printf '%s\n' "$point"; done | tr -d "'" | awk '/^[0-9]+$/' | sort -n | uniq)
  [[ -n "$sorted_points" ]] || return 0
  while read -r point; do
    [[ -z "$point" ]] && continue
    largest_static="$point"
    evaluate_sla_from_json "$(result_json_path_for_serving "$test_name" "$qps" "$point")"
    rc=$?
    if (( rc == 0 )); then
      static_last_pass="$point"
    elif (( rc == 1 )); then
      if [[ -n "$static_last_pass" ]]; then
        static_first_fail="$point"
        break
      fi
    fi
    if [[ -n "$previous_point" ]]; then
      step_hint=$(( point - previous_point ))
      if (( step_hint < 1 )); then step_hint=1; fi
    fi
    previous_point="$point"
  done <<< "$sorted_points"
  if [[ -z "$static_last_pass" ]]; then
    write_adaptive_summary_json "$summary_file" "$test_name" "$qps" "" "$static_first_fail" "" "$static_first_fail"
    return 0
  fi
  if [[ -n "$static_first_fail" ]]; then
    low=$static_last_pass
    high=$static_first_fail
    while (( low + 1 < high )) && (( probes < ADAPTIVE_MAX_PROBES )); do
      mid=$(( (low + high) / 2 ))
      probes=$(( probes + 1 ))
      run_single_serving_probe \
        "$test_name" "$qps" "$mid" "$tp" \
        "$compilation_config_mode" "$optimization_level" \
        "$client_args_effective" "$client_remote_args" "$server_command"
      rc=$?
      if (( rc == 0 )); then
        low=$mid
      elif (( rc == 1 )); then
        high=$mid
      else
        break
      fi
    done
    write_adaptive_summary_json "$summary_file" "$test_name" "$qps" "$static_last_pass" "$static_first_fail" "$low" "$high"
    return 0
  fi
  low=$largest_static
  high=""
  while (( probes < ADAPTIVE_MAX_PROBES )); do
    point=$(( low + step_hint ))
    if (( point > ADAPTIVE_MAX_CONCURRENCY )); then
      point=$ADAPTIVE_MAX_CONCURRENCY
    fi
    (( point > low )) || break
    probes=$(( probes + 1 ))
    run_single_serving_probe \
      "$test_name" "$qps" "$point" "$tp" \
      "$compilation_config_mode" "$optimization_level" \
      "$client_args_effective" "$client_remote_args" "$server_command"
    rc=$?
    if (( rc == 0 )); then
      low=$point
      (( point == ADAPTIVE_MAX_CONCURRENCY )) && break
      step_hint=$(( step_hint * 2 ))
      if (( step_hint < 1 )); then step_hint=1; fi
    elif (( rc == 1 )); then
      high=$point
      break
    else
      break
    fi
  done
  if [[ -n "$high" ]]; then
    while (( low + 1 < high )) && (( probes < ADAPTIVE_MAX_PROBES )); do
      mid=$(( (low + high) / 2 ))
      probes=$(( probes + 1 ))
      run_single_serving_probe \
        "$test_name" "$qps" "$mid" "$tp" \
        "$compilation_config_mode" "$optimization_level" \
        "$client_args_effective" "$client_remote_args" "$server_command"
      rc=$?
      if (( rc == 0 )); then
        low=$mid
      elif (( rc == 1 )); then
        high=$mid
      else
        break
      fi
    done
  fi
  write_adaptive_summary_json "$summary_file" "$test_name" "$qps" "$static_last_pass" "" "$low" "$high"
 }
 run_benchmark_tests() {
  # run benchmark tests using `vllm bench <test_type>` command
  # $1: test type (latency or throughput)
  # $2: a json file specifying test cases
  local test_type=$1
  local test_file=$2
  # Iterate over tests
  jq -c '.[]' "$test_file" | while read -r params; do
    # get the test name, and append the GPU type back to it.
    test_name=$(echo "$params" | jq -r '.test_name')
-    if [[ ! "$test_name" =~ ^${test_type}_ ]]; then
+    if [[ ! "$test_name" =~ ^latency_ ]]; then
-      echo "In ${test_type}-test.json, test_name must start with \"${test_type}_\"."
+      echo "In latency-test.json, test_name must start with \"latency_\"."
      exit 1
    fi
@@ -512,15 +199,15 @@ run_benchmark_tests() {
    fi
    # get arguments
-    bench_params=$(echo "$params" | jq -r '.parameters')
+    latency_params=$(echo "$params" | jq -r '.parameters')
-    bench_args=$(json2args "$bench_params")
+    latency_args=$(json2args "$latency_params")
-    bench_environment_variables=$(echo "$params" | jq -r '.environment_variables')
+    latency_environment_variables=$(echo "$params" | jq -r '.environment_variables')
-    bench_envs=$(json2envs "$bench_environment_variables")
+    latency_envs=$(json2envs "$latency_environment_variables")
    # check if there is enough GPU to run the test
-    tp=$(echo "$bench_params" | jq -r '.tensor_parallel_size')
+    tp=$(echo "$latency_params" | jq -r '.tensor_parallel_size')
-    if [[ "$ON_CPU" == "1" ]]; then
+    if [ "$ON_CPU" == "1" ]; then
-      pp=$(echo "$bench_params" | jq -r '.pipeline_parallel_size // 1')
+      pp=$(echo "$latency_params" | jq -r '.pipeline_parallel_size')
      world_size=$(($tp*$pp))
      if [[ $numa_count -lt $world_size  && -z "${REMOTE_HOST}" ]]; then
        echo "Required world-size $world_size but only $numa_count NUMA nodes found. Skip testcase $test_name."
@@ -533,107 +220,108 @@ run_benchmark_tests() {
      fi
    fi
-    bench_command=" $bench_envs vllm bench $test_type \
+    latency_command=" $latency_envs vllm bench latency \
      --output-json $RESULTS_FOLDER/${test_name}.json \
-      $bench_args"
+      $latency_args"
    echo "Running test case $test_name"
-    echo "${test_type^} command: $bench_command"
+    echo "Latency command: $latency_command"
-    # recording benchmarking command and GPU command
+    # recoding benchmarking command ang GPU command
    jq_output=$(jq -n \
-      --arg command "$bench_command" \
+      --arg latency "$latency_command" \
      --arg gpu "$gpu_type" \
      --arg test_type "$test_type" \
      '{
-        ($test_type + "_command"): $command,
+        latency_command: $latency,
        gpu_type: $gpu
      }')
    echo "$jq_output" >"$RESULTS_FOLDER/$test_name.commands"
    # run the benchmark
-    eval "$bench_command"
+    eval "$latency_command"
    kill_gpu_processes
  done
 }
-run_latency_tests() { run_benchmark_tests "latency" "$1"; }
+run_throughput_tests() {
-run_startup_tests() { run_benchmark_tests "startup" "$1"; }
+  # run throughput tests using `vllm bench throughput`
-run_throughput_tests() { run_benchmark_tests "throughput" "$1"; }
+  # $1: a json file specifying throughput test cases
-merge_serving_tests_stream() {
+  local throughput_test_file
-  # Emit merged serving test objects, optionally filtered by MODEL_FILTER/DTYPE_FILTER in DRY_RUN mode.
+  throughput_test_file=$1
-  # This helper does NOT modify JSON; it only filters the stream in dry-run mode.
+
-  local serving_test_file="$1"
+  # Iterate over throughput tests
-  # shellcheck disable=SC2016
+  jq -c '.[]' "$throughput_test_file" | while read -r params; do
-  local merged='
+    # get the test name, and append the GPU type back to it.
-    if type == "array" then
+    test_name=$(echo "$params" | jq -r '.test_name')
-      # Plain format: test cases array
+    if [[ ! "$test_name" =~ ^throughput_ ]]; then
-      .[]
+      echo "In throughput-test.json, test_name must start with \"throughput_\"."
-    elif (type == "object" and has("tests")) then
+      exit 1
-      # merge the default parameters into each test cases
+    fi
-      . as $root
+
-      | ($root.defaults // {}) as $d
+    # if TEST_SELECTOR is set, only run the test cases that match the selector
-      | ($root.tests // [])[]
+    if [[ -n "$TEST_SELECTOR" ]] && [[ ! "$test_name" =~ $TEST_SELECTOR ]]; then
-      # default qps / max_concurrency from defaults if missing
+      echo "Skip test case $test_name."
-      | .qps_list = (.qps_list // $d.qps_list)
+      continue
-      | .max_concurrency_list = (.max_concurrency_list // $d.max_concurrency_list)
+    fi
-      # merge envs / params: test overrides defaults
+
-      | .server_environment_variables =
+    # get arguments
-          (($d.server_environment_variables // {}) + (.server_environment_variables // {}))
+    throughput_params=$(echo "$params" | jq -r '.parameters')
-      | .server_parameters =
+    throughput_args=$(json2args "$throughput_params")
-          (($d.server_parameters // {}) + (.server_parameters // {}))
+    throughput_environment_variables=$(echo "$params" | jq -r '.environment_variables')
-      | .client_parameters =
+    throughput_envs=$(json2envs "$throughput_environment_variables")
-          (($d.client_parameters // {}) + (.client_parameters // {}))
+
    # check if there is enough GPU to run the test
    tp=$(echo "$throughput_params" | jq -r '.tensor_parallel_size')
    if [ "$ON_CPU" == "1" ]; then
      pp=$(echo "$throughput_params" | jq -r '.pipeline_parallel_size')
      world_size=$(($tp*$pp))
      if [[ $numa_count -lt $world_size  && -z "${REMOTE_HOST}" ]]; then
        echo "Required world-size $world_size but only $numa_count NUMA nodes found. Skip testcase $test_name."
        continue
      fi
    else
-      error("Unsupported serving test file format: must be array or object with .tests")
+      if [[ $gpu_count -lt $tp ]]; then
-    end
+        echo "Required tensor-parallel-size $tp but only $gpu_count GPU found. Skip testcase $test_name."
-  '
+        continue
      fi
    fi
-  jq -c "$merged" "$serving_test_file" | \
+    throughput_command=" $throughput_envs vllm bench throughput \
-  if [[ "${DRY_RUN:-0}" == "1" && ( "${MODEL_FILTER}${DTYPE_FILTER}" != "" ) ]]; then
+      --output-json $RESULTS_FOLDER/${test_name}.json \
-    jq -c --arg model "$MODEL_FILTER" --arg dtype "$DTYPE_FILTER" '
+      $throughput_args"
-      select((($model|length)==0)
+
-             or ((.server_parameters.model // "") == $model)
+    echo "Running test case $test_name"
-             or ((.client_parameters.model // "") == $model))
+    echo "Throughput command: $throughput_command"
-      | select((($dtype|length)==0) or ((.server_parameters.dtype // "") == $dtype))
+    # recoding benchmarking command ang GPU command
-    '
+    jq_output=$(jq -n \
-  else
+      --arg command "$throughput_command" \
-    cat
+      --arg gpu "$gpu_type" \
-  fi
+      '{
        throughput_command: $command,
        gpu_type: $gpu
      }')
    echo "$jq_output" >"$RESULTS_FOLDER/$test_name.commands"
    # run the benchmark
    eval "$throughput_command"
    kill_gpu_processes
  done
 }
 run_serving_tests() {
  # run serving tests using `vllm bench serve` command
  # $1: a json file specifying serving test cases
  #
  # Supported JSON formats:
  # 1) Plain format: top-level array
  #    [ { "test_name": "...", "server_parameters": {...}, ... }, ... ]
  #
  # 2) Default parameters field + plain format tests
  #    {
  #      "defaults": { ... },
  #      "tests": [ { "test_name": "...", "server_parameters": {...}, ... }, ... ]
  #    }
  local serving_test_file
  serving_test_file=$1
-  # In dry-run mode, if filters are provided but no tests match, fail fast.
+  # Iterate over serving tests
-  if [[ "${DRY_RUN:-0}" == "1" && ( "${MODEL_FILTER}${DTYPE_FILTER}" != "" ) ]]; then
+  jq -c '.[]' "$serving_test_file" | while read -r params; do
    local count
    count=$(merge_serving_tests_stream "$serving_test_file" | wc -l | tr -d ' ')
    if [[ "$count" -eq 0 ]]; then
      echo "No matching serving tests found in $serving_test_file for model='$MODEL_FILTER' dtype='$DTYPE_FILTER'." >&2
      return 0
    fi
  fi
  # Iterate over serving tests (merged + optional filtered stream)
  merge_serving_tests_stream "$serving_test_file" | while read -r params; do
    # get the test name, and append the GPU type back to it.
    test_name=$(echo "$params" | jq -r '.test_name')
    if [[ ! "$test_name" =~ ^serving_ ]]; then
@@ -647,71 +335,28 @@ run_serving_tests() {
      continue
    fi
-    # get client and server arguments (after merged the default parameters)
+    # get client and server arguments
    server_params=$(echo "$params" | jq -r '.server_parameters')
    server_envs=$(echo "$params" | jq -r '.server_environment_variables')
    client_params=$(echo "$params" | jq -r '.client_parameters')
-
+    server_args=$(json2args "$server_params")
    # vLLM serve CLI: model must be positional (no --model). Convert server_parameters accordingly.
    server_model=$(echo "$server_params" | jq -r '.model // empty')
    if [[ -z "$server_model" || "$server_model" == "null" ]]; then
      echo "Error: serving test '$test_name' is missing server_parameters.model" >&2
      exit 1
    fi
    server_params_no_model=$(echo "$server_params" | jq -c 'del(.model)')
    server_args=$(json2args "$server_params_no_model")
    server_envs=$(json2envs "$server_envs")
    client_args=$(json2args "$client_params")
    # ------------------------------------------------------------
    # Option 1: Dynamic num-prompts scaling based on max_concurrency
    #
    # If PROMPTS_PER_CONCURRENCY is set, override JSON num_prompts with:
    #   num_prompts = max_concurrency * PROMPTS_PER_CONCURRENCY
    #
    # If PROMPTS_PER_CONCURRENCY is NOT set, keep JSON num_prompts behavior
    # unchanged (i.e., whatever is in serving-tests-*.json).
    # ------------------------------------------------------------
    PROMPTS_PER_CONCURRENCY="${PROMPTS_PER_CONCURRENCY-}"  # no default on purpose
    MIN_NUM_PROMPTS="${MIN_NUM_PROMPTS:-1}"
    MAX_NUM_PROMPTS="${MAX_NUM_PROMPTS:-1000000}"
    if [[ -n "${PROMPTS_PER_CONCURRENCY}" ]]; then
      # Remove any fixed --num-prompts from JSON-derived args (avoid duplicates)
      # Remove any fixed --num-prompts from JSON-derived args (avoid duplicates)
      # Handles: --num-prompts 123   and   --num-prompts=123
      client_args_no_np="$(
        printf ' %s ' "$client_args" \
        | sed -E \
          -e 's/[[:space:]]--num-prompts=([^[:space:]]+)([[:space:]]|$)/ /g' \
          -e 's/[[:space:]]--num-prompts[[:space:]]+([^[:space:]]+)([[:space:]]|$)/ /g'
      )"
      # normalize whitespace
      client_args_no_np="$(echo "$client_args_no_np" | tr -s ' ' | sed -E 's/^ //; s/ $//')"
      client_args_no_np="$(echo "$client_args_no_np" | xargs)"
      client_args_effective="$client_args_no_np"
    else
      client_args_effective="$client_args"
    fi
    # qps_list
    qps_list=$(echo "$params" | jq -r '.qps_list')
    qps_list=$(echo "$qps_list" | jq -r '.[] | @sh')
    echo "Running over qps list $qps_list"
    # max_concurrency_list (fallback to num_prompts if missing)
    max_concurrency_list=$(echo "$params" | jq -r '.max_concurrency_list')
    if [[ -z "$max_concurrency_list" || "$max_concurrency_list" == "null" ]]; then
-      num_prompts=$(echo "$client_params" | jq -r '.num_prompts')
+        num_prompts=$(echo "$client_params" | jq -r '.num_prompts')
-      max_concurrency_list="[$num_prompts]"
+        max_concurrency_list="[$num_prompts]"
    fi
    max_concurrency_list=$(echo "$max_concurrency_list" | jq -r '.[] | @sh')
    echo "Running over max concurrency list $max_concurrency_list"
    # check if there is enough resources to run the test
    tp=$(echo "$server_params" | jq -r '.tensor_parallel_size')
-    if [[ "$ON_CPU" == "1" ]]; then
+    if [ "$ON_CPU" == "1" ]; then
-      pp=$(echo "$server_params" | jq -r '.pipeline_parallel_size // 1')
+      pp=$(echo "$server_params" | jq -r '.pipeline_parallel_size')
      world_size=$(($tp*$pp))
      if [[ $numa_count -lt $world_size  && -z "${REMOTE_HOST}" ]]; then
        echo "Required world-size $world_size but only $numa_count NUMA nodes found. Skip testcase $test_name."
@@ -725,13 +370,14 @@ run_serving_tests() {
    fi
    # check if server model and client model is aligned
    server_model=$(echo "$server_params" | jq -r '.model')
    client_model=$(echo "$client_params" | jq -r '.model')
    if [[ $server_model != "$client_model" ]]; then
      echo "Server model and client model must be the same. Skip testcase $test_name."
      continue
    fi
-    server_command="$server_envs vllm serve $server_model \
+    server_command="$server_envs vllm serve \
      $server_args"
    # run the server
@@ -739,7 +385,7 @@ run_serving_tests() {
    echo "Server command: $server_command"
    # support remote vllm server
    client_remote_args=""
-    if [[ -z "${REMOTE_HOST}" && "${DRY_RUN:-0}" != "1" ]]; then
+    if [[ -z "${REMOTE_HOST}" ]]; then
      bash -c "$server_command" &
      server_pid=$!
      # wait until the server is alive
@@ -750,9 +396,6 @@ run_serving_tests() {
        echo ""
        echo "vLLM failed to start within the timeout period."
      fi
    elif [[ "${DRY_RUN:-0}" == "1" ]]; then
        # dry-run: don't start server
        echo "Dry Run."
    else
      server_command="Using Remote Server $REMOTE_HOST $REMOTE_PORT"
      if [[ ${REMOTE_PORT} ]]; then
@@ -762,48 +405,34 @@ run_serving_tests() {
      fi
    fi
    # save the compilation mode and optimization level on the serving results
    # whenever they are set
    compilation_config_mode=$(echo "$server_params" | jq -r '."compilation_config.mode" // empty')
    optimization_level=$(echo "$server_params" | jq -r '.optimization_level // empty')
    # iterate over different QPS
    for qps in $qps_list; do
      # remove the surrounding single quote from qps
      if [[ "$qps" == *"inf"* ]]; then
        echo "qps was $qps"
        qps="inf"
        echo "now qps is $qps"
      fi
      # iterate over different max_concurrency
      for max_concurrency in $max_concurrency_list; do
-        new_test_name="${test_name}_qps_${qps}_concurrency_${max_concurrency}"
+        new_test_name=$test_name"_qps_"$qps"_concurrency_"$max_concurrency
        echo " new test name $new_test_name"
-        # If PROMPTS_PER_CONCURRENCY is set, compute per-concurrency --num-prompts.
+        # pass the tensor parallel size to the client so that it can be displayed
-        num_prompts_arg=""
+        # on the benchmark dashboard
        if [[ -n "${PROMPTS_PER_CONCURRENCY}" ]]; then
          num_prompts=$(( max_concurrency * PROMPTS_PER_CONCURRENCY ))
          if (( num_prompts < MIN_NUM_PROMPTS )); then num_prompts=$MIN_NUM_PROMPTS; fi
          if (( num_prompts > MAX_NUM_PROMPTS )); then num_prompts=$MAX_NUM_PROMPTS; fi
          num_prompts_arg="--num-prompts $num_prompts"
        fi
        # pass the tensor parallel size, the compilation mode, and the optimization
        # level to the client so that they can be used on the benchmark dashboard
        client_command="vllm bench serve \
          --save-result \
          --result-dir $RESULTS_FOLDER \
          --result-filename ${new_test_name}.json \
          --request-rate $qps \
          --max-concurrency $max_concurrency \
-          $num_prompts_arg \
+          --metadata "tensor_parallel_size=$tp" \
-          --metadata tensor_parallel_size=$tp compilation_config.mode=$compilation_config_mode optimization_level=$optimization_level \
+          $client_args $client_remote_args "
          $client_args_effective $client_remote_args "
        echo "Running test case $test_name with qps $qps"
        echo "Client command: $client_command"
-        if [[ "${DRY_RUN:-0}" != "1" ]]; then
+        bash -c "$client_command"
          bash -c "$client_command"
        fi
        # record the benchmarking commands
        jq_output=$(jq -n \
@@ -818,39 +447,25 @@ run_serving_tests() {
        echo "$jq_output" >"$RESULTS_FOLDER/${new_test_name}.commands"
      done
      adaptive_refine_from_static_results \
        "$test_name" "$qps" "$max_concurrency_list" "$tp" \
        "$compilation_config_mode" "$optimization_level" \
        "$client_args_effective" "$client_remote_args" "$server_command"
    done
    # clean up
-    if [[ "${DRY_RUN:-0}" != "1" ]]; then
+    kill -9 $server_pid
-      kill -9 "$server_pid"
+    kill_gpu_processes
      kill_gpu_processes
    fi
  done
 }
 main() {
  local ARCH
  ARCH=''
-  if [[ "$ON_CPU" == "1" ]]; then
+  if [ "$ON_CPU" == "1" ];then
-    check_cpus
+     check_cpus
-    ARCH="-$gpu_type"
+     ARCH='-cpu'
  else
     check_gpus
     ARCH="$arch_suffix"
  fi
-
+  check_hf_token
  # DRY_RUN does not execute vLLM; do not require HF_TOKEN.
  if [[ "${DRY_RUN:-0}" != "1" ]]; then
    check_hf_token
  else
    echo "DRY_RUN=1 -> skip HF_TOKEN validation"
  fi
  # dependencies
  (which wget && which curl) || (apt-get update && apt-get install -y wget curl)
@@ -871,24 +486,17 @@ main() {
  # dump vllm info via vllm collect-env
  env_output=$(vllm collect-env)
  echo "$env_output" >"$RESULTS_FOLDER/vllm_env.txt"
  # benchmarking
-  run_serving_tests $QUICK_BENCHMARK_ROOT/tests/"${SERVING_JSON:-serving-tests$ARCH.json}" || exit $?
+  run_serving_tests $QUICK_BENCHMARK_ROOT/tests/"${SERVING_JSON:-serving-tests$ARCH.json}"
  if [[ "${DRY_RUN:-0}" == "1" ]]; then
    echo "DRY_RUN=1 -> skip latency/startup/throughput suites"
    exit 0
  fi
  run_latency_tests $QUICK_BENCHMARK_ROOT/tests/"${LATENCY_JSON:-latency-tests$ARCH.json}"
  run_startup_tests $QUICK_BENCHMARK_ROOT/tests/"${STARTUP_JSON:-startup-tests$ARCH.json}"
  run_throughput_tests $QUICK_BENCHMARK_ROOT/tests/"${THROUGHPUT_JSON:-throughput-tests$ARCH.json}"
  # postprocess benchmarking results
  pip install tabulate pandas
  python3 $QUICK_BENCHMARK_ROOT/scripts/convert-results-json-to-markdown.py
  python3 $QUICK_BENCHMARK_ROOT/scripts/compare-json-results.py -f $RESULTS_FOLDER/benchmark_results.json
  upload_to_buildkite
 }
@@ -1,25 +0,0 @@
 [
    {
        "test_name": "latency_llama8B_tp1",
        "environment_variables": {
            "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
            "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
            "VLLM_CPU_KVCACHE_SPACE": 40
        },
        "parameters": {
            "model": "meta-llama/Llama-3.1-8B-Instruct",
            "tensor_parallel_size": 1,
            "load_format": "dummy",
            "dtype": "bfloat16",
            "distributed_executor_backend": "mp",
            "block_size": 128,
            "trust_remote_code": "",
            "disable_log_stats": "",
            "enforce_eager": "",
            "max_num_batched_tokens": 2048,
            "max_num_seqs": 256,
            "num_iters_warmup": 5,
            "num_iters": 15
        }
    }
 ]
@@ -2,6 +2,7 @@
    {
        "test_name": "latency_llama8B_tp2",
        "environment_variables": {
            "VLLM_RPC_TIMEOUT": 100000,
 	    "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
 	    "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
 	    "VLLM_CPU_SGL_KERNEL": 1,
@@ -51,56 +51,5 @@
            "max-model-len": 256,
            "async-scheduling": ""
        }
    },
    {
        "test_name": "latency_deepseek_r1",
        "environment_variables": {
            "PT_HPU_LAZY_MODE": 1,
            "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1,
            "VLLM_CONTIGUOUS_PA": 1,
            "VLLM_DEFRAG": 1
        },
        "parameters": {
            "model": "deepseek-ai/DeepSeek-R1",
            "tensor_parallel_size": 8,
            "load_format": "dummy",
            "max-model-len": 2048,
            "dtype": "bfloat16"
        }
    },
    {
        "test_name": "latency_llama4_maverick_17b128e_instruct_fp8",
        "environment_variables": {
            "PT_HPU_LAZY_MODE": 1,
            "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1,
            "VLLM_CONTIGUOUS_PA": 1,
            "VLLM_DEFRAG": 1
        },
        "parameters": {
            "model": "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8",
            "tensor_parallel_size": 8,
            "max-model-len": 512,
            "max-num-seqs": 128,
            "async-scheduling": "",
            "gpu-memory-utilization": 0.95,
            "enable_expert_parallel": ""
        }
    },
    {
        "test_name": "latency_qwen3_8b",
        "environment_variables": {
            "PT_HPU_LAZY_MODE": 1,
            "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1,
            "VLLM_CONTIGUOUS_PA": 1,
            "VLLM_DEFRAG": 1
        },
        "parameters": {
            "model": "Qwen/Qwen3-8B",
            "tensor_parallel_size": 1,
            "max-model-len": 2048,
            "max-num-seqs": 128,
            "dtype": "bfloat16",
            "async-scheduling": ""
        }
    }
 ]
@@ -1,130 +0,0 @@
 {
  "defaults": {
    "qps_list": [
      "inf"
    ],
    "max_concurrency_list": [
      12,
      16,
      24,
      32,
      64,
      128,
      200
    ],
    "server_environment_variables": {
      "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
      "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
      "VLLM_CPU_SGL_KERNEL": 1,
      "VLLM_CPU_KVCACHE_SPACE": 40
    },
    "server_parameters": {
      "model": "meta-llama/Llama-3.1-8B-Instruct",
      "tensor_parallel_size": 1,
      "dtype": "bfloat16",
      "distributed_executor_backend": "mp",
      "block_size": 128,
      "trust_remote_code": "",
      "disable_log_stats": "",
      "enforce_eager": "",
      "max_num_batched_tokens": 2048,
      "max_num_seqs": 256,
      "load_format": "dummy"
    },
    "client_parameters": {
      "model": "meta-llama/Llama-3.1-8B-Instruct",
      "backend": "vllm",
      "ignore-eos": "",
      "temperature": 0,
      "num_prompts": 200
    }
  },
  "tests": [
    {
      "test_name": "serving_llama8B_tp1_sharegpt",
      "server_parameters": {
        "tensor_parallel_size": 1
      },
      "client_parameters": {
        "dataset_name": "sharegpt",
        "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json"
      }
    },
    {
      "test_name": "serving_llama8B_tp2_sharegpt",
      "server_parameters": {
        "tensor_parallel_size": 2
      },
      "client_parameters": {
        "dataset_name": "sharegpt",
        "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json"
      }
    },
    {
      "test_name": "serving_llama8B_tp1_random_128_128",
      "server_parameters": {
        "tensor_parallel_size": 1
      },
      "client_parameters": {
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_llama8B_tp2_random_128_128",
      "server_parameters": {
        "tensor_parallel_size": 2
      },
      "client_parameters": {
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_llama8B_tp1_random_128_2048",
      "server_parameters": {
        "tensor_parallel_size": 1
      },
      "client_parameters": {
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 2048
      }
    },
    {
      "test_name": "serving_llama8B_tp2_random_128_2048",
      "server_parameters": {
        "tensor_parallel_size": 2
      },
      "client_parameters": {
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 2048
      }
    },
    {
      "test_name": "serving_llama8B_tp1_random_2048_128",
      "server_parameters": {
        "tensor_parallel_size": 1
      },
      "client_parameters": {
        "dataset_name": "random",
        "random-input-len": 2048,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_llama8B_tp2_random_2048_128",
      "server_parameters": {
        "tensor_parallel_size": 2
      },
      "client_parameters": {
        "dataset_name": "random",
        "random-input-len": 2048,
        "random-output-len": 128
      }
    }
  ]
 }
@@ -1,37 +0,0 @@
 {
  "defaults": {
    "qps_list": [
      "inf"
    ],
    "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
    "server_environment_variables": {
      "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120
    },
    "server_parameters": {
      "dtype": "bfloat16",
      "model": "openai/whisper-large-v3-turbo"
    },
    "client_parameters": {
      "model": "openai/whisper-large-v3-turbo",
      "backend": "openai-audio",
      "endpoint": "/v1/audio/transcriptions",
      "dataset_name": "hf",
      "dataset_path": "openslr/librispeech_asr",
      "hf_subset": "clean",
      "hf_split": "test",
      "no_stream": "",
      "no_oversample": "",
      "temperature": 0,
      "num_prompts": 200
    }
  },
  "tests": [
    {
      "test_name": "serving_whisper_large_v3_turbo_librispeech_clean_tp1",
      "server_parameters": {
        "tensor_parallel_size": 1
      },
      "client_parameters": {}
    }
  ]
 }
@@ -1,40 +0,0 @@
 {
  "defaults": {
    "qps_list": [
      "inf"
    ],
    "max_concurrency_list": [
      32,
      64,
      128
    ],
    "server_environment_variables": {
      "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
      "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
      "VLLM_CPU_SGL_KERNEL": 1,
      "VLLM_CPU_KVCACHE_SPACE": 40
    },
    "server_parameters": {
      "dtype": "bfloat16",
      "model": "jinaai/jina-embeddings-v3",
      "trust_remote_code": ""
    },
    "client_parameters": {
      "model": "jinaai/jina-embeddings-v3",
      "backend": "openai-embeddings",
      "endpoint": "/v1/embeddings",
      "dataset_name": "sharegpt",
      "dataset_path": "ShareGPT_V3_unfiltered_cleaned_split.json",
      "num_prompts": 200
    }
  },
  "tests": [
    {
      "test_name": "serving_jina_embed_v3_tp1_sharegpt",
      "server_parameters": {
        "tensor_parallel_size": 1
      },
      "client_parameters": {}
    }
  ]
 }
@@ -0,0 +1,610 @@
 [
    {
        "test_name": "serving_llama8B_bf16_tp1_sharegpt",
        "qps_list": ["inf"],
        "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
        "server_environment_variables": {
            "VLLM_RPC_TIMEOUT": 100000,
 	    "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
 	    "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
 	    "VLLM_CPU_SGL_KERNEL": 1,
 	    "VLLM_CPU_KVCACHE_SPACE": 40
        },
        "server_parameters": {
            "model": "meta-llama/Llama-3.1-8B-Instruct",
            "tensor_parallel_size": 1,
 	    "dtype": "bfloat16",
 	    "distributed_executor_backend": "mp",
 	    "block_size": 128,
 	    "trust_remote_code": "",
            "disable_log_stats": "",
 	    "enforce_eager": "",
 	    "max_num_batched_tokens": 2048,
 	    "max_num_seqs": 256,
            "load_format": "dummy"
        },
        "client_parameters": {
            "model": "meta-llama/Llama-3.1-8B-Instruct",
            "backend": "vllm",
            "dataset_name": "sharegpt",
            "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
            "num_prompts": 200
        }
    },
    {
        "test_name": "serving_llama8B_bf16_tp2_sharegpt",
        "qps_list": ["inf"],
        "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
        "server_environment_variables": {
            "VLLM_RPC_TIMEOUT": 100000,
 	    "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
 	    "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
 	    "VLLM_CPU_SGL_KERNEL": 1,
 	    "VLLM_CPU_KVCACHE_SPACE": 40
        },
        "server_parameters": {
            "model": "meta-llama/Llama-3.1-8B-Instruct",
            "tensor_parallel_size": 2,
 	    "dtype": "bfloat16",
 	    "distributed_executor_backend": "mp",
 	    "block_size": 128,
 	    "trust_remote_code": "",
            "disable_log_stats": "",
 	    "enforce_eager": "",
 	    "max_num_batched_tokens": 2048,
 	    "max_num_seqs": 256,
            "load_format": "dummy"
        },
        "client_parameters": {
            "model": "meta-llama/Llama-3.1-8B-Instruct",
            "backend": "vllm",
            "dataset_name": "sharegpt",
            "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
            "num_prompts": 200
        }
    },
    {
        "test_name": "serving_llama8B_bf16_tp4_sharegpt",
        "qps_list": ["inf"],
        "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
        "server_environment_variables": {
            "VLLM_RPC_TIMEOUT": 100000,
 	    "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
 	    "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
 	    "VLLM_CPU_SGL_KERNEL": 1,
 	    "VLLM_CPU_KVCACHE_SPACE": 40
        },
        "server_parameters": {
            "model": "meta-llama/Llama-3.1-8B-Instruct",
            "tensor_parallel_size": 4,
 	    "dtype": "bfloat16",
 	    "distributed_executor_backend": "mp",
 	    "block_size": 128,
 	    "trust_remote_code": "",
            "disable_log_stats": "",
 	    "enforce_eager": "",
 	    "max_num_batched_tokens": 2048,
 	    "max_num_seqs": 256,
            "load_format": "dummy"
        },
        "client_parameters": {
            "model": "meta-llama/Llama-3.1-8B-Instruct",
            "backend": "vllm",
            "dataset_name": "sharegpt",
            "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
            "num_prompts": 200
        }
    },
    {
        "test_name": "serving_llama8B_bf16_tp1_random_128_128",
        "qps_list": ["inf"],
        "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200, 1000],
        "server_environment_variables": {
            "VLLM_RPC_TIMEOUT": 100000,
 	    "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
 	    "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
 	    "VLLM_CPU_SGL_KERNEL": 1,
 	    "VLLM_CPU_KVCACHE_SPACE": 40
        },
        "server_parameters": {
            "model": "meta-llama/Llama-3.1-8B-Instruct",
            "tensor_parallel_size": 1,
 	    "dtype": "bfloat16",
 	    "distributed_executor_backend": "mp",
 	    "block_size": 128,
 	    "trust_remote_code": "",
 	    "enable_chunked_prefill": "",
            "disable_log_stats": "",
 	    "enforce_eager": "",
 	    "max_num_batched_tokens": 2048,
 	    "max_num_seqs": 256,
            "load_format": "dummy"
        },
        "client_parameters": {
            "model": "meta-llama/Llama-3.1-8B-Instruct",
            "backend": "vllm",
            "dataset_name": "random",
 	    "random-input-len": 128,
 	    "random-output-len": 128,
 	    "ignore-eos": "",
            "num_prompts": 1000
        }
    },
    {
        "test_name": "serving_llama8B_bf16_tp2_random_128_128",
        "qps_list": ["inf"],
        "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200, 1000],
        "server_environment_variables": {
            "VLLM_RPC_TIMEOUT": 100000,
 	    "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
 	    "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
 	    "VLLM_CPU_SGL_KERNEL": 1,
 	    "VLLM_CPU_KVCACHE_SPACE": 40
        },
        "server_parameters": {
            "model": "meta-llama/Llama-3.1-8B-Instruct",
            "tensor_parallel_size": 2,
 	    "dtype": "bfloat16",
 	    "distributed_executor_backend": "mp",
 	    "block_size": 128,
 	    "trust_remote_code": "",
 	    "enable_chunked_prefill": "",
            "disable_log_stats": "",
 	    "enforce_eager": "",
 	    "max_num_batched_tokens": 2048,
 	    "max_num_seqs": 256,
            "load_format": "dummy"
        },
        "client_parameters": {
            "model": "meta-llama/Llama-3.1-8B-Instruct",
            "backend": "vllm",
            "dataset_name": "random",
 	    "random-input-len": 128,
 	    "random-output-len": 128,
 	    "ignore-eos": "",
            "num_prompts": 1000
        }
    },
    {
        "test_name": "serving_llama8B_bf16_tp4_random_128_128",
        "qps_list": ["inf"],
        "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200, 1000],
        "server_environment_variables": {
            "VLLM_RPC_TIMEOUT": 100000,
 	    "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
 	    "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
 	    "VLLM_CPU_SGL_KERNEL": 1,
 	    "VLLM_CPU_KVCACHE_SPACE": 40
        },
        "server_parameters": {
            "model": "meta-llama/Llama-3.1-8B-Instruct",
            "tensor_parallel_size": 4,
 	    "dtype": "bfloat16",
 	    "distributed_executor_backend": "mp",
 	    "block_size": 128,
 	    "trust_remote_code": "",
 	    "enable_chunked_prefill": "",
            "disable_log_stats": "",
 	    "enforce_eager": "",
 	    "max_num_batched_tokens": 2048,
 	    "max_num_seqs": 256,
            "load_format": "dummy"
        },
        "client_parameters": {
            "model": "meta-llama/Llama-3.1-8B-Instruct",
            "backend": "vllm",
            "dataset_name": "random",
 	    "random-input-len": 128,
 	    "random-output-len": 128,
            "num_prompts": 1000
        }
    },
    {
        "test_name": "serving_llama8B_int8_tp1_sharegpt",
        "qps_list": ["inf"],
        "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
        "server_environment_variables": {
            "VLLM_RPC_TIMEOUT": 100000,
 	    "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
 	    "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
 	    "VLLM_CPU_SGL_KERNEL": 1,
 	    "VLLM_CPU_KVCACHE_SPACE": 40
        },
        "server_parameters": {
            "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
            "tensor_parallel_size": 1,
 	    "dtype": "bfloat16",
 	    "distributed_executor_backend": "mp",
 	    "block_size": 128,
 	    "trust_remote_code": "",
            "disable_log_stats": "",
 	    "enforce_eager": "",
 	    "max_num_batched_tokens": 2048,
 	    "max_num_seqs": 256,
            "load_format": "dummy"
        },
        "client_parameters": {
            "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
            "backend": "vllm",
            "dataset_name": "sharegpt",
            "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
            "num_prompts": 200
        }
    },
    {
        "test_name": "serving_llama8B_int8_tp2_sharegpt",
        "qps_list": ["inf"],
        "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
        "server_environment_variables": {
            "VLLM_RPC_TIMEOUT": 100000,
 	    "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
 	    "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
 	    "VLLM_CPU_SGL_KERNEL": 1,
 	    "VLLM_CPU_KVCACHE_SPACE": 40
        },
        "server_parameters": {
            "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
            "tensor_parallel_size": 2,
 	    "dtype": "bfloat16",
 	    "distributed_executor_backend": "mp",
 	    "block_size": 128,
 	    "trust_remote_code": "",
            "disable_log_stats": "",
 	    "enforce_eager": "",
 	    "max_num_batched_tokens": 2048,
 	    "max_num_seqs": 256,
            "load_format": "dummy"
        },
        "client_parameters": {
            "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
            "backend": "vllm",
            "dataset_name": "sharegpt",
            "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
            "num_prompts": 200
        }
    },
    {
        "test_name": "serving_llama8B_int8_tp4_sharegpt",
        "qps_list": ["inf"],
        "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
        "server_environment_variables": {
            "VLLM_RPC_TIMEOUT": 100000,
 	    "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
 	    "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
 	    "VLLM_CPU_SGL_KERNEL": 1,
 	    "VLLM_CPU_KVCACHE_SPACE": 40
        },
        "server_parameters": {
            "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
            "tensor_parallel_size": 4,
 	    "dtype": "bfloat16",
 	    "distributed_executor_backend": "mp",
 	    "block_size": 128,
 	    "trust_remote_code": "",
            "disable_log_stats": "",
 	    "enforce_eager": "",
 	    "max_num_batched_tokens": 2048,
 	    "max_num_seqs": 256,
            "load_format": "dummy"
        },
        "client_parameters": {
            "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
            "backend": "vllm",
            "dataset_name": "sharegpt",
            "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
            "num_prompts": 200
        }
    },
    {
        "test_name": "serving_llama8B_int8_tp1_random_128_128",
        "qps_list": ["inf"],
        "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200, 1000],
        "server_environment_variables": {
            "VLLM_RPC_TIMEOUT": 100000,
 	    "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
 	    "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
 	    "VLLM_CPU_SGL_KERNEL": 1,
 	    "VLLM_CPU_KVCACHE_SPACE": 40
        },
        "server_parameters": {
            "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
            "tensor_parallel_size": 1,
 	    "dtype": "bfloat16",
 	    "distributed_executor_backend": "mp",
 	    "block_size": 128,
 	    "trust_remote_code": "",
 	    "enable_chunked_prefill": "",
            "disable_log_stats": "",
 	    "enforce_eager": "",
 	    "max_num_batched_tokens": 2048,
 	    "max_num_seqs": 256,
            "load_format": "dummy"
        },
        "client_parameters": {
            "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
            "backend": "vllm",
            "dataset_name": "random",
 	    "random-input-len": 128,
 	    "random-output-len": 128,
 	    "ignore-eos": "",
            "num_prompts": 1000
        }
    },
    {
        "test_name": "serving_llama8B_int8_tp2_random_128_128",
        "qps_list": ["inf"],
        "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200, 1000],
        "server_environment_variables": {
            "VLLM_RPC_TIMEOUT": 100000,
 	    "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
 	    "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
 	    "VLLM_CPU_SGL_KERNEL": 1,
 	    "VLLM_CPU_KVCACHE_SPACE": 40
        },
        "server_parameters": {
            "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
            "tensor_parallel_size": 2,
 	    "dtype": "bfloat16",
 	    "distributed_executor_backend": "mp",
 	    "block_size": 128,
 	    "trust_remote_code": "",
 	    "enable_chunked_prefill": "",
            "disable_log_stats": "",
 	    "enforce_eager": "",
 	    "max_num_batched_tokens": 2048,
 	    "max_num_seqs": 256,
            "load_format": "dummy"
        },
        "client_parameters": {
            "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
            "backend": "vllm",
            "dataset_name": "random",
 	    "random-input-len": 128,
 	    "random-output-len": 128,
 	    "ignore-eos": "",
            "num_prompts": 1000
        }
    },
    {
        "test_name": "serving_llama8B_int8_tp4_random_128_128",
        "qps_list": ["inf"],
        "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200, 1000],
        "server_environment_variables": {
            "VLLM_RPC_TIMEOUT": 100000,
 	    "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
 	    "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
 	    "VLLM_CPU_SGL_KERNEL": 1,
 	    "VLLM_CPU_KVCACHE_SPACE": 40
        },
        "server_parameters": {
            "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
            "tensor_parallel_size": 4,
 	    "dtype": "bfloat16",
 	    "distributed_executor_backend": "mp",
 	    "block_size": 128,
 	    "trust_remote_code": "",
 	    "enable_chunked_prefill": "",
            "disable_log_stats": "",
 	    "enforce_eager": "",
 	    "max_num_batched_tokens": 2048,
 	    "max_num_seqs": 256,
            "load_format": "dummy"
        },
        "client_parameters": {
            "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
            "backend": "vllm",
            "dataset_name": "random",
 	    "random-input-len": 128,
 	    "random-output-len": 128,
 	    "ignore-eos": "",
            "num_prompts": 1000
        }
    },
    {
        "test_name": "serving_llama8B_int4_tp1_sharegpt",
        "qps_list": ["inf"],
        "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
        "server_environment_variables": {
            "VLLM_RPC_TIMEOUT": 100000,
 	    "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
 	    "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
 	    "VLLM_CPU_SGL_KERNEL": 1,
 	    "VLLM_CPU_KVCACHE_SPACE": 40
        },
        "server_parameters": {
            "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
 	    "quantization": "awq",
            "tensor_parallel_size": 1,
 	    "dtype": "bfloat16",
 	    "distributed_executor_backend": "mp",
 	    "block_size": 128,
 	    "trust_remote_code": "",
            "disable_log_stats": "",
 	    "enforce_eager": "",
 	    "max_num_batched_tokens": 2048,
 	    "max_num_seqs": 256,
            "load_format": "dummy"
        },
        "client_parameters": {
            "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
            "backend": "vllm",
            "dataset_name": "sharegpt",
            "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
            "num_prompts": 200
        }
    },
    {
        "test_name": "serving_llama8B_int4_tp2_sharegpt",
        "qps_list": ["inf"],
        "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
        "server_environment_variables": {
            "VLLM_RPC_TIMEOUT": 100000,
 	    "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
 	    "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
 	    "VLLM_CPU_SGL_KERNEL": 1,
 	    "VLLM_CPU_KVCACHE_SPACE": 40
        },
        "server_parameters": {
            "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
 	    "quantization": "awq",
            "tensor_parallel_size": 2,
 	    "dtype": "bfloat16",
 	    "distributed_executor_backend": "mp",
 	    "block_size": 128,
 	    "trust_remote_code": "",
            "disable_log_stats": "",
 	    "enforce_eager": "",
 	    "max_num_batched_tokens": 2048,
 	    "max_num_seqs": 256,
            "load_format": "dummy"
        },
        "client_parameters": {
            "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
            "backend": "vllm",
            "dataset_name": "sharegpt",
            "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
            "num_prompts": 200
        }
    },
    {
        "test_name": "serving_llama8B_int4_tp4_sharegpt",
        "qps_list": ["inf"],
        "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
        "server_environment_variables": {
            "VLLM_RPC_TIMEOUT": 100000,
 	    "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
 	    "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
 	    "VLLM_CPU_SGL_KERNEL": 1,
 	    "VLLM_CPU_KVCACHE_SPACE": 40
        },
        "server_parameters": {
            "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
 	    "quantization": "awq",
            "tensor_parallel_size": 4,
 	    "dtype": "bfloat16",
 	    "distributed_executor_backend": "mp",
 	    "block_size": 128,
 	    "trust_remote_code": "",
            "disable_log_stats": "",
 	    "enforce_eager": "",
 	    "max_num_batched_tokens": 2048,
 	    "max_num_seqs": 256,
            "load_format": "dummy"
        },
        "client_parameters": {
            "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
            "backend": "vllm",
            "dataset_name": "sharegpt",
            "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
            "num_prompts": 200
        }
    },
    {
        "test_name": "serving_llama8B_int4_tp1_random_128_128",
        "qps_list": ["inf"],
        "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200, 1000],
        "server_environment_variables": {
            "VLLM_RPC_TIMEOUT": 100000,
 	    "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
 	    "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
 	    "VLLM_CPU_SGL_KERNEL": 1,
 	    "VLLM_CPU_KVCACHE_SPACE": 40
        },
        "server_parameters": {
            "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
 	    "quantization": "awq",
            "tensor_parallel_size": 1,
 	    "dtype": "bfloat16",
 	    "distributed_executor_backend": "mp",
 	    "block_size": 128,
 	    "trust_remote_code": "",
 	    "enable_chunked_prefill": "",
            "disable_log_stats": "",
 	    "enforce_eager": "",
 	    "max_num_batched_tokens": 2048,
 	    "max_num_seqs": 256,
            "load_format": "dummy"
        },
        "client_parameters": {
            "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
            "backend": "vllm",
            "dataset_name": "random",
 	    "random-input-len": 128,
 	    "random-output-len": 128,
 	    "ignore-eos": "",
            "num_prompts": 1000
        }
    },
    {
        "test_name": "serving_llama8B_int4_tp2_random_128_128",
        "qps_list": ["inf"],
        "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200, 1000],
        "server_environment_variables": {
            "VLLM_RPC_TIMEOUT": 100000,
 	    "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
 	    "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
 	    "VLLM_CPU_SGL_KERNEL": 1,
 	    "VLLM_CPU_KVCACHE_SPACE": 40
        },
        "server_parameters": {
            "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
 	    "quantization": "awq",
            "tensor_parallel_size": 2,
 	    "dtype": "bfloat16",
 	    "distributed_executor_backend": "mp",
 	    "block_size": 128,
 	    "trust_remote_code": "",
 	    "enable_chunked_prefill": "",
            "disable_log_stats": "",
 	    "enforce_eager": "",
 	    "max_num_batched_tokens": 2048,
 	    "max_num_seqs": 256,
            "load_format": "dummy"
        },
        "client_parameters": {
            "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
            "backend": "vllm",
            "dataset_name": "random",
 	    "random-input-len": 128,
 	    "random-output-len": 128,
 	    "ignore-eos": "",
            "num_prompts": 1000
        }
    },
    {
        "test_name": "serving_llama8B_int4_tp4_random_128_128",
        "qps_list": ["inf"],
        "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200, 1000],
        "server_environment_variables": {
            "VLLM_RPC_TIMEOUT": 100000,
 	    "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
 	    "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
 	    "VLLM_CPU_SGL_KERNEL": 1,
 	    "VLLM_CPU_KVCACHE_SPACE": 40
        },
        "server_parameters": {
            "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
 	    "quantization": "awq",
            "tensor_parallel_size": 4,
 	    "dtype": "bfloat16",
 	    "distributed_executor_backend": "mp",
 	    "block_size": 128,
 	    "trust_remote_code": "",
 	    "enable_chunked_prefill": "",
            "disable_log_stats": "",
 	    "enforce_eager": "",
 	    "max_num_batched_tokens": 2048,
 	    "max_num_seqs": 256,
            "load_format": "dummy"
        },
        "client_parameters": {
            "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
            "backend": "vllm",
            "dataset_name": "random",
 	    "random-input-len": 128,
 	    "random-output-len": 128,
 	    "ignore-eos": "",
            "num_prompts": 1000
        }
    }
 ]
@@ -1,288 +0,0 @@
 {
  "defaults": {
    "qps_list": [
      "inf"
    ],
    "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
    "server_environment_variables": {
      "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
      "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
      "VLLM_CPU_SGL_KERNEL": 1,
      "VLLM_CPU_KVCACHE_SPACE": 40
    },
    "server_parameters": {
      "model": "meta-llama/Llama-3.1-8B-Instruct",
      "tensor_parallel_size": 1,
      "dtype": "bfloat16",
      "distributed_executor_backend": "mp",
      "block_size": 128,
      "trust_remote_code": "",
      "disable_log_stats": "",
      "max_num_batched_tokens": 2048,
      "max_num_seqs": 256
    },
    "client_parameters": {
      "model": "meta-llama/Llama-3.1-8B-Instruct",
      "backend": "vllm",
      "ignore-eos": "",
      "temperature": 0,
      "num_prompts": 200
    }
  },
  "tests": [
    {
      "test_name": "serving_llama8B_tp1_random_128_128",
      "server_parameters": {
      },
      "client_parameters": {
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_llama8B_int4_tp1_random_128_128",
      "server_parameters": {
        "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4"
      },
      "client_parameters": {
        "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_llama8B_int8_tp1_random_128_128",
      "server_parameters": {
        "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8"
      },
      "client_parameters": {
        "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_llama1B_tp1_random_128_128",
      "server_parameters": {
        "model": "meta-llama/Llama-3.2-1B"
      },
      "client_parameters": {
        "model": "meta-llama/Llama-3.2-1B",
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_llama3B_tp1_random_128_128",
      "server_parameters": {
        "model": "meta-llama/Llama-3.2-3B-Instruct"
      },
      "client_parameters": {
        "model": "meta-llama/Llama-3.2-3B-Instruct",
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_llama70B_tp1_random_128_128",
      "server_parameters": {
        "model": "meta-llama/Llama-3.3-70B-Instruct"
      },
      "client_parameters": {
        "model": "meta-llama/Llama-3.3-70B-Instruct",
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_granite2B_tp1_random_128_128",
      "server_parameters": {
        "model": "ibm-granite/granite-3.2-2b-instruct"
      },
      "client_parameters": {
        "model": "ibm-granite/granite-3.2-2b-instruct",
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_qwen1.7B_tp1_random_128_128",
      "server_parameters": {
        "model": "Qwen/Qwen3-1.7B"
      },
      "client_parameters": {
        "model": "Qwen/Qwen3-1.7B",
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_qwen4B_tp1_random_128_128",
      "server_parameters": {
        "model": "Qwen/Qwen3-4B"
      },
      "client_parameters": {
        "model": "Qwen/Qwen3-4B",
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_qwen8B_tp1_random_128_128",
      "server_parameters": {
        "model": "Qwen/Qwen3-8B"
      },
      "client_parameters": {
        "model": "Qwen/Qwen3-8B",
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_qwen14B_tp1_random_128_128",
      "server_parameters": {
        "model": "Qwen/Qwen3-14B"
      },
      "client_parameters": {
        "model": "Qwen/Qwen3-14B",
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_qwen30B_tp1_random_128_128",
      "server_parameters": {
        "model": "Qwen/Qwen3-30B-A3B"
      },
      "client_parameters": {
        "model": "Qwen/Qwen3-30B-A3B",
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_glm9B_tp1_random_128_128",
      "server_parameters": {
        "model": "zai-org/glm-4-9b-hf"
      },
      "client_parameters": {
        "model": "zai-org/glm-4-9b-hf",
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_gemma7B_tp1_random_128_128",
      "server_parameters": {
        "model": "google/gemma-7b"
      },
      "client_parameters": {
        "model": "google/gemma-7b",
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_gemma3-4b_tp1_random_128_128",
      "server_environment_variables": {
 	"VLLM_CPU_SGL_KERNEL": 0
      },
      "server_parameters": {
        "model": "google/gemma-3-4b-it"
      },
      "client_parameters": {
        "model": "google/gemma-3-4b-it",
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_gemma3-12b_tp1_random_128_128",
      "server_environment_variables": {
 	"VLLM_CPU_SGL_KERNEL": 0
      },
      "server_parameters": {
        "model": "google/gemma-3-12b-it"
      },
      "client_parameters": {
        "model": "google/gemma-3-12b-it",
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_gemma4-4b_tp1_random_128_128",
      "server_environment_variables": {
 	"VLLM_CPU_SGL_KERNEL": 0
      },
      "server_parameters": {
        "model": "google/gemma-4-E4B-it"
      },
      "client_parameters": {
        "model": "google/gemma-4-E4B-it",
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_gemma4-2b_tp1_random_128_128",
      "server_environment_variables": {
 	"VLLM_CPU_SGL_KERNEL": 0
      },
      "server_parameters": {
        "model": "google/gemma-4-E2B-it"
      },
      "client_parameters": {
        "model": "google/gemma-4-E2B-it",
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_gemma4-26b_tp1_random_128_128",
      "server_environment_variables": {
 	"VLLM_CPU_SGL_KERNEL": 0,
        "VLLM_CPU_ATTN_SPLIT_KV": 0
      },
      "server_parameters": {
        "model": "google/gemma-4-26B-A4B-it"
      },
      "client_parameters": {
        "model": "google/gemma-4-26B-A4B-it",
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_phi4_tp1_random_128_128",
      "server_parameters": {
        "model": "microsoft/Phi-4-reasoning"
      },
      "client_parameters": {
        "model": "microsoft/Phi-4-reasoning",
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    }
  ]
 }
@@ -1,142 +1,276 @@
-{
+[
-  "defaults": {
+    {
-    "qps_list": [
+        "test_name": "serving_llama8B_tp1_sharegpt",
-      "inf"
+        "qps_list": [1, 4, 16, "inf"],
-    ],
+        "max_concurrency_list": [32],
-    "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
+        "server_environment_variables": {
-    "server_environment_variables": {
+            "VLLM_RPC_TIMEOUT": 100000,
-      "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
+	    "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
-      "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
+	    "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
-      "VLLM_CPU_SGL_KERNEL": 1,
+	    "VLLM_CPU_SGL_KERNEL": 1,
-      "VLLM_CPU_KVCACHE_SPACE": 40
+	    "VLLM_CPU_KVCACHE_SPACE": 40
        },
        "server_parameters": {
            "model": "meta-llama/Llama-3.1-8B-Instruct",
            "tensor_parallel_size": 1,
 	    "dtype": "bfloat16",
 	    "distributed_executor_backend": "mp",
 	    "block_size": 128,
 	    "trust_remote_code": "",
            "disable_log_stats": "",
 	    "enforce_eager": "",
 	    "max_num_batched_tokens": 2048,
 	    "max_num_seqs": 256,
            "load_format": "dummy"
        },
        "client_parameters": {
            "model": "meta-llama/Llama-3.1-8B-Instruct",
            "backend": "vllm",
            "dataset_name": "sharegpt",
            "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
            "num_prompts": 32
        }
    },
-    "server_parameters": {
+    {
-      "model": "meta-llama/Llama-3.1-8B-Instruct",
+        "test_name": "serving_llama8B_tp2_sharegpt",
-      "tensor_parallel_size": 1,
+        "qps_list": [1, 4, 16, "inf"],
-      "dtype": "bfloat16",
+        "max_concurrency_list": [32],
-      "distributed_executor_backend": "mp",
+        "server_environment_variables": {
-      "block_size": 128,
+            "VLLM_RPC_TIMEOUT": 100000,
-      "trust_remote_code": "",
+	    "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
-      "disable_log_stats": "",
+	    "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
-      "max_num_batched_tokens": 2048,
+	    "VLLM_CPU_SGL_KERNEL": 1,
-      "max_num_seqs": 256
+	    "VLLM_CPU_KVCACHE_SPACE": 40
        },
        "server_parameters": {
            "model": "meta-llama/Llama-3.1-8B-Instruct",
            "tensor_parallel_size": 2,
 	    "dtype": "bfloat16",
 	    "distributed_executor_backend": "mp",
 	    "block_size": 128,
 	    "trust_remote_code": "",
            "disable_log_stats": "",
 	    "enforce_eager": "",
 	    "max_num_batched_tokens": 2048,
 	    "max_num_seqs": 256,
            "load_format": "dummy"
        },
        "client_parameters": {
            "model": "meta-llama/Llama-3.1-8B-Instruct",
            "backend": "vllm",
            "dataset_name": "sharegpt",
            "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
            "num_prompts": 32
        }
    },
-    "client_parameters": {
+    {
-      "model": "meta-llama/Llama-3.1-8B-Instruct",
+        "test_name": "serving_llama8B_tp1_random_128_128",
-      "backend": "vllm",
+        "qps_list": [1, 4, 16, "inf"],
-      "ignore-eos": "",
+        "max_concurrency_list": [32],
-      "temperature": 0,
+        "server_environment_variables": {
-      "num_prompts": 200
+            "VLLM_RPC_TIMEOUT": 100000,
 	    "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
 	    "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
 	    "VLLM_CPU_SGL_KERNEL": 1,
 	    "VLLM_CPU_KVCACHE_SPACE": 40
        },
        "server_parameters": {
            "model": "meta-llama/Llama-3.1-8B-Instruct",
            "tensor_parallel_size": 1,
 	    "dtype": "bfloat16",
 	    "distributed_executor_backend": "mp",
 	    "block_size": 128,
 	    "trust_remote_code": "",
 	    "enable_chunked_prefill": "",
            "disable_log_stats": "",
 	    "enforce_eager": "",
 	    "max_num_batched_tokens": 2048,
 	    "max_num_seqs": 256,
            "load_format": "dummy"
        },
        "client_parameters": {
            "model": "meta-llama/Llama-3.1-8B-Instruct",
            "backend": "vllm",
            "dataset_name": "random",
 	    "random-input-len": 128,
 	    "random-output-len": 128,
 	    "ignore-eos": "",
            "num_prompts": 32
        }
    },
    {
        "test_name": "serving_llama8B_tp2_random_128_128",
        "qps_list": [1, 4, 16, "inf"],
        "max_concurrency_list": [32],
        "server_environment_variables": {
            "VLLM_RPC_TIMEOUT": 100000,
 	    "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
 	    "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
 	    "VLLM_CPU_SGL_KERNEL": 1,
 	    "VLLM_CPU_KVCACHE_SPACE": 40
        },
        "server_parameters": {
            "model": "meta-llama/Llama-3.1-8B-Instruct",
            "tensor_parallel_size": 2,
 	    "dtype": "bfloat16",
 	    "distributed_executor_backend": "mp",
 	    "block_size": 128,
 	    "trust_remote_code": "",
 	    "enable_chunked_prefill": "",
            "disable_log_stats": "",
 	    "enforce_eager": "",
 	    "max_num_batched_tokens": 2048,
 	    "max_num_seqs": 256,
            "load_format": "dummy"
        },
        "client_parameters": {
            "model": "meta-llama/Llama-3.1-8B-Instruct",
            "backend": "vllm",
            "dataset_name": "random",
 	    "random-input-len": 128,
 	    "random-output-len": 128,
 	    "ignore-eos": "",
            "num_prompts": 32
        }
    },
    {
        "test_name": "serving_llama8B_tp1_random_128_2048",
        "qps_list": [1, 4, 16, "inf"],
        "max_concurrency_list": [32],
        "server_environment_variables": {
            "VLLM_RPC_TIMEOUT": 100000,
 	    "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
 	    "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
 	    "VLLM_CPU_SGL_KERNEL": 1,
 	    "VLLM_CPU_KVCACHE_SPACE": 40
        },
        "server_parameters": {
            "model": "meta-llama/Llama-3.1-8B-Instruct",
            "tensor_parallel_size": 1,
 	    "dtype": "bfloat16",
 	    "distributed_executor_backend": "mp",
 	    "block_size": 128,
 	    "trust_remote_code": "",
 	    "enable_chunked_prefill": "",
            "disable_log_stats": "",
 	    "enforce_eager": "",
 	    "max_num_batched_tokens": 2048,
 	    "max_num_seqs": 256,
            "load_format": "dummy"
        },
        "client_parameters": {
            "model": "meta-llama/Llama-3.1-8B-Instruct",
            "backend": "vllm",
            "dataset_name": "random",
 	    "random-input-len": 128,
 	    "random-output-len": 2048,
 	    "ignore-eos": "",
            "num_prompts": 32
        }
    },
    {
        "test_name": "serving_llama8B_tp2_random_128_2048",
        "qps_list": [1, 4, 16, "inf"],
        "max_concurrency_list": [32],
        "server_environment_variables": {
            "VLLM_RPC_TIMEOUT": 100000,
 	    "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
 	    "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
 	    "VLLM_CPU_SGL_KERNEL": 1,
 	    "VLLM_CPU_KVCACHE_SPACE": 40
        },
        "server_parameters": {
            "model": "meta-llama/Llama-3.1-8B-Instruct",
            "tensor_parallel_size": 2,
 	    "dtype": "bfloat16",
 	    "distributed_executor_backend": "mp",
 	    "block_size": 128,
 	    "trust_remote_code": "",
 	    "enable_chunked_prefill": "",
            "disable_log_stats": "",
 	    "enforce_eager": "",
 	    "max_num_batched_tokens": 2048,
 	    "max_num_seqs": 256,
            "load_format": "dummy"
        },
        "client_parameters": {
            "model": "meta-llama/Llama-3.1-8B-Instruct",
            "backend": "vllm",
            "dataset_name": "random",
 	    "random-input-len": 128,
 	    "random-output-len": 2048,
 	    "ignore-eos": "",
            "num_prompts": 32
        }
    },
    {
        "test_name": "serving_llama8B_tp1_random_2048_128",
        "qps_list": [1, 4, 16, "inf"],
        "max_concurrency_list": [32],
        "server_environment_variables": {
            "VLLM_RPC_TIMEOUT": 100000,
 	    "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
 	    "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
 	    "VLLM_CPU_SGL_KERNEL": 1,
 	    "VLLM_CPU_KVCACHE_SPACE": 40
        },
        "server_parameters": {
            "model": "meta-llama/Llama-3.1-8B-Instruct",
            "tensor_parallel_size": 1,
 	    "dtype": "bfloat16",
 	    "distributed_executor_backend": "mp",
 	    "block_size": 128,
 	    "trust_remote_code": "",
 	    "enable_chunked_prefill": "",
            "disable_log_stats": "",
 	    "enforce_eager": "",
 	    "max_num_batched_tokens": 2048,
 	    "max_num_seqs": 256,
            "load_format": "dummy"
        },
        "client_parameters": {
            "model": "meta-llama/Llama-3.1-8B-Instruct",
            "backend": "vllm",
            "dataset_name": "random",
 	    "random-input-len": 2048,
 	    "random-output-len": 128,
 	    "ignore-eos": "",
            "num_prompts": 32
        }
    },
    {
        "test_name": "serving_llama8B_tp2_random_2048_128",
        "qps_list": [1, 4, 16, "inf"],
        "max_concurrency_list": [32],
        "server_environment_variables": {
            "VLLM_RPC_TIMEOUT": 100000,
 	    "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
 	    "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
 	    "VLLM_CPU_SGL_KERNEL": 1,
 	    "VLLM_CPU_KVCACHE_SPACE": 40
        },
        "server_parameters": {
            "model": "meta-llama/Llama-3.1-8B-Instruct",
            "tensor_parallel_size": 2,
 	    "dtype": "bfloat16",
 	    "distributed_executor_backend": "mp",
 	    "block_size": 128,
 	    "trust_remote_code": "",
 	    "enable_chunked_prefill": "",
            "disable_log_stats": "",
 	    "enforce_eager": "",
 	    "max_num_batched_tokens": 2048,
 	    "max_num_seqs": 256,
            "load_format": "dummy"
        },
        "client_parameters": {
            "model": "meta-llama/Llama-3.1-8B-Instruct",
            "backend": "vllm",
            "dataset_name": "random",
 	    "random-input-len": 2048,
 	    "random-output-len": 128,
 	    "ignore-eos": "",
            "num_prompts": 32
        }
    }
-  },
+]
  "tests": [
    {
      "test_name": "serving_llama8B_tp1_sharegpt",
      "server_parameters": {
        "tensor_parallel_size": 1
      },
      "client_parameters": {
        "dataset_name": "sharegpt",
        "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json"
      }
    },
    {
      "test_name": "serving_llama8B_tp2_sharegpt",
      "server_parameters": {
        "tensor_parallel_size": 2
      },
      "client_parameters": {
        "dataset_name": "sharegpt",
        "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json"
      }
    },
    {
      "test_name": "serving_llama8B_tp1_random_128_128",
      "server_parameters": {
        "tensor_parallel_size": 1
      },
      "client_parameters": {
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_llama8B_tp2_random_128_128",
      "server_parameters": {
        "tensor_parallel_size": 2
      },
      "client_parameters": {
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_llama8B_tp1_random_128_2048",
      "server_parameters": {
        "tensor_parallel_size": 1
      },
      "client_parameters": {
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 2048
      }
    },
    {
      "test_name": "serving_llama8B_tp2_random_128_2048",
      "server_parameters": {
        "tensor_parallel_size": 2
      },
      "client_parameters": {
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 2048
      }
    },
    {
      "test_name": "serving_llama8B_tp1_random_2048_128",
      "server_parameters": {
        "tensor_parallel_size": 1
      },
      "client_parameters": {
        "dataset_name": "random",
        "random-input-len": 2048,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_llama8B_tp2_random_2048_128",
      "server_parameters": {
        "tensor_parallel_size": 2
      },
      "client_parameters": {
        "dataset_name": "random",
        "random-input-len": 2048,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_llama8B_tp1_random_2048_2048",
      "server_parameters": {
        "tensor_parallel_size": 1
      },
      "client_parameters": {
        "dataset_name": "random",
        "random-input-len": 2048,
        "random-output-len": 2048
      }
    },
    {
      "test_name": "serving_llama8B_tp2_random_2048_2048",
      "server_parameters": {
        "tensor_parallel_size": 2
      },
      "client_parameters": {
        "dataset_name": "random",
        "random-input-len": 2048,
        "random-output-len": 2048
      }
    }
  ]
 }
@@ -10,6 +10,7 @@
        "server_parameters": {
            "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
            "tensor_parallel_size": 1,
            "swap_space": 16,
            "disable_log_stats": "",
            "load_format": "dummy",
            "max-model-len": 2048,
@@ -21,7 +22,6 @@
            "backend": "vllm",
            "dataset_name": "sharegpt",
            "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
            "temperature": 0,
            "num_prompts": 200
        }
    },
@@ -37,6 +37,7 @@
        "server_parameters": {
            "model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
            "tensor_parallel_size": 4,
            "swap_space": 16,
            "disable_log_stats": "",
            "load_format": "dummy",
            "max-model-len": 2048,
@@ -48,7 +49,6 @@
            "backend": "vllm",
            "dataset_name": "sharegpt",
            "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
            "temperature": 0,
            "num_prompts": 200
        }
    },
@@ -64,6 +64,7 @@
        "server_parameters": {
            "model": "mistralai/Mixtral-8x7B-Instruct-v0.1",
            "tensor_parallel_size": 2,
            "swap_space": 16,
            "disable_log_stats": "",
            "load_format": "dummy",
            "max-model-len": 2048,
@@ -75,88 +76,6 @@
            "backend": "vllm",
            "dataset_name": "sharegpt",
            "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
            "temperature": 0,
            "num_prompts": 200
        }
    },
    {
        "test_name": "serving_deepseek_r1",
        "qps_list": [1, 4, 16, "inf"],
        "server_environment_variables": {
            "PT_HPU_LAZY_MODE": 1,
            "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1,
            "VLLM_CONTIGUOUS_PA": 1,
            "VLLM_DEFRAG": 1
        },
        "server_parameters": {
            "model": "deepseek-ai/DeepSeek-R1",
            "tensor_parallel_size": 8,
            "disable_log_stats": "",
            "load_format": "dummy",
            "max-model-len": 2048,
            "max-num-seqs": 200,
            "async-scheduling": "",
            "dtype": "bfloat16"
        },
        "client_parameters": {
            "model": "deepseek-ai/DeepSeek-R1",
            "backend": "vllm",
            "dataset_name": "sharegpt",
            "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
            "temperature": 0,
            "num_prompts": 200
        }
    },
    {
        "test_name": "serving_llama4_maverick_17b128e_instruct_fp8",
        "qps_list": [1, 4, 16, "inf"],
        "server_environment_variables": {
            "PT_HPU_LAZY_MODE": 1,
            "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1,
            "VLLM_CONTIGUOUS_PA": 1,
            "VLLM_DEFRAG": 1
        },
        "server_parameters": {
            "model": "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8",
            "tensor_parallel_size": 8,
            "disable_log_stats": "",
            "max-model-len": 2048,
            "max-num-seqs": 128,
            "async-scheduling": "",
            "enable_expert_parallel": "",
            "max-num-batched-tokens": 4096
        },
        "client_parameters": {
            "model": "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8",
            "backend": "vllm",
            "dataset_name": "sharegpt",
            "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
            "temperature": 0,
            "num_prompts": 200
        }
    },
    {
        "test_name": "serving_qwen3_8b",
        "qps_list": [1, 4, 10, "inf"],
        "server_environment_variables": {
            "PT_HPU_LAZY_MODE": 1,
            "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1,
            "VLLM_CONTIGUOUS_PA": 1,
            "VLLM_DEFRAG": 1
        },
        "server_parameters": {
            "model": "Qwen/Qwen-3-8B",
            "tensor_parallel_size": 1,
            "dtype": "bfloat16",
            "disable_log_stats": "",
            "async-scheduling": ""
        },
        "client_parameters": {
            "model": "Qwen/Qwen-3-8B",
            "backend": "vllm",
            "dataset_name": "sharegpt",
            "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
            "temperature": 0,
            "num_prompts": 200
        }
    }
@@ -1,112 +1,77 @@
-{
+[
-  "defaults": {
+    {
-    "qps_list": [
+        "test_name": "serving_llama8B_tp1_sharegpt",
-      "inf"
+        "qps_list": [1, 4, 16, "inf"],
-    ],
+        "server_parameters": {
-    "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
+            "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
-    "server_parameters": {
+            "tensor_parallel_size": 1,
-      "model": "meta-llama/Llama-3.1-8B-Instruct",
+            "swap_space": 16,
-      "tensor_parallel_size": 1,
+            "disable_log_stats": "",
-      "dtype": "bfloat16"
+            "load_format": "dummy"
        },
        "client_parameters": {
            "model": "meta-llama/Meta-Llama-3.1-8B-Instruct",
            "backend": "vllm",
            "dataset_name": "sharegpt",
            "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
            "num_prompts": 200
        }
    },
-    "client_parameters": {
+    {
-      "model": "meta-llama/Llama-3.1-8B-Instruct",
+        "test_name": "serving_llama70B_tp4_sharegpt",
-      "backend": "vllm",
+        "qps_list": [1, 4, 16, "inf"],
-      "ignore-eos": "",
+        "server_parameters": {
-      "temperature": 0,
+            "model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
-      "num_prompts": 200
+            "tensor_parallel_size": 4,
            "swap_space": 16,
            "disable_log_stats": "",
            "load_format": "dummy"
        },
        "client_parameters": {
            "model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
            "backend": "vllm",
            "dataset_name": "sharegpt",
            "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
            "num_prompts": 200
        }
    },
    {
        "test_name": "serving_mixtral8x7B_tp2_sharegpt",
        "qps_list": [1, 4, 16, "inf"],
        "server_parameters": {
            "model": "mistralai/Mixtral-8x7B-Instruct-v0.1",
            "tensor_parallel_size": 2,
            "swap_space": 16,
            "disable_log_stats": "",
            "load_format": "dummy"
        },
        "client_parameters": {
            "model": "mistralai/Mixtral-8x7B-Instruct-v0.1",
            "backend": "vllm",
            "dataset_name": "sharegpt",
            "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
            "num_prompts": 200
        }
    },
    {
        "test_name": "serving_llama70B_tp4_sharegpt_specdecode",
        "qps_list": [2],
        "server_parameters": {
            "model": "meta-llama/Meta-Llama-3.1-70B-Instruct", 
            "tensor_parallel_size": 4,
            "swap_space": 16,
            "speculative_config": {
                "model": "turboderp/Qwama-0.5B-Instruct",
                "num_speculative_tokens": 4,
                "draft_tensor_parallel_size": 1
            }
        },
        "client_parameters": {
            "model": "meta-llama/Meta-Llama-3.1-70B-Instruct",
            "backend": "vllm",
            "dataset_name": "sharegpt",
            "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
            "num_prompts": 200 
        }
    }
-  },
+]
  "tests": [
    {
      "test_name": "serving_llama8B_tp1_sharegpt",
      "server_parameters": {
        "tensor_parallel_size": 1
      },
      "client_parameters": {
        "dataset_name": "sharegpt",
        "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json"
      }
    },
    {
        "dataset_name": "sharegpt",
        "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json"
      }
    },
    {
      "test_name": "serving_llama8B_tp1_random_128_128",
      "server_parameters": {
        "tensor_parallel_size": 1
      },
      "client_parameters": {
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_llama8B_tp1_random_128_2048",
      "server_parameters": {
        "tensor_parallel_size": 1
      },
      "client_parameters": {
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 2048
      }
    },
    {
      "test_name": "serving_llama8B_tp1_random_2048_128",
      "server_parameters": {
        "tensor_parallel_size": 1
      },
      "client_parameters": {
        "dataset_name": "random",
        "random-input-len": 2048,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_llama8B_tp1_random_2048_2048",
      "server_parameters": {
        "tensor_parallel_size": 1
      },
      "client_parameters": {
        "dataset_name": "random",
        "random-input-len": 2048,
        "random-output-len": 2048
      }
    },
    {
      "test_name": "serving_llama70B_tp4_random_128_128",
      "server_parameters": {
        "model": "meta-llama/Llama-3.3-70B-Instruct",
        "async_scheduling": "",
 	"no_enable_prefix_caching": "",
        "max_num_batched_tokens": 8192
      },
      "client_parameters": {
        "model": "meta-llama/Llama-3.3-70B-Instruct",
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    },
    {
      "test_name": "serving_gemma4-e4b_tp1_random_128_128",
      "server_parameters": {
        "model": "google/gemma-4-E4B-it",
 	"enable_auto_tool_choice": "",
 	"tool_call_parser": "gemma4",
 	"chat_template": "examples/tool_chat_template_gemma4.jinja",
 	"reasoning_parser": "gemma4"
      },
      "client_parameters": {
        "model": "google/gemma-4-E4B-it",
        "dataset_name": "random",
        "random-input-len": 128,
        "random-output-len": 128
      }
    }
  ]
 }
@@ -1,26 +0,0 @@
 [
    {
        "test_name": "throughput_llama8B_tp1",
        "environment_variables": {
            "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
            "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
            "VLLM_CPU_KVCACHE_SPACE": 40
        },
        "parameters": {
            "model": "meta-llama/Llama-3.1-8B-Instruct",
            "tensor_parallel_size": 1,
            "load_format": "dummy",
            "dtype": "bfloat16",
            "distributed_executor_backend": "mp",
            "block_size": 128,
            "trust_remote_code": "",
            "disable_log_stats": "",
            "enforce_eager": "",
            "max_num_batched_tokens": 2048,
            "max_num_seqs": 256,
            "dataset": "./ShareGPT_V3_unfiltered_cleaned_split.json",
            "num_prompts": 200,
            "backend": "vllm"
        }
    }
 ]
@@ -2,6 +2,7 @@
    {
        "test_name": "throughput_llama8B_tp2",
        "environment_variables": {
            "VLLM_RPC_TIMEOUT": 100000,
 	    "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
 	    "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
 	    "VLLM_CPU_SGL_KERNEL": 1,
@@ -57,67 +57,5 @@
            "max-num-seqs": 512,
            "async-scheduling": ""
        }
    },
    {
        "test_name": "throughput_deepseek_r1",
        "environment_variables": {
            "PT_HPU_LAZY_MODE": 1,
            "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1,
            "VLLM_CONTIGUOUS_PA": 1,
            "VLLM_DEFRAG": 1
        },
        "parameters": {
            "model": "deepseek-ai/DeepSeek-R1",
            "tensor_parallel_size": 8,
            "load_format": "dummy",
            "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
            "dataset_name": "sharegpt",
            "num_prompts": 1000,
            "backend": "vllm",
            "max-model-len": 2048,
            "max-num-seqs": 384,
            "async-scheduling": ""
        }
    },
    {
        "test_name": "throughput_llama4_maverick_17b128e_instruct_fp8",
        "environment_variables": {
            "PT_HPU_LAZY_MODE": 1,
            "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1,
            "VLLM_CONTIGUOUS_PA": 1,
            "VLLM_DEFRAG": 1
        },
        "parameters": {
            "model": "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8",
            "tensor_parallel_size": 8,
            "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
            "dataset_name": "sharegpt",
            "num_prompts": 1000,
            "backend": "vllm",
            "max-model-len": 2048,
            "max-num-seqs": 512,
            "async-scheduling": "",
            "enable_expert_parallel": ""
        }
    },
    {
        "test_name": "throughput_qwen3_8b",
        "environment_variables": {
            "PT_HPU_LAZY_MODE": 1,
            "PT_HPU_ENABLE_LAZY_COLLECTIVES": 1,
            "VLLM_CONTIGUOUS_PA": 1,
            "VLLM_DEFRAG": 1
        },
        "parameters": {
            "model": "Qwen/Qwen-3-8B",
            "tensor_parallel_size": 1,
            "load_format": "dummy",
            "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json",
            "dataset_name": "sharegpt",
            "num_prompts": 1000,
            "max-num-seqs": 512,
            "backend": "vllm",
            "async-scheduling": ""
        }
    }
 ]
@@ -1,10 +0,0 @@
 #!/bin/bash
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 #
 # Append a build artifact line to the Buildkite annotation.
 # Usage: annotate-build-artifact.sh <label> <value> <context>
 set -e
 echo "- **${1}**: \`${2}\`" | \
  buildkite-agent annotate --append --style 'info' \
    --context "${3:?context is required}"
@@ -1,36 +0,0 @@
 #!/bin/bash
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 #
 # Append the Docker image tag(s) an image-build step pushed to a Buildkite
 # annotation, so the built image tags show up on the build page instead of
 # being buried in the job logs.
 #
 # Usage: annotate-image-build.sh <image_tag> [<image_tag> ...]
 set -euo pipefail
 # buildkite-agent only exists on Buildkite agents; no-op elsewhere so the
 # image build scripts stay runnable locally.
 if ! command -v buildkite-agent >/dev/null 2>&1; then
    echo "buildkite-agent not found; skipping image tag annotation"
    exit 0
 fi
 label="${BUILDKITE_LABEL:-Image build}"
 content=""
 for image in "$@"; do
    [[ -n "$image" ]] || continue
    content+="- **${label}**: \`${image}\`"$'\n'
 done
 if [[ -z "$content" ]]; then
    echo "No image tags provided; nothing to annotate"
    exit 0
 fi
 # Best-effort: a flaky annotation must never fail an otherwise successful
 # (and expensive) image build.
 if ! printf '%s' "$content" | \
    buildkite-agent annotate --append --style 'info' --context 'docker-images'; then
    echo "warning: failed to annotate build with image tags"
 fi
@@ -0,0 +1,54 @@
 #!/bin/bash
 set -ex
 # Get release version, default to 1.0.0.dev for nightly/per-commit builds
 RELEASE_VERSION=$(buildkite-agent meta-data get release-version 2>/dev/null | sed 's/^v//')
 if [ -z "${RELEASE_VERSION}" ]; then
  RELEASE_VERSION="1.0.0.dev"
 fi
 buildkite-agent annotate --style 'info' --context 'release-workflow' << EOF
 To download the wheel (by commit):
 \`\`\`
 aws s3 cp s3://vllm-wheels/${BUILDKITE_COMMIT}/vllm-${RELEASE_VERSION}-cp38-abi3-manylinux1_x86_64.whl .
 aws s3 cp s3://vllm-wheels/${BUILDKITE_COMMIT}/vllm-${RELEASE_VERSION}-cp38-abi3-manylinux2014_aarch64.whl .
 aws s3 cp s3://vllm-wheels/${BUILDKITE_COMMIT}/vllm-${RELEASE_VERSION}+cu129-cp38-abi3-manylinux1_x86_64.whl .
 aws s3 cp s3://vllm-wheels/${BUILDKITE_COMMIT}/vllm-${RELEASE_VERSION}+cu129-cp38-abi3-manylinux1_x86_64.whl .
 \`\`\`
 To download the wheel (by version):
 \`\`\`
 aws s3 cp s3://vllm-wheels/${RELEASE_VERSION}/vllm-${RELEASE_VERSION}-cp38-abi3-manylinux1_x86_64.whl .
 aws s3 cp s3://vllm-wheels/${RELEASE_VERSION}/vllm-${RELEASE_VERSION}-cp38-abi3-manylinux2014_aarch64.whl .
 aws s3 cp s3://vllm-wheels/${RELEASE_VERSION}+cu129/vllm-${RELEASE_VERSION}+cu129-cp38-abi3-manylinux1_x86_64.whl .
 aws s3 cp s3://vllm-wheels/${RELEASE_VERSION}+cu130/vllm-${RELEASE_VERSION}+cu130-cp38-abi3-manylinux1_x86_64.whl .
 \`\`\`
 To download and upload the image:
 \`\`\`
 docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-x86_64
 docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-aarch64
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-x86_64 vllm/vllm-openai:x86_64
 docker tag vllm/vllm-openai:x86_64 vllm/vllm-openai:latest-x86_64
 docker tag vllm/vllm-openai:x86_64 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64
 docker push vllm/vllm-openai:latest-x86_64
 docker push vllm/vllm-openai:v${RELEASE_VERSION}-x86_64
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-aarch64 vllm/vllm-openai:aarch64
 docker tag vllm/vllm-openai:aarch64 vllm/vllm-openai:latest-aarch64
 docker tag vllm/vllm-openai:aarch64 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64
 docker push vllm/vllm-openai:latest-aarch64
 docker push vllm/vllm-openai:v${RELEASE_VERSION}-aarch64
 docker manifest rm vllm/vllm-openai:latest
 docker manifest create vllm/vllm-openai:latest vllm/vllm-openai:latest-x86_64 vllm/vllm-openai:latest-aarch64
 docker manifest create vllm/vllm-openai:v${RELEASE_VERSION} vllm/vllm-openai:v${RELEASE_VERSION}-x86_64 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64
 docker manifest push vllm/vllm-openai:latest
 docker manifest push vllm/vllm-openai:v${RELEASE_VERSION}
 \`\`\`
 EOF 
@@ -1,113 +0,0 @@
 #!/bin/bash
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 #
 # Generate Buildkite annotation for ROCm wheel release
 set -ex
 # Extract build configuration from Dockerfile.rocm_base (single source of truth)
 # Extract ROCm version dynamically from Dockerfile.rocm_base
 # BASE_IMAGE format: rocm/dev-ubuntu-22.04:7.0-complete -> extracts "7.0"
 ROCM_VERSION=$(grep -E '^ARG BASE_IMAGE=' docker/Dockerfile.rocm_base | sed -E 's/.*:([0-9]+\.[0-9]+).*/\1/' || echo "unknown")
 PYTHON_VERSION=$(grep '^ARG PYTHON_VERSION=' docker/Dockerfile.rocm_base | sed 's/^ARG PYTHON_VERSION=//')
 PYTORCH_ROCM_ARCH=$(grep '^ARG PYTORCH_ROCM_ARCH=' docker/Dockerfile.rocm_base | sed 's/^ARG PYTORCH_ROCM_ARCH=//')
 # Get release version, default to 1.0.0.dev for nightly/per-commit builds
 RELEASE_VERSION=$(buildkite-agent meta-data get release-version 2>/dev/null || echo "")
 if [ -z "${RELEASE_VERSION}" ]; then
  RELEASE_VERSION="1.0.0.dev"
 fi
 ROCM_BASE_CACHE_KEY=$(.buildkite/scripts/cache-rocm-base-wheels.sh key)
 # S3 URLs
 S3_BUCKET="${S3_BUCKET:-vllm-wheels}"
 S3_REGION="${AWS_DEFAULT_REGION:-us-west-2}"
 S3_URL="http://${S3_BUCKET}.s3-website-${S3_REGION}.amazonaws.com"
 # Format ROCm version for path (e.g., "7.1" -> "rocm710")
 ROCM_VERSION_PATH="rocm$(echo "${ROCM_VERSION}" | tr -d '.')"
 ROCM_PATH="rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}"
 buildkite-agent annotate --style 'success' --context 'rocm-release-workflow' << EOF
 ## ROCm Wheel and Docker Image Releases
 ### Build Configuration
 | Setting | Value |
 |---------|-------|
 | **ROCm Version** | ${ROCM_VERSION} |
 | **Python Version** | ${PYTHON_VERSION} |
 | **GPU Architectures** | ${PYTORCH_ROCM_ARCH} |
 | **Branch** | \`${BUILDKITE_BRANCH}\` |
 | **Commit** | \`${BUILDKITE_COMMIT}\` |
 ### :package: Installation
 **Install from this build (by commit):**
 \`\`\`bash
 pip install vllm --extra-index-url ${S3_URL}/${ROCM_PATH}/ --trusted-host ${S3_BUCKET}.s3-website-${S3_REGION}.amazonaws.com
 # Example for ROCm ${ROCM_VERSION}:
 pip install vllm --extra-index-url ${S3_URL}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/ --trusted-host ${S3_BUCKET}.s3-website-${S3_REGION}.amazonaws.com
 \`\`\`
 **Install from nightly (if published):**
 \`\`\`bash
 pip install vllm --extra-index-url ${S3_URL}/rocm/nightly/ --trusted-host ${S3_BUCKET}.s3-website-${S3_REGION}.amazonaws.com
 \`\`\`
 ### :floppy_disk: Download Wheels Directly
 \`\`\`bash
 # List all ROCm wheels
 aws s3 ls s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/
 # Download specific wheels
 aws s3 cp s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/vllm-*.whl .
 aws s3 cp s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/torch-*.whl .
 aws s3 cp s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/triton-*.whl .
 aws s3 cp s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/triton-kernels-*.whl .
 aws s3 cp s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/torchvision-*.whl .
 aws s3 cp s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/torchaudio-*.whl .
 aws s3 cp s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/amdsmi-*.whl .
 aws s3 cp s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/amd_aiter-*.whl .
 aws s3 cp s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/flash-attn-*.whl .
 \`\`\`
 ### :gear: Included Packages
 - **vllm**: vLLM with ROCm support
 - **torch**: PyTorch built for ROCm ${ROCM_VERSION}
 - **triton**: Triton
 - **triton-kernels**: Triton kernels
 - **torchvision**: TorchVision for ROCm PyTorch
 - **torchaudio**: Torchaudio for ROCm PyTorch
 - **amdsmi**: AMD SMI Python bindings
 - **amd_aiter**: Aiter for ROCm
 - **flash-attn**: Flash Attention for ROCm
 ### :warning: Notes
 - These wheels are built for **ROCm ${ROCM_VERSION}** and will NOT work with CUDA GPUs
 - Supported GPU architectures: ${PYTORCH_ROCM_ARCH}
 - Platform: Linux x86_64 only
 ### :package: Docker Image Release
 To download and upload the image:
 \`\`\`
 docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-rocm-base
 docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-rocm
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${ROCM_BASE_CACHE_KEY}-rocm-base vllm/vllm-openai-rocm:${BUILDKITE_COMMIT}-base
 docker tag vllm/vllm-openai-rocm:${BUILDKITE_COMMIT}-base vllm/vllm-openai-rocm:latest-base
 docker tag vllm/vllm-openai-rocm:${BUILDKITE_COMMIT}-base vllm/vllm-openai-rocm:v${RELEASE_VERSION}-base
 docker push vllm/vllm-openai-rocm:latest-base
 docker push vllm/vllm-openai-rocm:v${RELEASE_VERSION}-base
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-rocm vllm/vllm-openai-rocm:${BUILDKITE_COMMIT}
 docker tag vllm/vllm-openai-rocm:${BUILDKITE_COMMIT} vllm/vllm-openai-rocm:latest
 docker tag vllm/vllm-openai-rocm:${BUILDKITE_COMMIT} vllm/vllm-openai-rocm:v${RELEASE_VERSION}
 docker push vllm/vllm-openai-rocm:latest
 docker push vllm/vllm-openai-rocm:v${RELEASE_VERSION}
 \`\`\`
 EOF
@@ -1,32 +0,0 @@
 #!/usr/bin/env bash
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 #
 # Build the macOS arm64 CPU wheel natively on a macOS agent (the `macmini`
 # queue) into artifacts/dist/ for upload-nightly-wheels.sh.
 set -euo pipefail
 # The Rust frontend build needs protoc.
 if ! command -v protoc >/dev/null 2>&1; then
  brew install protobuf
 fi
 # upload-nightly-wheels.sh expects exactly one wheel.
 rm -rf artifacts/dist
 mkdir -p artifacts/dist
 export VLLM_TARGET_DEVICE=cpu
 export VLLM_REQUIRE_RUST_FRONTEND=1
 export MACOSX_DEPLOYMENT_TARGET=11.0
 # uv's CPython is universal2; force an arm64-only build and tag so the wheel
 # isn't mislabelled universal2 and installed on Intel Macs where import fails.
 export ARCHFLAGS="-arch arm64"
 export _PYTHON_HOST_PLATFORM="macosx-11.0-arm64"
 export CMAKE_BUILD_PARALLEL_LEVEL="${CMAKE_BUILD_PARALLEL_LEVEL:-4}"
 uv venv --python 3.12
 uv pip install -r requirements/build/cpu.txt --index-strategy unsafe-best-match
 uv build --wheel --no-build-isolation -o artifacts/dist
 ls -l artifacts/dist/*.whl
@@ -1,131 +0,0 @@
 #!/usr/bin/env bash
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 #
 # Cache helper for ROCm base wheels
 #
 # This script manages caching of pre-built ROCm base wheels (torch, triton, etc.)
 # to avoid rebuilding them when Dockerfile.rocm_base hasn't changed.
 #
 # Usage:
 #   cache-rocm-base-wheels.sh check    - Check if cache exists, outputs "hit" or "miss"
 #   cache-rocm-base-wheels.sh upload   - Upload wheels to cache
 #   cache-rocm-base-wheels.sh download - Download wheels from cache
 #   cache-rocm-base-wheels.sh key      - Output the cache key
 #
 # Environment variables:
 #   S3_BUCKET          - S3 bucket name (default: vllm-wheels)
 #
 # Note: ROCm version is determined by BASE_IMAGE in Dockerfile.rocm_base,
 #       so changes to ROCm version are captured by the Dockerfile hash.
 set -euo pipefail
 BUCKET="${S3_BUCKET:-vllm-wheels}"
 DOCKERFILE="docker/Dockerfile.rocm_base"
 CACHE_PREFIX="rocm/cache"
 # Generate hash from Dockerfile content + build args
 generate_cache_key() {
    # Include Dockerfile content
    if [[ ! -f "$DOCKERFILE" ]]; then
        echo "ERROR: Dockerfile not found: $DOCKERFILE" >&2
        exit 1
    fi
    local dockerfile_hash=$(sha256sum "$DOCKERFILE" | cut -c1-16)
    echo "${dockerfile_hash}"
 }
 CACHE_KEY=$(generate_cache_key)
 CACHE_PATH="s3://${BUCKET}/${CACHE_PREFIX}/${CACHE_KEY}/"
 case "${1:-}" in
    check)
        echo "Checking cache for key: ${CACHE_KEY}" >&2
        echo "Cache path: ${CACHE_PATH}" >&2
        # Check if cache exists by listing objects
        # We look for at least one .whl file
        echo "Running: aws s3 ls ${CACHE_PATH}" >&2
        S3_OUTPUT=$(aws s3 ls "${CACHE_PATH}" 2>&1) || true
        echo "S3 ls output:" >&2
        echo "$S3_OUTPUT" | head -5 >&2
        if echo "$S3_OUTPUT" | grep -q "\.whl"; then
            echo "hit"
        else
            echo "miss"
        fi
        ;;
    upload)
        echo "========================================"
        echo "Uploading wheels to cache"
        echo "========================================"
        echo "Cache key: ${CACHE_KEY}"
        echo "Cache path: ${CACHE_PATH}"
        echo ""
        if [[ ! -d "artifacts/rocm-base-wheels" ]]; then
            echo "ERROR: artifacts/rocm-base-wheels directory not found" >&2
            exit 1
        fi
        WHEEL_COUNT=$(find artifacts/rocm-base-wheels -maxdepth 1 -name '*.whl' 2>/dev/null | wc -l)
        if [[ "$WHEEL_COUNT" -eq 0 ]]; then
            echo "ERROR: No wheels found in artifacts/rocm-base-wheels/" >&2
            exit 1
        fi
        echo "Uploading $WHEEL_COUNT wheels..."
        aws s3 cp --recursive artifacts/rocm-base-wheels/ "${CACHE_PATH}"
        echo ""
        echo "Cache upload complete!"
        echo "========================================"
        ;;
    download)
        echo "========================================"
        echo "Downloading wheels from cache"
        echo "========================================"
        echo "Cache key: ${CACHE_KEY}"
        echo "Cache path: ${CACHE_PATH}"
        echo ""
        mkdir -p artifacts/rocm-base-wheels
        # Use sync with include/exclude to only download .whl files
        aws s3 sync "${CACHE_PATH}" artifacts/rocm-base-wheels/ \
            --exclude "*" \
            --include "*.whl"
        echo ""
        echo "Downloaded wheels:"
        find artifacts/rocm-base-wheels -maxdepth 1 -name '*.whl' -exec ls -lh {} \;
        WHEEL_COUNT=$(find artifacts/rocm-base-wheels -maxdepth 1 -name '*.whl' 2>/dev/null | wc -l)
        echo ""
        echo "Total: $WHEEL_COUNT wheels"
        echo "========================================"
        ;;
    key)
        echo "${CACHE_KEY}"
        ;;
    path)
        echo "${CACHE_PATH}"
        ;;
    *)
        echo "Usage: $0 {check|upload|download|key|path}" >&2
        echo "" >&2
        echo "Commands:" >&2
        echo "  check    - Check if cache exists, outputs 'hit' or 'miss'" >&2
        echo "  upload   - Upload wheels from artifacts/rocm-base-wheels/ to cache" >&2
        echo "  download - Download wheels from cache to artifacts/rocm-base-wheels/" >&2
        echo "  key      - Output the cache key" >&2
        echo "  path     - Output the full S3 cache path" >&2
        exit 1
        ;;
 esac
@@ -1,239 +0,0 @@
 #!/bin/bash
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 #
 # Check if Ray LLM can generate lock files that are compatible with this
 # version of vllm. Downloads Ray's requirement files and runs a full
 # dependency resolution with the installed vllm's constraints to see if
 # a valid lock file can be produced.
 #
 # See: https://github.com/vllm-project/vllm/issues/33599
 set -eo pipefail
 RAY_BASE_URL="https://raw.githubusercontent.com/ray-project/ray/master/python"
 WORK_DIR=$(mktemp -d)
 trap 'rm -rf "$WORK_DIR"' EXIT
 # ── Detect PyTorch index URL ─────────────────────────────────────────────
 if python3 -c "import torch; assert torch.version.hip" 2>/dev/null; then
    ROCM_VER=$(python3 -c "import torch; print(torch.version.hip.rsplit('.', 1)[0])")
    CANDIDATE_URL="https://download.pytorch.org/whl/rocm${ROCM_VER}"
    if curl -fsSL --head "${CANDIDATE_URL}/" >/dev/null 2>&1; then
        TORCH_INDEX_URL="${CANDIDATE_URL}"
    else
        echo ">>> WARNING: ROCm ${ROCM_VER} wheel index not found at ${CANDIDATE_URL}"
        echo ">>>          Falling back to default PyPI (resolution may be incomplete)"
        TORCH_INDEX_URL=""
    fi
 else
    if [ "${TORCH_NIGHTLY:-0}" = "1" ]; then
        TORCH_INDEX_URL="https://download.pytorch.org/whl/nightly/cu130"
    else
        TORCH_INDEX_URL="https://download.pytorch.org/whl/cu130"
    fi
 fi
 echo ">>> Using PyTorch index: ${TORCH_INDEX_URL:-PyPI default}"
 # Fetch all Ray requirement files used in the LLM depset pipeline
 echo ">>> Fetching Ray requirement files"
 RAY_FILES=(
    "requirements.txt"
    "requirements/cloud-requirements.txt"
    "requirements/base-test-requirements.txt"
    "requirements/llm/llm-requirements.txt"
    "requirements/llm/llm-test-requirements.txt"
 )
 for FILE in "${RAY_FILES[@]}"; do
    LOCAL_PATH="${WORK_DIR}/$(basename "$FILE")"
    echo "    ${FILE}"
    curl -fsSL -o "$LOCAL_PATH" "${RAY_BASE_URL}/${FILE}"
 done
 # Extract installed vllm deps
 echo ">>> Extracting installed vllm dependency constraints"
 python3 - "${WORK_DIR}/vllm-constraints.txt" <<'PYEOF'
 """Write out the installed vllm's dependencies as pip constraint lines.
 Ray uses vllm[audio], so audio-extra deps are included with their extra
 markers stripped. The resolver cannot evaluate extra markers for a
 package that is not itself being resolved from an index, so we activate
 them manually here.
 """
 import importlib.metadata
 import re
 import sys
 out_path = sys.argv[1]
 raw_reqs = importlib.metadata.requires("vllm") or []
 # Ray uses vllm[audio] – activate that extra.
 ACTIVE_EXTRAS = {"audio"}
 EXTRA_RE = re.compile(r"""extra\s*==\s*['"]([^'"]+)['"]""")
 lines = []
 for r in raw_reqs:
    if ";" not in r:
        # Unconditional dep — always include.
        lines.append(r.strip())
        continue
    req_part, _, marker_part = r.partition(";")
    marker_part = marker_part.strip()
    extra_matches = EXTRA_RE.findall(marker_part)
    if not extra_matches:
        # Non-extra marker (python_version, etc.) — keep as-is.
        lines.append(r.strip())
        continue
    if not ACTIVE_EXTRAS.intersection(extra_matches):
        continue  # Skip inactive extras (tensorizer, bench, …).
    # Strip the extra== conditions but keep any remaining markers
    # (e.g. python_version).
    cleaned = EXTRA_RE.sub("", marker_part)
    cleaned = re.sub(r"\band\b\s*\band\b", "and", cleaned)
    cleaned = re.sub(r"^\s*and\s+|\s+and\s*$", "", cleaned).strip()
    if cleaned:
        lines.append(f"{req_part.strip()} ; {cleaned}")
    else:
        lines.append(req_part.strip())
 with open(out_path, "w") as f:
    for line in lines:
        f.write(line + "\n")
 print(f"Wrote {len(lines)} constraints to {out_path}")
 PYEOF
 echo ">>> Installed vllm deps (first 20 lines):"
 head -20 "${WORK_DIR}/vllm-constraints.txt"
 # Remove Ray's vllm pin — the installed vllm's transitive deps
 # (written above) replace it in the resolution. vllm itself cannot
 # be resolved from PyPI for in-development versions, so we test
 # whether Ray's requirements can coexist with vllm's dependency
 # constraints instead.
 sed -i '/^vllm/d' "${WORK_DIR}/llm-requirements.txt"
 # Install uv if needed
 if ! command -v uv &>/dev/null; then
    echo ">>> Installing uv"
    pip install uv -q
 fi
 # Resolve: given vllm's constraints, can Ray compile a lock file?
 #
 # vllm's dependency constraints are the fixed side — Ray is flexible and
 # can regenerate its lock files. We pass vllm's constraints via -c so
 # the resolver treats them as non-negotiable bounds, then check whether
 # Ray's own requirements can still be satisfied within those bounds.
 echo ""
 echo "============================================================"
 echo ">>> Resolving: Can Ray generate compatible lock files?"
 echo "============================================================"
 EXTRA_INDEX_ARGS=()
 if [[ -n "${TORCH_INDEX_URL}" ]]; then
    EXTRA_INDEX_ARGS+=(--extra-index-url "${TORCH_INDEX_URL}")
 fi
 set +e
 uv pip compile \
    "${WORK_DIR}/requirements.txt" \
    "${WORK_DIR}/cloud-requirements.txt" \
    "${WORK_DIR}/base-test-requirements.txt" \
    "${WORK_DIR}/llm-requirements.txt" \
    "${WORK_DIR}/llm-test-requirements.txt" \
    -c "${WORK_DIR}/vllm-constraints.txt" \
    --python-version 3.12 \
    --python-platform x86_64-manylinux_2_31 \
    "${EXTRA_INDEX_ARGS[@]}" \
    --index-strategy unsafe-best-match \
    --unsafe-package setuptools \
    --unsafe-package ray \
    --no-header \
    -o "${WORK_DIR}/resolved.txt" \
    2>&1
 EXIT_CODE=$?
 set -e
 echo ""
 echo "=========================================="
 if [ $EXIT_CODE -eq 0 ]; then
    echo "SUCCESS: Ray can generate lock files compatible with this vllm."
    echo ""
    echo "Key resolved versions:"
    grep -E '^(protobuf|torch|numpy|transformers)==' \
        "${WORK_DIR}/resolved.txt" | sort || true
    echo "=========================================="
    exit 0
 fi
 echo "FAILURE: Ray cannot generate lock files compatible with this vllm."
 echo "This means a fundamental dependency conflict exists that Ray"
 echo "cannot resolve by regenerating its lock files."
 echo "See: https://github.com/vllm-project/vllm/issues/33599"
 echo "=========================================="
 # Buildkite annotation
 if [ -f /usr/bin/buildkite-agent ]; then
    buildkite-agent annotate --style 'warning' --context 'ray-compat' << EOF
 ### :warning: Ray Dependency Compatibility Warning
 This PR introduces dependencies that **cannot** be resolved with Ray's requirements.
 Ray would not be able to regenerate its lock files to accommodate this vllm version.
 Please check the **Ray Dependency Compatibility Check** step logs for details.
 See [issue #33599](https://github.com/vllm-project/vllm/issues/33599) for context.
 EOF
 fi
 # Notify Slack if webhook is configured and PR/branch are valid.
 if [ -n "$RAY_COMPAT_SLACK_WEBHOOK_URL" ]; then
    PR="${BUILDKITE_PULL_REQUEST:-}"
    BRANCH="${BUILDKITE_BRANCH:-}"
    # Skip notification if PR is invalid or branch is empty
    if [[ "$PR" = "false" || -z "$PR" || -z "$BRANCH" ]]; then
        echo ">>> Skipping Slack notification (invalid PR or empty branch: PR=$PR, branch=$BRANCH)"
    else
        echo ">>> Sending Slack notification"
        # Single quotes are intentional: the f-string expressions are Python, not shell.
        # shellcheck disable=SC2016
        PAYLOAD=$(python3 -c '
 import json, os, sys
 pr = os.getenv("BUILDKITE_PULL_REQUEST", "N/A")
 branch = os.getenv("BUILDKITE_BRANCH", "unknown")
 url = os.getenv("BUILDKITE_BUILD_URL", "#")
 data = {
    "text": ":warning: Ray Dependency Compatibility Check Failed",
    "blocks": [{
        "type": "section",
        "text": {
            "type": "mrkdwn",
            "text": (
                "*:warning: Ray Dependency Compatibility Check Failed*\n"
                f"PR #{pr} on branch `{branch}` introduces dependencies "
                f"that cannot be resolved with Ray'\''s requirements.\n"
                f"<{url}|View Build>"
            ),
        },
    }],
 }
 print(json.dumps(data))
 ')
        HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -X POST "$RAY_COMPAT_SLACK_WEBHOOK_URL" \
            -H 'Content-type: application/json' \
            -d "$PAYLOAD")
        echo "    Slack webhook response: $HTTP_CODE"
    fi
 else
    echo ">>> Skipping Slack notification (RAY_COMPAT_SLACK_WEBHOOK_URL not set)"
 fi
 exit 1
@@ -1,242 +0,0 @@
 #!/bin/bash
 #
 # cherry-pick-from-milestone.sh
 # Find commits from a GitHub milestone that are missing from the current branch
 # and output them in chronological order for cherry-picking.
 #
 # Usage: ./cherry-pick-from-milestone.sh <milestone> [--dry-run] [--execute]
 #
 set -euo pipefail
 # Colors for output
 RED='\033[0;31m'
 GREEN='\033[0;32m'
 YELLOW='\033[1;33m'
 BLUE='\033[0;34m'
 NC='\033[0m' # No Color
 usage() {
    cat <<EOF
 Usage: $(basename "$0") <milestone> [options]
 Find commits from a GitHub milestone that need to be cherry-picked into the current branch.
 Arguments:
    milestone       The GitHub milestone name (e.g., v0.14.0)
 Options:
    --dry-run       Show the cherry-pick commands without executing (default)
    --execute       Actually execute the cherry-picks
    --main-branch   Specify the main branch name (default: main)
    --help          Show this help message
 Examples:
    $(basename "$0") v0.14.0
    $(basename "$0") v0.14.0 --dry-run
    $(basename "$0") v0.14.0 --execute
    $(basename "$0") v0.14.0 --main-branch master
 EOF
    exit 1
 }
 log_info() {
    echo -e "${BLUE}[INFO]${NC} $1"
 }
 log_success() {
    echo -e "${GREEN}[OK]${NC} $1"
 }
 log_warn() {
    echo -e "${YELLOW}[WARN]${NC} $1"
 }
 log_error() {
    echo -e "${RED}[ERROR]${NC} $1" >&2
 }
 # Default values
 MILESTONE=""
 DRY_RUN=true
 MAIN_BRANCH="main"
 # Parse arguments
 while [[ $# -gt 0 ]]; do
    case $1 in
        --dry-run)
            DRY_RUN=true
            shift
            ;;
        --execute)
            DRY_RUN=false
            shift
            ;;
        --main-branch)
            MAIN_BRANCH="$2"
            shift 2
            ;;
        --help|-h)
            usage
            ;;
        -*)
            log_error "Unknown option: $1"
            usage
            ;;
        *)
            if [[ -z "$MILESTONE" ]]; then
                MILESTONE="$1"
            else
                log_error "Unexpected argument: $1"
                usage
            fi
            shift
            ;;
    esac
 done
 # Validate milestone argument
 if [[ -z "$MILESTONE" ]]; then
    log_error "Milestone is required"
    usage
 fi
 # Check if we're in a git repository
 if ! git rev-parse --is-inside-work-tree &>/dev/null; then
    log_error "Not in a git repository"
    exit 1
 fi
 # Check if gh CLI is available
 if ! command -v gh &>/dev/null; then
    log_error "GitHub CLI (gh) is not installed"
    exit 1
 fi
 # Check if authenticated with gh
 if ! gh auth status &>/dev/null; then
    log_error "Not authenticated with GitHub CLI. Run 'gh auth login' first."
    exit 1
 fi
 CURRENT_BRANCH=$(git branch --show-current)
 log_info "Current branch: ${CURRENT_BRANCH}"
 log_info "Main branch: ${MAIN_BRANCH}"
 log_info "Milestone: ${MILESTONE}"
 echo ""
 # Fetch latest from remote
 log_info "Fetching latest from remote..."
 git fetch origin "$MAIN_BRANCH" --quiet
 # Get merged PRs from the milestone, sorted by merge date
 log_info "Fetching merged PRs from milestone '${MILESTONE}'..."
 # Store PR data in a temp file
 PR_DATA=$(mktemp)
 trap 'rm -f "$PR_DATA"' EXIT
 if ! gh pr list --state merged --search "milestone:${MILESTONE}" \
    --limit 1000 \
    --json number,title,mergeCommit,mergedAt \
    --jq 'sort_by(.mergedAt) | .[] | "\(.mergeCommit.oid)\t\(.number)\t\(.title)"' > "$PR_DATA" 2>/dev/null; then
    log_error "Failed to fetch PRs from milestone '${MILESTONE}'"
    log_error "This could be due to:"
    log_error "  - Milestone does not exist"
    log_error "  - Network/authentication issues"
    log_error "  - Invalid milestone name format"
    exit 1
 fi
 if [[ ! -s "$PR_DATA" ]]; then
    log_warn "No merged PRs found for milestone '${MILESTONE}'"
    exit 0
 fi
 TOTAL_PRS=$(wc -l < "$PR_DATA")
 log_info "Found ${TOTAL_PRS} merged PR(s) in milestone"
 echo ""
 # Find commits that are missing from current branch
 MISSING_COMMITS=()
 MISSING_INFO=()
 while IFS=$'\t' read -r sha pr_number title; do
    # Skip if SHA is empty or null
    if [[ -z "$sha" || "$sha" == "null" ]]; then
        log_warn "PR #${pr_number} has no merge commit SHA, skipping"
        continue
    fi
    # Check if this commit is already in the current branch
    if git merge-base --is-ancestor "$sha" HEAD 2>/dev/null; then
        log_success "PR #${pr_number} already in branch: ${title:0:60}"
    else
        log_warn "PR #${pr_number} MISSING: ${title:0:60}"
        MISSING_COMMITS+=("$sha")
        MISSING_INFO+=("$sha PR #${pr_number}: ${title}")
    fi
 done < "$PR_DATA"
 echo ""
 if [[ ${#MISSING_COMMITS[@]} -eq 0 ]]; then
    log_success "All PRs from milestone '${MILESTONE}' are already in the current branch!"
    exit 0
 fi
 log_info "Found ${#MISSING_COMMITS[@]} missing commit(s) to cherry-pick"
 echo ""
 # Output the cherry-pick commands
 echo "=========================================="
 echo "Cherry-pick commands (in chronological order):"
 echo "=========================================="
 echo ""
 for info in "${MISSING_INFO[@]}"; do
    echo "# $info"
 done
 echo ""
 echo "# Run these commands to cherry-pick all missing commits:"
 echo "git cherry-pick ${MISSING_COMMITS[*]}"
 echo ""
 # Or one by one
 echo "# Or cherry-pick one at a time:"
 for sha in "${MISSING_COMMITS[@]}"; do
    echo "git cherry-pick $sha"
 done
 echo ""
 # Execute if requested
 if [[ "$DRY_RUN" == false ]]; then
    echo "=========================================="
    log_info "Executing cherry-picks..."
    echo "=========================================="
    for i in "${!MISSING_COMMITS[@]}"; do
        sha="${MISSING_COMMITS[$i]}"
        info="${MISSING_INFO[$i]}"
        echo ""
        log_info "Cherry-picking: $info"
        if git cherry-pick "$sha"; then
            log_success "Successfully cherry-picked $sha"
        else
            log_error "Failed to cherry-pick $sha"
            log_error "Resolve conflicts and run 'git cherry-pick --continue', or 'git cherry-pick --abort' to cancel"
            exit 1
        fi
    done
    echo ""
    log_success "All cherry-picks completed successfully!"
 else
    echo "=========================================="
    echo -e "${YELLOW}Dry run mode - no changes made${NC}"
    echo "Run with --execute to perform the cherry-picks"
    echo "=========================================="
 fi
@@ -13,8 +13,5 @@ INPUT_FILE="$1"
 # Strip timestamps
 sed -i 's/^\[[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}T[0-9]\{2\}:[0-9]\{2\}:[0-9]\{2\}Z\] //' "$INPUT_FILE"
 # Strip Buildkite inline timestamp markers (ESC _bk;t=<ms> BEL)
 sed -i 's/\x1B_bk;t=[0-9]*\x07//g' "$INPUT_FILE"
 # Strip colorization
 sed -i -r 's/\x1B\[[0-9;]*[mK]//g' "$INPUT_FILE"
@@ -1,178 +0,0 @@
 #!/bin/bash
 # Fetch vLLM Buildkite CI logs (public; no login required).
 #
 # Usage:
 #   ci-fetch-log.sh [--soft|--all] --pr [<PR>]  failed jobs in the PR's latest
 #                                               build (current branch if omitted)
 #   ci-fetch-log.sh [--soft|--all] <build_url>  failed jobs in that build
 #   ci-fetch-log.sh <job_url> [output]          one job; both #<job_uuid> and
 #                                               ?sid=<id> URL forms work
 #   ci-fetch-log.sh <build> <job_uuid> [output]
 #
 # --soft also fetches soft-failed jobs; --all fetches every finished job.
 # Saves each log as ci-<build>-<job-name>.log (ANSI/timestamps stripped) and
 # prints "<file>\t<job name>" per job. [output] is single-job only; "-"
 # streams to stdout. Existing files are kept; CI_FETCH_LOG_FORCE=1 refetches.
 set -euo pipefail
 ORG="vllm"
 PIPELINE="ci"
 UA="vllm-ci-fetch-log"
 UUID_RE='[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'
 usage() {
    sed -n '2,15p' "$0" | sed 's/^# \{0,1\}//'
    exit 1
 }
 die() {
    echo "$1" >&2
    exit 1
 }
 BUILD="" JOB="" SID="" OUT=""
 SCOPE="failed"
 while :; do
    case "${1:-}" in
    --soft) SCOPE="soft" ;;
    --all) SCOPE="all" ;;
    *) break ;;
    esac
    shift
 done
 case "${1:-}" in
 --pr)
    PR="${2:-}"
    # gh pr checks exits non-zero when checks are failing; that is the
    # expected case here.
    URL=$(gh pr checks ${PR:+"$PR"} --repo vllm-project/vllm 2>/dev/null |
        grep -oE "https://buildkite.com/${ORG}/${PIPELINE}/builds/[0-9]+" |
        sort -t/ -k7 -n | tail -1 || true)
    [ -n "$URL" ] || die "No Buildkite build found via: gh pr checks ${PR:-<current branch>}"
    BUILD="${URL##*/}"
    ;;
 https://*)
    BUILD=$(echo "$1" | sed -nE 's#.*/builds/([0-9]+).*#\1#p')
    JOB=$(echo "$1" | grep -oE "#${UUID_RE}" | head -n 1 | cut -c2- || true)
    SID=$(echo "$1" | grep -oE "[?&]sid=${UUID_RE}" | head -n 1 | sed 's/.*sid=//' || true)
    OUT="${2:-}"
    [ -n "$BUILD" ] || die "Could not parse build number from: $1"
    ;;
 [0-9]*)
    [ $# -ge 2 ] || usage
    BUILD="$1"
    JOB="$2"
    OUT="${3:-}"
    ;;
 *)
    usage
    ;;
 esac
 COOKIES=$(mktemp)
 JOBS_TSV=$(mktemp)
 trap 'rm -f "$COOKIES" "$JOBS_TSV"' EXIT
 # Buildkite issues a session cookie on first hit; later requests need it.
 curl -fsSL -c "$COOKIES" -A "$UA" \
    "https://buildkite.com/${ORG}/${PIPELINE}/builds/${BUILD}" -o /dev/null
 # The build's job list (id, step uuid, state, name) is served as JSON from
 # the user-facing /data/jobs endpoint. Flatten it to TSV for easy filtering:
 #   job_id  step_uuid  failed  soft_failed  finished  slug  name
 curl -fsSL -b "$COOKIES" -A "$UA" \
    "https://buildkite.com/${ORG}/${PIPELINE}/builds/${BUILD}/data/jobs" |
    python3 -c '
 import json, re, sys
 data = json.load(sys.stdin)
 if data.get("has_next_page"):
    print("warning: job list is paginated; some jobs not shown", file=sys.stderr)
 for r in data["records"]:
    if r.get("type") != "script":
        continue
    name = (r.get("name") or "").replace("\t", " ").replace("\n", " ")
    slug = re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-")[:60]
    print("\t".join([
        r["id"],
        r.get("step_uuid") or "",
        str(r.get("passed") is False),
        str(bool(r.get("soft_failed"))),
        str(bool(r.get("finished_at"))),
        slug,
        name,
    ]))
 ' >"$JOBS_TSV" || die "Could not list jobs for build ${BUILD}"
 if [ -n "$SID" ] && [ -z "$JOB" ]; then
    # The ?sid= in builds/<N>/list URLs is the *step* uuid, not the job uuid.
    JOB=$(awk -F'\t' -v s="$SID" '$1 == s || $2 == s {print $1; exit}' "$JOBS_TSV")
    [ -n "$JOB" ] || die "No job matching sid=${SID} in build ${BUILD}"
 fi
 fetch_job() { # <job_uuid> <output_file>
    curl -fsSL -b "$COOKIES" -A "$UA" \
        "https://buildkite.com/organizations/${ORG}/pipelines/${PIPELINE}/builds/${BUILD}/jobs/$1/download" \
        -o "$2"
    bash "$(dirname "$0")/ci-clean-log.sh" "$2"
 }
 if [ -n "$JOB" ]; then
    # Single-job mode.
    NAME=$(awk -F'\t' -v j="$JOB" '$1 == j {print $7; exit}' "$JOBS_TSV")
    SLUG=$(awk -F'\t' -v j="$JOB" '$1 == j {print $6; exit}' "$JOBS_TSV")
    [ -n "$OUT" ] || OUT="ci-${BUILD}-${SLUG:-${JOB:0:13}}.log"
    if [ "$OUT" = "-" ]; then
        TMP=$(mktemp)
        fetch_job "$JOB" "$TMP"
        cat "$TMP"
        rm -f "$TMP"
        exit 0
    fi
    if [ -e "$OUT" ] && [ -z "${CI_FETCH_LOG_FORCE:-}" ]; then
        die "Refusing to overwrite existing ${OUT} (set CI_FETCH_LOG_FORCE=1 or pass an output path)."
    fi
    fetch_job "$JOB" "$OUT"
    printf '%s\t%s\n' "$OUT" "${NAME:-$JOB}"
    exit 0
 fi
 # Build-wide mode: fetch finished jobs matching $SCOPE.
 [ -z "$OUT" ] || die "[output_file] is only valid when fetching a single job."
 case "$SCOPE" in
 failed) FILTER='$3 == "True" && $4 == "False" && $5 == "True"' ;;
 soft) FILTER='$3 == "True" && $5 == "True"' ;;
 all) FILTER='$5 == "True"' ;;
 esac
 if [ "$SCOPE" = "failed" ]; then
    SOFT=$(awk -F'\t' '$3 == "True" && $4 == "True"' "$JOBS_TSV" | wc -l)
    [ "$SOFT" -eq 0 ] || echo "Skipping ${SOFT} soft-failed job(s); use --soft to include them." >&2
 fi
 FOUND=0
 EMITTED=" "
 while IFS=$'\t' read -r job_id _ _ _ _ slug name; do
    FOUND=$((FOUND + 1))
    out="ci-${BUILD}-${slug:-${job_id:0:13}}.log"
    # Retries share a name with the original job; disambiguate by uuid.
    case "$EMITTED" in
    *" $out "*) out="ci-${BUILD}-${slug:-job}-${job_id:0:13}.log" ;;
    esac
    EMITTED="${EMITTED}${out} "
    if [ -e "$out" ] && [ -z "${CI_FETCH_LOG_FORCE:-}" ]; then
        echo "Keeping existing ${out} (set CI_FETCH_LOG_FORCE=1 to refetch)." >&2
    elif ! fetch_job "$job_id" "$out"; then
        echo "Failed to download log for job ${job_id} (${name})." >&2
        continue
    fi
    printf '%s\t%s\n' "$out" "$name"
 done < <(awk -F'\t' "$FILTER" "$JOBS_TSV")
 if [ "$FOUND" -eq 0 ]; then
    echo "No matching jobs in build ${BUILD} (scope: ${SCOPE})." >&2
 fi
@@ -3,20 +3,10 @@
 set -ex
 # Clean up old nightly builds from DockerHub, keeping only the last 14 builds
-# This script uses DockerHub API to list and delete old tags with specified prefix
+# This script uses DockerHub API to list and delete old tags with "nightly-" prefix
 # Usage: cleanup-nightly-builds.sh [TAG_PREFIX] [REPO]
 # Example: cleanup-nightly-builds.sh "nightly-"
 # Example: cleanup-nightly-builds.sh "cu130-nightly-"
 # Example: cleanup-nightly-builds.sh "nightly-" "vllm/vllm-openai-rocm"
-# Get tag prefix and repo from arguments
+# DockerHub API endpoint for vllm/vllm-openai repository
-TAG_PREFIX="${1:-nightly-}"
+REPO_API_URL="https://hub.docker.com/v2/repositories/vllm/vllm-openai/tags"
 REPO="${2:-vllm/vllm-openai}"
 echo "Cleaning up tags with prefix: $TAG_PREFIX in repository: $REPO"
 # DockerHub API endpoint for the repository
 REPO_API_URL="https://hub.docker.com/v2/repositories/${REPO}/tags"
 # Get DockerHub credentials from environment
 if [ -z "$DOCKERHUB_TOKEN" ]; then
@@ -55,7 +45,7 @@ get_all_tags() {
        set -x
        # Get both last_updated timestamp and tag name, separated by |
-        local tags=$(echo "$response" | jq -r --arg prefix "$TAG_PREFIX" '.results[] | select(.name | startswith($prefix)) | "\(.last_updated)|\(.name)"')
+        local tags=$(echo "$response" | jq -r '.results[] | select(.name | startswith("nightly-")) | "\(.last_updated)|\(.name)"')
        if [ -z "$tags" ]; then
            break
@@ -73,7 +63,7 @@ delete_tag() {
    local tag_name="$1"
    echo "Deleting tag: $tag_name"
-    local delete_url="https://hub.docker.com/v2/repositories/${REPO}/tags/$tag_name"
+    local delete_url="https://hub.docker.com/v2/repositories/vllm/vllm-openai/tags/$tag_name"
    set +x
    local response=$(curl -s -X DELETE -H "Authorization: Bearer $BEARER_TOKEN" "$delete_url")
    set -x
@@ -1,142 +0,0 @@
 #!/usr/bin/env python3
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 """Detect the manylinux platform tag for a wheel and rename it in place.
 vLLM's build images produce wheels with the generic ``linux_<arch>`` platform
 tag, which installers like ``pip`` won't accept off PyPI/our index. We need to
 rewrite the platform tag to the appropriate ``manylinux_<major>_<minor>_<arch>``
 before uploading.
 Historically the tag was hard-coded per build (``manylinux_2_31`` for the
 Ubuntu 20.04-based image, ``manylinux_2_35`` for the Ubuntu 22.04-based
 images). That is brittle: bumping the base image silently produces wheels
 labelled with the wrong glibc requirement. This script asks ``auditwheel``
 to derive the tag from the symbol versions actually referenced by the
 binaries inside the wheel, so the label tracks reality.
 We can't simply call ``auditwheel repair`` -- it tries to graft external
 shared libraries into the wheel and fails on vLLM's CUDA/cuBLAS dependencies.
 Instead we use ``auditwheel.wheel_abi.analyze_wheel_abi`` directly, which is
 the same call that powers ``auditwheel show``, and read off
 ``winfo.sym_policy.name``.
 Usage:
    detect-manylinux-tag.py <wheel_path>
 The wheel is renamed in place; the new path is printed on stdout. All
 diagnostics go to stderr so callers can capture stdout safely.
 """
 from __future__ import annotations
 import argparse
 import sys
 from pathlib import Path
 from auditwheel.error import (
    AuditwheelError,
    NonPlatformWheelError,
    WheelToolsError,
 )
 from auditwheel.wheel_abi import analyze_wheel_abi
 from auditwheel.wheeltools import get_wheel_architecture, get_wheel_libc
 def detect_platform_tag(wheel_path: Path) -> str:
    """Return the most precise platform tag the wheel is consistent with.
    Mirrors ``auditwheel show`` but returns ``sym_policy`` rather than
    ``overall_policy``: we only care about the glibc symbol versions used,
    not about other policy axes (ISA extensions, blacklist, etc.) that
    ``overall_policy`` folds in.
    """
    fn = wheel_path.name
    try:
        arch = get_wheel_architecture(fn)
    except (WheelToolsError, NonPlatformWheelError):
        # Architecture isn't deducible from the filename; let auditwheel
        # infer it from the ELF binaries inside the wheel.
        arch = None
    try:
        libc = get_wheel_libc(fn)
    except WheelToolsError:
        # An unrepaired wheel uses ``linux_<arch>``, which doesn't encode
        # libc. Let auditwheel infer it from the ELF binaries.
        libc = None
    winfo = analyze_wheel_abi(
        libc,
        arch,
        wheel_path,
        frozenset(),
        disable_isa_ext_check=False,
        allow_graft=False,
    )
    return winfo.sym_policy.name
 def rename_wheel(wheel_path: Path, new_platform_tag: str) -> Path:
    """Rename the wheel in place, replacing only its platform tag."""
    # Wheel filename per PEP 427:
    #   {distribution}-{version}(-{build})?-{python}-{abi}-{platform}.whl
    # The platform tag is always the last ``-``-separated token before
    # ``.whl``. Compound tags like ``manylinux_2_31_x86_64`` use ``_`` as the
    # internal separator, so ``-``-splitting is unambiguous.
    parts = wheel_path.stem.split("-")
    if len(parts) < 5:
        raise ValueError(f"Unrecognised wheel filename: {wheel_path.name}")
    parts[-1] = new_platform_tag
    new_path = wheel_path.with_name("-".join(parts) + ".whl")
    if new_path != wheel_path:
        wheel_path.rename(new_path)
    return new_path
 def main() -> int:
    parser = argparse.ArgumentParser(
        description="Detect a wheel's manylinux platform tag with "
        "auditwheel and rename the wheel in place."
    )
    parser.add_argument(
        "wheel",
        type=Path,
        help="Path to the wheel to inspect and rename.",
    )
    args = parser.parse_args()
    wheel_path: Path = args.wheel
    if not wheel_path.is_file():
        print(f"error: {wheel_path} is not a file", file=sys.stderr)
        return 1
    # Catch the things that ``analyze_wheel_abi`` and ``rename_wheel`` can
    # raise: any subclass of ``AuditwheelError`` (pure-Python wheels,
    # invalid libc, malformed wheels), filesystem errors, or our own
    # ``ValueError`` for an unrecognised wheel filename. Print a single
    # ``ERROR_TYPE: message`` line to stderr instead of a Python
    # traceback, which is much friendlier in CI logs.
    try:
        new_tag = detect_platform_tag(wheel_path)
        print(f"detected platform tag: {new_tag}", file=sys.stderr)
        new_path = rename_wheel(wheel_path, new_tag)
    except (AuditwheelError, ValueError, OSError) as e:
        print(
            f"error: failed to retag {wheel_path.name}: {type(e).__name__}: {e}",
            file=sys.stderr,
        )
        return 2
    if new_path != wheel_path:
        print(f"renamed {wheel_path.name} -> {new_path.name}", file=sys.stderr)
    else:
        print(f"wheel already tagged {new_tag}", file=sys.stderr)
    print(new_path)
    return 0
 if __name__ == "__main__":
    sys.exit(main())
@@ -1,54 +0,0 @@
 #!/bin/bash
 # Emit docker build flags for release image provenance metadata.
 # Keep this helper best-effort: missing Buildkite metadata should fall back to
 # local/default values instead of blocking the Docker build.
 # Variant examples: "", "cu129", "ubuntu2404", "cu129-ubuntu2404".
 variant="${1:-}"
 variant_suffix="${variant:+-${variant}}"
 image_name="${VLLM_DOCKER_IMAGE_NAME:-vllm/vllm-openai}"
 staging_repo="${VLLM_STAGING_IMAGE_REPO:-public.ecr.aws/q9t5s3a7/vllm-release-repo}"
 build_commit="${VLLM_BUILD_COMMIT:-${BUILDKITE_COMMIT:-unknown}}"
 build_pipeline="${VLLM_BUILD_PIPELINE:-${BUILDKITE_PIPELINE_ID:-${BUILDKITE_PIPELINE_SLUG:-local}}}"
 build_url="${VLLM_BUILD_URL:-${BUILDKITE_BUILD_URL:-}}"
 tag_commit="${BUILDKITE_COMMIT:-${build_commit}}"
 if [[ -n "${BUILDKITE:-}" || -n "${BUILDKITE_COMMIT:-}" ]]; then
  release_version="${RELEASE_VERSION:-}"
  if command -v buildkite-agent >/dev/null 2>&1; then
    release_version="${release_version:-$(buildkite-agent meta-data get release-version 2>/dev/null)}"
  fi
  release_version="${release_version#v}"
  release_version="${release_version:-${tag_commit}}"
  staging_image_ref="${staging_repo}:${tag_commit}-$(uname -m)${variant_suffix}"
  if [[ "${NIGHTLY:-}" == "1" ]]; then
    if [[ -z "${variant}" ]]; then
      image_tag="${image_name}:nightly-${tag_commit}"
    elif [[ "${variant}" == cu* ]]; then
      cuda_variant="${variant%%-*}"
      remaining_variant="${variant#${cuda_variant}}"
      image_tag="${image_name}:${cuda_variant}-nightly-${tag_commit}${remaining_variant}"
    else
      image_tag="${image_name}:nightly-${tag_commit}${variant_suffix}"
    fi
  else
    image_tag="${image_name}:v${release_version}${variant_suffix}"
  fi
 else
  image_tag="${VLLM_IMAGE_TAG:-local/vllm-openai:dev}"
  staging_image_ref="${image_tag}"
 fi
 emit_arg() {
  printf -- "--build-arg %s=%s " "$1" "$2"
 }
 emit_arg VLLM_BUILD_COMMIT "${build_commit}"
 emit_arg VLLM_BUILD_PIPELINE "${build_pipeline}"
 emit_arg VLLM_BUILD_URL "${build_url}"
 # This is the intended public tag. The final digest is only known after push.
 emit_arg VLLM_IMAGE_TAG "${image_tag}"
 printf -- "--tag %s " "${staging_image_ref}"
@@ -1,79 +0,0 @@
 #!/usr/bin/env bash
 set -ex
 # Generate and upload wheel indices for all wheels in the commit directory.
 # This script should run once after all wheels have been built and uploaded.
 # ======== setup ========
 BUCKET="vllm-wheels"
 INDICES_OUTPUT_DIR="indices"
 DEFAULT_VARIANT_ALIAS="cu130" # align with vLLM_MAIN_CUDA_VERSION in vllm/envs.py
 SUBPATH=$BUILDKITE_COMMIT
 S3_COMMIT_PREFIX="s3://$BUCKET/$SUBPATH/"
 # Select python3 (>= 3.12) -- local if available, else a docker fallback.
 # shellcheck source=lib/select-python.sh
 source .buildkite/scripts/lib/select-python.sh
 select_python
 # ======== generate and upload indices ========
 # list all wheels in the commit directory
 echo "Existing wheels on S3:"
 aws s3 ls "$S3_COMMIT_PREFIX"
 obj_json="objects.json"
 aws s3api list-objects-v2 --bucket "$BUCKET" --prefix "$SUBPATH/" --delimiter / --output json > "$obj_json"
 mkdir -p "$INDICES_OUTPUT_DIR"
 # call script to generate indices for all existing wheels
 # these indices have relative paths that work as long as they are next to the wheel directory in s3
 # i.e., the wheels are always in s3://vllm-wheels/<commit>/
 # and indices can be placed in /<commit>/, or /nightly/, or /<version>/
 alias_args=()
 if [[ -n "$DEFAULT_VARIANT_ALIAS" ]]; then
    alias_args=(--alias-to-default "$DEFAULT_VARIANT_ALIAS")
 fi
 # HACK: we do not need regex module here, but it is required by pre-commit hook
 # To avoid any external dependency, we simply replace it back to the stdlib re module
 sed -i 's/import regex as re/import re/g' .buildkite/scripts/generate-nightly-index.py
 $PYTHON .buildkite/scripts/generate-nightly-index.py --version "$SUBPATH" --current-objects "$obj_json" --output-dir "$INDICES_OUTPUT_DIR" --comment "commit $BUILDKITE_COMMIT" "${alias_args[@]}"
 # copy indices to /<commit>/ unconditionally
 echo "Uploading indices to $S3_COMMIT_PREFIX"
 aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "$S3_COMMIT_PREFIX"
 # copy to /nightly/ only when enabled for a main branch build that is not a PR
 if [[ "${UPDATE_NIGHTLY_INDEX:-1}" == "1" && \
      "$BUILDKITE_BRANCH" == "main" && \
      "$BUILDKITE_PULL_REQUEST" == "false" ]]; then
    echo "Uploading indices to overwrite /nightly/"
    aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "s3://$BUCKET/nightly/"
 fi
 # detect version from any wheel in the commit directory
 # download the first wheel we find to extract version metadata
 first_wheel_key=$($PYTHON -c "import json; obj=json.load(open('$obj_json')); print(next((c['Key'] for c in obj.get('Contents', []) if c['Key'].endswith('.whl')), ''))")
 if [[ -z "$first_wheel_key" ]]; then
    echo "Error: No wheels found in $S3_COMMIT_PREFIX"
    exit 1
 fi
 first_wheel=$(basename "$first_wheel_key")
 aws s3 cp "s3://$BUCKET/${first_wheel_key}" "/tmp/${first_wheel}"
 version=$(unzip -p "/tmp/${first_wheel}" '**/METADATA' | grep '^Version: ' | cut -d' ' -f2)
 rm -f "/tmp/${first_wheel}"
 echo "Version in wheel: $version"
 pure_version="${version%%+*}"
 echo "Pure version (without variant): $pure_version"
 # re-generate and copy to /<pure_version>/ only if it does not have "dev" in the version
 if [[ "${UPDATE_VERSION_INDEX:-1}" == "1" && "$version" != *"dev"* ]]; then
    echo "Re-generating indices for /$pure_version/"
    rm -rf "${INDICES_OUTPUT_DIR:?}"
    mkdir -p "$INDICES_OUTPUT_DIR"
    # wheel-dir is overridden to be the commit directory, so that the indices point to the correct wheel path
    $PYTHON .buildkite/scripts/generate-nightly-index.py --version "$pure_version" --wheel-dir "$SUBPATH" --current-objects "$obj_json" --output-dir "$INDICES_OUTPUT_DIR" --comment "version $pure_version" "${alias_args[@]}"
    aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "s3://$BUCKET/$pure_version/"
 fi
@@ -1,468 +0,0 @@
 #!/usr/bin/env python3
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 # do not complain about line length (for docstring)
 # ruff: noqa: E501
 import argparse
 import json
 import sys
 from dataclasses import asdict, dataclass
 from datetime import datetime
 from pathlib import Path
 from typing import Any
 from urllib.parse import quote
 import regex as re
 def normalize_package_name(name: str) -> str:
    """
    Normalize package name according to PEP 503.
    https://peps.python.org/pep-0503/#normalized-names
    Replace runs of underscores, hyphens, and periods with a single hyphen,
    and lowercase the result.
    """
    return re.sub(r"[-_.]+", "-", name).lower()
 if not sys.version_info >= (3, 12):
    raise RuntimeError("This script requires Python 3.12 or higher.")
 INDEX_HTML_TEMPLATE = """<!DOCTYPE html>
 <html>
  <!-- {comment} -->
  <meta name="pypi:repository-version" content="1.0">
  <body>
 {items}
  </body>
 </html>
 """
@dataclass
 class WheelFileInfo:
    package_name: str
    version: str
    build_tag: str | None
    python_tag: str
    abi_tag: str
    platform_tag: str
    variant: str | None
    filename: str
 def parse_from_filename(file: str) -> WheelFileInfo:
    """
    Parse wheel file name to extract metadata.
    The format of wheel names:
        {package_name}-{version}(-{build_tag})?-{python_tag}-{abi_tag}-{platform_tag}.whl
    All versions could contain a variant like '+cu129' or '.cpu' or `.rocm` (or not).
    Example:
        vllm-0.11.0-cp38-abi3-manylinux1_x86_64.whl
        vllm-0.10.2rc2+cu129-cp38-abi3-manylinux2014_aarch64.whl
        vllm-0.11.1rc8.dev14+gaa384b3c0-cp38-abi3-manylinux2014_aarch64.whl
        vllm-0.11.1rc8.dev14+gaa384b3c0.cu130-cp38-abi3-manylinux1_x86_64.whl
    """
    wheel_file_re = re.compile(
        r"^(?P<package_name>.+)-(?P<version>[^-]+?)(-(?P<build_tag>[^-]+))?-(?P<python_tag>[^-]+)-(?P<abi_tag>[^-]+)-(?P<platform_tag>[^-]+)\.whl$"
    )
    match = wheel_file_re.match(file)
    if not match:
        raise ValueError(f"Invalid wheel file name: {file}")
    package_name = match.group("package_name")
    version = match.group("version")
    build_tag = match.group("build_tag")
    python_tag = match.group("python_tag")
    abi_tag = match.group("abi_tag")
    platform_tag = match.group("platform_tag")
    # extract variant from version
    variant = None
    if "dev" in version:
        ver_after_dev = version.split("dev")[-1]
        if "." in ver_after_dev:
            variant = ver_after_dev.split(".")[-1]
            version = version.removesuffix("." + variant)
    else:
        if "+" in version:
            version_part, suffix = version.split("+", 1)
            # Only treat known patterns as variants (rocmXXX, cuXXX, cpu)
            # Git hashes and other suffixes are NOT variants
            if suffix.startswith(("rocm", "cu", "cpu")):
                variant = suffix
                version = version_part
            # Otherwise keep the full version string (variant stays None)
    return WheelFileInfo(
        package_name=package_name,
        version=version,
        build_tag=build_tag,
        python_tag=python_tag,
        abi_tag=abi_tag,
        platform_tag=platform_tag,
        variant=variant,
        filename=file,
    )
 def generate_project_list(subdir_names: list[str], comment: str = "") -> str:
    """
    Generate project list HTML content linking to each project & variant subdirectory.
    """
    href_tags = []
    for name in sorted(subdir_names):
        name = name.strip("/").strip(".")
        href_tags.append(f'    <a href="{name}/">{name}/</a><br/>')
    return INDEX_HTML_TEMPLATE.format(items="\n".join(href_tags), comment=comment)
 def generate_package_index_and_metadata(
    wheel_files: list[WheelFileInfo],
    wheel_base_dir: Path,
    index_base_dir: Path,
    comment: str = "",
 ) -> tuple[str, str]:
    """
    Generate package index HTML content for a specific package, linking to actual wheel files.
    """
    href_tags = []
    metadata = []
    for file in sorted(wheel_files, key=lambda x: x.filename):
        relative_path = (
            wheel_base_dir.relative_to(index_base_dir, walk_up=True) / file.filename
        )
        # handle with '+' in URL, and avoid double-encoding '/' and already-encoded '%2B'
        # NOTE: this is AWS S3 specific behavior!
        file_path_quoted = quote(relative_path.as_posix(), safe=":%/")
        href_tags.append(f'    <a href="{file_path_quoted}">{file.filename}</a><br/>')
        file_meta = asdict(file)
        file_meta["path"] = file_path_quoted
        metadata.append(file_meta)
    index_str = INDEX_HTML_TEMPLATE.format(items="\n".join(href_tags), comment=comment)
    metadata_str = json.dumps(metadata, indent=2)
    return index_str, metadata_str
 def generate_index_and_metadata(
    whl_files: list[str],
    wheel_base_dir: Path,
    index_base_dir: Path,
    default_variant: str | None = None,
    alias_to_default: str | None = None,
    comment: str = "",
 ):
    """
    Generate index for all wheel files.
    Args:
        whl_files (list[str]): List of wheel files (must be directly under `wheel_base_dir`).
        wheel_base_dir (Path): Base directory for wheel files.
        index_base_dir (Path): Base directory to store index files.
        default_variant (str | None): The default variant name, if any.
        alias_to_default (str | None): Alias variant name for the default variant, if any.
        comment (str | None): Optional comment to include in the generated HTML files.
    First, parse all wheel files to extract metadata.
    We need to collect all wheel files for each variant, and generate an index for it (in a subdirectory).
    The index for the default variant (if any) is generated in the root index directory.
    If `default_variant` is provided, all wheels must have variant suffixes, and the default variant index
    is purely a copy of the corresponding variant index, with only the links adjusted.
    Otherwise, all wheels without variant suffixes are treated as the default variant.
    If `alias_to_default` is provided, an additional alias subdirectory is created, it has the same content
    as the default variant index, but the links are adjusted accordingly.
    Index directory structure:
        index_base_dir/ (hosted at wheels.vllm.ai/{nightly,$commit,$version}/)
            index.html  # project list, linking to "vllm/" and other packages, and all variant subdirectories
            vllm/
                index.html # package index, pointing to actual files in wheel_base_dir (relative path)
                metadata.json # machine-readable metadata for all wheels in this package
            cpu/ # cpu variant subdirectory
                index.html
                vllm/
                    index.html
                    metadata.json
            cu129/ # cu129 is actually the alias to default variant
                index.html
                vllm/
                    index.html
                    metadata.json
            cu130/ # cu130 variant subdirectory
                index.html
                vllm/
                    index.html
                    metadata.json
            ...
    metadata.json stores a dump of all wheel files' metadata in a machine-readable format:
        [
            {
                "package_name": "vllm",
                "version": "0.10.2rc2",
                "build_tag": null,
                "python_tag": "cp38",
                "abi_tag": "abi3",
                "platform_tag": "manylinux2014_aarch64",
                "variant": "cu129",
                "filename": "vllm-0.10.2rc2+cu129-cp38-abi3-manylinux2014_aarch64.whl",
                "path": "../vllm-0.10.2rc2%2Bcu129-cp38-abi3-manylinux2014_aarch64.whl" # to be concatenated with the directory URL and URL-encoded
            },
            ...
        ]
    """
    parsed_files = [parse_from_filename(f) for f in whl_files]
    if not parsed_files:
        print("No wheel files found, skipping index generation.")
        return
    # For ROCm builds: inherit variant from vllm wheel
    # All ROCm wheels should share the same variant as vllm
    rocm_variant = None
    for file in parsed_files:
        if (
            file.package_name == "vllm"
            and file.variant
            and file.variant.startswith("rocm")
        ):
            rocm_variant = file.variant
            print(f"Detected ROCm variant from vllm: {rocm_variant}")
            break
    # Apply ROCm variant to all wheels without a variant
    if rocm_variant:
        for file in parsed_files:
            if file.variant is None:
                file.variant = rocm_variant
                print(f"Inherited variant '{rocm_variant}' for {file.filename}")
    # Group by variant
    variant_to_files: dict[str, list[WheelFileInfo]] = {}
    for file in parsed_files:
        variant = file.variant or "default"
        if variant not in variant_to_files:
            variant_to_files[variant] = []
        variant_to_files[variant].append(file)
    print(f"Found variants: {list(variant_to_files.keys())}")
    # sanity check for default variant
    if default_variant:
        if "default" in variant_to_files:
            raise ValueError(
                "All wheel files must have variant suffixes when `default_variant` is specified."
            )
        if default_variant not in variant_to_files:
            raise ValueError(
                f"Default variant '{default_variant}' not found among wheel files."
            )
    if alias_to_default:
        if "default" not in variant_to_files:
            # e.g. only some wheels are uploaded to S3 currently
            print(
                "[WARN] Alias to default variant specified, but no default variant found."
            )
        elif alias_to_default in variant_to_files:
            raise ValueError(
                f"Alias variant name '{alias_to_default}' already exists among wheel files."
            )
        else:
            variant_to_files[alias_to_default] = variant_to_files["default"].copy()
            print(f"Alias variant '{alias_to_default}' created for default variant.")
    # Generate comment in HTML header
    comment_str = f" ({comment})" if comment else ""
    comment_tmpl = f"Generated on {datetime.now().isoformat()}{comment_str}"
    # Generate index for each variant
    subdir_names = set()
    for variant, files in variant_to_files.items():
        if variant == "default":
            variant_dir = index_base_dir
        else:
            variant_dir = index_base_dir / variant
            subdir_names.add(variant)
        variant_dir.mkdir(parents=True, exist_ok=True)
        # gather all package names in this variant (normalized per PEP 503)
        packages = set(normalize_package_name(f.package_name) for f in files)
        if variant == "default":
            # these packages should also appear in the "project list"
            # generate after all variants are processed
            subdir_names = subdir_names.union(packages)
        else:
            # generate project list for this variant directly
            project_list_str = generate_project_list(sorted(packages), comment_tmpl)
            with open(variant_dir / "index.html", "w") as f:
                f.write(project_list_str)
        for package in packages:
            # filter files belonging to this package only (compare normalized names)
            package_files = [
                f for f in files if normalize_package_name(f.package_name) == package
            ]
            package_dir = variant_dir / package
            package_dir.mkdir(parents=True, exist_ok=True)
            index_str, metadata_str = generate_package_index_and_metadata(
                package_files, wheel_base_dir, package_dir, comment
            )
            with open(package_dir / "index.html", "w") as f:
                f.write(index_str)
            with open(package_dir / "metadata.json", "w") as f:
                f.write(metadata_str)
    # Generate top-level project list index
    project_list_str = generate_project_list(sorted(subdir_names), comment_tmpl)
    with open(index_base_dir / "index.html", "w") as f:
        f.write(project_list_str)
 if __name__ == "__main__":
    """
    Arguments:
        --version <version> : version string for the current build (e.g., commit hash)
        --wheel-dir <wheel_directory> : directory containing wheel files (default to be same as `version`)
        --current-objects <path_to_json> : path to JSON file containing current S3 objects listing in this version directory
        --output-dir <output_directory> : directory to store generated index files
        --alias-to-default <alias_variant_name> : (optional) alias variant name for the default variant
        --comment <comment_string> : (optional) comment string to include in generated HTML files
    """
    parser = argparse.ArgumentParser(
        description="Process nightly build wheel files to generate indices."
    )
    parser.add_argument(
        "--version",
        type=str,
        required=True,
        help="Version string for the current build (e.g., commit hash)",
    )
    parser.add_argument(
        "--current-objects",
        type=str,
        required=True,
        help="Path to JSON file containing current S3 objects listing in this version directory",
    )
    parser.add_argument(
        "--output-dir",
        type=str,
        required=True,
        help="Directory to store generated index files",
    )
    parser.add_argument(
        "--wheel-dir",
        type=str,
        default=None,
        help="Directory containing wheel files (default to be same as `version`)",
    )
    parser.add_argument(
        "--alias-to-default",
        type=str,
        default=None,
        help="Alias variant name for the default variant",
    )
    parser.add_argument(
        "--comment",
        type=str,
        default="",
        help="Optional comment string to include in generated HTML files",
    )
    args = parser.parse_args()
    version = args.version
    # Allow rocm/ prefix, reject other slashes and all backslashes
    if "\\" in version:
        raise ValueError("Version string must not contain backslashes.")
    if "/" in version and not version.startswith("rocm/"):
        raise ValueError(
            "Version string must not contain slashes (except for 'rocm/' prefix)."
        )
    current_objects_path = Path(args.current_objects)
    output_dir = Path(args.output_dir)
    if not output_dir.exists():
        output_dir.mkdir(parents=True, exist_ok=True)
    # Read current objects JSON
    with open(current_objects_path) as f:
        current_objects: dict[str, list[dict[str, Any]]] = json.load(f)
    # current_objects looks like from list_objects_v2 S3 API:
    """
    "Contents": [
        {
            "Key": "e2f56c309d2a28899c68975a7e104502d56deb8f/vllm-0.11.2.dev363+ge2f56c309-cp38-abi3-manylinux1_x86_64.whl",
            "LastModified": "2025-11-28T14:00:32+00:00",
            "ETag": "\"37a38339c7cdb61ca737021b968075df-52\"",
            "ChecksumAlgorithm": [
                "CRC64NVME"
            ],
            "ChecksumType": "FULL_OBJECT",
            "Size": 435649349,
            "StorageClass": "STANDARD"
        },
        ...
    ]
    """
    # Extract wheel file keys
    wheel_files = []
    for item in current_objects.get("Contents", []):
        key: str = item["Key"]
        if key.endswith(".whl"):
            wheel_files.append(key.split("/")[-1])  # only the filename is used
    print(f"Found {len(wheel_files)} wheel files for version {version}: {wheel_files}")
    # keep only "official" files for a non-nightly version (specified by cli args)
    PY_VERSION_RE = re.compile(r"^\d+\.\d+\.\d+([a-zA-Z0-9.+-]*)?$")
    if PY_VERSION_RE.match(version):
        # upload-wheels.sh ensures no "dev" is in args.version
        wheel_files = list(
            filter(lambda x: version in x and "dev" not in x, wheel_files)
        )
        print(f"Non-nightly version detected, wheel files used: {wheel_files}")
    else:
        print("Nightly version detected, keeping all wheel files.")
    # Generate index and metadata, assuming wheels and indices are stored as:
    # s3://vllm-wheels/{wheel_dir}/<wheel files>
    # s3://vllm-wheels/<anything>/<index files>
    #
    # For ROCm builds, version is "rocm/{commit}" and indices are uploaded to:
    #   - rocm/{commit}/  (same as wheels)
    #   - rocm/nightly/
    #   - rocm/{version}/
    # All these are under the "rocm/" prefix, so relative paths should be
    # relative to "rocm/", not the bucket root.
    if args.wheel_dir:
        # Explicit wheel-dir provided (e.g., for version-specific indices pointing to commit dir)
        wheel_dir = args.wheel_dir.strip().rstrip("/")
    elif version.startswith("rocm/"):
        # For rocm/commit, wheel_base_dir should be just the commit part
        # so relative path from rocm/0.12.0/rocm710/vllm/ -> ../../../{commit}/
        wheel_dir = version.split("/", 1)[1]
    else:
        wheel_dir = version
    wheel_base_dir = Path(output_dir).parent / wheel_dir
    index_base_dir = Path(output_dir)
    generate_index_and_metadata(
        whl_files=wheel_files,
        wheel_base_dir=wheel_base_dir,
        index_base_dir=index_base_dir,
        default_variant=None,
        alias_to_default=args.alias_to_default,
        comment=args.comment.strip(),
    )
    print(f"Successfully generated index and metadata in {output_dir}")
@@ -1,65 +0,0 @@
 #!/bin/bash
 set -euox pipefail
 export VLLM_CPU_KVCACHE_SPACE=1 
 export VLLM_CPU_CI_ENV=1
 # Reduce sub-processes for acceleration
 export TORCH_COMPILE_DISABLE=1 
 export VLLM_ENABLE_V1_MULTIPROCESSING=0
 SDE_ARCHIVE="sde-external-10.7.0-2026-02-18-lin.tar.xz"
 SDE_CHECKSUM="CA3D4086DE4ACB3FAEDF9F57B541C6936B7D5E19AE2BF763B6EA933573A0A217"
 wget "https://downloadmirror.intel.com/913594/${SDE_ARCHIVE}"
 echo "${SDE_CHECKSUM}  ${SDE_ARCHIVE}" | sha256sum --check
 mkdir -p sde
 tar -xvf "./${SDE_ARCHIVE}" --strip-components=1 -C ./sde/
 wait_for_pid_and_check_log() {
    local pid="$1"
    local log_file="$2"
    local exit_status
    if [ -z "$pid" ] || [ -z "$log_file" ]; then
        echo "Usage: wait_for_pid_and_check_log <PID> <LOG_FILE>"
        return 1
    fi
    echo "Waiting for process $pid to finish..."
    # Use the 'wait' command to pause the script until the specific PID exits.
    # The 'wait' command's own exit status will be that of the waited-for process.
    if wait "$pid"; then
        exit_status=$?
        echo "Process $pid finished with exit status $exit_status (Success)."
    else
        exit_status=$?
        echo "Process $pid finished with exit status $exit_status (Failure)."
    fi
    if [ "$exit_status" -ne 0 ]; then
        echo "Process exited with a non-zero status."
        echo "--- Last few lines of log file: $log_file ---"
        tail -n 50 "$log_file"
        echo "---------------------------------------------"
        return 1 # Indicate failure based on exit status
    fi
    echo "No errors detected in log file and process exited successfully."
    return 0
 }
 # Test Sky Lake (AVX512F)
 ./sde/sde64 -skl -- python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --dtype bfloat16 > test_0.log 2>&1 &
 PID_TEST_0=$!
 # Test Cascade Lake (AVX512F + VNNI)
 ./sde/sde64 -clx -- python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --dtype bfloat16 > test_1.log 2>&1 &
 PID_TEST_1=$!
 # Test Cooper Lake (AVX512F + VNNI + BF16)
 ./sde/sde64 -cpx -- python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --dtype bfloat16 > test_2.log 2>&1 &
 PID_TEST_2=$!
 wait_for_pid_and_check_log $PID_TEST_0 test_0.log
 wait_for_pid_and_check_log $PID_TEST_1 test_1.log
 wait_for_pid_and_check_log $PID_TEST_2 test_2.log
@@ -1,39 +0,0 @@
 #!/bin/bash
 set -euox pipefail
 export VLLM_CPU_CI_ENV=0
 export VLLM_CPU_KVCACHE_SPACE=1 # avoid OOM
 MODE=${1:-all}
 run_scenario() {
    local label="$1" result_file="$2"
    shift 2
    echo "--- $label"
    vllm serve meta-llama/Llama-3.2-3B-Instruct "$@" --max-model-len=4096 &
    local server_pid=$!
    timeout 600 bash -c "until curl localhost:8000/v1/models > /dev/null 2>&1; do sleep 1; done" || exit 1
    vllm bench serve \
        --backend vllm \
        --dataset-name random \
        --model meta-llama/Llama-3.2-3B-Instruct \
        --num-prompts 20 \
        --result-dir ./test_results \
        --result-filename "$result_file" \
        --save-result \
        --endpoint /v1/completions
    kill -s SIGTERM "$server_pid"; wait "$server_pid" || true
    if [ "$(jq '.failed' "./test_results/$result_file")" -ne 0 ]; then
        echo "Some requests were failed in $label!"
        exit 1
    fi
 }
 case "$MODE" in
    tp_pp) run_scenario "PP+TP" tp_pp.json -tp=2 -pp=2 ;;
    dp_tp) run_scenario "DP+TP" dp_tp.json -tp=2 -dp=2 ;;
    all)
        run_scenario "PP+TP" tp_pp.json -tp=2 -pp=2
        run_scenario "DP+TP" dp_tp.json -tp=2 -dp=2
        ;;
    *) echo "ERROR: unknown mode '$MODE' (expected: tp_pp | dp_tp | all)" >&2; exit 1 ;;
 esac
@@ -5,10 +5,10 @@
 set -ex
 # allow to bind to different cores
-CORE_RANGE=${CORE_RANGE:-0-31}
+CORE_RANGE=${CORE_RANGE:-0-16}
-OMP_CORE_RANGE=${OMP_CORE_RANGE:-0-31}
+OMP_CORE_RANGE=${OMP_CORE_RANGE:-0-16}
-export CMAKE_BUILD_PARALLEL_LEVEL=32
+export CMAKE_BUILD_PARALLEL_LEVEL=16
 # Setup cleanup
 remove_docker_container() {
@@ -31,40 +31,16 @@ function cpu_tests() {
    set -e
    pip list"
  # offline inference
  docker exec cpu-test bash -c "
    set -e
    python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m"
  # Run kernel tests
  docker exec cpu-test bash -c "
    set -e
    pytest -x -v -s tests/kernels/test_onednn.py
-    pytest -x -v -s tests/kernels/attention/test_cpu_attn.py
+    pytest -x -v -s tests/kernels/attention/test_cpu_attn.py"
    pytest -x -v -s tests/kernels/core/test_cpu_activation.py
    pytest -x -v -s tests/kernels/moe/test_cpu_fused_moe.py
    pytest -x -v -s tests/kernels/mamba/cpu/test_cpu_gdn_ops.py
    pytest -x -v -s tests/kernels/moe/test_cpu_int4_moe.py
    pytest -x -v -s tests/kernels/mamba/test_cpu_short_conv.py
    pytest -x -v -s tests/kernels/mamba/test_causal_conv1d.py
    pytest -x -v -s tests/kernels/mamba/test_mamba_ssm.py"
  # skip tests requiring model downloads if HF_TOKEN is not set
  # due to rate-limits
  if [ -z "$HF_TOKEN" ]; then
    echo "Warning: HF_TOKEN is not set. Skipping tests that require model downloads."
    return
  fi
  # offline inference
  docker exec cpu-test bash -c "
    set -e
    python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m"
  # Run model tests
  docker exec cpu-test bash -c "
    set -e
    pytest -x -v -s tests/models/multimodal/generation/test_whisper.py -m cpu_model"
  # Run quantized model tests
  docker exec cpu-test bash -c "
    set -e
    pytest -x -v -s tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_logprobs"
  # basic online serving
  docker exec cpu-test bash -c '
@@ -79,24 +55,8 @@ function cpu_tests() {
      --num-prompts 20 \
      --endpoint /v1/completions
    kill -s SIGTERM $server_pid &'
  # smoke test for Gated DeltaNet
  docker exec cpu-test bash -c '
    set -e
    VLLM_CPU_OMP_THREADS_BIND=$E2E_OMP_THREADS vllm serve Qwen/Qwen3.5-0.8B --max-model-len 2048 &
    server_pid=$!
    timeout 600 bash -c "until curl localhost:8000/v1/models; do sleep 1; done" || exit 1
    vllm bench serve \
      --backend vllm \
      --dataset-name random \
      --model Qwen/Qwen3.5-0.8B \
      --num-prompts 20 \
      --endpoint /v1/completions
    kill -s SIGTERM $server_pid &'
 }
 # All of CPU tests are expected to be finished less than 40 mins.
 export -f cpu_tests
 timeout 2h bash -c cpu_tests
@@ -27,7 +27,7 @@ function cpu_tests() {
  podman exec -it "$container_id" bash -c "
    export TORCH_COMPILE_DISABLE=1
    set -xve
-    python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m" >> "$HOME"/test_basic.log
+    python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m" >> $HOME/test_basic.log
  # Run basic model test
  podman exec -it "$container_id" bash -c "
@@ -43,7 +43,7 @@ function cpu_tests() {
    pytest -v -s tests/models/language/generation/test_common.py::test_models[False-False-5-32-google/gemma-1.1-2b-it]
    pytest -v -s tests/models/language/pooling/test_classification.py::test_models[float-jason9693/Qwen2.5-1.5B-apeach]
    # TODO: Below test case tests/models/language/pooling/test_embedding.py::test_models[True-ssmits/Qwen2-7B-Instruct-embed-base] fails on ppc64le. Disabling it for time being.
-    # pytest -v -s tests/models/language/pooling/test_embedding.py -m cpu_model" >> "$HOME"/test_rest.log
+    # pytest -v -s tests/models/language/pooling/test_embedding.py -m cpu_model" >> $HOME/test_rest.log
 }
 # All of CPU tests are expected to be finished less than 40 mins.
@@ -2,58 +2,118 @@
 # This script build the CPU docker image and run the offline inference inside the container.
 # It serves a sanity check for compilation and basic model usage.
-set -euox pipefail
+set -ex
 # allow to bind to different cores
 CORE_RANGE=${CORE_RANGE:-48-95}
 # used for TP/PP E2E test
 OMP_CORE_RANGE=${OMP_CORE_RANGE:-48-95}
 NUMA_NODE=${NUMA_NODE:-1}
 AGENT_SLOT=${AGENT_SLOT:-}
 IMAGE_NAME="cpu-test-${NUMA_NODE}${AGENT_SLOT:+-${AGENT_SLOT}}"
 TIMEOUT_VAL=$1
 TEST_COMMAND=$2
-# Disk hygiene knobs. Reclaim space only once the Docker root filesystem crosses
+export CMAKE_BUILD_PARALLEL_LEVEL=32
 # DISK_USAGE_THRESHOLD percent, and cap the shared BuildKit cache at
 # BUILDKIT_CACHE_MAX so subsequent builds keep reusing the hottest layers.
 DISK_USAGE_THRESHOLD=${DISK_USAGE_THRESHOLD:-70}
 BUILDKIT_CACHE_MAX=${BUILDKIT_CACHE_MAX:-80GB}
-# Reclaim disk only when the host is under pressure. We trim (not purge) the
+# Setup cleanup
-# shared BuildKit cache so cross-job/cross-agent reuse stays intact, and only
+remove_docker_container() {
-# touch dangling images; other agents' uniquely tagged images are left alone.
+    set -e;
-prune_if_disk_pressure() {
+    docker rm -f cpu-test-"$NUMA_NODE" cpu-test-"$NUMA_NODE"-avx2 || true;
    local docker_root disk_usage
    docker_root=$(docker info -f '{{.DockerRootDir}}' 2>/dev/null || true)
    if [ -z "$docker_root" ]; then
        return 0
    fi
    disk_usage=$(df "$docker_root" 2>/dev/null | tail -1 | awk '{print $5}' | tr -d '%')
    if [ "${disk_usage:-0}" -gt "$DISK_USAGE_THRESHOLD" ]; then
        echo "--- :broom: Disk usage ${disk_usage}% exceeds ${DISK_USAGE_THRESHOLD}%, reclaiming space"
        docker image prune -f || true
        docker builder prune -f --keep-storage="$BUILDKIT_CACHE_MAX" || true
    else
        echo "Disk usage ${disk_usage:-unknown}% within ${DISK_USAGE_THRESHOLD}% threshold; skipping prune"
    fi
 }
 trap remove_docker_container EXIT
 remove_docker_container
-# Always drop this agent's image once the job ends (the default builder never
+# Try building the docker image
-# uses it as a cache source, so removing it costs no rebuild speed), then
+numactl -C "$CORE_RANGE" -N "$NUMA_NODE" docker build --progress plain --tag cpu-test-"$NUMA_NODE" --target vllm-test -f docker/Dockerfile.cpu .
-# reclaim space if needed. Guard every docker call with `|| true` so the trap
+numactl -C "$CORE_RANGE" -N "$NUMA_NODE" docker build --progress plain --build-arg VLLM_CPU_DISABLE_AVX512="true" --tag cpu-test-"$NUMA_NODE"-avx2 --target vllm-test -f docker/Dockerfile.cpu .
 # never overrides the test's exit code.
 cleanup() {
    docker image rm -f "$IMAGE_NAME" || true
    prune_if_disk_pressure
 }
 trap cleanup EXIT
 # Free space up front so a nearly-full host doesn't fail the build.
 prune_if_disk_pressure
 # building the docker image
 echo "--- :docker: Building Docker image"
 docker build --progress plain --tag "$IMAGE_NAME" --target vllm-test -f docker/Dockerfile.cpu .
 # Run the image, setting --shm-size=4g for tensor parallel.
-docker run --rm --cpuset-cpus="$CORE_RANGE" --cpuset-mems="$NUMA_NODE" -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true -e HF_TOKEN -e VLLM_CPU_KVCACHE_SPACE=16 -e VLLM_CPU_CI_ENV=1 -e VLLM_CPU_SIM_MULTI_NUMA=1 -e VLLM_CPU_ATTN_SPLIT_KV=0 --shm-size=4g "$IMAGE_NAME" \
+docker run -itd --cpuset-cpus="$CORE_RANGE" --cpuset-mems="$NUMA_NODE" --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=16 --env VLLM_CPU_CI_ENV=1 -e E2E_OMP_THREADS="$OMP_CORE_RANGE" --shm-size=4g --name cpu-test-"$NUMA_NODE" cpu-test-"$NUMA_NODE"
-        timeout "$TIMEOUT_VAL" bash -c "set -euox pipefail; echo \"--- Print packages\"; pip list; echo \"--- Running tests\"; ${TEST_COMMAND}"
+docker run -itd --cpuset-cpus="$CORE_RANGE" --cpuset-mems="$NUMA_NODE" --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=16 --env VLLM_CPU_CI_ENV=1 -e E2E_OMP_THREADS="$OMP_CORE_RANGE" --shm-size=4g --name cpu-test-"$NUMA_NODE"-avx2 cpu-test-"$NUMA_NODE"-avx2
 function cpu_tests() {
  set -e
  export NUMA_NODE=$2
  # list packages
  docker exec cpu-test-"$NUMA_NODE"-avx2 bash -c "
    set -e
    pip list"
  docker exec cpu-test-"$NUMA_NODE" bash -c "
    set -e
    pip list"
  # offline inference
  docker exec cpu-test-"$NUMA_NODE"-avx2 bash -c "
    set -e
    python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m"
  # Run kernel tests
  docker exec cpu-test-"$NUMA_NODE" bash -c "
    set -e
    pytest -x -v -s tests/kernels/attention/test_cpu_attn.py
    pytest -x -v -s tests/kernels/test_onednn.py"
  # Run basic model test
  docker exec cpu-test-"$NUMA_NODE" bash -c "
    set -e
    # Note: disable until supports V1
    # pytest -x -v -s tests/kernels/attention/test_cache.py -m cpu_model
    # pytest -x -v -s tests/kernels/attention/test_mla_decode_cpu.py -m cpu_model
    pytest -x -v -s tests/models/language/generation -m cpu_model
    VLLM_CPU_SGL_KERNEL=1 pytest -x -v -s tests/models/language/generation -m cpu_model
    pytest -x -v -s tests/models/language/pooling -m cpu_model
    pytest -x -v -s tests/models/multimodal/generation \
                --ignore=tests/models/multimodal/generation/test_pixtral.py \
                -m cpu_model"
  # Run compressed-tensor test
  docker exec cpu-test-"$NUMA_NODE" bash -c "
    set -e
    pytest -x -s -v \
    tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_logprobs"
  # Run AWQ/GPTQ test
  docker exec cpu-test-"$NUMA_NODE" bash -c "
    set -e
    pytest -x -s -v \
    tests/quantization/test_cpu_wna16.py"
  # Run multi-lora tests
  docker exec cpu-test-"$NUMA_NODE" bash -c "
    set -e
    pytest -x -s -v \
    tests/lora/test_qwen2vl.py"
  # online serving: tp+pp
  docker exec cpu-test-"$NUMA_NODE" bash -c '
    set -e
    VLLM_CPU_OMP_THREADS_BIND=$E2E_OMP_THREADS VLLM_CPU_SGL_KERNEL=1 vllm serve meta-llama/Llama-3.2-3B-Instruct -tp=2 -pp=2 &
    server_pid=$!
    timeout 600 bash -c "until curl localhost:8000/v1/models; do sleep 1; done" || exit 1
    vllm bench serve \
      --backend vllm \
      --dataset-name random \
      --model meta-llama/Llama-3.2-3B-Instruct \
      --num-prompts 20 \
      --endpoint /v1/completions
    kill -s SIGTERM $server_pid &'
  # online serving: tp+dp
  docker exec cpu-test-"$NUMA_NODE" bash -c '
    set -e
    VLLM_CPU_OMP_THREADS_BIND=$E2E_OMP_THREADS VLLM_CPU_SGL_KERNEL=1 vllm serve meta-llama/Llama-3.2-3B-Instruct -tp=2 -dp=2 &
    server_pid=$!
    timeout 600 bash -c "until curl localhost:8000/v1/models; do sleep 1; done" || exit 1
    vllm bench serve \
      --backend vllm \
      --dataset-name random \
      --model meta-llama/Llama-3.2-3B-Instruct \
      --num-prompts 20 \
      --endpoint /v1/completions
    kill -s SIGTERM $server_pid &'
 }
 # All of CPU tests are expected to be finished less than 40 mins.
 export -f cpu_tests
 timeout 2.5h bash -c "cpu_tests $CORE_RANGE $NUMA_NODE"
@@ -25,5 +25,5 @@ remove_docker_container
 # Run the image and test offline inference
 docker run -e HF_TOKEN -e VLLM_WORKER_MULTIPROC_METHOD=spawn -v /root/.cache/huggingface:/root/.cache/huggingface --name gh200-test --gpus=all --entrypoint="" gh200-test bash -c '
-    python3 examples/basic/offline_inference/generate.py --model meta-llama/Llama-3.2-1B
+    python3 examples/offline_inference/basic/generate.py --model meta-llama/Llama-3.2-1B
 '
@@ -1,49 +1,21 @@
 #!/bin/bash
-# This script builds the HPU docker image and runs the offline inference inside the container.
+# This script build the CPU docker image and run the offline inference inside the container.
 # It serves a sanity check for compilation and basic model usage.
 #
 # vllm-gaudi compatibility pinning:
 #   The vllm-gaudi plugin is installed on top of the vllm upstream checkout used by this CI job.
 #   When upstream vllm changes its API, the plugin may break before it has been updated.
 #   To handle this, the vllm-gaudi repository maintains a file:
 #     vllm/last-good-commit-for-vllm-gaudi/VLLM_COMMUNITY_COMMIT
 #   The first line of that file controls what version of vllm is used inside the Docker image:
 #     - "latest"        : no checkout override; the current Buildkite CI commit is used as-is.
 #     - "<commit SHA>"  : vllm is checked out to that specific commit before building, pinning
 #                         the test to a known-compatible baseline.
 #   To unpin (resume testing against the live vllm tip), set the file content back to "latest".
 set -exuo pipefail
 # Fetch the vllm community commit reference from vllm-gaudi (first line only).
 VLLM_COMMUNITY_COMMIT=$(curl -s \
  https://raw.githubusercontent.com/vllm-project/vllm-gaudi/vllm/last-good-commit-for-vllm-gaudi/VLLM_COMMUNITY_COMMIT \
  | head -1 | tr -d '\n')
 echo "Using vllm community commit: ${VLLM_COMMUNITY_COMMIT}"
 # Try building the docker image
-image_name="hpu/upstream-vllm-ci:${BUILDKITE_COMMIT}"
+cat <<EOF | docker build -t hpu-plugin-v1-test-env -f - .
 container_name="hpu-upstream-vllm-ci-${BUILDKITE_COMMIT}-container"
 cat <<EOF | docker build -t "${image_name}" -f - .
 FROM gaudi-base-image:latest
 COPY ./ /workspace/vllm
 # If VLLM_COMMUNITY_COMMIT is a specific commit (not "latest"), check it out to pin vllm
 # to the version known to be compatible with vllm-gaudi. When the value is "latest",
 # the current checkout (the Buildkite CI commit) is used unchanged.
 RUN if [ "${VLLM_COMMUNITY_COMMIT}" != "latest" ]; then \
      cd /workspace/vllm && git fetch --unshallow 2>/dev/null || true && git checkout ${VLLM_COMMUNITY_COMMIT}; \
    fi
 WORKDIR /workspace/vllm
 ENV no_proxy=localhost,127.0.0.1
 ENV PT_HPU_ENABLE_LAZY_COLLECTIVES=true
-RUN bash -c 'pip install -r <(sed "/^torch/d" requirements/build/cuda.txt)'
+RUN VLLM_TARGET_DEVICE=empty pip install .
 RUN VLLM_TARGET_DEVICE=empty pip install --no-build-isolation -e .
 RUN pip install git+https://github.com/vllm-project/vllm-gaudi.git
 # install development dependencies (for testing)
@@ -64,20 +36,15 @@ EOF
 # functions, while other platforms only need one remove_docker_container
 # function.
 EXITCODE=1
-remove_docker_containers() { docker rm -f "${container_name}" || true; }
+remove_docker_containers() { docker rm -f hpu-plugin-v1-test || true; }
 trap 'remove_docker_containers; exit $EXITCODE;' EXIT
 remove_docker_containers
 echo "Running HPU plugin v1 test"
-docker run --rm --runtime=habana --name="${container_name}" --network=host \
+docker run --rm --runtime=habana --name=hpu-plugin-v1-test --network=host \
  -e HABANA_VISIBLE_DEVICES=all \
-  -e VLLM_SKIP_WARMUP=true \
+  hpu-plugin-v1-test-env \
-  -e PT_HPU_ENABLE_LAZY_COLLECTIVES=true \
+  /bin/bash "/workspace/vllm-gaudi/tests/upstream_tests/ci_tests.sh"
  -e PT_HPU_LAZY_MODE=1 \
  "${image_name}" \
  /bin/bash -c '
  cd vllm; timeout 120s python -u examples/basic/offline_inference/generate.py --model facebook/opt-125m
 '
 EXITCODE=$?
 if [ $EXITCODE -eq 0 ]; then
@@ -1,53 +0,0 @@
 #!/bin/bash
 set -euo pipefail
 test_suite="${1:-}"
 if [[ -z "${test_suite}" ]]; then
  echo "Usage: $0 <example|v1|server>" >&2
  exit 1
 fi
 case "${test_suite}" in
  example)
    pip install tblib==3.1.0
    python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager
    python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 -O3 -cc.cudagraph_mode=NONE
    python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager -tp 2 --distributed-executor-backend mp
    python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --attention-backend=TRITON_ATTN
    python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --quantization fp8
    python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --kv-cache-dtype fp8
    python3 examples/basic/offline_inference/generate.py --model nvidia/Llama-3.1-8B-Instruct-FP8 --block-size 64 --enforce-eager --quantization modelopt --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --max-model-len 4096
    python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --block-size 64 --enforce-eager --max-model-len 8192
    python3 examples/basic/offline_inference/generate.py --model TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ --block-size 64 --enforce-eager 
    python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2
    python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 --enable-expert-parallel
    python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --max-model-len 8192
    ;;
  v1)
    cd tests
    pytest -v -s v1/core --ignore=v1/core/test_reset_prefix_cache_e2e.py --ignore=v1/core/test_scheduler_e2e.py
    pytest -v -s v1/engine --ignore=v1/engine/test_output_processor.py
    pytest -v -s v1/sample --ignore=v1/sample/test_logprobs.py --ignore=v1/sample/test_logprobs_e2e.py -k "not test_topk_only and not test_topp_only and not test_topk_and_topp"
    pytest -v -s v1/worker --ignore=v1/worker/test_gpu_model_runner.py --ignore=v1/worker/test_worker_memory_snapshot.py
    pytest -v -s v1/structured_output
    pytest -v -s v1/test_serial_utils.py
    pytest -v -s v1/e2e/general/test_correctness_sliding_window.py --deselect="tests/v1/e2e/general/test_correctness_sliding_window.py::test_sliding_window_retrieval[True-1-5-google/gemma-3-1b-it]"
    pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py --ignore=v1/spec_decode/test_speculators_correctness.py
    pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_multi_connector.py --ignore=v1/kv_connector/unit/test_example_connector.py --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py --ignore=v1/kv_connector/unit/test_offloading_connector.py
    ;;
  server)
    pip install av
    cd tests
    pytest -v -s entrypoints/multimodal/openai/chat_completion/test_audio_in_video.py
    pytest -v -s benchmarks/test_serve_cli.py
    ;;
  *)
    echo "Unknown Intel test suite: ${test_suite}" >&2
    exit 1
    ;;
 esac
@@ -1,376 +0,0 @@
 #!/bin/bash
 # This script runs tests inside the Intel XPU docker container.
 # It mirrors the structure of run-amd-test.sh while keeping Intel-specific
 # container setup and allowing commands to be sourced from YAML or env.
 #
 # Command sources (in priority order):
 #   1) VLLM_TEST_COMMANDS env var (preferred, preserves quoting)
 #   2) Positional args (legacy)
 #   3) One or more YAML files with a commands list (test-area style)
 ###############################################################################
 set -o pipefail
 DRY_RUN=${DRY_RUN:-0}
 if [[ "${1:-}" == "--dry-run" ]]; then
  DRY_RUN=1
  shift
 fi
 # Export Python path
 export PYTHONPATH=".."
 ###############################################################################
 # Helper Functions
 ###############################################################################
 cleanup_docker() {
  # Share the same lock with image pull to avoid cleanup/pull races on one node.
  local docker_lock="/tmp/docker-pull.lock"
  exec 9>"$docker_lock"
  flock 9
  docker_root=$(docker info -f '{{.DockerRootDir}}')
  if [ -z "$docker_root" ]; then
    echo "Failed to determine Docker root directory." >&2
    flock -u 9
    return 1
  fi
  echo "Docker root directory: $docker_root"
  disk_usage=$(df "$docker_root" | tail -1 | awk '{print $5}' | sed 's/%//')
  threshold=70
  if [ "$disk_usage" -gt "$threshold" ]; then
    echo "Disk usage is above $threshold%. Running aggressive CI image cleanup..."
    cleanup_old_ci_images "${REGISTRY}/${REPO}" "${image_name}" "${DOCKER_IMAGE_CLEANUP_HOURS:-72}" 1
  else
    echo "Disk usage is below $threshold%. Checking old CI images anyway."
    cleanup_old_ci_images "${REGISTRY}/${REPO}" "${image_name}" "${DOCKER_IMAGE_CLEANUP_HOURS:-72}" 0
  fi
  echo "Old CI image cleanup completed."
  flock -u 9
 }
 cleanup_old_ci_images() {
  local repo_prefix="$1"
  local current_image_ref="$2"
  local ttl_hours="$3"
  local aggressive_cleanup="$4"
  if [[ -z "$repo_prefix" || "$repo_prefix" == "/" ]]; then
    echo "Skip old-image cleanup: invalid repo prefix '${repo_prefix}'"
    return 0
  fi
  if ! [[ "$ttl_hours" =~ ^[0-9]+$ ]]; then
    echo "Invalid DOCKER_IMAGE_CLEANUP_HOURS='${ttl_hours}', fallback to 72"
    ttl_hours=72
  fi
  local now_epoch cutoff_epoch
  now_epoch=$(date +%s)
  cutoff_epoch=$((now_epoch - ttl_hours * 3600))
  local -a used_image_ids
  mapfile -t used_image_ids < <(docker ps -aq | xargs -r docker inspect --format '{{.Image}}' | sort -u)
  local removed_count=0
  local examined_count=0
  declare -A seen_ids=()
  while read -r image_ref image_id; do
    [[ -z "$image_ref" || -z "$image_id" ]] && continue
    ((examined_count++))
    # Keep the image this job is going to use.
    if [[ "$image_ref" == "$current_image_ref" ]]; then
      continue
    fi
    # Avoid duplicate deletes when multiple tags point to same image id.
    if [[ -n "${seen_ids[$image_id]:-}" ]]; then
      continue
    fi
    seen_ids[$image_id]=1
    # Never delete images that are used by any container on this node.
    if printf '%s\n' "${used_image_ids[@]}" | grep -qx "$image_id"; then
      continue
    fi
    local created created_epoch
    created=$(docker image inspect -f '{{.Created}}' "$image_id" 2>/dev/null || true)
    [[ -z "$created" ]] && continue
    created_epoch=$(date -d "$created" +%s 2>/dev/null || true)
    [[ -z "$created_epoch" ]] && continue
    if (( created_epoch < cutoff_epoch )) || [[ "$aggressive_cleanup" == "1" ]]; then
      if docker image rm -f "$image_id" >/dev/null 2>&1; then
        ((removed_count++))
      fi
    fi
  done < <(docker image ls --no-trunc "$repo_prefix" --format '{{.Repository}}:{{.Tag}} {{.ID}}')
  # Also trim old dangling layers; this is safe and does not remove referenced images.
  docker image prune -f --filter "until=${ttl_hours}h" >/dev/null 2>&1 || true
  if [[ "$aggressive_cleanup" == "1" ]]; then
    echo "Examined ${examined_count} images under ${repo_prefix}, removed ${removed_count} unused images under disk pressure."
  else
    echo "Examined ${examined_count} images under ${repo_prefix}, removed ${removed_count} old images (>${ttl_hours}h)."
  fi
 }
 re_quote_pytest_markers() {
  local input="$1"
  local output=""
  local collecting=false
  local marker_buf=""
  local flat="${input//$'\n'/ }"
  local restore_glob
  restore_glob="$(shopt -p -o noglob 2>/dev/null || true)"
  set -o noglob
  local -a words
  read -ra words <<< "$flat"
  eval "$restore_glob"
  for word in "${words[@]}"; do
    if $collecting; then
      if [[ "$word" == *"'"* ]]; then
        if [[ -n "$marker_buf" ]]; then
          output+="${marker_buf} "
          marker_buf=""
        fi
        output+="${word} "
        collecting=false
        continue
      fi
      local is_boundary=false
      case "$word" in
        "&&"|"||"|";"|"|")
          is_boundary=true ;;
        --*)
          is_boundary=true ;;
        -[a-zA-Z])
          is_boundary=true ;;
        */*)
          is_boundary=true ;;
        *.py|*.py::*)
          is_boundary=true ;;
        *=*)
          if [[ "$word" =~ ^[A-Z_][A-Z0-9_]*= ]]; then
            is_boundary=true
          fi
          ;;
      esac
      if $is_boundary; then
        if [[ "$marker_buf" == *" "* || "$marker_buf" == *"("* ]]; then
          output+="'${marker_buf}' "
        else
          output+="${marker_buf} "
        fi
        collecting=false
        marker_buf=""
        if [[ "$word" == "-m" || "$word" == "-k" ]]; then
          output+="${word} "
          collecting=true
        else
          output+="${word} "
        fi
      else
        if [[ -n "$marker_buf" ]]; then
          marker_buf+=" ${word}"
        else
          marker_buf="${word}"
        fi
      fi
    elif [[ "$word" == "-m" || "$word" == "-k" ]]; then
      output+="${word} "
      collecting=true
      marker_buf=""
    else
      output+="${word} "
    fi
  done
  if $collecting && [[ -n "$marker_buf" ]]; then
    if [[ "$marker_buf" == *" "* || "$marker_buf" == *"("* ]]; then
      output+="'${marker_buf}'"
    else
      output+="${marker_buf}"
    fi
  fi
  echo "${output% }"
 }
 apply_intel_test_overrides() {
  local cmds="$1"
  # Placeholder for Intel-specific exclusions/overrides.
  echo "$cmds"
 }
 is_yaml_file() {
  local p="$1"
  [[ -f "$p" && "$p" == *.yaml ]]
 }
 extract_yaml_commands() {
  local yaml_path="$1"
  awk '
    $1 == "commands:" { in_cmds=1; next }
    in_cmds && $0 ~ /^[[:space:]]*-[[:space:]]/ {
      sub(/^[[:space:]]*-[[:space:]]/, "");
      print;
      next
    }
    in_cmds && $0 ~ /^[^[:space:]]/ { exit }
  ' "$yaml_path"
 }
 ###############################################################################
 # Main
 ###############################################################################
 default_image_name="${REGISTRY}/${REPO}:${BUILDKITE_COMMIT}-xpu"
 #default_image_name="public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:${BUILDKITE_COMMIT}-xpu"
 image_name="${IMAGE_TAG_XPU:-${default_image_name}}"
 container_name="xpu_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10; echo)"
 # ---- Command source selection ----
 commands=""
 commands_source=""
 if [[ -n "${VLLM_TEST_COMMANDS:-}" ]]; then
  commands="${VLLM_TEST_COMMANDS}"
  commands_source="env"
  echo "Commands sourced from VLLM_TEST_COMMANDS (quoting preserved)"
 elif [[ $# -gt 0 ]]; then
  all_yaml=true
  for arg in "$@"; do
    if ! is_yaml_file "$arg"; then
      all_yaml=false
      break
    fi
  done
  if $all_yaml; then
    for yaml in "$@"; do
      mapfile -t COMMANDS < <(extract_yaml_commands "$yaml")
      if [[ ${#COMMANDS[@]} -eq 0 ]]; then
        echo "Error: No commands found in ${yaml}" >&2
        exit 1
      fi
      for cmd in "${COMMANDS[@]}"; do
        if [[ -z "$commands" ]]; then
          commands="${cmd}"
        else
          commands+=" && ${cmd}"
        fi
      done
    done
    echo "Commands sourced from YAML files: $*"
  else
    commands="$*"
    echo "Commands sourced from positional args (legacy mode)"
  fi
 else
  SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
  DEFAULT_YAML="${SCRIPT_DIR}/intel-test.yaml"
  if [[ ! -f "${DEFAULT_YAML}" ]]; then
    echo "Error: YAML file not found: ${DEFAULT_YAML}" >&2
    exit 1
  fi
  mapfile -t COMMANDS < <(extract_yaml_commands "${DEFAULT_YAML}")
  if [[ ${#COMMANDS[@]} -eq 0 ]]; then
    echo "Error: No commands found in ${DEFAULT_YAML}" >&2
    exit 1
  fi
  for cmd in "${COMMANDS[@]}"; do
    if [[ -z "$commands" ]]; then
      commands="${cmd}"
    else
      commands+=" && ${cmd}"
    fi
  done
  echo "Commands sourced from default YAML: ${DEFAULT_YAML}"
 fi
 if [[ -z "$commands" ]]; then
  echo "Error: No test commands provided." >&2
  exit 1
 fi
 echo "Raw commands: $commands"
 if [[ "$commands_source" != "env" ]]; then
  commands=$(re_quote_pytest_markers "$commands")
  echo "After re-quoting: $commands"
 else
  echo "Skipping re-quoting for VLLM_TEST_COMMANDS input"
 fi
 commands=$(apply_intel_test_overrides "$commands")
 echo "Final commands: $commands"
 # Dry-run mode prints final commands and exits before Docker.
 if [[ "$DRY_RUN" == "1" ]]; then
  echo "DRY_RUN=1 set, skipping Docker execution."
  exit 0
 fi
 # --- Docker housekeeping ---
 cleanup_docker
 aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY"
 # --- Build or pull test image ---
 IMAGE="${IMAGE_TAG_XPU:-${image_name}}"
 echo "Using image: ${IMAGE}"
 remove_docker_container() {
  docker rm -f "${container_name}" || true
 }
 trap remove_docker_container EXIT
 # --- Single-node job ---
 if [[ -z "${ZE_AFFINITY_MASK:-}" ]]; then
  echo "Warning: ZE_AFFINITY_MASK is not set. Proceeding without device affinity." >&2
 fi
 export CMDS="${commands}"
 export HF_TOKEN ZE_AFFINITY_MASK
 {
  flock 9
  if docker image inspect "${IMAGE}" >/dev/null 2>&1; then
    echo "Image already exists locally, skipping pull"
  else
    echo "Image not found locally, pulling image..."
    timeout 900 docker pull "${IMAGE}"
    echo "Pull step completed"
  fi
  docker create \
    --device /dev/dri:/dev/dri \
    --net=host \
    --ipc=host \
    --privileged \
    -v /dev/dri/by-path:/dev/dri/by-path \
    -v "/data/huggingface:/root/.cache/huggingface" \
    --entrypoint='' \
    -e HF_TOKEN \
    -e ZE_AFFINITY_MASK \
    -e BUILDKITE_PARALLEL_JOB \
    -e BUILDKITE_PARALLEL_JOB_COUNT \
    -e CMDS \
    --name "${container_name}" \
    "${IMAGE}" \
    bash -c 'set -e; source /opt/intel/oneapi/setvars.sh --force; source /opt/intel/oneapi/ccl/2021.15/env/vars.sh --force; echo "ZE_AFFINITY_MASK is ${ZE_AFFINITY_MASK:-}"; eval "$CMDS"' \
    >/dev/null
 } 9>/tmp/docker-pull.lock
 docker start -a "${container_name}"
@@ -41,7 +41,6 @@ get_config() {
        echo "Error: file '${TEST_RUN_CONFIG_FILE}' does not exist in the warehouse" >&2
        exit 1
    fi
    # shellcheck source=/dev/null
    source "${TEST_RUN_CONFIG_FILE}"
    echo "Base docker image name that get from configuration: ${BASE_IMAGE_NAME}"
    return 0
@@ -49,8 +48,9 @@ get_config() {
 # get test running configuration.
 fetch_vllm_test_cfg
 get_config
 # Check if the function call was successful. If not, exit the script.
-if ! get_config; then
+if [ $? -ne 0 ]; then
  exit 1
 fi
@@ -62,19 +62,18 @@ agent_idx=$(echo "${BUILDKITE_AGENT_NAME}" | awk -F'-' '{print $(NF-1)}')
 echo "agent_idx: ${agent_idx}"
 builder_name="cachebuilder${agent_idx}"
 builder_cache_dir="/mnt/docker-cache${agent_idx}"
-mkdir -p "${builder_cache_dir}"
+mkdir -p ${builder_cache_dir}
 # Try building the docker image
 cat <<EOF | DOCKER_BUILDKIT=1 docker build \
-    --add-host cache-service-vllm.nginx-pypi-cache.svc.cluster.local:"${PYPI_CACHE_HOST}" \
+    --add-host cache-service-vllm.nginx-pypi-cache.svc.cluster.local:${PYPI_CACHE_HOST} \
-    --builder "${builder_name}" --cache-from type=local,src="${builder_cache_dir}" \
+    --builder ${builder_name} --cache-from type=local,src=${builder_cache_dir} \
-                           --cache-to type=local,dest="${builder_cache_dir}",mode=max \
+                           --cache-to type=local,dest=${builder_cache_dir},mode=max \
-    --progress=plain --load -t "${image_name}" -f - .
+    --progress=plain --load -t ${image_name} -f - .
 FROM ${BASE_IMAGE_NAME}
 # Define environments
 ENV DEBIAN_FRONTEND=noninteractive
 ENV SOC_VERSION="ascend910b1"
 RUN pip config set global.index-url http://cache-service-vllm.nginx-pypi-cache.svc.cluster.local:${PYPI_CACHE_PORT}/pypi/simple && \
    pip config set global.trusted-host cache-service-vllm.nginx-pypi-cache.svc.cluster.local && \
@@ -85,7 +84,7 @@ RUN pip config set global.index-url http://cache-service-vllm.nginx-pypi-cache.s
 # Install for pytest to make the docker build cache layer always valid
 RUN --mount=type=cache,target=/root/.cache/pip \
-    pip install pytest>=6.0  'modelscope<1.38'
+    pip install pytest>=6.0  modelscope
 WORKDIR /workspace/vllm
@@ -116,7 +115,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
    export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi && \
    source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
    source /usr/local/Ascend/nnal/atb/set_env.sh && \
-    export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/$(uname -i)-linux/devlib && \
+    export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
    python3 -m pip install -v -e /workspace/vllm-ascend/ --extra-index https://download.pytorch.org/whl/cpu/
 ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
@@ -139,7 +138,7 @@ trap remove_docker_container EXIT
 # Generate corresponding --device args based on BUILDKITE_AGENT_NAME
 # Ascend NPU BUILDKITE_AGENT_NAME format is {hostname}-{agent_idx}-{npu_card_num}cards, and agent_idx starts from 1.
 #   e.g. atlas-a2-001-1-2cards means this is the 1-th agent on atlas-a2-001 host, and it has 2 NPU cards.
-#   returns one argument per line: --device, /dev/davinciX, ...
+#   returns --device /dev/davinci0 --device /dev/davinci1
 parse_and_gen_devices() {
    local input="$1"
    local index cards_num
@@ -151,24 +150,29 @@ parse_and_gen_devices() {
        return 1
    fi
    local devices=""
    local i=0
    while (( i < cards_num )); do
        local dev_idx=$(((index - 1)*cards_num + i ))
-        printf '%s\n' "--device"
+        devices="$devices --device /dev/davinci${dev_idx}"
        printf '%s\n' "/dev/davinci${dev_idx}"
        ((i++))
    done
    # trim leading space
    devices="${devices#"${devices%%[![:space:]]*}"}"
    # Output devices: assigned to the caller variable
    printf '%s' "$devices"
 }
-mapfile -t device_args < <(parse_and_gen_devices "${BUILDKITE_AGENT_NAME}") || exit 1
+devices=$(parse_and_gen_devices "${BUILDKITE_AGENT_NAME}") || exit 1
 # Run the image and execute the Out-Of-Tree (OOT) platform interface test case on Ascend NPU hardware.
 # This test checks whether the OOT platform interface is functioning properly in conjunction with
 # the hardware plugin vllm-ascend.
 model_cache_dir=/mnt/modelscope${agent_idx}
-mkdir -p "${model_cache_dir}"
+mkdir -p ${model_cache_dir}
 docker run \
-    "${device_args[@]}" \
+    ${devices} \
    --device /dev/davinci_manager \
    --device /dev/devmm_svm \
    --device /dev/hisi_hdc \
@@ -177,7 +181,7 @@ docker run \
    -v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \
    -v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \
    -v /etc/ascend_install.info:/etc/ascend_install.info \
-    -v "${model_cache_dir}":/root/.cache/modelscope \
+    -v ${model_cache_dir}:/root/.cache/modelscope \
    --entrypoint="" \
    --name "${container_name}" \
    "${image_name}" \
@@ -61,7 +61,7 @@ echo "Results will be stored in: $RESULTS_DIR"
 echo "--- Installing Python dependencies ---"
 python3 -m pip install --progress-bar off git+https://github.com/thuml/depyf.git \
    && python3 -m pip install --progress-bar off pytest pytest-asyncio tpu-info \
-    && python3 -m pip install --progress-bar off "lm-eval[api]>=0.4.12" \
+    && python3 -m pip install --progress-bar off "lm-eval @ git+https://github.com/EleutherAI/lm-evaluation-harness.git@206b7722158f58c35b7ffcd53b035fdbdda5126d" \
    && python3 -m pip install --progress-bar off hf-transfer tblib==3.1.0
 echo "--- Python dependencies installed ---"
@@ -127,7 +127,7 @@ run_and_track_test() {
 # --- Actual Test Execution ---
 run_and_track_test 1 "test_struct_output_generate.py" \
-    "python3 -m pytest -s -v /workspace/vllm/tests/entrypoints/llm/test_struct_output_generate.py -k \"not test_structured_output_with_reasoning_matrices\""
+    "python3 -m pytest -s -v /workspace/vllm/tests/v1/entrypoints/llm/test_struct_output_generate.py -k \"not test_structured_output_with_reasoning_matrices\""
 run_and_track_test 2 "test_moe_pallas.py" \
    "python3 -m pytest -s -v /workspace/vllm/tests/tpu/test_moe_pallas.py"
 run_and_track_test 3 "test_lora.py" \
@@ -61,7 +61,7 @@ echo "Results will be stored in: $RESULTS_DIR"
 echo "--- Installing Python dependencies ---"
 python3 -m pip install --progress-bar off git+https://github.com/thuml/depyf.git \
    && python3 -m pip install --progress-bar off pytest pytest-asyncio tpu-info \
-    && python3 -m pip install --progress-bar off "lm-eval[api]>=0.4.12" \
+    && python3 -m pip install --progress-bar off "lm-eval @ git+https://github.com/EleutherAI/lm-evaluation-harness.git@206b7722158f58c35b7ffcd53b035fdbdda5126d" \
    && python3 -m pip install --progress-bar off hf-transfer tblib==3.1.0
 echo "--- Python dependencies installed ---"
@@ -136,6 +136,8 @@ run_and_track_test 3 "test_accuracy.py::test_lm_eval_accuracy_v1_engine" \
    "python3 -m pytest -s -v /workspace/vllm/tests/entrypoints/llm/test_accuracy.py::test_lm_eval_accuracy_v1_engine"
 run_and_track_test 4 "test_quantization_accuracy.py" \
    "python3 -m pytest -s -v /workspace/vllm/tests/tpu/test_quantization_accuracy.py"
 run_and_track_test 5 "examples/offline_inference/tpu.py" \
    "python3 /workspace/vllm/examples/offline_inference/tpu.py"
 run_and_track_test 6 "test_tpu_model_runner.py" \
    "python3 -m pytest -s -v /workspace/vllm/tests/v1/tpu/worker/test_tpu_model_runner.py"
 run_and_track_test 7 "test_sampler.py" \
@@ -0,0 +1,51 @@
 #!/bin/bash
 # This script build the CPU docker image and run the offline inference inside the container.
 # It serves a sanity check for compilation and basic model usage.
 set -ex
 image_name="xpu/vllm-ci:${BUILDKITE_COMMIT}"
 container_name="xpu_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10; echo)"
 # Try building the docker image
 docker build -t ${image_name} -f docker/Dockerfile.xpu .
 # Setup cleanup
 remove_docker_container() {
  docker rm -f "${container_name}" || true;
  docker image rm -f "${image_name}" || true;
  docker system prune -f || true;
 }
 trap remove_docker_container EXIT
 # Run the image and test offline inference/tensor parallel
 docker run \
    --device /dev/dri:/dev/dri \
    --net=host \
    --ipc=host \
    --privileged \
    -v /dev/dri/by-path:/dev/dri/by-path \
    --entrypoint="" \
    -e "HF_TOKEN=${HF_TOKEN}" \
    -e "ZE_AFFINITY_MASK=${ZE_AFFINITY_MASK}" \
    --name "${container_name}" \
    "${image_name}" \
    bash -c '
    set -e
    echo $ZE_AFFINITY_MASK
    pip install tblib==3.1.0
    python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager
    python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m --block-size 64 -O3 -cc.cudagraph_mode=NONE
    python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager -tp 2 --distributed-executor-backend ray
    python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager -tp 2 --distributed-executor-backend mp
    VLLM_ATTENTION_BACKEND=TRITON_ATTN python3 examples/offline_inference/basic/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager
    cd tests
    pytest -v -s v1/core
    pytest -v -s v1/engine
    pytest -v -s v1/sample --ignore=v1/sample/test_logprobs.py --ignore=v1/sample/test_logprobs_e2e.py
    pytest -v -s v1/worker --ignore=v1/worker/test_gpu_model_runner.py
    pytest -v -s v1/structured_output
    pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_tree_attention.py --ignore=v1/spec_decode/test_speculators_eagle3.py
    pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_multi_connector.py --ignore=v1/kv_connector/unit/test_nixl_connector.py --ignore=v1/kv_connector/unit/test_shared_storage_connector.py --ignore=v1/kv_connector/unit/test_lmcache_integration.py
    pytest -v -s v1/test_serial_utils.py
 '
@@ -1,44 +0,0 @@
 #!/bin/bash
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 set -euo pipefail
 if python3 -c "import torch; raise SystemExit(0 if torch.version.hip is not None else 1)"; then
    uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
    exit 0
 fi
 REQUIREMENTS_FILE="${KV_CONNECTORS_REQUIREMENTS:-/vllm-workspace/requirements/kv_connectors.txt}"
 uv pip install --system -r "${REQUIREMENTS_FILE}"
 NIXL_METADATA=$(python3 - <<'PY'
 import importlib.metadata as metadata
 import torch
 cuda_version = torch.version.cuda
 if cuda_version is None:
    raise SystemExit("torch.version.cuda is not set")
 print(cuda_version.split(".", 1)[0], metadata.version("nixl"))
 PY
 )
 read -r CUDA_MAJOR NIXL_VERSION <<<"${NIXL_METADATA}"
 # nixl>=1.1.0 can install multiple CUDA wheel variants. Keep only the variant
 # matching this CI image so nixl_ep_cpp links against the available libcudart.
 uv pip uninstall --system nixl-cu12 nixl-cu13 2>/dev/null || true
 uv pip install --system --no-deps "nixl-cu${CUDA_MAJOR}==${NIXL_VERSION}"
 python3 - <<'PY'
 import importlib.metadata as metadata
 for package_name in ("nixl", "nixl-cu12", "nixl-cu13"):
    try:
        version = metadata.version(package_name)
    except metadata.PackageNotFoundError:
        version = "not installed"
    print(f"{package_name}: {version}")
 PY
@@ -1,127 +0,0 @@
 #!/usr/bin/env bash
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 #
 # Shared helper for rewriting a wheel's platform tag from the generic
 # ``linux_<arch>`` to the correct ``manylinux_<major>_<minor>_<arch>``.
 # After sourcing, call ``apply_manylinux_tag <wheel>`` on each wheel
 # that still carries the generic tag; the renamed path is printed on
 # stdout (logs go to stderr).
 #
 # Why a pinned Docker container instead of using whatever Python
 # happens to be on the agent:
 #   - vLLM's release agents are heterogeneous -- they don't agree on
 #     a Python minor version, and we can't rely on a particular
 #     ``auditwheel`` being installed.
 #   - ``detect-manylinux-tag.py`` reads ``auditwheel.wheel_abi`` and
 #     ``Policy.sym_policy``, which are *internal* APIs without a
 #     stability promise. Pinning both Python and auditwheel makes the
 #     detected tag a function of the inputs alone, and shifts version
 #     bumps from "implicit drift" to "deliberate, retested change".
 #   - Other release scripts (``generate-and-upload-nightly-index.sh``,
 #     ``upload-rocm-wheels.sh``) already use the python:3-slim image
 #     when the agent's interpreter is too old; this is the same idea
 #     made stricter.
 #
 # To keep the per-wheel cost down (the ROCm upload retags ~10 wheels
 # each run), we install auditwheel into a long-lived helper container
 # once on source, then ``docker exec`` into it for each call.
 #
 # Trap behaviour:
 # - Sourcing installs an EXIT trap that calls ``manylinux_cleanup`` to
 #   tear down the helper container. Any EXIT trap that was already in
 #   place when this file was sourced is captured and run AFTER our
 #   cleanup, so we don't silently clobber it.
 # - If a caller sets a new EXIT trap *after* sourcing, that trap will
 #   replace ours; in that case the caller should call
 #   ``manylinux_cleanup`` from their own handler.
 if [[ -n "${_MANYLINUX_LIB_SOURCED:-}" ]]; then
    return 0
 fi
 _MANYLINUX_LIB_SOURCED=1
 # Pin both sides. Bump these deliberately and re-run a representative
 # wheel from each build target through the detection.
 _MANYLINUX_PYTHON_IMAGE="python:3.12-slim"
 _MANYLINUX_AUDITWHEEL_VERSION="6.6.0"
 # Resolve our own directory (and the sibling detect script) using the
 # canonical, symlink-resolved path. The container mounts cwd at the
 # same absolute path on both sides, so all paths we hand to it -- the
 # script, the wheel -- must canonicalise to a location under cwd.
 _MANYLINUX_LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
 _MANYLINUX_DETECT_SCRIPT="$(cd "${_MANYLINUX_LIB_DIR}/.." && pwd -P)/detect-manylinux-tag.py"
 _MANYLINUX_CWD="$(pwd -P)"
 docker pull --quiet "$_MANYLINUX_PYTHON_IMAGE" >/dev/null
 # Spin up a long-lived helper container so we install auditwheel once
 # and then ``docker exec`` into it for each wheel.
 #
 # The container runs as root so ``pip install`` can write into the
 # system site-packages; individual ``docker exec`` calls below pin
 # themselves to the host UID so any file rename happens with host
 # ownership, not root.
 _MANYLINUX_CONTAINER="$(docker run -d --rm \
    -v "$_MANYLINUX_CWD:$_MANYLINUX_CWD" \
    -w "$_MANYLINUX_CWD" \
    "$_MANYLINUX_PYTHON_IMAGE" \
    sleep infinity)"
 docker exec "$_MANYLINUX_CONTAINER" \
    pip install --quiet --disable-pip-version-check \
    --root-user-action=ignore \
    "auditwheel==${_MANYLINUX_AUDITWHEEL_VERSION}"
 # Public cleanup -- safe to call multiple times.
 manylinux_cleanup() {
    if [[ -n "${_MANYLINUX_CONTAINER:-}" ]]; then
        docker rm -f "$_MANYLINUX_CONTAINER" >/dev/null 2>&1 || true
        _MANYLINUX_CONTAINER=""
    fi
 }
 # Capture any EXIT trap that was already in place so we can chain to
 # it rather than overwrite it. ``trap -p EXIT`` prints the handler in
 # eval-able form (``trap -- 'CMD' EXIT``) or nothing if unset; we
 # strip the wrapper to recover ``CMD``. Handles the common case --
 # CMDs without embedded single quotes -- and degrades gracefully (we
 # still run our own cleanup) for the pathological case.
 _manylinux_prev_exit_trap_cmd=""
 _manylinux_existing_exit_trap="$(trap -p EXIT)"
 if [[ -n "$_manylinux_existing_exit_trap" ]]; then
    _tmp="${_manylinux_existing_exit_trap#trap -- \'}"
    _manylinux_prev_exit_trap_cmd="${_tmp%\' EXIT}"
    unset _tmp
 fi
 unset _manylinux_existing_exit_trap
 _manylinux_run_exit_chain() {
    manylinux_cleanup
    if [[ -n "$_manylinux_prev_exit_trap_cmd" ]]; then
        eval "$_manylinux_prev_exit_trap_cmd"
    fi
 }
 trap _manylinux_run_exit_chain EXIT
 # Detect the manylinux platform tag for a single wheel and rename it
 # in place, printing the renamed wheel path on stdout. Returns
 # non-zero on failure (which under ``set -e`` propagates to caller).
 #
 # The wheel must be reachable via a path under the host cwd so it's
 # visible inside the helper container; in CI the wheels always live
 # under ``artifacts/`` so this is fine.
 apply_manylinux_tag() {
    local wheel="$1"
    local abs_wheel
    abs_wheel="$(realpath "$wheel")"
    local new_wheel
    new_wheel="$(docker exec -u "$(id -u):$(id -g)" \
        "$_MANYLINUX_CONTAINER" \
        python "$_MANYLINUX_DETECT_SCRIPT" "$abs_wheel")"
    if [[ -z "$new_wheel" || ! -f "$new_wheel" ]]; then
        echo "apply_manylinux_tag: detect-manylinux-tag.py did not produce a valid wheel path for $wheel" >&2
        return 1
    fi
    printf '%s\n' "$new_wheel"
 }
@@ -1,41 +0,0 @@
 #!/usr/bin/env bash
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 #
 # Pick a Python interpreter for buildkite scripts: prefer a local
 # ``python3`` if it is recent enough (>= 3.12), otherwise fall back to
 # a one-shot Docker container running ``python:3-slim``. After
 # ``select_python`` returns, ``$PYTHON`` is set in the caller's shell
 # and is safe to use as a command (e.g. ``$PYTHON some_script.py``).
 #
 # The 3.12 threshold matches what the existing nightly-index work
 # expects -- typing features used by ``generate-nightly-index.py``.
 # This helper does not pin the *minor* version; if you need stricter
 # reproducibility (e.g. relying on auditwheel internals), invoke
 # Docker yourself with a pinned tag rather than calling this.
 if [[ -n "${_SELECT_PYTHON_LIB_SOURCED:-}" ]]; then
    return 0
 fi
 _SELECT_PYTHON_LIB_SOURCED=1
 # Sets ``PYTHON`` in the caller's shell and exports it. Idempotent --
 # calling twice is safe and the second call simply re-runs the probe.
 select_python() {
    local py="${PYTHON_PROG:-python3}"
    local has_new_python
    has_new_python=$("$py" -c \
        "print(1 if __import__('sys').version_info >= (3,12) else 0)" \
        2>/dev/null || echo 0)
    if [[ "$has_new_python" -eq 0 ]]; then
        # ``-u $(id -u):$(id -g)`` so files created via the container
        # end up owned by the host user, not root.
        docker pull python:3-slim
        PYTHON="docker run --rm -u $(id -u):$(id -g) -v $(pwd):/app -w /app python:3-slim python3"
    else
        PYTHON="$py"
    fi
    export PYTHON
    echo "Using python interpreter: $PYTHON"
    echo "Python version: $($PYTHON --version)"
 }
@@ -1,196 +0,0 @@
 #!/bin/bash
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 #
 # Publish release Docker images from ECR to DockerHub.
 # Pulls per-arch images, tags with latest and versioned tags, pushes them,
 # then creates and pushes multi-arch manifests.
 set -euo pipefail
 RELEASE_VERSION=$(buildkite-agent meta-data get release-version --default "" | sed 's/^v//')
 if [ -z "${RELEASE_VERSION}" ]; then
  echo "ERROR: release-version metadata not set"
  exit 1
 fi
 COMMIT="$BUILDKITE_COMMIT"
 ROCM_BASE_CACHE_KEY=$(.buildkite/scripts/cache-rocm-base-wheels.sh key)
 echo "========================================"
 echo "Publishing release images v${RELEASE_VERSION}"
 echo "  Commit: ${COMMIT}"
 echo "  ROCm base cache key: ${ROCM_BASE_CACHE_KEY}"
 echo "========================================"
 # Login to ECR to pull staging images
 aws ecr-public get-login-password --region us-east-1 | \
  docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
 # ---- CUDA (default: 13.0) ----
 docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64
 docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64 vllm/vllm-openai:latest-x86_64
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64
 docker push vllm/vllm-openai:latest-x86_64
 docker push vllm/vllm-openai:v${RELEASE_VERSION}-x86_64
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64 vllm/vllm-openai:latest-aarch64
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64
 docker push vllm/vllm-openai:latest-aarch64
 docker push vllm/vllm-openai:v${RELEASE_VERSION}-aarch64
 docker manifest rm vllm/vllm-openai:latest || true
 docker manifest rm vllm/vllm-openai:v${RELEASE_VERSION} || true
 docker manifest create vllm/vllm-openai:latest vllm/vllm-openai:latest-x86_64 vllm/vllm-openai:latest-aarch64
 docker manifest create vllm/vllm-openai:v${RELEASE_VERSION} vllm/vllm-openai:v${RELEASE_VERSION}-x86_64 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64
 docker manifest push vllm/vllm-openai:latest
 docker manifest push vllm/vllm-openai:v${RELEASE_VERSION}
 # ---- CUDA 12.9 ----
 docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-cu129
 docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64-cu129
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-cu129 vllm/vllm-openai:latest-x86_64-cu129
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-cu129 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu129
 docker push vllm/vllm-openai:latest-x86_64-cu129
 docker push vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu129
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64-cu129 vllm/vllm-openai:latest-aarch64-cu129
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64-cu129 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu129
 docker push vllm/vllm-openai:latest-aarch64-cu129
 docker push vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu129
 docker manifest rm vllm/vllm-openai:latest-cu129 || true
 docker manifest rm vllm/vllm-openai:v${RELEASE_VERSION}-cu129 || true
 docker manifest create vllm/vllm-openai:latest-cu129 vllm/vllm-openai:latest-x86_64-cu129 vllm/vllm-openai:latest-aarch64-cu129
 docker manifest create vllm/vllm-openai:v${RELEASE_VERSION}-cu129 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu129 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu129
 docker manifest push vllm/vllm-openai:latest-cu129
 docker manifest push vllm/vllm-openai:v${RELEASE_VERSION}-cu129
 # ---- Ubuntu 24.04 (CUDA 13.0) ----
 docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-ubuntu2404
 docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64-ubuntu2404
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-ubuntu2404 vllm/vllm-openai:latest-x86_64-ubuntu2404
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-ubuntu2404 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-ubuntu2404
 docker push vllm/vllm-openai:latest-x86_64-ubuntu2404
 docker push vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-ubuntu2404
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64-ubuntu2404 vllm/vllm-openai:latest-aarch64-ubuntu2404
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64-ubuntu2404 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-ubuntu2404
 docker push vllm/vllm-openai:latest-aarch64-ubuntu2404
 docker push vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-ubuntu2404
 docker manifest rm vllm/vllm-openai:latest-ubuntu2404 || true
 docker manifest rm vllm/vllm-openai:v${RELEASE_VERSION}-ubuntu2404 || true
 docker manifest create vllm/vllm-openai:latest-ubuntu2404 vllm/vllm-openai:latest-x86_64-ubuntu2404 vllm/vllm-openai:latest-aarch64-ubuntu2404
 docker manifest create vllm/vllm-openai:v${RELEASE_VERSION}-ubuntu2404 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-ubuntu2404 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-ubuntu2404
 docker manifest push vllm/vllm-openai:latest-ubuntu2404
 docker manifest push vllm/vllm-openai:v${RELEASE_VERSION}-ubuntu2404
 # ---- Ubuntu 24.04 (CUDA 12.9) ----
 docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-cu129-ubuntu2404
 docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64-cu129-ubuntu2404
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-cu129-ubuntu2404 vllm/vllm-openai:latest-x86_64-cu129-ubuntu2404
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-cu129-ubuntu2404 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu129-ubuntu2404
 docker push vllm/vllm-openai:latest-x86_64-cu129-ubuntu2404
 docker push vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu129-ubuntu2404
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64-cu129-ubuntu2404 vllm/vllm-openai:latest-aarch64-cu129-ubuntu2404
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-aarch64-cu129-ubuntu2404 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu129-ubuntu2404
 docker push vllm/vllm-openai:latest-aarch64-cu129-ubuntu2404
 docker push vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu129-ubuntu2404
 docker manifest rm vllm/vllm-openai:latest-cu129-ubuntu2404 || true
 docker manifest rm vllm/vllm-openai:v${RELEASE_VERSION}-cu129-ubuntu2404 || true
 docker manifest create vllm/vllm-openai:latest-cu129-ubuntu2404 vllm/vllm-openai:latest-x86_64-cu129-ubuntu2404 vllm/vllm-openai:latest-aarch64-cu129-ubuntu2404
 docker manifest create vllm/vllm-openai:v${RELEASE_VERSION}-cu129-ubuntu2404 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu129-ubuntu2404 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu129-ubuntu2404
 docker manifest push vllm/vllm-openai:latest-cu129-ubuntu2404
 docker manifest push vllm/vllm-openai:v${RELEASE_VERSION}-cu129-ubuntu2404
 # ---- ROCm ----
 docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-rocm
 docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${ROCM_BASE_CACHE_KEY}-rocm-base
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-rocm vllm/vllm-openai-rocm:latest
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-rocm vllm/vllm-openai-rocm:v${RELEASE_VERSION}
 docker push vllm/vllm-openai-rocm:latest
 docker push vllm/vllm-openai-rocm:v${RELEASE_VERSION}
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${ROCM_BASE_CACHE_KEY}-rocm-base vllm/vllm-openai-rocm:latest-base
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${ROCM_BASE_CACHE_KEY}-rocm-base vllm/vllm-openai-rocm:v${RELEASE_VERSION}-base
 docker push vllm/vllm-openai-rocm:latest-base
 docker push vllm/vllm-openai-rocm:v${RELEASE_VERSION}-base
 # ---- XPU ----
 docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-xpu
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-xpu vllm/vllm-openai-xpu:latest-x86_64
 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-xpu vllm/vllm-openai-xpu:v${RELEASE_VERSION}-x86_64
 docker push vllm/vllm-openai-xpu:latest-x86_64
 docker push vllm/vllm-openai-xpu:v${RELEASE_VERSION}-x86_64
 docker manifest rm vllm/vllm-openai-xpu:latest || true
 docker manifest rm vllm/vllm-openai-xpu:v${RELEASE_VERSION} || true
 docker manifest create vllm/vllm-openai-xpu:latest vllm/vllm-openai-xpu:latest-x86_64 --amend
 docker manifest create vllm/vllm-openai-xpu:v${RELEASE_VERSION} vllm/vllm-openai-xpu:v${RELEASE_VERSION}-x86_64 --amend
 docker manifest push vllm/vllm-openai-xpu:latest
 docker manifest push vllm/vllm-openai-xpu:v${RELEASE_VERSION}
 # ---- CPU ----
 # CPU images are behind separate block steps and may not have been built.
 # All-or-nothing: inspect both arches first, then either publish everything
 # (per-arch + multi-arch manifest) or skip everything. Publishing only one
 # arch would leave `:latest-x86_64` pointing at the new release while the
 # `:latest` multi-arch manifest still resolves to the previous release.
 CPU_X86_TAG=public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:v${RELEASE_VERSION}
 CPU_ARM_TAG=public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:v${RELEASE_VERSION}
 CPU_X86_AVAILABLE=false
 CPU_ARM_AVAILABLE=false
 docker manifest inspect "${CPU_X86_TAG}" >/dev/null 2>&1 && CPU_X86_AVAILABLE=true
 docker manifest inspect "${CPU_ARM_TAG}" >/dev/null 2>&1 && CPU_ARM_AVAILABLE=true
 if [ "$CPU_X86_AVAILABLE" = "true" ] && [ "$CPU_ARM_AVAILABLE" = "true" ]; then
  docker pull "${CPU_X86_TAG}"
  docker tag "${CPU_X86_TAG}" vllm/vllm-openai-cpu:latest-x86_64
  docker tag "${CPU_X86_TAG}" vllm/vllm-openai-cpu:v${RELEASE_VERSION}-x86_64
  docker push vllm/vllm-openai-cpu:latest-x86_64
  docker push vllm/vllm-openai-cpu:v${RELEASE_VERSION}-x86_64
  docker pull "${CPU_ARM_TAG}"
  docker tag "${CPU_ARM_TAG}" vllm/vllm-openai-cpu:latest-arm64
  docker tag "${CPU_ARM_TAG}" vllm/vllm-openai-cpu:v${RELEASE_VERSION}-arm64
  docker push vllm/vllm-openai-cpu:latest-arm64
  docker push vllm/vllm-openai-cpu:v${RELEASE_VERSION}-arm64
  docker manifest rm vllm/vllm-openai-cpu:latest || true
  docker manifest rm vllm/vllm-openai-cpu:v${RELEASE_VERSION} || true
  docker manifest create vllm/vllm-openai-cpu:latest vllm/vllm-openai-cpu:latest-x86_64 vllm/vllm-openai-cpu:latest-arm64
  docker manifest create vllm/vllm-openai-cpu:v${RELEASE_VERSION} vllm/vllm-openai-cpu:v${RELEASE_VERSION}-x86_64 vllm/vllm-openai-cpu:v${RELEASE_VERSION}-arm64
  docker manifest push vllm/vllm-openai-cpu:latest
  docker manifest push vllm/vllm-openai-cpu:v${RELEASE_VERSION}
 elif [ "$CPU_X86_AVAILABLE" = "false" ] && [ "$CPU_ARM_AVAILABLE" = "false" ]; then
  echo "WARNING: Neither CPU image found in ECR, skipping CPU publish (ensure block-cpu-release-image-build and block-arm64-cpu-release-image-build were unblocked and the builds finished pushing)"
 else
  # Partial state: one arch built, the other did not. Fail loudly rather than
  # ship a Docker Hub state where `:latest-${arch}` and `:latest` (multi-arch)
  # disagree on which release they point at.
  echo "ERROR: Partial CPU build detected (x86_64=${CPU_X86_AVAILABLE}, arm64=${CPU_ARM_AVAILABLE})."
  echo "       Refusing to publish to avoid split-tag drift between per-arch and multi-arch tags."
  echo "       Re-run the missing CPU build and retry, or manually publish if a single-arch release is intended."
  exit 1
 fi
 echo ""
 echo "Successfully published release images for v${RELEASE_VERSION}"
--- a/Show More
+++ b/Show More
`@@ -1,2 +1 @@`
	`Qwen3-235B-A22B-Instruct-2507-FP8.yaml`	`Qwen3-235B-A22B-Instruct-2507-FP8.yaml`
	`NVIDIA-Nemotron-3-Nano-30B-A3B-FP8.yaml`
		`@@ -1 +0,0 @@`
			`Meta-Llama-4-Maverick-17B-128E-Instruct-FP8.yaml`