Compare commits

..
Author SHA1 Message Date
Tyler Michael SmithandClaude 5b4f6d5284 Use cumem allocator for the KV cache by default
Enable the cumem (CUDA VMM) allocator by default on CUDA and ROCm
platforms so users get stable physical pages for KV cache without
needing to set --enable-cumem-allocator explicitly. This is required
for MNNVL KV transfers.

Changes:
- ModelConfig.enable_cumem_allocator defaults to None, resolved to
  True when the cumem C extension is available
- Cached fabric handle probe in csrc/cumem_allocator.cpp uses a real
  cuMemCreate probe instead of trusting cuDeviceGetAttribute, with
  fallback to POSIX FD when fabric handles aren't available
- Safety-net fallback: if fabric was probed as available but a real
  allocation still fails, updates the cache and retries with POSIX FD
- Weight loading bypasses cumem pool when sleep mode is off (cumem is
  only needed for KV cache stability in that case)
- Guard against stale engine allocations corrupting the singleton

Co-authored-by: Claude <noreply@anthropic.com>
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
2026-06-10 15:25:35 -04:00
2900 changed files with 91120 additions and 337190 deletions
+3 -3
View File
@@ -2,17 +2,17 @@ name: vllm_intel_ci
job_dirs:
- ".buildkite/intel_jobs"
run_all_patterns:
- ".buildkite/ci_config_intel.yaml"
- ".buildkite/scripts/hardware_ci/run-intel-test.sh"
- "docker/Dockerfile"
- "docker/Dockerfile.xpu"
- "CMakeLists.txt"
- "requirements/common.txt"
- "requirements/xpu.txt"
- "requirements/build/cuda.txt"
- "requirements/test/cuda.txt"
- "setup.py"
- "csrc/"
- "cmake/"
run_all_exclude_patterns:
- "docker/Dockerfile."
- "csrc/cpu/"
- "csrc/rocm/"
- "cmake/hipify.py"
-1
View File
@@ -8,7 +8,6 @@ run_all_patterns:
- "docker/docker-bake-rocm.hcl"
- ".buildkite/hardware_tests/amd.yaml"
- ".buildkite/scripts/ci-bake-rocm.sh"
- ".buildkite/scripts/rocm/"
- ".buildkite/scripts/hardware_ci/run-amd-test.py"
- ".buildkite/scripts/hardware_ci/run-amd-test.sh"
- "CMakeLists.txt"
+29 -40
View File
@@ -1,55 +1,24 @@
group: Hardware - AMD Build
# ROCm image flow:
# 1. Refresh the long-lived ROCm base image only when Dockerfile.rocm_base changes.
# 2. Build ci_base from either the stable base or the freshly refreshed base.
# 3. Build the per-commit ROCm CI image and smoke-test it before GPU jobs run.
steps:
- label: "AMD: :docker: refresh ROCm base"
key: refresh-rocm-base-amd
depends_on: []
device: amd_cpu
no_plugin: true
commands:
- bash .buildkite/scripts/rocm/refresh-base-image.sh
env:
DOCKER_BUILDKIT: "1"
BUILDKIT_PROGRESS: "tty"
TERM: "xterm-256color"
retry:
automatic:
- exit_status: 1 # Transient Docker/BuildKit failure
limit: 1
- exit_status: -1 # Agent was lost
limit: 1
- exit_status: -10 # Agent was lost
limit: 1
# Ensure ci_base is up-to-date before building the test image.
# Compares a content hash of ci_base-affecting files against the remote
# image label. If hashes match the build is skipped (< 30 s); if they
# differ ci_base is rebuilt and pushed automatically.
- label: "AMD: :docker: ensure ci_base"
key: ensure-ci-base-amd
soft_fail: false
depends_on:
- refresh-rocm-base-amd
depends_on: []
device: amd_cpu
no_plugin: true
commands:
- bash .buildkite/scripts/rocm/build-ci-base.sh
- bash .buildkite/scripts/ci-bake-rocm.sh ci-base-rocm-ci-with-deps
env:
DOCKER_BUILDKIT: "1"
BUILDKIT_PROGRESS: "tty"
TERM: "xterm-256color"
VLLM_BAKE_FILE: "docker/docker-bake-rocm.hcl"
PYTORCH_ROCM_ARCH: "gfx90a;gfx942;gfx950"
REMOTE_VLLM: "1"
VLLM_BRANCH: "$BUILDKITE_COMMIT"
retry:
automatic:
- exit_status: 1 # Transient Docker/BuildKit failure
limit: 1
- exit_status: -1 # Agent was lost
limit: 1
- exit_status: -10 # Agent was lost
@@ -57,18 +26,40 @@ steps:
- label: "AMD: :docker: build test image and artifacts"
key: image-build-amd
soft_fail: false
depends_on:
- ensure-ci-base-amd
device: amd_cpu
no_plugin: true
commands:
- bash .buildkite/scripts/rocm/build-test-image.sh
- bash .buildkite/scripts/rocm/smoke-test-image.sh
- |
if [[ "${ROCM_CI_ARTIFACT_ONLY:-0}" == "1" ]]; then
echo "ROCM_CI_ARTIFACT_ONLY=1; building ROCm wheel artifact only"
IMAGE_TAG="" bash .buildkite/scripts/ci-bake-rocm.sh test-rocm-ci-with-artifacts
else
bash .buildkite/scripts/ci-bake-rocm.sh test-rocm-ci-with-wheel
fi
- |
docker run --rm --network=none --entrypoint /bin/bash "rocm/vllm-ci:${BUILDKITE_COMMIT}" -ec '
if [ ! -d /vllm-workspace ]; then echo Missing directory: /vllm-workspace >&2; exit 1; fi
if [ ! -d /vllm-workspace/tests ]; then echo Missing directory: /vllm-workspace/tests >&2; exit 1; fi
if [ ! -d /vllm-workspace/src/vllm ]; then echo Missing directory: /vllm-workspace/src/vllm >&2; exit 1; fi
if [ ! -x /vllm-workspace/src/vllm/vllm-rs ]; then echo Missing executable: /vllm-workspace/src/vllm/vllm-rs >&2; exit 1; fi
command -v python3
command -v uv
command -v pytest
if ! command -v amd-smi >/dev/null 2>&1 && ! command -v rocminfo >/dev/null 2>&1; then
echo No ROCm CLI found in image >&2
exit 1
fi
python3 - <<PY
import torch, vllm
print(torch.__version__)
print(vllm.__version__)
PY
echo AMD image smoke OK
'
env:
DOCKER_BUILDKIT: "1"
BUILDKIT_PROGRESS: "tty"
TERM: "xterm-256color"
VLLM_BAKE_FILE: "docker/docker-bake-rocm.hcl"
PYTORCH_ROCM_ARCH: "gfx90a;gfx942;gfx950"
IMAGE_TAG: "rocm/vllm-ci:$BUILDKITE_COMMIT"
@@ -76,8 +67,6 @@ steps:
VLLM_BRANCH: "$BUILDKITE_COMMIT"
retry:
automatic:
- exit_status: 1 # Transient Docker/BuildKit failure
limit: 1
- exit_status: -1 # Agent was lost
limit: 1
- exit_status: -10 # Agent was lost
+7 -30
View File
@@ -17,22 +17,16 @@ steps:
- tests/kernels/test_awq_int4_to_int8.py
- tests/kernels/quantization/test_cpu_fp8_scaled_mm.py
- tests/kernels/mamba/cpu/test_cpu_gdn_ops.py
- tests/kernels/mamba/test_cpu_short_conv.py
- tests/kernels/mamba/test_causal_conv1d.py
- tests/kernels/mamba/test_mamba_ssm.py
commands:
- |
bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 30m "
pytest -x -v -s tests/kernels/attention/test_cpu_attn.py
pytest -x -v -s tests/kernels/moe/test_cpu_fused_moe.py
pytest -x -v -s tests/kernels/moe/test_cpu_quant_fused_moe.py
pytest -x -v -s tests/kernels/mamba/test_cpu_short_conv.py
pytest -x -v -s tests/kernels/test_onednn.py
pytest -x -v -s tests/kernels/test_awq_int4_to_int8.py
pytest -x -v -s tests/kernels/quantization/test_cpu_fp8_scaled_mm.py
pytest -x -v -s tests/kernels/mamba/cpu/test_cpu_gdn_ops.py
pytest -x -v -s tests/kernels/mamba/test_causal_conv1d.py
pytest -x -v -s tests/kernels/mamba/test_mamba_ssm.py"
pytest -x -v -s tests/kernels/mamba/cpu/test_cpu_gdn_ops.py"
# Note: SDE can't be downloaded from CI host because of AWS WAF
# - label: CPU-Compatibility Tests
@@ -59,7 +53,7 @@ steps:
- tests/models/language/pooling/
commands:
- |
bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 50m "
bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 40m "
pytest -x -v -s tests/models/language/generation -m cpu_model
pytest -x -v -s tests/models/language/pooling -m cpu_model"
@@ -74,15 +68,13 @@ steps:
- vllm/v1/sample/ops/topk_topp_triton.py
- vllm/v1/sample/ops/topk_topp_sampler.py
- tests/v1/sample/test_topk_topp_sampler.py
- tests/v1/e2e/test_cpu_linear_attn_chunked_prefix.py
commands:
- |
bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 45m "
uv pip install git+https://github.com/triton-lang/triton-cpu.git@270e696d
VLLM_USE_V2_MODEL_RUNNER=1 pytest -x -v -s tests/models/language/generation/test_granite.py -m cpu_model
# TODO: move to CPU-Kernel Tests once triton-cpu has a pre-built wheel
pytest -x -v -s tests/v1/sample/test_topk_topp_sampler.py::TestTritonTopkTopp
pytest -x -v -s tests/v1/e2e/test_cpu_linear_attn_chunked_prefix.py"
pytest -x -v -s tests/v1/sample/test_topk_topp_sampler.py::TestTritonTopkTopp"
- label: CPU-Quantization Model Tests
depends_on: []
@@ -97,13 +89,11 @@ steps:
- vllm/model_executor/layers/fused_moe/experts/cpu_moe.py
- tests/quantization/test_compressed_tensors.py
- tests/quantization/test_cpu_wna16.py
- tests/quantization/test_cpu_w8a8.py
commands:
- |
bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 45m "
bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 30m "
pytest -x -v -s tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_logprobs
pytest -x -v -s tests/quantization/test_cpu_wna16.py
pytest -x -v -s tests/quantization/test_cpu_w8a8.py"
pytest -x -v -s tests/quantization/test_cpu_wna16.py"
- label: CPU-Distributed Tests (PP+TP)
depends_on: []
@@ -145,21 +135,8 @@ steps:
commands:
- |
bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 45m "
pytest -x -v -s tests/models/multimodal/generation --ignore=tests/models/multimodal/generation/test_pixtral.py --ignore=tests/models/multimodal/generation/test_qwen2_5_vl.py -m cpu_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
parallelism: 4
- label: CPU-Qwen2.5-VL Multimodal Tests
depends_on: []
device: intel_cpu
no_plugin: true
source_file_dependencies:
# - vllm/
- vllm/model_executor/layers/rotary_embedding
- tests/models/multimodal/generation/
commands:
- |
bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 40m "
VLLM_CI_ENV=0 pytest -x -v -s tests/models/multimodal/generation/test_qwen2_5_vl.py"
pytest -x -v -s tests/models/multimodal/generation --ignore=tests/models/multimodal/generation/test_pixtral.py -m cpu_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
parallelism: 3
- label: "Arm CPU Test"
depends_on: []
@@ -1,80 +0,0 @@
group: Intel
steps:
- label: ":docker: Build XPU image"
soft_fail: true
optional: true
depends_on: []
key: image-build-xpu
commands:
- bash -lc '.buildkite/image_build/image_build_xpu.sh "public.ecr.aws/q9t5s3a7" "vllm-ci-test-repo" "$BUILDKITE_COMMIT"'
env:
DOCKER_BUILDKIT: "1"
retry:
automatic:
- exit_status: -1 # Agent was lost
limit: 2
- exit_status: -10 # Agent was lost
limit: 2
- label: "XPU example Test"
depends_on:
- image-build-xpu
timeout_in_minutes: 50
optional: true
device: intel_gpu
agent_tags:
label: production
gpu: 2+
mem: 24+
no_plugin: true
env:
REGISTRY: "public.ecr.aws/q9t5s3a7"
REPO: "vllm-ci-test-repo"
source_file_dependencies:
- .buildkite/hardware_tests/intel_xpu_ci/test-intel.yaml
- .buildkite/scripts/hardware_ci/run-intel-ci-test.sh
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'bash .buildkite/scripts/hardware_ci/run-intel-ci-test.sh example'
- label: "XPU V1 test"
depends_on:
- image-build-xpu
timeout_in_minutes: 70
optional: true
device: intel_gpu
agent_tags:
label: production
gpu: 1+
mem: 24+
no_plugin: true
env:
REGISTRY: "public.ecr.aws/q9t5s3a7"
REPO: "vllm-ci-test-repo"
source_file_dependencies:
- .buildkite/hardware_tests/intel_xpu_ci/test-intel.yaml
- .buildkite/scripts/hardware_ci/run-intel-ci-test.sh
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'bash .buildkite/scripts/hardware_ci/run-intel-ci-test.sh v1'
- label: "XPU server test"
depends_on:
- image-build-xpu
timeout_in_minutes: 45
optional: true
device: intel_gpu
agent_tags:
label: production
gpu: 1+
mem: 16+
no_plugin: true
env:
REGISTRY: "public.ecr.aws/q9t5s3a7"
REPO: "vllm-ci-test-repo"
source_file_dependencies:
- .buildkite/hardware_tests/intel_xpu_ci/test-intel.yaml
- .buildkite/scripts/hardware_ci/run-intel-ci-test.sh
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'bash .buildkite/scripts/hardware_ci/run-intel-ci-test.sh server'
-20
View File
@@ -79,18 +79,12 @@ setup_buildx_builder() {
docker buildx ls | grep -E '^\*|^NAME' || docker buildx ls
}
annotate_image_tags() {
.buildkite/scripts/annotate-image-build.sh \
"${IMAGE_TAG:-}" "${IMAGE_TAG_LATEST:-}"
}
check_and_skip_if_image_exists() {
if [[ -n "${IMAGE_TAG:-}" ]]; then
echo "--- :mag: Checking if image exists"
if docker manifest inspect "${IMAGE_TAG}" >/dev/null 2>&1; then
echo "Image already exists: ${IMAGE_TAG}"
echo "Skipping build"
annotate_image_tags
exit 0
fi
echo "Image not found, proceeding with build"
@@ -177,18 +171,6 @@ BRANCH=$4
IMAGE_TAG=$5
IMAGE_TAG_LATEST=${6:-} # only used for main branch, optional
# When TORCH_NIGHTLY=1, build the base CI image against PyTorch nightly so the
# entire existing pipeline runs on nightly torch (CUDA/GPU lane only). Delegate
# to the dedicated nightly build (PYTORCH_NIGHTLY=1, CUDA 13.0) and tag it at the
# normal IMAGE_TAG that every test step already pulls -- no separate image tag,
# no duplicate "vLLM Against PyTorch Nightly" pipeline section.
if [[ "${TORCH_NIGHTLY:-0}" == "1" ]]; then
echo "--- :warning: TORCH_NIGHTLY=1 -- building base image on PyTorch nightly"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
exec "${SCRIPT_DIR}/image_build_torch_nightly.sh" \
"${REGISTRY}" "${REPO}" "${BUILDKITE_COMMIT}" "${BRANCH}" "${IMAGE_TAG}"
fi
# build config
TARGET="test-ci"
VLLM_BAKE_FILE_PATH="${VLLM_BAKE_FILE_PATH:-docker/docker-bake.hcl}"
@@ -272,5 +254,3 @@ echo "--- :docker: Building ${TARGET}"
docker --debug buildx bake -f "${VLLM_BAKE_FILE_PATH}" -f "${CI_HCL_PATH}" --progress plain "${TARGET}"
echo "--- :white_check_mark: Build complete"
annotate_image_tags
+18 -20
View File
@@ -9,31 +9,29 @@ fi
REGISTRY=$1
REPO=$2
BUILDKITE_COMMIT=$3
IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-arm64"
# authenticate with AWS ECR
aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true
# skip build if image already exists
if docker manifest inspect "$IMAGE" >/dev/null 2>&1; then
echo "Image found"
else
if [[ -z $(docker manifest inspect "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64) ]]; then
echo "Image not found, proceeding with build..."
# build for arm64 GPU targets: Grace/GH200 (sm_90),
# Blackwell/Thor (sm_100/sm_103/sm_110), and DGX Spark/GB10
# (sm_121, family-covered by 12.0 under CUDA 13)
docker build --file docker/Dockerfile \
--platform linux/arm64 \
--build-arg max_jobs=16 \
--build-arg nvcc_threads=4 \
--build-arg torch_cuda_arch_list="9.0 10.0 11.0 12.0" \
--build-arg USE_SCCACHE=1 \
--build-arg buildkite_commit="$BUILDKITE_COMMIT" \
--tag "$IMAGE" \
--target test \
--progress plain .
# push
docker push "$IMAGE"
else
echo "Image found"
exit 0
fi
.buildkite/scripts/annotate-image-build.sh "$IMAGE"
# build (Grace/GH200 is the arm64 GPU target; sm_90)
docker build --file docker/Dockerfile \
--platform linux/arm64 \
--build-arg max_jobs=16 \
--build-arg nvcc_threads=4 \
--build-arg torch_cuda_arch_list="9.0" \
--build-arg USE_SCCACHE=1 \
--build-arg buildkite_commit="$BUILDKITE_COMMIT" \
--tag "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64 \
--target test \
--progress plain .
# push
docker push "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64
+15 -15
View File
@@ -9,26 +9,26 @@ fi
REGISTRY=$1
REPO=$2
BUILDKITE_COMMIT=$3
IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-cpu"
# authenticate with AWS ECR
aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true
# skip build if image already exists
if docker manifest inspect "$IMAGE" >/dev/null 2>&1; then
echo "Image found"
else
if [[ -z $(docker manifest inspect "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-cpu) ]]; then
echo "Image not found, proceeding with build..."
# build
docker build --file docker/Dockerfile.cpu \
--build-arg max_jobs=16 \
--build-arg buildkite_commit="$BUILDKITE_COMMIT" \
--build-arg VLLM_CPU_X86=true \
--tag "$IMAGE" \
--target vllm-test \
--progress plain .
# push
docker push "$IMAGE"
else
echo "Image found"
exit 0
fi
.buildkite/scripts/annotate-image-build.sh "$IMAGE"
# build
docker build --file docker/Dockerfile.cpu \
--build-arg max_jobs=16 \
--build-arg buildkite_commit="$BUILDKITE_COMMIT" \
--build-arg VLLM_CPU_X86=true \
--tag "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-cpu \
--target vllm-test \
--progress plain .
# push
docker push "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-cpu
+14 -14
View File
@@ -9,25 +9,25 @@ fi
REGISTRY=$1
REPO=$2
BUILDKITE_COMMIT=$3
IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-arm64-cpu"
# authenticate with AWS ECR
aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true
# skip build if image already exists
if docker manifest inspect "$IMAGE" >/dev/null 2>&1; then
echo "Image found"
else
if [[ -z $(docker manifest inspect "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64-cpu) ]]; then
echo "Image not found, proceeding with build..."
# build
docker build --file docker/Dockerfile.cpu \
--build-arg max_jobs=16 \
--build-arg buildkite_commit="$BUILDKITE_COMMIT" \
--tag "$IMAGE" \
--target vllm-test \
--progress plain .
# push
docker push "$IMAGE"
else
echo "Image found"
exit 0
fi
.buildkite/scripts/annotate-image-build.sh "$IMAGE"
# build
docker build --file docker/Dockerfile.cpu \
--build-arg max_jobs=16 \
--build-arg buildkite_commit="$BUILDKITE_COMMIT" \
--tag "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64-cpu \
--target vllm-test \
--progress plain .
# push
docker push "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64-cpu
+15 -15
View File
@@ -9,26 +9,26 @@ fi
REGISTRY=$1
REPO=$2
BUILDKITE_COMMIT=$3
IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-hpu"
# authenticate with AWS ECR
aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true
# skip build if image already exists
if docker manifest inspect "$IMAGE" >/dev/null 2>&1; then
echo "Image found"
else
if [[ -z $(docker manifest inspect "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-hpu) ]]; then
echo "Image not found, proceeding with build..."
# build
docker build \
--file tests/pytorch_ci_hud_benchmark/Dockerfile.hpu \
--build-arg max_jobs=16 \
--build-arg buildkite_commit="$BUILDKITE_COMMIT" \
--tag "$IMAGE" \
--progress plain \
https://github.com/vllm-project/vllm-gaudi.git
# push
docker push "$IMAGE"
else
echo "Image found"
exit 0
fi
.buildkite/scripts/annotate-image-build.sh "$IMAGE"
# build
docker build \
--file tests/pytorch_ci_hud_benchmark/Dockerfile.hpu \
--build-arg max_jobs=16 \
--build-arg buildkite_commit="$BUILDKITE_COMMIT" \
--tag "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-hpu \
--progress plain \
https://github.com/vllm-project/vllm-gaudi.git
# push
docker push "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-hpu
@@ -40,7 +40,6 @@ docker buildx ls
echo "--- :mag: Checking if image already exists"
if docker manifest inspect "$IMAGE_TAG" >/dev/null 2>&1; then
echo "Image found: $IMAGE_TAG — skipping build"
.buildkite/scripts/annotate-image-build.sh "$IMAGE_TAG"
exit 0
fi
echo "Image not found, proceeding with build..."
@@ -67,5 +66,3 @@ docker buildx build --file docker/Dockerfile \
--progress plain .
echo "--- :white_check_mark: Torch nightly image build complete: $IMAGE_TAG"
.buildkite/scripts/annotate-image-build.sh "$IMAGE_TAG"
+14 -14
View File
@@ -9,26 +9,26 @@ fi
REGISTRY=$1
REPO=$2
BUILDKITE_COMMIT=$3
IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-xpu"
# authenticate with AWS ECR
aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true
aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 936637512419.dkr.ecr.us-east-1.amazonaws.com || true
# skip build if image already exists
if docker manifest inspect "$IMAGE" &> /dev/null; then
echo "Image found"
else
if ! docker manifest inspect "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-xpu &> /dev/null; then
echo "Image not found, proceeding with build..."
# build
docker build \
--file docker/Dockerfile.xpu \
--build-arg max_jobs=16 \
--build-arg buildkite_commit="$BUILDKITE_COMMIT" \
--tag "$IMAGE" \
--progress plain .
# push
docker push "$IMAGE"
else
echo "Image found"
exit 0
fi
.buildkite/scripts/annotate-image-build.sh "$IMAGE"
# build
docker build \
--file docker/Dockerfile.xpu \
--build-arg max_jobs=16 \
--build-arg buildkite_commit="$BUILDKITE_COMMIT" \
--tag "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-xpu \
--progress plain .
# push
docker push "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-xpu
+1 -6
View File
@@ -3,12 +3,8 @@ depends_on:
- image-build-xpu
steps:
- label: XPU Sleep Mode
timeout_in_minutes: 45
timeout_in_minutes: 30
device: intel_gpu
agent_tags:
label: production
gpu: 1+
mem: 16+
no_plugin: true
working_dir: "."
env:
@@ -23,5 +19,4 @@ steps:
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'cd tests &&
export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
pytest -v -s basic_correctness/test_cpu_offload.py &&
pytest -v -s basic_correctness/test_mem.py::test_end_to_end'
-4
View File
@@ -5,10 +5,6 @@ steps:
- label: Engine (1 GPU)
timeout_in_minutes: 30
device: intel_gpu
agent_tags:
label: production
gpu: 1+
mem: 16+
no_plugin: true
working_dir: "."
env:
@@ -1,15 +1,11 @@
group: Expert Parallelism
depends_on:
depends_on:
- image-build-xpu
steps:
- label: EPLB Algorithm
key: eplb-algorithm
timeout_in_minutes: 45
device: intel_gpu
agent_tags:
label: production
gpu: 1+
mem: 16+
no_plugin: true
working_dir: "."
env:
-4
View File
@@ -5,10 +5,6 @@ steps:
- label: vLLM IR Tests
timeout_in_minutes: 30
device: intel_gpu
agent_tags:
label: production
gpu: 1+
mem: 16+
no_plugin: true
working_dir: "."
env:
+3 -29
View File
@@ -5,10 +5,6 @@ steps:
- label: LoRA Runtime + Utils
timeout_in_minutes: 45
device: intel_gpu
agent_tags:
label: production
gpu: 1+
mem: 24+
no_plugin: true
working_dir: "."
env:
@@ -38,10 +34,6 @@ steps:
- label: LoRA Fused/MoE Kernels
timeout_in_minutes: 45
device: intel_gpu
agent_tags:
label: production
gpu: 1+
mem: 16+
no_plugin: true
working_dir: "."
env:
@@ -62,10 +54,6 @@ steps:
- label: LoRA Punica Kernels
timeout_in_minutes: 45
device: intel_gpu
agent_tags:
label: production
gpu: 1+
mem: 16+
no_plugin: true
working_dir: "."
env:
@@ -81,17 +69,11 @@ steps:
'cd tests &&
export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
set -o pipefail &&
pytest -v -s lora/test_punica_ops.py::test_kernels &&
pytest -v -s lora/test_punica_ops.py::test_kernels_hidden_size &&
pytest -v -s lora/test_punica_ops.py::test_add_lora_fused_moe_early_exit'
pytest -v -s lora/test_punica_ops.py --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[expand-0-xpu:0-dtype0-3-43264-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype1-1-2049-64-128-16]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-128-1-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-256-1-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-256-8-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels[expand-0-xpu:0-dtype0-3-2049-128-8-16]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-128-8-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels[expand-0-xpu:0-dtype1-1-2049-256-128-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype0-3-64256-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype1-2-29696-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype1-3-49408-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype0-2-16384-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[expand-0-xpu:0-dtype0-2-51328-32-4-4]"'
- label: LoRA Punica FP8/XPU Ops
timeout_in_minutes: 60
timeout_in_minutes: 45
device: intel_gpu
agent_tags:
label: production
gpu: 1+
mem: 16+
no_plugin: true
working_dir: "."
env:
@@ -112,10 +94,6 @@ steps:
- label: LoRA Models
timeout_in_minutes: 45
device: intel_gpu
agent_tags:
label: production
gpu: 2+
mem: 24+
no_plugin: true
working_dir: "."
env:
@@ -130,19 +108,15 @@ steps:
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'cd tests &&
export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
(pytest -v -s lora/test_mixtral.py --deselect="tests/lora/test_mixtral.py::test_mixtral_lora[4]" || true) &&
pytest -v -s lora/test_quant_model.py --deselect="tests/lora/test_quant_model.py::test_quant_model_lora[model0]" --deselect="tests/lora/test_quant_model.py::test_quant_model_lora[model1]" --deselect="tests/lora/test_quant_model.py::test_quant_model_tp_equality[model0]" &&
pytest -v -s lora/test_transformers_model.py &&
pytest -v -s lora/test_chatglm3_tp.py &&
pytest -v -s lora/test_llama_tp.py::test_llama_lora &&
pytest -s -v lora/test_minicpmv_tp.py'
- label: LoRA Multimodal
timeout_in_minutes: 45
device: intel_gpu
agent_tags:
label: production
gpu: 1+
mem: 16+
no_plugin: true
working_dir: "."
env:
+8 -82
View File
@@ -3,12 +3,8 @@ depends_on:
- image-build-xpu
steps:
- label: V1 Core + KV + Metrics
timeout_in_minutes: 45
timeout_in_minutes: 30
device: intel_gpu
agent_tags:
label: production
gpu: 1+
mem: 16+
no_plugin: true
working_dir: "."
env:
@@ -33,12 +29,8 @@ steps:
pytest -v -s v1/executor'
- label: V1 Sample + Logits
timeout_in_minutes: 90
timeout_in_minutes: 30
device: intel_gpu
agent_tags:
label: production
gpu: 1+
mem: 24+
no_plugin: true
working_dir: "."
env:
@@ -65,46 +57,17 @@ steps:
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'pip install lm_eval[api]>=0.4.12 &&
export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
'export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
cd tests &&
pytest -v -s v1/logits_processors --ignore=v1/logits_processors/test_custom_online.py --ignore=v1/logits_processors/test_custom_offline.py &&
pytest -v -s v1/test_oracle.py &&
pytest -v -s v1/test_request.py &&
pytest -v -s v1/test_outputs.py &&
pytest -v -s v1/sample'
- label: Basic Models Tests (Initialization)
timeout_in_minutes: 60
device: intel_gpu
agent_tags:
label: production
gpu: 1+
mem: 16+
no_plugin: true
working_dir: "."
env:
REGISTRY: "public.ecr.aws/q9t5s3a7"
REPO: "vllm-ci-test-repo"
VLLM_TEST_DEVICE: "xpu"
source_file_dependencies:
- vllm/
- tests/models/test_initialization.py
- tests/models/registry.py
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'export VLLM_XPU_FUSED_MOE_USE_REF=1 &&
cd tests &&
pytest -v -s models/test_initialization.py::test_can_initialize_large_subset[Eagle3MiniMaxM2ForCausalLM]'
pytest -v -s v1/sample/test_topk_topp_sampler.py'
- label: XPU CPU Offload
timeout_in_minutes: 60
device: intel_gpu
agent_tags:
label: production
gpu: 1+
mem: 16+
no_plugin: true
working_dir: "."
env:
@@ -125,39 +88,10 @@ steps:
pytest -v -s v1/kv_offload &&
pytest -v -s v1/kv_connector/unit/test_offloading_connector.py'
- label: NixlConnector PD accuracy (2 GPUs)
timeout_in_minutes: 60
num_devices: 2
device: intel_gpu
agent_tags:
label: production
gpu: 2+
mem: 16+
no_plugin: true
working_dir: "."
env:
REGISTRY: "public.ecr.aws/q9t5s3a7"
REPO: "vllm-ci-test-repo"
VLLM_TEST_DEVICE: "xpu"
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- vllm/v1/worker/kv_connector_model_runner_mixin.py
- tests/v1/kv_connector/nixl_integration/
- vllm/platforms/xpu.py
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'cd tests &&
bash v1/kv_connector/nixl_integration/run_xpu_disagg_accuracy_test.sh'
- label: Regression
key: regression
timeout_in_minutes: 50
timeout_in_minutes: 30
device: intel_gpu
agent_tags:
label: production
gpu: 1+
mem: 16+
no_plugin: true
working_dir: "."
env:
@@ -180,19 +114,15 @@ steps:
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'pip install modelscope\<1.38 &&
'pip install modelscope &&
cd tests &&
pytest -v -s test_regression.py'
- label: Metrics, Tracing (2 GPUs)
key: metrics-tracing-2-gpus
timeout_in_minutes: 45
timeout_in_minutes: 30
num_devices: 2
device: intel_gpu
agent_tags:
label: production
gpu: 2+
mem: 16+
no_plugin: true
working_dir: "."
env:
@@ -222,12 +152,8 @@ steps:
- label: Async Engine, Inputs, Utils, Worker
key: async-engine-inputs-utils-worker
timeout_in_minutes: 55
timeout_in_minutes: 30
device: intel_gpu
agent_tags:
label: production
gpu: 1+
mem: 24+
no_plugin: true
working_dir: "."
env:
@@ -1,62 +0,0 @@
group: Model Runner V2 Intel
depends_on:
- image-build-xpu
steps:
- label: Model Runner V2 Core Tests (Intel)
timeout_in_minutes: 45
device: intel_gpu
agent_tags:
label: production
gpu: 2+
mem: 16+
no_plugin: true
working_dir: "."
env:
REGISTRY: "public.ecr.aws/q9t5s3a7"
REPO: "vllm-ci-test-repo"
VLLM_TEST_DEVICE: "xpu"
source_file_dependencies:
- vllm/v1/worker/gpu/
- vllm/v1/worker/gpu_worker.py
- vllm/v1/core/sched/
- vllm/v1/attention/
- tests/v1/engine/test_llm_engine.py
- tests/v1/e2e/
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'export VLLM_USE_V2_MODEL_RUNNER=1 &&
cd tests &&
pytest -v -s v1/engine/test_llm_engine.py -k "not test_engine_metrics" &&
ENFORCE_EAGER=1 pytest -v -s v1/e2e/general/test_async_scheduling.py -k "not ngram" &&
pytest -v -s v1/e2e/general/test_min_tokens.py'
- label: Model Runner V2 Examples (Intel)
timeout_in_minutes: 45
device: intel_gpu
agent_tags:
label: production
gpu: 1+
mem: 24+
no_plugin: true
working_dir: "."
env:
REGISTRY: "public.ecr.aws/q9t5s3a7"
REPO: "vllm-ci-test-repo"
VLLM_TEST_DEVICE: "xpu"
source_file_dependencies:
- vllm/v1/worker/gpu/
- vllm/v1/core/sched/
- vllm/v1/worker/gpu_worker.py
- examples/basic/offline_inference/
- examples/generate/multimodal/
- examples/features/
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'export VLLM_USE_V2_MODEL_RUNNER=1 &&
cd examples &&
python3 basic/offline_inference/chat.py &&
python3 basic/offline_inference/generate.py --model facebook/opt-125m &&
python3 generate/multimodal/vision_language_offline.py --seed 0 &&
python3 features/automatic_prefix_caching/prefix_caching_offline.py'
@@ -1,27 +0,0 @@
group: Models - Distributed
depends_on:
- image-build-xpu
steps:
- label: Distributed Model Tests (2 GPUs)
key: distributed-model-tests-2-gpus
timeout_in_minutes: 65
device: intel_gpu
agent_tags:
label: production
gpu: 2+
mem: 16+
no_plugin: true
working_dir: "."
env:
REGISTRY: "public.ecr.aws/q9t5s3a7"
REPO: "vllm-ci-test-repo"
VLLM_TEST_DEVICE: "xpu"
source_file_dependencies:
- vllm/model_executor/model_loader/sharded_state_loader.py
- vllm/model_executor/models/
- tests/model_executor/model_loader/test_sharded_state_loader.py
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'cd tests &&
pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m "not slow_test"'
@@ -1,15 +1,11 @@
group: Models - Multimodal
depends_on:
depends_on:
- image-build-xpu
steps:
- label: "Multi-Modal Models (Standard) 1: qwen2"
key: multi-modal-models-standard-1-qwen2
timeout_in_minutes: 70
timeout_in_minutes: 45
device: intel_gpu
agent_tags:
label: production
gpu: 1+
mem: 24+
no_plugin: true
working_dir: "."
env:
@@ -22,19 +18,15 @@ steps:
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'pip install av &&
'pip install av git+https://github.com/TIGER-AI-Lab/Mantis.git &&
cd tests &&
pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2" &&
pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model'
- label: "Multi-Modal Models (Standard) 2: qwen3 + gemma"
key: multi-modal-models-standard-2-qwen3-gemma
timeout_in_minutes: 70
timeout_in_minutes: 45
device: intel_gpu
agent_tags:
label: production
gpu: 1+
mem: 16+
no_plugin: true
working_dir: "."
env:
@@ -47,17 +39,14 @@ steps:
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'cd tests &&
'pip install git+https://github.com/TIGER-AI-Lab/Mantis.git &&
cd tests &&
pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model'
- label: "Multi-Modal Models (Standard) 3: llava + qwen2_vl"
key: multi-modal-models-standard-3-llava-qwen2-vl
timeout_in_minutes: 65
timeout_in_minutes: 45
device: intel_gpu
agent_tags:
label: production
gpu: 1+
mem: 24+
no_plugin: true
working_dir: "."
env:
@@ -70,7 +59,8 @@ steps:
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'cd tests &&
'pip install git+https://github.com/TIGER-AI-Lab/Mantis.git &&
cd tests &&
pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma" &&
pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model'
@@ -78,10 +68,6 @@ steps:
key: multi-modal-models-standard-4-other-whisper
timeout_in_minutes: 45
device: intel_gpu
agent_tags:
label: production
gpu: 1+
mem: 24+
no_plugin: true
working_dir: "."
env:
@@ -94,18 +80,14 @@ steps:
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'pip install av &&
'pip install av git+https://github.com/TIGER-AI-Lab/Mantis.git &&
cd tests &&
pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/processing'
- label: Multi-Modal Processor # 44min
key: multi-modal-processor
timeout_in_minutes: 60
timeout_in_minutes: 45
device: intel_gpu
agent_tags:
label: production
gpu: 1+
mem: 24+
no_plugin: true
working_dir: "."
env:
@@ -119,9 +101,11 @@ steps:
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'pip install av matplotlib ftfy &&
'pip install av matplotlib ftfy git+https://github.com/TIGER-AI-Lab/Mantis.git &&
pip install open-clip-torch --no-deps &&
cd tests &&
pytest -v -s models/multimodal/processing/test_tensor_schema.py
--deselect "tests/models/multimodal/processing/test_tensor_schema.py::test_model_tensor_schema[mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4]"
--deselect "tests/models/multimodal/processing/test_tensor_schema.py::test_model_tensor_schema[Qwen/Qwen2.5-Omni-7B-AWQ]"
--num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB'
parallelism: 4
-28
View File
@@ -1,28 +0,0 @@
group: Quantization
depends_on:
- image-build-xpu
steps:
- label: Quantization
key: quantization
timeout_in_minutes: 30
env:
REGISTRY: "public.ecr.aws/q9t5s3a7"
REPO: "vllm-ci-test-repo"
VLLM_TEST_DEVICE: "xpu"
no_plugin: true
working_dir: "."
device: intel_gpu
agent_tags:
label: production
gpu: 1+
mem: 16+
source_file_dependencies:
- csrc/
- vllm/model_executor/layers/quantization
- tests/quantization
commands:
# - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s tests/quantization/test_per_token_kv_cache.py --deselect="tests/quantization/test_per_token_kv_cache.py::test_triton_unified_attention_per_token_head_scale[int4-16-128-num_heads0-seq_lens1]"'
+4 -86
View File
@@ -17,12 +17,8 @@ steps:
- label: "XPU example Test"
depends_on:
- image-build-xpu
timeout_in_minutes: 50
timeout_in_minutes: 30
device: intel_gpu
agent_tags:
label: production
gpu: 2+
mem: 24+
no_plugin: true
env:
REGISTRY: "public.ecr.aws/q9t5s3a7"
@@ -42,46 +38,17 @@ steps:
python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --kv-cache-dtype fp8 &&
python3 examples/basic/offline_inference/generate.py --model nvidia/Llama-3.1-8B-Instruct-FP8 --block-size 64 --enforce-eager --quantization modelopt --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --max-model-len 4096 &&
python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --block-size 64 --enforce-eager --max-model-len 8192 &&
python3 examples/basic/offline_inference/generate.py --model TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ --block-size 64 --enforce-eager &&
python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 &&
python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 --enable-expert-parallel &&
python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --max-model-len 8192 &&
VLLM_XPU_FUSED_MOE_USE_REF=1 python3 examples/basic/offline_inference/generate.py --model Qwen/Qwen3-30B-A3B-Instruct-2507-FP8 --enforce-eager -tp 2 --max-model-len 8192 &&
python3 examples/basic/offline_inference/generate.py --model INCModel/Qwen3-30B-A3B-Instruct-2507-MXFP4-LLMC --enforce-eager -tp 2 --max-model-len 8192
'
- label: "XPU W8A8 FP8 Linear Examples"
depends_on:
- image-build-xpu
timeout_in_minutes: 60
device: intel_gpu
agent_tags:
label: production
gpu: 1+
mem: 24+
no_plugin: true
env:
REGISTRY: "public.ecr.aws/q9t5s3a7"
REPO: "vllm-ci-test-repo"
VLLM_TEST_DEVICE: "xpu"
source_file_dependencies:
- vllm/
- .buildkite/intel_jobs/test-intel.yaml
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8 --enforce-eager --max-model-len 4096 &&
python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model neuralmagic/Llama-3.2-1B-Instruct-FP8-dynamic --enforce-eager --max-model-len 4096 &&
python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8 --enforce-eager --max-model-len 4096
'
- label: "XPU V1 test"
depends_on:
- image-build-xpu
timeout_in_minutes: 70
timeout_in_minutes: 30
device: intel_gpu
agent_tags:
label: production
gpu: 1+
mem: 24+
no_plugin: true
env:
REGISTRY: "public.ecr.aws/q9t5s3a7"
@@ -99,18 +66,13 @@ steps:
pytest -v -s v1/worker --ignore=v1/worker/test_gpu_model_runner.py --ignore=v1/worker/test_worker_memory_snapshot.py &&
pytest -v -s v1/structured_output &&
pytest -v -s v1/test_serial_utils.py &&
pytest -v -s v1/e2e/general/test_correctness_sliding_window.py --deselect="tests/v1/e2e/general/test_correctness_sliding_window.py::test_sliding_window_retrieval[True-1-5-google/gemma-3-1b-it]" &&
pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py --ignore=v1/spec_decode/test_speculators_correctness.py &&
pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py &&
pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_multi_connector.py --ignore=v1/kv_connector/unit/test_example_connector.py --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py --ignore=v1/kv_connector/unit/test_offloading_connector.py'
- label: "XPU server test"
depends_on:
- image-build-xpu
timeout_in_minutes: 45
timeout_in_minutes: 30
device: intel_gpu
agent_tags:
label: production
gpu: 1+
mem: 16+
no_plugin: true
env:
REGISTRY: "public.ecr.aws/q9t5s3a7"
@@ -125,47 +87,3 @@ steps:
cd tests &&
pytest -v -s entrypoints/multimodal/openai/chat_completion/test_audio_in_video.py &&
pytest -v -s benchmarks/test_serve_cli.py'
- label: "XPU quantization test"
depends_on:
- image-build-xpu
timeout_in_minutes: 30
device: intel_gpu
agent_tags:
label: production
gpu: 1+
mem: 16+
no_plugin: true
env:
REGISTRY: "public.ecr.aws/q9t5s3a7"
REPO: "vllm-ci-test-repo"
source_file_dependencies:
- vllm/
- .buildkite/intel_jobs/test-intel.yaml
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'cd tests &&
pytest -v -s quantization/test_auto_round.py'
- label: "XPU compressed tensors FP8 test"
depends_on:
- image-build-xpu
timeout_in_minutes: 60
device: intel_gpu
agent_tags:
label: production
gpu: 1+
mem: 16+
no_plugin: true
env:
REGISTRY: "public.ecr.aws/q9t5s3a7"
REPO: "vllm-ci-test-repo"
VLLM_TEST_DEVICE: "xpu"
source_file_dependencies:
- vllm/
- tests/quantization/test_compressed_tensors.py
- .buildkite/intel_jobs/test-intel.yaml
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'cd tests &&
pytest -v -s quantization/test_compressed_tensors.py::test_compressed_tensors_fp8'
@@ -1,7 +1,6 @@
# For hf script, without -t option (tensor parallel size).
# bash .buildkite/lm-eval-harness/run-lm-eval-mmlupro-vllm-baseline.sh -m meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 -l 250 -t 8 -f 5
model_name: "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8"
rocm_safetensors_load_strategy: lazy
required_gpu_arch:
- gfx942
- gfx950
@@ -6,7 +6,9 @@ tasks:
value: 0.7142
- name: "exact_match,flexible-extract"
value: 0.4579
moe_backend: "flashinfer_cutlass"
env_vars:
VLLM_USE_FLASHINFER_MOE_FP8: "1"
VLLM_FLASHINFER_MOE_BACKEND: "throughput"
limit: 1319
num_fewshot: 5
max_model_len: 262144
@@ -68,15 +68,6 @@ def launch_lm_eval(eval_config, tp_size):
if current_platform.is_rocm() and "Nemotron-3" in eval_config["model_name"]:
model_args += "attention_backend=TRITON_ATTN"
moe_backend = eval_config.get("moe_backend", None)
if moe_backend is not None:
model_args += f"moe_backend={moe_backend},"
if current_platform.is_rocm():
rocm_load_strategy = eval_config.get("rocm_safetensors_load_strategy")
if rocm_load_strategy is not None:
model_args += f"safetensors_load_strategy={rocm_load_strategy},"
env_vars = eval_config.get("env_vars", None)
with scoped_env_vars(env_vars):
results = lm_eval.simple_evaluate(
File diff suppressed because it is too large Load Diff
@@ -3,8 +3,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
#
# Append a build artifact line to the Buildkite annotation.
# Usage: annotate-build-artifact.sh <label> <value> <context>
# Usage: annotate-build-artifact.sh <label> <value>
set -e
echo "- **${1}**: \`${2}\`" | \
buildkite-agent annotate --append --style 'info' \
--context "${3:?context is required}"
buildkite-agent annotate --append --style 'info' --context 'release-artifacts'
@@ -1,36 +0,0 @@
#!/bin/bash
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
#
# Append the Docker image tag(s) an image-build step pushed to a Buildkite
# annotation, so the built image tags show up on the build page instead of
# being buried in the job logs.
#
# Usage: annotate-image-build.sh <image_tag> [<image_tag> ...]
set -euo pipefail
# buildkite-agent only exists on Buildkite agents; no-op elsewhere so the
# image build scripts stay runnable locally.
if ! command -v buildkite-agent >/dev/null 2>&1; then
echo "buildkite-agent not found; skipping image tag annotation"
exit 0
fi
label="${BUILDKITE_LABEL:-Image build}"
content=""
for image in "$@"; do
[[ -n "$image" ]] || continue
content+="- **${label}**: \`${image}\`"$'\n'
done
if [[ -z "$content" ]]; then
echo "No image tags provided; nothing to annotate"
exit 0
fi
# Best-effort: a flaky annotation must never fail an otherwise successful
# (and expensive) image build.
if ! printf '%s' "$content" | \
buildkite-agent annotate --append --style 'info' --context 'docker-images'; then
echo "warning: failed to annotate build with image tags"
fi
-32
View File
@@ -1,32 +0,0 @@
#!/usr/bin/env bash
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
#
# Build the macOS arm64 CPU wheel natively on a macOS agent (the `macmini`
# queue) into artifacts/dist/ for upload-nightly-wheels.sh.
set -euo pipefail
# The Rust frontend build needs protoc.
if ! command -v protoc >/dev/null 2>&1; then
brew install protobuf
fi
# upload-nightly-wheels.sh expects exactly one wheel.
rm -rf artifacts/dist
mkdir -p artifacts/dist
export VLLM_TARGET_DEVICE=cpu
export VLLM_REQUIRE_RUST_FRONTEND=1
export MACOSX_DEPLOYMENT_TARGET=11.0
# uv's CPython is universal2; force an arm64-only build and tag so the wheel
# isn't mislabelled universal2 and installed on Intel Macs where import fails.
export ARCHFLAGS="-arch arm64"
export _PYTHON_HOST_PLATFORM="macosx-11.0-arm64"
export CMAKE_BUILD_PARALLEL_LEVEL="${CMAKE_BUILD_PARALLEL_LEVEL:-4}"
uv venv --python 3.12
uv pip install -r requirements/build/cpu.txt --index-strategy unsafe-best-match
uv build --wheel --no-build-isolation -o artifacts/dist
ls -l artifacts/dist/*.whl
@@ -29,11 +29,7 @@ if python3 -c "import torch; assert torch.version.hip" 2>/dev/null; then
TORCH_INDEX_URL=""
fi
else
if [ "${TORCH_NIGHTLY:-0}" = "1" ]; then
TORCH_INDEX_URL="https://download.pytorch.org/whl/nightly/cu130"
else
TORCH_INDEX_URL="https://download.pytorch.org/whl/cu130"
fi
TORCH_INDEX_URL="https://download.pytorch.org/whl/cu130"
fi
echo ">>> Using PyTorch index: ${TORCH_INDEX_URL:-PyPI default}"
+53 -480
View File
@@ -15,10 +15,9 @@ set -euo pipefail
DEFAULT_REPO_SLUG="vllm-project/vllm"
DEFAULT_CI_HCL_SOURCE="docker/ci-rocm.hcl"
DEFAULT_CI_BASE_CONTENT_FILES="requirements/common.txt requirements/rocm.txt requirements/test/rocm.txt docker/Dockerfile.rocm_base docker/ci-rocm.hcl docker/docker-bake-rocm.hcl tools/install_torchcodec_rocm.sh tools/install_protoc.sh rust-toolchain.toml tests/vllm_test_utils .buildkite/scripts/ci-bake-rocm.sh .buildkite/scripts/rocm/build-ci-base.sh"
DEFAULT_CI_BASE_CONTENT_FILES="requirements/common.txt requirements/rocm.txt requirements/test/rocm.txt docker/Dockerfile.rocm_base tools/install_torchcodec_rocm.sh tests/vllm_test_utils"
DEFAULT_CI_BASE_DOCKERFILE="docker/Dockerfile.rocm"
DEFAULT_CI_BASE_DOCKERFILE_STAGES="base rust_toolchain_input_0 rust_toolchain_input_1 rust-toolchain-input rust-toolchain build_rixl build_rocshmem build_deepep mori_base ci_base"
DEFAULT_CI_BASE_METADATA_VERSION="1"
DEFAULT_CI_BASE_DOCKERFILE_STAGES="base build_rixl build_rocshmem build_deepep mori_base ci_base"
IMAGE_EXISTED_BEFORE_BUILD=0
TARGET=""
@@ -285,7 +284,7 @@ get_content_arg_names() {
fi | awk 'NF && !seen[$0]++'
}
compute_ci_base_content_hash_once() {
compute_ci_base_content_hash() {
local -a content_paths=()
local -a content_args=()
local dockerfile="${CI_BASE_DOCKERFILE:-}"
@@ -301,8 +300,7 @@ compute_ci_base_content_hash_once() {
if [[ -n "${dockerfile}" ]]; then
printf 'dockerfile:%s\n' "${dockerfile}"
printf 'resolved-build-args:\n'
hash_dockerfile_arg_values "${dockerfile}" "${content_args[@]}" \
|| return 1
hash_dockerfile_arg_values "${dockerfile}" "${content_args[@]}"
if [[ -n "${stages}" ]]; then
printf 'dockerfile-stages:%s\n' "${stages}"
if [[ -f "${dockerfile}" ]]; then
@@ -315,53 +313,6 @@ compute_ci_base_content_hash_once() {
} | sha256sum | cut -d' ' -f1
}
compute_ci_base_content_hash() {
local attempts="${CI_BASE_HASH_ATTEMPTS:-3}"
local delay_secs="${CI_BASE_HASH_RETRY_DELAY:-5}"
local attempt=0
local hash=""
local failed=0
local -a hashes=()
if [[ ! "${attempts}" =~ ^[1-9][0-9]*$ ]]; then
echo "Invalid CI_BASE_HASH_ATTEMPTS: ${attempts}" >&2
return 1
fi
if [[ ! "${delay_secs}" =~ ^[0-9]+$ ]]; then
echo "Invalid CI_BASE_HASH_RETRY_DELAY: ${delay_secs}" >&2
return 1
fi
for ((attempt = 1; attempt <= attempts; attempt++)); do
if ! hash=$(compute_ci_base_content_hash_once); then
echo "ci_base content hash calculation ${attempt}/${attempts} failed" >&2
failed=1
else
hashes+=("${hash}")
echo "ci_base content hash calculation ${attempt}/${attempts}: ${hash}" >&2
fi
if ((attempt < attempts)); then
sleep "${delay_secs}"
fi
done
if ((failed)) || ((${#hashes[@]} != attempts)); then
echo "Could not calculate a reliable ci_base content hash" >&2
return 1
fi
for hash in "${hashes[@]:1}"; do
if [[ "${hash}" != "${hashes[0]}" ]]; then
echo "ci_base content hash changed between calculations" >&2
printf ' observed: %s\n' "${hashes[@]}" >&2
return 1
fi
done
printf '%s\n' "${hashes[0]}"
}
extract_dockerfile_arg_default() {
local dockerfile="$1"
local arg_name="$2"
@@ -414,11 +365,7 @@ hash_dockerfile_arg_values() {
printf 'arg:%s=%s\n' "${arg_name}" "${arg_value:-<empty>}"
if [[ "${arg_name}" == "BASE_IMAGE" && -n "${arg_value}" ]]; then
digest=$(resolve_image_digest "${arg_value}")
if [[ -z "${digest}" ]]; then
echo "Failed to resolve digest for BASE_IMAGE=${arg_value}" >&2
return 1
fi
printf 'arg:%s.digest=%s\n' "${arg_name}" "${digest}"
printf 'arg:%s.digest=%s\n' "${arg_name}" "${digest:-unknown}"
fi
done
}
@@ -445,16 +392,6 @@ should_upload_wheel_artifacts() {
|| "${TARGET}" == *"artifact"* ]]
}
set_buildkite_metadata() {
local key="$1"
local value="$2"
[[ -n "${value}" ]] || return 0
if command -v buildkite-agent >/dev/null 2>&1; then
buildkite-agent meta-data set "${key}" "${value}" || true
fi
}
get_remote_image_label() {
local image_ref="$1"
local label_key="$2"
@@ -588,22 +525,6 @@ get_remote_image_label_with_retry() {
return 0
}
remote_ci_base_metadata_is_current() {
local image_ref="$1"
local metadata_version=""
metadata_version=$(get_remote_image_label "${image_ref}" "vllm.ci_base.metadata_version")
[[ "${metadata_version}" == "${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}" ]]
}
remote_ci_base_metadata_is_current_with_retry() {
local image_ref="$1"
local metadata_version=""
metadata_version=$(get_remote_image_label_with_retry "${image_ref}" "vllm.ci_base.metadata_version")
[[ "${metadata_version}" == "${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}" ]]
}
remote_image_exists() {
local image_ref="$1"
docker manifest inspect "${image_ref}" >/dev/null 2>&1
@@ -660,7 +581,6 @@ init_config() {
CI_BASE_CONTENT_FILES="${CI_BASE_CONTENT_FILES:-${DEFAULT_CI_BASE_CONTENT_FILES}}"
CI_BASE_DOCKERFILE="${CI_BASE_DOCKERFILE:-${DEFAULT_CI_BASE_DOCKERFILE}}"
CI_BASE_DOCKERFILE_STAGES="${CI_BASE_DOCKERFILE_STAGES:-${DEFAULT_CI_BASE_DOCKERFILE_STAGES}}"
CI_BASE_METADATA_VERSION="${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}"
CI_BASE_IMAGE_TAG="${CI_BASE_IMAGE_TAG:-rocm/vllm-dev:ci_base}"
export PYTORCH_ROCM_ARCH
@@ -715,10 +635,6 @@ load_ci_hcl() {
echo "Copied ${CI_HCL_SOURCE} to ${CI_HCL_PATH}"
}
init_bake_files() {
BAKE_FILES=(-f "${VLLM_BAKE_FILE}" -f "${CI_HCL_PATH}")
}
compute_ci_base_hash_if_needed() {
if [[ -z "${CI_BASE_CONTENT_FILES:-}" ]]; then
return 0
@@ -760,14 +676,12 @@ configure_ci_base_image_refs() {
fi
content_tag=$(ci_base_tag_with_suffix "${stable_tag}" "${CI_BASE_CONTENT_HASH}")
CI_BASE_IMAGE_TAG_CONTENT_REF="${content_tag}"
if [[ -n "${BUILDKITE_COMMIT:-}" ]]; then
commit_tag=$(ci_base_tag_with_suffix "${stable_tag}" "${BUILDKITE_COMMIT}")
CI_BASE_IMAGE_TAG_COMMIT="${commit_tag}"
export CI_BASE_IMAGE_TAG_COMMIT
fi
CI_BASE_IMAGE_TAG_COMMIT_REF="${commit_tag}"
# *_REF is the logical tag recorded in metadata. *_EXTRA is only passed to
# bake when that tag is not already the primary tag, avoiding duplicates.
if should_push_stable_ci_base_tag; then
primary_tag="${content_tag}"
CI_BASE_IMAGE_TAG_STABLE="${stable_tag}"
@@ -777,35 +691,19 @@ configure_ci_base_image_refs() {
fi
CI_BASE_IMAGE_TAG="${primary_tag}"
if [[ "${primary_tag}" == "${content_tag}" ]]; then
CI_BASE_IMAGE_TAG_CONTENT_EXTRA=""
CI_BASE_IMAGE_TAG_CONTENT=""
else
CI_BASE_IMAGE_TAG_CONTENT_EXTRA="${content_tag}"
CI_BASE_IMAGE_TAG_CONTENT="${content_tag}"
fi
if [[ -n "${commit_tag}" && "${commit_tag}" != "${primary_tag}" ]]; then
CI_BASE_IMAGE_TAG_COMMIT_EXTRA="${commit_tag}"
else
CI_BASE_IMAGE_TAG_COMMIT_EXTRA=""
fi
export CI_BASE_IMAGE_TAG
export CI_BASE_IMAGE_TAG_COMMIT_EXTRA
export CI_BASE_IMAGE_TAG_CONTENT_EXTRA
export CI_BASE_IMAGE_TAG_CONTENT_REF
export CI_BASE_IMAGE_TAG_COMMIT_REF
export CI_BASE_IMAGE_TAG_STABLE
export CI_BASE_IMAGE_TAG CI_BASE_IMAGE_TAG_CONTENT CI_BASE_IMAGE_TAG_STABLE
if is_ci_base_target; then
IMAGE_TAG="${primary_tag}"
CI_BASE_IMAGE="${primary_tag}"
export CI_BASE_IMAGE
export IMAGE_TAG
echo "ci_base primary image tag: ${CI_BASE_IMAGE_TAG}"
if [[ -n "${commit_tag}" ]]; then
if [[ "${commit_tag}" == "${primary_tag}" ]]; then
echo "ci_base commit image tag: ${commit_tag} (primary)"
else
echo "ci_base commit image tag: ${commit_tag}"
fi
if [[ -n "${CI_BASE_IMAGE_TAG_COMMIT:-}" ]]; then
echo "ci_base commit image tag: ${CI_BASE_IMAGE_TAG_COMMIT}"
fi
echo "ci_base content image tag: ${content_tag}"
if [[ -n "${CI_BASE_IMAGE_TAG_STABLE}" ]]; then
@@ -814,10 +712,6 @@ configure_ci_base_image_refs() {
echo "ci_base stable alias will not be pushed for this build"
echo "Set NIGHTLY=1 on ${CI_BASE_STABLE_BRANCH:-main} to refresh ${stable_tag}"
fi
set_buildkite_metadata "rocm-ci-base-image" "${CI_BASE_IMAGE_TAG}"
set_buildkite_metadata "rocm-ci-base-image-content" "${content_tag}"
set_buildkite_metadata "rocm-ci-base-image-commit" "${CI_BASE_IMAGE_TAG_COMMIT_REF:-}"
set_buildkite_metadata "rocm-ci-base-image-stable" "${CI_BASE_IMAGE_TAG_STABLE:-}"
return 0
fi
@@ -834,8 +728,8 @@ ci_base_candidate_refs() {
printf '%s\n' \
"${IMAGE_TAG:-}" \
"${CI_BASE_IMAGE_TAG:-}" \
"${CI_BASE_IMAGE_TAG_COMMIT_EXTRA:-}" \
"${CI_BASE_IMAGE_TAG_CONTENT_EXTRA:-}" \
"${CI_BASE_IMAGE_TAG_COMMIT:-}" \
"${CI_BASE_IMAGE_TAG_CONTENT:-}" \
"${CI_BASE_IMAGE_TAG_STABLE:-}" \
| awk 'NF && !seen[$0]++'
}
@@ -849,10 +743,6 @@ find_matching_ci_base_ref() {
remote_image_exists "${candidate}" || continue
candidate_hash=$(get_remote_image_label "${candidate}" "vllm.ci_base.content_hash")
if [[ "${candidate_hash}" == "${CI_BASE_CONTENT_HASH}" ]]; then
if ! remote_ci_base_metadata_is_current "${candidate}"; then
echo "Found matching ci_base content hash but stale metadata: ${candidate}" >&2
continue
fi
printf '%s\n' "${candidate}"
return 0
fi
@@ -927,10 +817,6 @@ maybe_skip_existing_image() {
if [[ -n "${remote_hash}" ]]; then
echo "Remote ci_base content hash: ${remote_hash:0:16}..."
if [[ "${remote_hash}" == "${CI_BASE_CONTENT_HASH}" ]]; then
if ! remote_ci_base_metadata_is_current "${IMAGE_TAG}"; then
echo "Content hashes match but ci_base metadata is stale; rebuilding to refresh metadata"
return 0
fi
if ! refresh_ci_base_tags_from_ref "${IMAGE_TAG}"; then
echo "ci_base tag refresh failed; rebuilding to push expected tags"
return 0
@@ -1112,104 +998,12 @@ prepare_git_cache_metadata() {
fi
}
ci_base_metadata_pairs() {
local dockerfile="${CI_BASE_DOCKERFILE:-${DEFAULT_CI_BASE_DOCKERFILE}}"
local stages="${CI_BASE_DOCKERFILE_STAGES:-${DEFAULT_CI_BASE_DOCKERFILE_STAGES}}"
local content_files="${CI_BASE_CONTENT_FILES:-${DEFAULT_CI_BASE_CONTENT_FILES}}"
local content_files_hash=""
local base_image=""
local base_image_digest=""
local git_branch=""
local -a content_paths=()
local -a content_args=()
read -r -a content_paths <<< "${content_files}"
if [[ ${#content_paths[@]} -gt 0 ]]; then
content_files_hash=$(compute_content_hash "${content_paths[@]}")
fi
mapfile -t content_args < <(
get_content_arg_names "${dockerfile}" "${stages}" "${CI_BASE_CONTENT_ARGS:-}"
)
base_image=$(resolve_dockerfile_arg_value "${dockerfile}" "BASE_IMAGE")
if [[ -n "${base_image}" ]]; then
base_image_digest=$(resolve_image_digest "${base_image}")
fi
git_branch="${BUILDKITE_BRANCH:-${VLLM_BRANCH:-}}"
metadata_pair "vllm.ci_base.metadata_version" "${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}"
metadata_pair "vllm.ci_base.content_hash" "${CI_BASE_CONTENT_HASH:-}"
metadata_pair "vllm.ci_base.content_files_hash" "${content_files_hash}"
metadata_pair "vllm.ci_base.content_files" "${content_files}"
metadata_pair "vllm.ci_base.content_args" "$(join_words "${content_args[@]}")"
metadata_pair "vllm.ci_base.dockerfile" "${dockerfile}"
metadata_pair "vllm.ci_base.dockerfile_stages" "${stages}"
metadata_pair "vllm.ci_base.image.primary" "${CI_BASE_IMAGE_TAG:-}"
metadata_pair "vllm.ci_base.image.content" "${CI_BASE_IMAGE_TAG_CONTENT_REF:-${CI_BASE_IMAGE_TAG_CONTENT_EXTRA:-}}"
metadata_pair "vllm.ci_base.image.commit" "${CI_BASE_IMAGE_TAG_COMMIT_REF:-${CI_BASE_IMAGE_TAG_COMMIT_EXTRA:-}}"
metadata_pair "vllm.ci_base.image.stable" "${CI_BASE_IMAGE_TAG_STABLE:-}"
metadata_pair "vllm.ci_base.git_commit" "${BUILDKITE_COMMIT:-}"
metadata_pair "vllm.ci_base.git_branch" "${git_branch}"
metadata_pair "vllm.ci_base.vllm_branch" "${VLLM_BRANCH:-}"
metadata_pair "vllm.ci_base.stable_branch" "${CI_BASE_STABLE_BRANCH:-main}"
metadata_pair "vllm.rocm.base_image" "${base_image}"
metadata_pair "vllm.rocm.base_image_digest" "${base_image_digest}"
metadata_pair "vllm.rocm.pytorch_rocm_arch" "${PYTORCH_ROCM_ARCH:-}"
metadata_pair "vllm.rocm.nic_backend" "$(resolve_dockerfile_arg_value "${dockerfile}" "NIC_BACKEND")"
metadata_pair "vllm.rocm.ainic_version" "$(resolve_dockerfile_arg_value "${dockerfile}" "AINIC_VERSION")"
metadata_pair "vllm.rocm.ubuntu_codename" "$(resolve_dockerfile_arg_value "${dockerfile}" "UBUNTU_CODENAME")"
metadata_pair "vllm.rocm.rixl_repo" "$(resolve_dockerfile_arg_value "${dockerfile}" "RIXL_REPO")"
metadata_pair "vllm.rocm.rixl_commit" "${RIXL_BRANCH:-$(resolve_dockerfile_arg_value "${dockerfile}" "RIXL_BRANCH")}"
metadata_pair "vllm.rocm.ucx_repo" "$(resolve_dockerfile_arg_value "${dockerfile}" "UCX_REPO")"
metadata_pair "vllm.rocm.ucx_commit" "${UCX_BRANCH:-$(resolve_dockerfile_arg_value "${dockerfile}" "UCX_BRANCH")}"
metadata_pair "vllm.rocm.rocshmem_repo" "$(resolve_dockerfile_arg_value "${dockerfile}" "ROCSHMEM_REPO")"
metadata_pair "vllm.rocm.rocshmem_commit" "${ROCSHMEM_BRANCH:-$(resolve_dockerfile_arg_value "${dockerfile}" "ROCSHMEM_BRANCH")}"
metadata_pair "vllm.rocm.deepep_repo" "$(resolve_dockerfile_arg_value "${dockerfile}" "DEEPEP_REPO")"
metadata_pair "vllm.rocm.deepep_commit" "${DEEPEP_BRANCH:-$(resolve_dockerfile_arg_value "${dockerfile}" "DEEPEP_BRANCH")}"
metadata_pair "vllm.rocm.deepep_nic" "$(resolve_dockerfile_arg_value "${dockerfile}" "DEEPEP_NIC")"
metadata_pair "vllm.rocm.deepep_rocm_arch" "$(resolve_dockerfile_arg_value "${dockerfile}" "DEEPEP_ROCM_ARCH")"
metadata_pair "vllm.rocm.rixl_cache_key" "${RIXL_CACHE_KEY:-}"
metadata_pair "vllm.rocm.rocshmem_cache_key" "${ROCSHMEM_CACHE_KEY:-}"
metadata_pair "vllm.rocm.deepep_cache_key" "${DEEPEP_CACHE_KEY:-}"
metadata_pair "vllm.buildkite.build_number" "${BUILDKITE_BUILD_NUMBER:-}"
metadata_pair "vllm.buildkite.build_id" "${BUILDKITE_BUILD_ID:-}"
}
write_ci_base_metadata_annotations() {
local metadata="$1"
local key=""
local value=""
local annotation=""
[[ -n "${metadata}" ]] || return 0
while IFS=$'\t' read -r key value; do
[[ -n "${key}" && -n "${value}" ]] || continue
annotation="manifest:${key}=${value}"
printf ' "%s",\n' "$(hcl_escape_string "${annotation}")"
done <<< "${metadata}"
}
write_ci_base_metadata_labels() {
local metadata="$1"
local key=""
local value=""
[[ -n "${metadata}" ]] || return 0
while IFS=$'\t' read -r key value; do
[[ -n "${key}" && -n "${value}" ]] || continue
printf ' "%s" = "%s"\n' \
"$(hcl_escape_string "${key}")" \
"$(hcl_escape_string "${value}")"
done <<< "${metadata}"
}
write_ci_base_label_override() {
local target_name=""
local metadata=""
local -a ci_base_targets=()
BAKE_FILES=(-f "${VLLM_BAKE_FILE}" -f "${CI_HCL_PATH}")
if [[ -z "${CI_BASE_CONTENT_HASH:-}" ]]; then
return 0
fi
@@ -1225,23 +1019,16 @@ write_ci_base_label_override() {
return 0
fi
metadata=$(ci_base_metadata_pairs)
: > "${CI_BASE_LABEL_OVERRIDE_PATH}"
for target_name in "${ci_base_targets[@]}"; do
cat >> "${CI_BASE_LABEL_OVERRIDE_PATH}" <<EOF
target "${target_name}" {
annotations = [
"manifest:org.opencontainers.image.revision=",
EOF
write_ci_base_metadata_annotations "${metadata}" >> "${CI_BASE_LABEL_OVERRIDE_PATH}"
cat >> "${CI_BASE_LABEL_OVERRIDE_PATH}" <<EOF
]
labels = {
"org.opencontainers.image.revision" = ""
EOF
write_ci_base_metadata_labels "${metadata}" >> "${CI_BASE_LABEL_OVERRIDE_PATH}"
cat >> "${CI_BASE_LABEL_OVERRIDE_PATH}" <<EOF
"vllm.ci_base.content_hash" = "${CI_BASE_CONTENT_HASH}"
}
}
@@ -1249,7 +1036,7 @@ EOF
done
BAKE_FILES+=(-f "${CI_BASE_LABEL_OVERRIDE_PATH}")
echo "Appended ci_base metadata label override for targets: ${ci_base_targets[*]}"
echo "Appended ci_base content-hash label override for targets: ${ci_base_targets[*]}"
}
uses_rocm_csrc_cache() {
@@ -1263,24 +1050,12 @@ uses_rocm_csrc_cache() {
esac
}
uses_rocm_rust_cache() {
case "${TARGET}" in
rust-rocm-ci|test-rocm-ci|test-rocm-ci-with-wheel|test-rocm-ci-with-artifacts|export-wheel-rocm)
return 0
;;
*)
return 1
;;
esac
}
compute_rocm_csrc_content_hash() {
local bake_dir=""
local dockerfile_rocm=""
local -a content_paths=(
"requirements/common.txt"
"requirements/rocm.txt"
"pyproject.toml"
"setup.py"
"CMakeLists.txt"
"cmake"
@@ -1324,56 +1099,6 @@ compute_rocm_csrc_content_hash_if_needed() {
echo "ROCm csrc content cache ref: ${ROCM_CSRC_CONTENT_CACHE_REF}"
}
compute_rocm_rust_content_hash() {
local bake_dir=""
local dockerfile_rocm=""
local -a content_paths=(
"requirements/build/rust.txt"
"rust/Cargo.lock"
"rust/Cargo.toml"
"rust/proto"
"rust/src"
"rust-toolchain.toml"
"tools/build_rust.py"
"tools/install_protoc.sh"
"build_rust.sh"
)
local -a content_args=()
bake_dir=$(dirname "${VLLM_BAKE_FILE}")
dockerfile_rocm="${bake_dir}/Dockerfile.rocm"
mapfile -t content_args < <(
get_content_arg_names "${dockerfile_rocm}" "base rust_toolchain_input_0 rust_toolchain_input_1 rust-toolchain-input rust_input_0 rust_input_1 rust-input rust-toolchain rust-build" "${ROCM_RUST_CONTENT_ARGS:-}"
)
{
printf 'rust-input-files-hash:%s\n' "$(compute_content_hash "${content_paths[@]}")"
printf 'dockerfile:%s\n' "${dockerfile_rocm}"
printf 'resolved-build-args:\n'
hash_dockerfile_arg_values "${dockerfile_rocm}" "${content_args[@]}"
printf 'dockerfile-stages:base rust_toolchain_input_0 rust_toolchain_input_1 rust-toolchain-input rust_input_0 rust_input_1 rust-input rust-toolchain rust-build\n'
if [[ -f "${dockerfile_rocm}" ]]; then
hash_dockerfile_stages "${dockerfile_rocm}" "base rust_toolchain_input_0 rust_toolchain_input_1 rust-toolchain-input rust_input_0 rust_input_1 rust-input rust-toolchain rust-build"
else
printf 'missing:%s\n' "${dockerfile_rocm}"
fi
} | sha256sum | cut -d' ' -f1
}
compute_rocm_rust_content_hash_if_needed() {
local cache_repo="${DOCKERHUB_CACHE_REPO:-rocm/vllm-ci-cache}"
if [[ "${ROCM_RUST_CONTENT_CACHE:-1}" == "0" ]] || ! uses_rocm_rust_cache; then
return 0
fi
ROCM_RUST_CONTENT_HASH=$(compute_rocm_rust_content_hash)
ROCM_RUST_CONTENT_CACHE_REF="${cache_repo}:rust-rocm-input-${ROCM_RUST_CONTENT_HASH}"
export ROCM_RUST_CONTENT_HASH
export ROCM_RUST_CONTENT_CACHE_REF
echo "ROCm Rust content cache ref: ${ROCM_RUST_CONTENT_CACHE_REF}"
}
write_hcl_string_list_entries() {
local indent="$1"
local value=""
@@ -1394,18 +1119,6 @@ hcl_escape_string() {
printf '%s' "${value}"
}
join_words() {
local IFS=" "
printf '%s' "$*"
}
metadata_pair() {
local key="$1"
local value="${2:-}"
printf '%s\t%s\n' "${key}" "${value}"
}
write_hcl_string_list() {
local indent="$1"
shift
@@ -1431,7 +1144,6 @@ write_rocm_build_arg_override() {
"${CI_BASE_DOCKERFILE_STAGES:-${DEFAULT_CI_BASE_DOCKERFILE_STAGES}}" \
"${CI_BASE_CONTENT_ARGS:-}"
get_content_arg_names "${dockerfile_rocm}" "base csrc-build" "${ROCM_CSRC_CONTENT_ARGS:-}"
get_content_arg_names "${dockerfile_rocm}" "base rust_toolchain_input_0 rust_toolchain_input_1 rust-toolchain-input rust_input_0 rust_input_1 rust-input rust-toolchain rust-build" "${ROCM_RUST_CONTENT_ARGS:-}"
} | awk 'NF && !seen[$0]++'
)
@@ -1480,133 +1192,46 @@ validate_cache_export_mode() {
esac
}
validate_content_cache_export_mode() {
local mode="$1"
local env_name="$2"
case "${mode}" in
missing|always|never)
;;
*)
echo "Error: ${env_name} must be one of: missing, always, never"
exit 1
;;
esac
}
should_export_content_cache_ref() {
local cache_ref="$1"
local cache_name="$2"
local mode="${ROCM_CONTENT_CACHE_EXPORT_MODE:-missing}"
case "${mode}" in
always)
echo "${cache_name} content cache export mode is always; exporting ${cache_ref}"
return 0
;;
never)
echo "${cache_name} content cache export mode is never; not exporting ${cache_ref}"
return 1
;;
missing|"")
if docker buildx imagetools inspect "${cache_ref}" >/dev/null 2>&1; then
echo "${cache_name} content cache exists; not re-exporting ${cache_ref}"
return 1
fi
echo "${cache_name} content cache missing; will export ${cache_ref}"
return 0
;;
*)
echo "Error: ROCM_CONTENT_CACHE_EXPORT_MODE must be one of: missing, always, never"
exit 1
;;
esac
}
write_rocm_cache_override() {
local cache_repo="${DOCKERHUB_CACHE_REPO:-rocm/vllm-ci-cache}"
local content_cache_export_mode="${ROCM_CONTENT_CACHE_EXPORT_MODE:-missing}"
local csrc_cache_to_mode="${ROCM_CSRC_CACHE_TO_MODE:-max}"
local rust_cache_to_mode="${ROCM_RUST_CACHE_TO_MODE:-max}"
local rocm_cache_to_mode="${ROCM_FINAL_CACHE_TO_MODE:-min}"
local -a csrc_content_cache_from=()
local -a rust_content_cache_from=()
local -a combined_content_cache_from=()
local -a content_cache_from=()
local -a csrc_cache_to=()
local -a rust_cache_to=()
local -a rocm_cache_to=()
local -a export_wheel_cache_to=()
local export_csrc_cache=1
local export_rust_cache=1
if ! uses_rocm_csrc_cache && ! uses_rocm_rust_cache; then
if ! uses_rocm_csrc_cache; then
return 0
fi
validate_content_cache_export_mode \
"${content_cache_export_mode}" \
"ROCM_CONTENT_CACHE_EXPORT_MODE"
validate_cache_export_mode "${csrc_cache_to_mode}" "ROCM_CSRC_CACHE_TO_MODE"
validate_cache_export_mode "${rust_cache_to_mode}" "ROCM_RUST_CACHE_TO_MODE"
validate_cache_export_mode "${rocm_cache_to_mode}" "ROCM_FINAL_CACHE_TO_MODE"
echo "ROCm content cache export mode: ${content_cache_export_mode}"
echo "ROCm csrc cache export mode: ${csrc_cache_to_mode}"
echo "ROCm Rust cache export mode: ${rust_cache_to_mode}"
echo "ROCm final image cache export mode: ${rocm_cache_to_mode}"
if [[ -n "${ROCM_CSRC_CONTENT_CACHE_REF:-}" ]]; then
csrc_content_cache_from+=("type=registry,ref=${ROCM_CSRC_CONTENT_CACHE_REF}")
if should_export_content_cache_ref "${ROCM_CSRC_CONTENT_CACHE_REF}" "ROCm csrc"; then
csrc_cache_to+=(
"type=registry,ref=${ROCM_CSRC_CONTENT_CACHE_REF},mode=${csrc_cache_to_mode},ignore-error=true"
)
else
export_csrc_cache=0
fi
content_cache_from+=("type=registry,ref=${ROCM_CSRC_CONTENT_CACHE_REF}")
csrc_cache_to+=(
"type=registry,ref=${ROCM_CSRC_CONTENT_CACHE_REF},mode=${csrc_cache_to_mode},ignore-error=true"
)
fi
if [[ -n "${ROCM_RUST_CONTENT_CACHE_REF:-}" ]]; then
rust_content_cache_from+=("type=registry,ref=${ROCM_RUST_CONTENT_CACHE_REF}")
if should_export_content_cache_ref "${ROCM_RUST_CONTENT_CACHE_REF}" "ROCm Rust"; then
rust_cache_to+=(
"type=registry,ref=${ROCM_RUST_CONTENT_CACHE_REF},mode=${rust_cache_to_mode},ignore-error=true"
)
else
export_rust_cache=0
fi
fi
combined_content_cache_from=("${csrc_content_cache_from[@]}" "${rust_content_cache_from[@]}")
# Docker Hub cache exports are best-effort. A cache-only target failure can
# otherwise cancel the sibling image target before its manifest is pushed.
if [[ -n "${BUILDKITE_COMMIT:-}" ]]; then
if [[ ${export_csrc_cache} -eq 1 ]]; then
csrc_cache_to+=(
"type=registry,ref=${cache_repo}:csrc-rocm-${BUILDKITE_COMMIT},mode=${csrc_cache_to_mode},ignore-error=true"
)
fi
if [[ ${export_rust_cache} -eq 1 ]]; then
rust_cache_to+=(
"type=registry,ref=${cache_repo}:rust-rocm-${BUILDKITE_COMMIT},mode=${rust_cache_to_mode},ignore-error=true"
)
fi
csrc_cache_to+=(
"type=registry,ref=${cache_repo}:csrc-rocm-${BUILDKITE_COMMIT},mode=${csrc_cache_to_mode},ignore-error=true"
)
rocm_cache_to+=(
"type=registry,ref=${cache_repo}:rocm-${BUILDKITE_COMMIT},mode=${rocm_cache_to_mode},ignore-error=true"
)
fi
if [[ -n "${ROCM_CACHE_BRANCH_TAG:-}" ]]; then
if [[ ${export_csrc_cache} -eq 1 ]]; then
csrc_cache_to+=(
"type=registry,ref=${cache_repo}:csrc-rocm-branch-${ROCM_CACHE_BRANCH_TAG},mode=${csrc_cache_to_mode},ignore-error=true"
)
fi
if [[ ${export_rust_cache} -eq 1 ]]; then
rust_cache_to+=(
"type=registry,ref=${cache_repo}:rust-rocm-branch-${ROCM_CACHE_BRANCH_TAG},mode=${rust_cache_to_mode},ignore-error=true"
)
fi
csrc_cache_to+=(
"type=registry,ref=${cache_repo}:csrc-rocm-branch-${ROCM_CACHE_BRANCH_TAG},mode=${csrc_cache_to_mode},ignore-error=true"
)
rocm_cache_to+=(
"type=registry,ref=${cache_repo}:rocm-branch-${ROCM_CACHE_BRANCH_TAG},mode=${rocm_cache_to_mode},ignore-error=true"
)
@@ -1624,7 +1249,7 @@ target "csrc-rocm-ci" {
cache-from = concat(
get_cache_from_rocm_csrc(),
EOF
write_hcl_string_list " " "${csrc_content_cache_from[@]}"
write_hcl_string_list " " "${content_cache_from[@]}"
cat <<EOF
)
EOF
@@ -1632,23 +1257,11 @@ EOF
cat <<EOF
}
target "rust-rocm-ci" {
cache-from = concat(
get_cache_from_rocm_rust(),
EOF
write_hcl_string_list " " "${rust_content_cache_from[@]}"
cat <<EOF
)
EOF
write_hcl_string_list_attr " " "cache-to" "${rust_cache_to[@]}"
cat <<EOF
}
target "test-rocm-ci" {
cache-from = concat(
get_cache_from_rocm(),
EOF
write_hcl_string_list " " "${combined_content_cache_from[@]}"
write_hcl_string_list " " "${content_cache_from[@]}"
cat <<EOF
)
EOF
@@ -1660,7 +1273,7 @@ target "export-wheel-rocm" {
cache-from = concat(
get_cache_from_rocm(),
EOF
write_hcl_string_list " " "${combined_content_cache_from[@]}"
write_hcl_string_list " " "${content_cache_from[@]}"
cat <<EOF
)
EOF
@@ -1928,13 +1541,7 @@ confirm_remote_image_push() {
remote_hash=$(get_remote_image_label_with_retry "${image_ref}" "vllm.ci_base.content_hash")
if [[ -n "${remote_hash}" && "${remote_hash}" == "${CI_BASE_CONTENT_HASH}" ]]; then
if remote_ci_base_metadata_is_current_with_retry "${image_ref}"; then
return 0
fi
echo "Remote image exists with the expected ci_base content hash but stale metadata."
echo " expected metadata version: ${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}"
return 1
return 0
fi
echo "Remote image exists but does not have the expected ci_base content hash."
@@ -1950,8 +1557,8 @@ confirm_remote_image_push() {
fi
if [[ -z "${remote_revision}" \
&& ${IMAGE_EXISTED_BEFORE_BUILD} -eq 0 ]] \
&& image_tag_is_commit_scoped; then
&& ${IMAGE_EXISTED_BEFORE_BUILD} -eq 0 \
&& image_tag_is_commit_scoped ]]; then
echo "Remote image exists under a commit-scoped tag; accepting push despite missing revision label."
return 0
fi
@@ -2009,10 +1616,7 @@ seed_dependency_caches_if_needed() {
echo "--- :docker: Seeding ${target}"
echo "Expected cache ref: ${cache_ref}"
docker buildx bake \
"${BAKE_FILES[@]}" \
--progress "${BUILDKIT_PROGRESS:-plain}" \
"${target}"
docker buildx bake "${BAKE_FILES[@]}" --progress plain "${target}"
verify_dependency_cache_ref "${cache_ref}"
done
}
@@ -2040,10 +1644,7 @@ run_bake() {
local build_rc=0
echo "--- :docker: Building ${TARGET}"
docker buildx bake \
"${BAKE_FILES[@]}" \
--progress "${BUILDKIT_PROGRESS:-plain}" \
"${BAKE_TARGETS[@]}" || build_rc=$?
docker buildx bake "${BAKE_FILES[@]}" --progress plain "${BAKE_TARGETS[@]}" || build_rc=$?
if [[ ${build_rc} -eq 0 ]]; then
echo "--- :white_check_mark: Build complete"
@@ -2081,57 +1682,36 @@ upload_wheel_artifacts_if_present() {
local wheel_dir="./wheel-export"
local artifact_dir="artifacts/vllm-rocm-install"
local archive_name="vllm-rocm-install.tar.gz"
local metadata_dir="${wheel_dir}/.vllm-ci-artifact"
local native_base_image=""
local whl=""
local whl_name=""
local -a wheels=()
if ! should_upload_wheel_artifacts; then
return 0
fi
if [[ -d "${wheel_dir}" ]]; then
mapfile -t wheels < <(find "${wheel_dir}" -maxdepth 1 -type f -name '*.whl' -print)
fi
if [[ ${#wheels[@]} -ne 1 ]]; then
echo "Expected exactly one ROCm wheel in ${wheel_dir}; found ${#wheels[@]}" >&2
return 1
fi
whl="${wheels[0]}"
whl_name=$(basename "${whl}")
native_base_image="${CI_BASE_IMAGE_TAG_COMMIT_REF:-${CI_BASE_IMAGE:-}}"
if [[ -z "${native_base_image}" ]]; then
echo "Native ROCm artifact requires a ci_base image reference" >&2
return 1
if [[ ! -d "${wheel_dir}" ]] || ! ls "${wheel_dir}"/*.whl >/dev/null 2>&1; then
echo "No ROCm wheel artifacts found in ${wheel_dir}"
return 0
fi
echo "--- :package: Uploading ROCm vLLM install artifact"
rm -rf "${artifact_dir}" "${metadata_dir}"
mkdir -p "${artifact_dir}" "${metadata_dir}"
printf '%s\n' "${BUILDKITE_COMMIT:-local}" > "${metadata_dir}/commit.txt"
printf '%s\n' "${native_base_image}" > "${metadata_dir}/native-base-image.txt"
printf '%s\n' "${CI_BASE_IMAGE:-}" > "${metadata_dir}/ci-base-image.txt"
printf '%s\n' "${IMAGE_TAG:-}" > "${metadata_dir}/fallback-image.txt"
printf '%s\n' "${whl_name}" > "${metadata_dir}/wheel-filename.txt"
mkdir -p "${artifact_dir}"
tar -C "${wheel_dir}" -czf "${artifact_dir}/${archive_name}" .
(
cd "${artifact_dir}"
sha256sum "${archive_name}" > "${archive_name}.sha256"
)
echo "Created ${archive_name}: $(du -sh "${artifact_dir}/${archive_name}" | cut -f1)"
cp "${metadata_dir}"/*.txt "${artifact_dir}/"
cp "${whl}" "${artifact_dir}/${whl_name}"
echo "Copied ${whl_name}: $(du -sh "${artifact_dir}/${whl_name}" | cut -f1)"
printf '%s\n' "${CI_BASE_IMAGE:-}" > "${artifact_dir}/ci-base-image.txt"
printf '%s\n' "${IMAGE_TAG:-}" > "${artifact_dir}/fallback-image.txt"
for whl in "${wheel_dir}"/*.whl; do
[[ -f "${whl}" ]] || continue
whl_name=$(basename "${whl}")
cp "${whl}" "${artifact_dir}/${whl_name}"
echo "Copied ${whl_name}: $(du -sh "${artifact_dir}/${whl_name}" | cut -f1)"
done
if command -v buildkite-agent >/dev/null 2>&1; then
buildkite-agent artifact upload "${artifact_dir}/*" || return 1
buildkite-agent artifact upload "${artifact_dir}/*"
echo "ROCm vLLM install artifacts uploaded to ${artifact_dir}/"
elif [[ "${BUILDKITE:-false}" == "true" ]]; then
echo "buildkite-agent not found; cannot upload required ROCm artifacts" >&2
return 1
else
echo "Not in Buildkite, skipping artifact upload"
fi
@@ -2144,18 +1724,16 @@ main() {
print_header
validate_inputs
load_ci_hcl
init_bake_files
compute_ci_base_hash_if_needed
configure_ci_base_image_refs
maybe_skip_existing_image
setup_builder
prepare_git_cache_metadata
write_ci_base_label_override
extract_dependency_pins
write_rocm_build_arg_override
compute_dependency_cache_keys
write_ci_base_label_override
compute_rocm_csrc_content_hash_if_needed
compute_rocm_rust_content_hash_if_needed
write_rocm_cache_override
resolve_ci_base_dependency_targets
print_bake_config
@@ -2163,11 +1741,6 @@ main() {
echo "BAKE_PRINT_ONLY=1 set; skipping build"
return 0
fi
if should_upload_wheel_artifacts; then
# wheel-export is an output directory, not a BuildKit cache. Starting
# clean prevents a failed/retried export from packaging a stale wheel.
rm -rf ./wheel-export
fi
seed_dependency_caches_if_needed
run_bake
upload_wheel_artifacts_if_present
-3
View File
@@ -13,8 +13,5 @@ INPUT_FILE="$1"
# Strip timestamps
sed -i 's/^\[[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}T[0-9]\{2\}:[0-9]\{2\}:[0-9]\{2\}Z\] //' "$INPUT_FILE"
# Strip Buildkite inline timestamp markers (ESC _bk;t=<ms> BEL)
sed -i 's/\x1B_bk;t=[0-9]*\x07//g' "$INPUT_FILE"
# Strip colorization
sed -i -r 's/\x1B\[[0-9;]*[mK]//g' "$INPUT_FILE"
+47 -151
View File
@@ -1,178 +1,74 @@
#!/bin/bash
# Fetch vLLM Buildkite CI logs (public; no login required).
# Usage: ./ci-fetch-log.sh <buildkite_job_url> [output_file]
# ./ci-fetch-log.sh <build_number> <job_uuid> [output_file]
#
# Usage:
# ci-fetch-log.sh [--soft|--all] --pr [<PR>] failed jobs in the PR's latest
# build (current branch if omitted)
# ci-fetch-log.sh [--soft|--all] <build_url> failed jobs in that build
# ci-fetch-log.sh <job_url> [output] one job; both #<job_uuid> and
# ?sid=<id> URL forms work
# ci-fetch-log.sh <build> <job_uuid> [output]
# Downloads the raw log for a Buildkite job from the public, unauthenticated
# /organizations/<org>/pipelines/<pipeline>/builds/<n>/jobs/<uuid>/download
# endpoint, then strips ANSI/timestamps via ci-clean-log.sh.
#
# --soft also fetches soft-failed jobs; --all fetches every finished job.
# Saves each log as ci-<build>-<job-name>.log (ANSI/timestamps stripped) and
# prints "<file>\t<job name>" per job. [output] is single-job only; "-"
# streams to stdout. Existing files are kept; CI_FETCH_LOG_FORCE=1 refetches.
# Find <build_number> and <job_uuid> via:
# gh pr checks <PR> --repo vllm-project/vllm
# Each failing row's URL is .../builds/<build_number>#<job_uuid>.
#
# Default output path: ci-<build>-<uuid_first_13_chars>.log (e.g.
# ci-68478-019e6b07-daae.log). Jobs in the same build share the UUID's
# first 8 chars, so the second segment is needed for uniqueness when
# fetching multiple jobs in parallel. The script refuses to overwrite an
# existing output file; pass an explicit path or set CI_FETCH_LOG_FORCE=1
# to override.
set -euo pipefail
ORG="vllm"
PIPELINE="ci"
UA="vllm-ci-fetch-log"
UUID_RE='[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}'
usage() {
sed -n '2,15p' "$0" | sed 's/^# \{0,1\}//'
echo "Usage: $0 <buildkite_job_url> [output_file]"
echo " $0 <build_number> <job_uuid> [output_file]"
exit 1
}
die() {
echo "$1" >&2
exit 1
}
if [ $# -lt 1 ]; then usage; fi
BUILD="" JOB="" SID="" OUT=""
SCOPE="failed"
while :; do
case "${1:-}" in
--soft) SCOPE="soft" ;;
--all) SCOPE="all" ;;
*) break ;;
esac
shift
done
case "${1:-}" in
--pr)
PR="${2:-}"
# gh pr checks exits non-zero when checks are failing; that is the
# expected case here.
URL=$(gh pr checks ${PR:+"$PR"} --repo vllm-project/vllm 2>/dev/null |
grep -oE "https://buildkite.com/${ORG}/${PIPELINE}/builds/[0-9]+" |
sort -t/ -k7 -n | tail -1 || true)
[ -n "$URL" ] || die "No Buildkite build found via: gh pr checks ${PR:-<current branch>}"
BUILD="${URL##*/}"
;;
https://*)
if [[ "$1" == https://* ]]; then
BUILD=$(echo "$1" | sed -nE 's#.*/builds/([0-9]+).*#\1#p')
JOB=$(echo "$1" | grep -oE "#${UUID_RE}" | head -n 1 | cut -c2- || true)
SID=$(echo "$1" | grep -oE "[?&]sid=${UUID_RE}" | head -n 1 | sed 's/.*sid=//' || true)
JOB=$(echo "$1" | grep -oE '[0-9a-f]{8}-[0-9a-f-]+' | head -n 1)
OUT="${2:-}"
[ -n "$BUILD" ] || die "Could not parse build number from: $1"
;;
[0-9]*)
[ $# -ge 2 ] || usage
else
if [ $# -lt 2 ]; then usage; fi
BUILD="$1"
JOB="$2"
OUT="${3:-}"
;;
*)
fi
if [ -z "$BUILD" ] || [ -z "$JOB" ]; then
echo "Could not parse build number or job UUID from: $1" >&2
usage
;;
esac
fi
# Jobs in the same build share the UUID's first segment, so include the
# second segment (chars 9-13, e.g. "019e6b07-daae") to keep default filenames
# unique when fetching multiple jobs from one build in parallel.
if [ -z "$OUT" ]; then
OUT="ci-${BUILD}-${JOB:0:13}.log"
fi
if [ -e "$OUT" ] && [ -z "${CI_FETCH_LOG_FORCE:-}" ]; then
echo "Refusing to overwrite existing $OUT (set CI_FETCH_LOG_FORCE=1 or pass an explicit output path)." >&2
exit 1
fi
COOKIES=$(mktemp)
JOBS_TSV=$(mktemp)
trap 'rm -f "$COOKIES" "$JOBS_TSV"' EXIT
trap 'rm -f "$COOKIES"' EXIT
# Buildkite issues a session cookie on first hit; later requests need it.
curl -fsSL -c "$COOKIES" -A "$UA" \
# Buildkite issues a session cookie on first hit; subsequent /download needs it.
curl -fsSL -c "$COOKIES" -A "vllm-ci-fetch-log" \
"https://buildkite.com/${ORG}/${PIPELINE}/builds/${BUILD}" -o /dev/null
# The build's job list (id, step uuid, state, name) is served as JSON from
# the user-facing /data/jobs endpoint. Flatten it to TSV for easy filtering:
# job_id step_uuid failed soft_failed finished slug name
curl -fsSL -b "$COOKIES" -A "$UA" \
"https://buildkite.com/${ORG}/${PIPELINE}/builds/${BUILD}/data/jobs" |
python3 -c '
import json, re, sys
curl -fsSL -b "$COOKIES" -A "vllm-ci-fetch-log" \
"https://buildkite.com/organizations/${ORG}/pipelines/${PIPELINE}/builds/${BUILD}/jobs/${JOB}/download" \
-o "$OUT"
data = json.load(sys.stdin)
if data.get("has_next_page"):
print("warning: job list is paginated; some jobs not shown", file=sys.stderr)
for r in data["records"]:
if r.get("type") != "script":
continue
name = (r.get("name") or "").replace("\t", " ").replace("\n", " ")
slug = re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-")[:60]
print("\t".join([
r["id"],
r.get("step_uuid") or "",
str(r.get("passed") is False),
str(bool(r.get("soft_failed"))),
str(bool(r.get("finished_at"))),
slug,
name,
]))
' >"$JOBS_TSV" || die "Could not list jobs for build ${BUILD}"
bash "$(dirname "$0")/ci-clean-log.sh" "$OUT"
if [ -n "$SID" ] && [ -z "$JOB" ]; then
# The ?sid= in builds/<N>/list URLs is the *step* uuid, not the job uuid.
JOB=$(awk -F'\t' -v s="$SID" '$1 == s || $2 == s {print $1; exit}' "$JOBS_TSV")
[ -n "$JOB" ] || die "No job matching sid=${SID} in build ${BUILD}"
fi
fetch_job() { # <job_uuid> <output_file>
curl -fsSL -b "$COOKIES" -A "$UA" \
"https://buildkite.com/organizations/${ORG}/pipelines/${PIPELINE}/builds/${BUILD}/jobs/$1/download" \
-o "$2"
bash "$(dirname "$0")/ci-clean-log.sh" "$2"
}
if [ -n "$JOB" ]; then
# Single-job mode.
NAME=$(awk -F'\t' -v j="$JOB" '$1 == j {print $7; exit}' "$JOBS_TSV")
SLUG=$(awk -F'\t' -v j="$JOB" '$1 == j {print $6; exit}' "$JOBS_TSV")
[ -n "$OUT" ] || OUT="ci-${BUILD}-${SLUG:-${JOB:0:13}}.log"
if [ "$OUT" = "-" ]; then
TMP=$(mktemp)
fetch_job "$JOB" "$TMP"
cat "$TMP"
rm -f "$TMP"
exit 0
fi
if [ -e "$OUT" ] && [ -z "${CI_FETCH_LOG_FORCE:-}" ]; then
die "Refusing to overwrite existing ${OUT} (set CI_FETCH_LOG_FORCE=1 or pass an output path)."
fi
fetch_job "$JOB" "$OUT"
printf '%s\t%s\n' "$OUT" "${NAME:-$JOB}"
exit 0
fi
# Build-wide mode: fetch finished jobs matching $SCOPE.
[ -z "$OUT" ] || die "[output_file] is only valid when fetching a single job."
case "$SCOPE" in
failed) FILTER='$3 == "True" && $4 == "False" && $5 == "True"' ;;
soft) FILTER='$3 == "True" && $5 == "True"' ;;
all) FILTER='$5 == "True"' ;;
esac
if [ "$SCOPE" = "failed" ]; then
SOFT=$(awk -F'\t' '$3 == "True" && $4 == "True"' "$JOBS_TSV" | wc -l)
[ "$SOFT" -eq 0 ] || echo "Skipping ${SOFT} soft-failed job(s); use --soft to include them." >&2
fi
FOUND=0
EMITTED=" "
while IFS=$'\t' read -r job_id _ _ _ _ slug name; do
FOUND=$((FOUND + 1))
out="ci-${BUILD}-${slug:-${job_id:0:13}}.log"
# Retries share a name with the original job; disambiguate by uuid.
case "$EMITTED" in
*" $out "*) out="ci-${BUILD}-${slug:-job}-${job_id:0:13}.log" ;;
esac
EMITTED="${EMITTED}${out} "
if [ -e "$out" ] && [ -z "${CI_FETCH_LOG_FORCE:-}" ]; then
echo "Keeping existing ${out} (set CI_FETCH_LOG_FORCE=1 to refetch)." >&2
elif ! fetch_job "$job_id" "$out"; then
echo "Failed to download log for job ${job_id} (${name})." >&2
continue
fi
printf '%s\t%s\n' "$out" "$name"
done < <(awk -F'\t' "$FILTER" "$JOBS_TSV")
if [ "$FOUND" -eq 0 ]; then
echo "No matching jobs in build ${BUILD} (scope: ${SCOPE})." >&2
fi
echo "$OUT"
@@ -45,10 +45,8 @@ $PYTHON .buildkite/scripts/generate-nightly-index.py --version "$SUBPATH" --curr
echo "Uploading indices to $S3_COMMIT_PREFIX"
aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "$S3_COMMIT_PREFIX"
# copy to /nightly/ only when enabled for a main branch build that is not a PR
if [[ "${UPDATE_NIGHTLY_INDEX:-1}" == "1" && \
"$BUILDKITE_BRANCH" == "main" && \
"$BUILDKITE_PULL_REQUEST" == "false" ]]; then
# copy to /nightly/ only if it is on the main branch and not a PR
if [[ "$BUILDKITE_BRANCH" == "main" && "$BUILDKITE_PULL_REQUEST" == "false" ]]; then
echo "Uploading indices to overwrite /nightly/"
aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "s3://$BUCKET/nightly/"
fi
@@ -69,7 +67,7 @@ pure_version="${version%%+*}"
echo "Pure version (without variant): $pure_version"
# re-generate and copy to /<pure_version>/ only if it does not have "dev" in the version
if [[ "${UPDATE_VERSION_INDEX:-1}" == "1" && "$version" != *"dev"* ]]; then
if [[ "$version" != *"dev"* ]]; then
echo "Re-generating indices for /$pure_version/"
rm -rf "${INDICES_OUTPUT_DIR:?}"
mkdir -p "$INDICES_OUTPUT_DIR"
+4 -429
View File
@@ -1,7 +1,7 @@
#!/bin/bash
# This script runs ROCm tests either directly in a native CI pod or inside the
# corresponding Docker container. Multi-node tests continue to use Docker.
# This script runs tests inside the corresponding ROCm docker container.
# It handles both single-node and multi-node test configurations.
#
# Multi-node detection: Instead of matching on fragile group names, we detect
# multi-node jobs structurally by looking for the bracket command syntax
@@ -28,34 +28,6 @@
###############################################################################
set -o pipefail
: "${BUILDKIT_PROGRESS:=plain}"
: "${TERM:=xterm-256color}"
: "${FORCE_COLOR:=1}"
: "${CLICOLOR_FORCE:=1}"
: "${PY_COLORS:=1}"
: "${ROCM_DOCKER_TTY:=1}"
: "${PYTHONFAULTHANDLER:=1}"
: "${PYTEST_TIMEOUT:=2100}"
if [[ " ${PYTEST_ADDOPTS:-} " != *" --color"* ]]; then
PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--color=yes"
fi
if [[ " ${PYTEST_ADDOPTS:-} " != *" --durations="* ]]; then
PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--durations=25"
fi
if [[ " ${PYTEST_ADDOPTS:-} " != *" --durations-min="* ]]; then
PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--durations-min=1.0"
fi
# Dump stacks after 15 minutes, then stop an individual test after 35 minutes.
if [[ " ${PYTEST_ADDOPTS:-} " != *" faulthandler_timeout="* ]]; then
PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }-o faulthandler_timeout=900"
fi
if [[ " ${PYTEST_ADDOPTS:-} " != *" --timeout-method="* &&
" ${PYTEST_ADDOPTS:-} " != *" --timeout-method "* ]]; then
PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--timeout-method=thread"
fi
export BUILDKIT_PROGRESS TERM FORCE_COLOR CLICOLOR_FORCE PY_COLORS PYTEST_ADDOPTS PYTEST_TIMEOUT ROCM_DOCKER_TTY
export PYTHONFAULTHANDLER
# Export Python path for commands that run directly on the host. Containerized
# tests set this to /vllm-workspace below so spawned Python processes do not
# depend on their current working directory.
@@ -70,28 +42,6 @@ report_docker_usage() {
docker system df || true
}
clear_ci_orchestration_env() {
unset -v \
VLLM_TEST_GROUP_NAME \
VLLM_CI_REQUIRE_PERSISTENT_HF_CACHE \
VLLM_CI_ARTIFACT_STEP \
VLLM_TEST_CACHE \
VLLM_CI_EXECUTION_MODE \
VLLM_CI_WORKSPACE \
VLLM_CI_REQUIRE_WORKSPACE_MOUNT \
VLLM_TEST_COMMANDS \
VLLM_CI_BRANCH \
VLLM_CI_BASE_IMAGE \
VLLM_CI_FALLBACK_IMAGE \
VLLM_CI_DOCKER_DISABLED \
VLLM_CI_ARTIFACT_GLOB \
VLLM_CI_ARTIFACT_CHECKSUM_GLOB \
VLLM_CI_EXPECTED_GPU_COUNT \
VLLM_CI_USE_ARTIFACTS \
VLLM_CI_RESULTS_ROOT \
VLLM_ALLOW_DEPRECATED_BEAM_SEARCH
}
cleanup_network() {
local max_nodes=${NUM_NODES:-2}
for node in $(seq 0 $((max_nodes - 1))); do
@@ -184,11 +134,7 @@ prepare_artifact_image() {
fi
cp "${wheel_dir}"/*.whl "${context_dir}/wheels/" || return 1
tar -C "${wheel_dir}" \
--exclude='*.whl' \
--exclude='.vllm-ci-artifact' \
--exclude='./.vllm-ci-artifact' \
-cf - . \
tar -C "${wheel_dir}" --exclude='*.whl' -cf - . \
| tar -C "${workspace_dir}" -xf - || return 1
cat > "${context_dir}/Dockerfile" <<'EOF'
ARG BASE_IMAGE
@@ -203,7 +149,6 @@ EOF
echo "--- Building local ROCm test image"
docker build \
--pull=false \
--progress "${BUILDKIT_PROGRESS}" \
--build-arg "BASE_IMAGE=${base_image}" \
-t "${artifact_image}" \
"${context_dir}" || return 1
@@ -211,257 +156,6 @@ EOF
return 0
}
is_native_runtime() {
[[ "${AMD_CI_RUNTIME:-}" == "native" || "${NATIVE_CI:-}" == "true" ]]
}
validate_native_workspace() {
local workspace_dir="${VLLM_CI_WORKSPACE:-/vllm-workspace}"
local workspace_real=""
local checkout_real=""
local workspace_mount=""
mkdir -p "${workspace_dir}" || return 1
workspace_real=$(readlink -m "${workspace_dir}") || return 1
if [[ -n "${BUILDKITE_BUILD_CHECKOUT_PATH:-}" ]]; then
checkout_real=$(readlink -m "${BUILDKITE_BUILD_CHECKOUT_PATH}") || return 1
if [[ "${checkout_real}" == "${workspace_real}" \
|| "${checkout_real}" == "${workspace_real}/"* \
|| "${workspace_real}" == "${checkout_real}/"* ]]; then
echo "Refusing to replace ${workspace_real}; it overlaps the Buildkite checkout ${checkout_real}" >&2
return 1
fi
fi
if [[ "${VLLM_CI_REQUIRE_WORKSPACE_MOUNT:-1}" == "1" ]]; then
if ! command -v findmnt >/dev/null 2>&1; then
echo "findmnt is required to verify the native workspace mount" >&2
return 1
fi
workspace_mount=$(findmnt -n -T "${workspace_real}" -o TARGET 2>/dev/null || true)
if [[ "$(readlink -m "${workspace_mount:-/}")" != "${workspace_real}" ]]; then
echo "Native CI requires a dedicated volume mounted at ${workspace_real}" >&2
return 1
fi
fi
}
prepare_native_workspace() {
if [[ "${VLLM_CI_USE_ARTIFACTS:-0}" != "1" ]]; then
echo "Native CI requires VLLM_CI_USE_ARTIFACTS=1"
return 1
fi
if ! command -v buildkite-agent >/dev/null 2>&1; then
echo "buildkite-agent not found; cannot download ROCm wheel artifact"
return 1
fi
validate_native_workspace || return 1
local artifact_glob="${VLLM_CI_ARTIFACT_GLOB:-artifacts/vllm-rocm-install/vllm-rocm-install.tar.gz}"
local artifact_checksum_glob="${VLLM_CI_ARTIFACT_CHECKSUM_GLOB:-${artifact_glob}.sha256}"
local artifact_step="${VLLM_CI_ARTIFACT_STEP:-image-build-amd}"
local archive=""
local checksum=""
local download_dir=""
local metadata_dir=""
local recorded_base=""
local recorded_commit=""
local recorded_wheel=""
local workspace_dir="${VLLM_CI_WORKSPACE:-/vllm-workspace}"
local wheel_dir=""
local attempt=0
local attempt_dir=""
local -a archives=()
local -a checksums=()
local -a wheels=()
artifact_work_dir=$(mktemp -d -t vllm-rocm-artifact.XXXXXX) || return 1
wheel_dir="${artifact_work_dir}/wheels"
mkdir -p "${wheel_dir}" || return 1
echo "--- Downloading ROCm wheel artifact from ${artifact_step} (native in-pod)"
for attempt in 1 2 3; do
attempt_dir="${artifact_work_dir}/download-${attempt}"
rm -rf "${attempt_dir}" || return 1
mkdir -p "${attempt_dir}" || return 1
if buildkite-agent artifact download \
"${artifact_glob}" "${attempt_dir}" --step "${artifact_step}" \
&& buildkite-agent artifact download \
"${artifact_checksum_glob}" "${attempt_dir}" --step "${artifact_step}"; then
download_dir="${attempt_dir}"
break
fi
echo "Artifact download attempt ${attempt}/3 failed"
if [[ "${attempt}" -lt 3 ]]; then
sleep $((attempt * 2))
fi
done
if [[ -z "${download_dir}" ]]; then
echo "Failed to download ${artifact_glob} and ${artifact_checksum_glob} from ${artifact_step}"
return 1
fi
mapfile -t archives < <(
find "${download_dir}" -name "vllm-rocm-install.tar.gz" -type f -print
)
mapfile -t checksums < <(
find "${download_dir}" -name "vllm-rocm-install.tar.gz.sha256" -type f -print
)
if [[ ${#archives[@]} -ne 1 || ${#checksums[@]} -ne 1 ]]; then
echo "Expected exactly one ROCm archive and checksum; found ${#archives[@]} archive(s) and ${#checksums[@]} checksum(s)" >&2
return 1
fi
archive="${archives[0]}"
checksum="${checksums[0]}"
if [[ "$(dirname "${archive}")" != "$(dirname "${checksum}")" ]]; then
echo "ROCm archive and checksum were downloaded to different directories" >&2
return 1
fi
(
cd "$(dirname "${archive}")"
sha256sum -c "$(basename "${checksum}")"
) || return 1
tar --no-same-owner -xzf "${archive}" -C "${wheel_dir}" || return 1
mapfile -t wheels < <(
find "${wheel_dir}" -maxdepth 1 -type f -name '*.whl' -print
)
if [[ ${#wheels[@]} -ne 1 ]]; then
echo "ROCm artifact must contain exactly one top-level wheel; found ${#wheels[@]}" >&2
return 1
fi
metadata_dir="${wheel_dir}/.vllm-ci-artifact"
for metadata_file in commit.txt native-base-image.txt wheel-filename.txt; do
if [[ ! -s "${metadata_dir}/${metadata_file}" ]]; then
echo "ROCm artifact metadata is missing ${metadata_file}" >&2
return 1
fi
done
for metadata_file in ci-base-image.txt fallback-image.txt; do
if [[ ! -f "${metadata_dir}/${metadata_file}" ]]; then
echo "ROCm artifact metadata is missing ${metadata_file}" >&2
return 1
fi
done
recorded_commit=$(tr -d '\r\n' < "${metadata_dir}/commit.txt")
recorded_base=$(tr -d '\r\n' < "${metadata_dir}/native-base-image.txt")
recorded_wheel=$(tr -d '\r\n' < "${metadata_dir}/wheel-filename.txt")
if [[ -z "${BUILDKITE_COMMIT:-}" || "${recorded_commit}" != "${BUILDKITE_COMMIT}" ]]; then
echo "ROCm artifact commit ${recorded_commit} does not match ${BUILDKITE_COMMIT:-unset}" >&2
return 1
fi
if [[ -z "${VLLM_CI_BASE_IMAGE:-}" || "${recorded_base}" != "${VLLM_CI_BASE_IMAGE}" ]]; then
echo "ROCm artifact base ${recorded_base} does not match ${VLLM_CI_BASE_IMAGE:-unset}" >&2
return 1
fi
if [[ "${recorded_wheel}" != "$(basename "${wheels[0]}")" ]]; then
echo "ROCm artifact wheel manifest ${recorded_wheel} does not match $(basename "${wheels[0]}")" >&2
return 1
fi
for required_dir in tests .buildkite requirements; do
if [[ ! -d "${wheel_dir}/${required_dir}" ]]; then
echo "ROCm wheel artifact did not contain ${required_dir}/" >&2
return 1
fi
done
echo "--- Installing ROCm wheel into pod environment"
python3 -m pip install --no-deps --force-reinstall "${wheels[0]}" || return 1
echo "--- Preparing ${workspace_dir} from artifact"
find "${workspace_dir}" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + || return 1
tar -C "${wheel_dir}" \
--exclude='*.whl' \
--exclude='.vllm-ci-artifact' \
--exclude='./.vllm-ci-artifact' \
-cf - . | tar --no-same-owner -C "${workspace_dir}" -xf - || return 1
if [[ ! -d "${workspace_dir}/tests" ]]; then
echo "Failed to stage the native test workspace" >&2
return 1
fi
return 0
}
initialize_native_environment() {
local job_id="${BUILDKITE_JOB_ID:-${BUILDKITE_PARALLEL_JOB:-local}}"
local job_id_suffix=""
local native_root=""
local hf_mount=""
if [[ "$(id -u)" -ne 0 ]]; then
echo "Native ROCm CI currently requires the ci_base container to run as root" >&2
return 1
fi
job_id="${job_id//[^A-Za-z0-9_.-]/_}"
job_id_suffix="${job_id##*-}"
job_id_suffix="${job_id_suffix:0:12}"
native_root="/tmp/vllm-native-${job_id}"
TMPDIR="/tmp/vllm-${job_id_suffix}/tmp"
VLLM_RPC_BASE_PATH="/tmp"
: "${TORCHINDUCTOR_CACHE_DIR:=${native_root}/cache/torchinductor}"
: "${TRITON_CACHE_DIR:=${native_root}/cache/triton}"
: "${VLLM_CACHE_ROOT:=${native_root}/cache/vllm}"
: "${XDG_CACHE_HOME:=${native_root}/cache/xdg}"
: "${HF_HOME:=/home/buildkite-agent/huggingface}"
: "${HF_HUB_DOWNLOAD_TIMEOUT:=300}"
: "${HF_HUB_ETAG_TIMEOUT:=60}"
export TMPDIR VLLM_RPC_BASE_PATH
export TORCHINDUCTOR_CACHE_DIR TRITON_CACHE_DIR VLLM_CACHE_ROOT XDG_CACHE_HOME
export HF_HOME HF_HUB_DOWNLOAD_TIMEOUT HF_HUB_ETAG_TIMEOUT
export PYTORCH_ROCM_ARCH=""
mkdir -p "${TMPDIR}" \
"${TORCHINDUCTOR_CACHE_DIR}" \
"${TRITON_CACHE_DIR}" \
"${VLLM_CACHE_ROOT}" \
"${XDG_CACHE_HOME}" \
"${HF_HOME}" || return 1
if [[ "${VLLM_CI_REQUIRE_PERSISTENT_HF_CACHE:-0}" == "1" ]]; then
if ! command -v findmnt >/dev/null 2>&1; then
echo "findmnt is required to verify the native Hugging Face cache mount" >&2
return 1
fi
hf_mount=$(findmnt -n -T "${HF_HOME}" -o TARGET 2>/dev/null || true)
if [[ -z "${hf_mount}" || "${hf_mount}" == "/" ]]; then
echo "Native CI requires a persistent volume mounted at or above ${HF_HOME}" >&2
return 1
fi
fi
}
run_native_preflight() {
local expected_gpus="${VLLM_CI_EXPECTED_GPU_COUNT:-1}"
if [[ ! "${expected_gpus}" =~ ^[0-9]+$ ]]; then
echo "Invalid VLLM_CI_EXPECTED_GPU_COUNT=${expected_gpus}" >&2
return 1
fi
python3 -c "import encodings, importlib.metadata as im, importlib.util as iu; [im.version(d) for d in ('transformers', 'torch', 'ray', 'sympy', 'markupsafe', 'vllm')]; missing=[m for m in ('torch.utils.model_zoo', 'transformers.models.nomic_bert', 'ray.dag', 'sympy.physics', 'markupsafe._speedups') if iu.find_spec(m) is None]; assert not missing, missing" || return 1
if [[ "${expected_gpus}" == "0" ]]; then
echo "Native CPU-only AMD job: skipping ROCm device validation"
return 0
fi
echo "--- ROCm info"
rocminfo || return 1
VLLM_CI_EXPECTED_GPU_COUNT="${expected_gpus}" python3 - <<'PY'
import os
import torch
expected = int(os.environ["VLLM_CI_EXPECTED_GPU_COUNT"])
assert torch.version.hip, "PyTorch is not a ROCm build"
assert torch.cuda.is_available(), "ROCm GPU is not available to PyTorch"
actual = torch.cuda.device_count()
assert actual == expected, f"Expected {expected} ROCm GPU(s), found {actual}"
PY
}
is_multi_node() {
local cmds="$1"
# Primary signal: NUM_NODES environment variable set by the pipeline
@@ -644,58 +338,7 @@ re_quote_pytest_markers() {
# Main
###############################################################################
if is_native_runtime; then
echo "--- Native in-pod ROCm CI (AMD_CI_RUNTIME=${AMD_CI_RUNTIME:-unset}, NATIVE_CI=${NATIVE_CI:-unset})"
artifact_work_dir=""
cleanup_native_workspace() {
if [[ -n "${artifact_work_dir}" ]]; then
rm -rf "${artifact_work_dir}"
fi
}
trap cleanup_native_workspace EXIT
if [[ -n "${VLLM_TEST_COMMANDS:-}" ]]; then
commands="${VLLM_TEST_COMMANDS}"
commands_source="env"
else
commands="$*"
commands_source="argv"
if [[ -z "$commands" ]]; then
echo "Error: No test commands provided for native CI." >&2
exit 1
fi
fi
if [[ "$commands_source" == "argv" ]]; then
commands=$(re_quote_pytest_markers "$commands")
fi
if is_multi_node "$commands"; then
echo "Native CI does not support multi-node jobs yet."
exit 1
fi
if ! initialize_native_environment; then
echo "Failed to initialize the native test environment"
exit 1
fi
if ! prepare_native_workspace; then
echo "Failed to prepare native test workspace"
exit 1
fi
export PYTHONPATH="${VLLM_CI_WORKSPACE:-/vllm-workspace}"
echo "Native test commands: $commands"
run_native_preflight || exit 1
# Keep AMD CI orchestration variables out of vLLM's runtime environment.
clear_ci_orchestration_env
/bin/bash -o pipefail -c "${commands}"
handle_pytest_exit "$?"
fi
# --- GPU initialization for legacy Docker execution ---
# --- GPU initialization ---
echo "--- ROCm info"
rocminfo
@@ -724,20 +367,6 @@ remove_docker_container() {
}
trap remove_docker_container EXIT
# python_only_compile.sh runs `python setup.py develop` and needs the full repo tree
# under /vllm-workspace (Dockerfile.rocm test stage: mkdir src && mv vllm).
# The ROCm wheel artifact tarball only ships a thin tree (tests, etc.), so
# artifact images cannot satisfy that test — use the full rocm/vllm-ci image.
_cmd_probe="${VLLM_TEST_COMMANDS:-}"
if [[ -z "${_cmd_probe}" ]]; then
_cmd_probe="$*"
fi
if [[ "${VLLM_CI_USE_ARTIFACTS:-0}" == "1" && "${_cmd_probe}" == *python_only_compile.sh* ]]; then
echo "INFO: disabling VLLM_CI_USE_ARTIFACTS for python_only_compile (requires full /vllm-workspace tree)"
export VLLM_CI_USE_ARTIFACTS=0
fi
unset -v _cmd_probe
if ! prepare_artifact_image; then
echo "Using full ROCm CI image: ${image_name}"
docker pull "${image_name}" || exit 1
@@ -797,28 +426,6 @@ fi
echo "Final commands: $commands"
standalone_merge_base_env=()
if [[ "$commands" == *python_only_compile.sh* ]]; then
# The ROCm test image often ships /vllm-workspace without .git. Resolve the
# wheels.vllm.ai commit from the agent checkout for this test only.
vllm_standalone_merge_base=""
checkout="${BUILDKITE_BUILD_CHECKOUT_PATH:-}"
if [[ -z "${checkout}" || ! -d "${checkout}" ]]; then
checkout="."
fi
# Pass safe.directory per-command because Buildkite uses mixed user IDs.
if git -c "safe.directory=${checkout}" -C "${checkout}" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
vllm_standalone_merge_base="$(
git -c "safe.directory=${checkout}" -C "${checkout}" merge-base HEAD origin/main 2>/dev/null || true
)"
fi
if [[ -z "${vllm_standalone_merge_base}" ]]; then
vllm_standalone_merge_base="${BUILDKITE_COMMIT:-}"
fi
echo "INFO: passing CI_STANDALONE_MERGE_BASE into container: ${vllm_standalone_merge_base}"
standalone_merge_base_env=(-e "CI_STANDALONE_MERGE_BASE=${vllm_standalone_merge_base}")
fi
MYPYTHONPATH="/vllm-workspace"
container_job_id="${BUILDKITE_JOB_ID:-${BUILDKITE_PARALLEL_JOB:-0}}"
@@ -848,7 +455,6 @@ else
fi
# --- Route: multi-node vs single-node ---
clear_ci_orchestration_env
if is_multi_node "$commands"; then
echo "--- Multi-node job detected"
export DCKR_VER=$(docker --version | sed 's/Docker version \(.*\), build .*/\1/')
@@ -895,37 +501,14 @@ if is_multi_node "$commands"; then
else
echo "--- Single-node job"
echo "Render devices: $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES"
docker_run_terminal_args=(-i)
if [[ "${ROCM_DOCKER_TTY}" == "1" ]]; then
docker_run_terminal_args+=(-t)
echo "Docker interactive stdin: enabled; TTY allocation: enabled"
else
echo "Docker interactive stdin: enabled; TTY allocation: disabled"
fi
ulimit_core_hard=$(ulimit -H -c)
if [[ "$ulimit_core_hard" == "unlimited" ]]; then
# docker run can't pass "unlimited" to --ulimit
ulimit_core_hard="-1"
fi
# Disable core dumps in the ROCm test container unless the ROCm debug agent is enabled
coredump_flags="--ulimit core=0:$ulimit_core_hard"
if [[ "$commands" == *"ROCm debug agent enabled"* ]]; then
# Works around https://github.com/rocm/rocm-systems/issues/6206
coredump_flags='-e HSA_COREDUMP_PATTERN="/tmp/gpucore.%p"'
else
echo "ROCm debug agent not enabled, coredumps are disabled in the test container."
fi
docker run \
"${docker_run_terminal_args[@]}" \
--device /dev/kfd $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES \
$RDMA_FLAGS \
--network=host \
--shm-size=16gb \
--group-add "$render_gid" \
--rm \
$coredump_flags \
-e HF_TOKEN \
-e "HF_HUB_DOWNLOAD_TIMEOUT=${HF_HUB_DOWNLOAD_TIMEOUT}" \
-e "HF_HUB_ETAG_TIMEOUT=${HF_HUB_ETAG_TIMEOUT}" \
@@ -933,13 +516,6 @@ else
-e AWS_SECRET_ACCESS_KEY \
-e BUILDKITE_PARALLEL_JOB \
-e BUILDKITE_PARALLEL_JOB_COUNT \
-e TERM \
-e FORCE_COLOR \
-e CLICOLOR_FORCE \
-e PY_COLORS \
-e PYTHONFAULTHANDLER \
-e PYTEST_ADDOPTS \
-e PYTEST_TIMEOUT \
-v "${HF_CACHE}:${HF_MOUNT}" \
-e "HF_HOME=${HF_MOUNT}" \
-e "PYTHONPATH=${MYPYTHONPATH}" \
@@ -949,7 +525,6 @@ else
-e "VLLM_CACHE_ROOT=${CONTAINER_CACHE_ROOT}/vllm" \
-e "XDG_CACHE_HOME=${CONTAINER_CACHE_ROOT}/xdg" \
-e "PYTORCH_ROCM_ARCH=" \
"${standalone_merge_base_env[@]}" \
--name "${container_name}" \
"${image_name}" \
/bin/bash -c "${CONTAINER_PREFLIGHT} && ${commands}"
@@ -8,7 +8,7 @@ set -ex
CORE_RANGE=${CORE_RANGE:-0-31}
OMP_CORE_RANGE=${OMP_CORE_RANGE:-0-31}
export CMAKE_BUILD_PARALLEL_LEVEL=32
export CMAKE_BUILD_PARALLEL_LEVEL=16
# Setup cleanup
remove_docker_container() {
@@ -37,12 +37,8 @@ function cpu_tests() {
pytest -x -v -s tests/kernels/test_onednn.py
pytest -x -v -s tests/kernels/attention/test_cpu_attn.py
pytest -x -v -s tests/kernels/core/test_cpu_activation.py
pytest -x -v -s tests/kernels/moe/test_cpu_fused_moe.py
pytest -x -v -s tests/kernels/mamba/cpu/test_cpu_gdn_ops.py
pytest -x -v -s tests/kernels/moe/test_cpu_int4_moe.py
pytest -x -v -s tests/kernels/mamba/test_cpu_short_conv.py
pytest -x -v -s tests/kernels/mamba/test_causal_conv1d.py
pytest -x -v -s tests/kernels/mamba/test_mamba_ssm.py"
pytest -x -v -s tests/kernels/moe/test_moe.py -k test_cpu_fused_moe_basic
pytest -x -v -s tests/kernels/mamba/cpu/test_cpu_gdn_ops.py"
# skip tests requiring model downloads if HF_TOKEN is not set
# due to rate-limits
@@ -66,6 +62,7 @@ function cpu_tests() {
set -e
pytest -x -v -s tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_logprobs"
# basic online serving
docker exec cpu-test bash -c '
set -e
@@ -99,4 +96,3 @@ function cpu_tests() {
# All of CPU tests are expected to be finished less than 40 mins.
export -f cpu_tests
timeout 2h bash -c cpu_tests
+1 -40
View File
@@ -7,49 +7,10 @@ set -euox pipefail
# allow to bind to different cores
CORE_RANGE=${CORE_RANGE:-48-95}
NUMA_NODE=${NUMA_NODE:-1}
AGENT_SLOT=${AGENT_SLOT:-}
IMAGE_NAME="cpu-test-${NUMA_NODE}${AGENT_SLOT:+-${AGENT_SLOT}}"
IMAGE_NAME="cpu-test-$NUMA_NODE"
TIMEOUT_VAL=$1
TEST_COMMAND=$2
# Disk hygiene knobs. Reclaim space only once the Docker root filesystem crosses
# DISK_USAGE_THRESHOLD percent, and cap the shared BuildKit cache at
# BUILDKIT_CACHE_MAX so subsequent builds keep reusing the hottest layers.
DISK_USAGE_THRESHOLD=${DISK_USAGE_THRESHOLD:-70}
BUILDKIT_CACHE_MAX=${BUILDKIT_CACHE_MAX:-80GB}
# Reclaim disk only when the host is under pressure. We trim (not purge) the
# shared BuildKit cache so cross-job/cross-agent reuse stays intact, and only
# touch dangling images; other agents' uniquely tagged images are left alone.
prune_if_disk_pressure() {
local docker_root disk_usage
docker_root=$(docker info -f '{{.DockerRootDir}}' 2>/dev/null || true)
if [ -z "$docker_root" ]; then
return 0
fi
disk_usage=$(df "$docker_root" 2>/dev/null | tail -1 | awk '{print $5}' | tr -d '%')
if [ "${disk_usage:-0}" -gt "$DISK_USAGE_THRESHOLD" ]; then
echo "--- :broom: Disk usage ${disk_usage}% exceeds ${DISK_USAGE_THRESHOLD}%, reclaiming space"
docker image prune -f || true
docker builder prune -f --keep-storage="$BUILDKIT_CACHE_MAX" || true
else
echo "Disk usage ${disk_usage:-unknown}% within ${DISK_USAGE_THRESHOLD}% threshold; skipping prune"
fi
}
# Always drop this agent's image once the job ends (the default builder never
# uses it as a cache source, so removing it costs no rebuild speed), then
# reclaim space if needed. Guard every docker call with `|| true` so the trap
# never overrides the test's exit code.
cleanup() {
docker image rm -f "$IMAGE_NAME" || true
prune_if_disk_pressure
}
trap cleanup EXIT
# Free space up front so a nearly-full host doesn't fail the build.
prune_if_disk_pressure
# building the docker image
echo "--- :docker: Building Docker image"
docker build --progress plain --tag "$IMAGE_NAME" --target vllm-test -f docker/Dockerfile.cpu .
@@ -1,53 +0,0 @@
#!/bin/bash
set -euo pipefail
test_suite="${1:-}"
if [[ -z "${test_suite}" ]]; then
echo "Usage: $0 <example|v1|server>" >&2
exit 1
fi
case "${test_suite}" in
example)
pip install tblib==3.1.0
python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager
python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 -O3 -cc.cudagraph_mode=NONE
python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager -tp 2 --distributed-executor-backend mp
python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --attention-backend=TRITON_ATTN
python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --quantization fp8
python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --kv-cache-dtype fp8
python3 examples/basic/offline_inference/generate.py --model nvidia/Llama-3.1-8B-Instruct-FP8 --block-size 64 --enforce-eager --quantization modelopt --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --max-model-len 4096
python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --block-size 64 --enforce-eager --max-model-len 8192
python3 examples/basic/offline_inference/generate.py --model TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ --block-size 64 --enforce-eager
python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2
python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 --enable-expert-parallel
python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --max-model-len 8192
;;
v1)
cd tests
pytest -v -s v1/core --ignore=v1/core/test_reset_prefix_cache_e2e.py --ignore=v1/core/test_scheduler_e2e.py
pytest -v -s v1/engine --ignore=v1/engine/test_output_processor.py
pytest -v -s v1/sample --ignore=v1/sample/test_logprobs.py --ignore=v1/sample/test_logprobs_e2e.py -k "not test_topk_only and not test_topp_only and not test_topk_and_topp"
pytest -v -s v1/worker --ignore=v1/worker/test_gpu_model_runner.py --ignore=v1/worker/test_worker_memory_snapshot.py
pytest -v -s v1/structured_output
pytest -v -s v1/test_serial_utils.py
pytest -v -s v1/e2e/general/test_correctness_sliding_window.py --deselect="tests/v1/e2e/general/test_correctness_sliding_window.py::test_sliding_window_retrieval[True-1-5-google/gemma-3-1b-it]"
pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py --ignore=v1/spec_decode/test_speculators_correctness.py
pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_multi_connector.py --ignore=v1/kv_connector/unit/test_example_connector.py --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py --ignore=v1/kv_connector/unit/test_offloading_connector.py
;;
server)
pip install av
cd tests
pytest -v -s entrypoints/multimodal/openai/chat_completion/test_audio_in_video.py
pytest -v -s benchmarks/test_serve_cli.py
;;
*)
echo "Unknown Intel test suite: ${test_suite}" >&2
exit 1
;;
esac
@@ -243,10 +243,8 @@ container_name="xpu_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head
# ---- Command source selection ----
commands=""
commands_source=""
if [[ -n "${VLLM_TEST_COMMANDS:-}" ]]; then
commands="${VLLM_TEST_COMMANDS}"
commands_source="env"
echo "Commands sourced from VLLM_TEST_COMMANDS (quoting preserved)"
elif [[ $# -gt 0 ]]; then
all_yaml=true
@@ -305,12 +303,8 @@ if [[ -z "$commands" ]]; then
fi
echo "Raw commands: $commands"
if [[ "$commands_source" != "env" ]]; then
commands=$(re_quote_pytest_markers "$commands")
echo "After re-quoting: $commands"
else
echo "Skipping re-quoting for VLLM_TEST_COMMANDS input"
fi
commands=$(re_quote_pytest_markers "$commands")
echo "After re-quoting: $commands"
commands=$(apply_intel_test_overrides "$commands")
echo "Final commands: $commands"
@@ -360,7 +354,7 @@ export HF_TOKEN ZE_AFFINITY_MASK
--ipc=host \
--privileged \
-v /dev/dri/by-path:/dev/dri/by-path \
-v "/data/huggingface:/root/.cache/huggingface" \
-v "${HOME}/.cache/huggingface:/root/.cache/huggingface" \
--entrypoint='' \
-e HF_TOKEN \
-e ZE_AFFINITY_MASK \
@@ -369,7 +363,7 @@ export HF_TOKEN ZE_AFFINITY_MASK
-e CMDS \
--name "${container_name}" \
"${IMAGE}" \
bash -c 'set -e; source /opt/intel/oneapi/setvars.sh --force; source /opt/intel/oneapi/ccl/2021.15/env/vars.sh --force; echo "ZE_AFFINITY_MASK is ${ZE_AFFINITY_MASK:-}"; eval "$CMDS"' \
bash -c 'set -e; echo "ZE_AFFINITY_MASK is ${ZE_AFFINITY_MASK:-}"; eval "$CMDS"' \
>/dev/null
} 9>/tmp/docker-pull.lock
@@ -85,7 +85,7 @@ RUN pip config set global.index-url http://cache-service-vllm.nginx-pypi-cache.s
# Install for pytest to make the docker build cache layer always valid
RUN --mount=type=cache,target=/root/.cache/pip \
pip install pytest>=6.0 'modelscope<1.38'
pip install pytest>=6.0 modelscope
WORKDIR /workspace/vllm
@@ -4,11 +4,6 @@
set -euo pipefail
if python3 -c "import torch; raise SystemExit(0 if torch.version.hip is not None else 1)"; then
uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
exit 0
fi
REQUIREMENTS_FILE="${KV_CONNECTORS_REQUIREMENTS:-/vllm-workspace/requirements/kv_connectors.txt}"
uv pip install --system -r "${REQUIREMENTS_FILE}"
@@ -130,22 +130,6 @@ docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${ROCM_BASE_CACHE_KEY}-rocm
docker push vllm/vllm-openai-rocm:latest-base
docker push vllm/vllm-openai-rocm:v${RELEASE_VERSION}-base
# ---- XPU ----
docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-xpu
docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-xpu vllm/vllm-openai-xpu:latest-x86_64
docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-xpu vllm/vllm-openai-xpu:v${RELEASE_VERSION}-x86_64
docker push vllm/vllm-openai-xpu:latest-x86_64
docker push vllm/vllm-openai-xpu:v${RELEASE_VERSION}-x86_64
docker manifest rm vllm/vllm-openai-xpu:latest || true
docker manifest rm vllm/vllm-openai-xpu:v${RELEASE_VERSION} || true
docker manifest create vllm/vllm-openai-xpu:latest vllm/vllm-openai-xpu:latest-x86_64 --amend
docker manifest create vllm/vllm-openai-xpu:v${RELEASE_VERSION} vllm/vllm-openai-xpu:v${RELEASE_VERSION}-x86_64 --amend
docker manifest push vllm/vllm-openai-xpu:latest
docker manifest push vllm/vllm-openai-xpu:v${RELEASE_VERSION}
# ---- CPU ----
# CPU images are behind separate block steps and may not have been built.
# All-or-nothing: inspect both arches first, then either publish everything
-32
View File
@@ -1,32 +0,0 @@
#!/usr/bin/env bash
# Build the ROCm ci_base image, optionally from a freshly rebuilt ROCm base.
set -euo pipefail
metadata_get() {
local key="$1"
if command -v buildkite-agent >/dev/null 2>&1; then
buildkite-agent meta-data get "${key}" 2>/dev/null || true
fi
}
main() {
local base_refreshed=""
base_refreshed="$(metadata_get rocm-base-refresh)"
if [[ "${base_refreshed}" == "1" ]]; then
export BASE_IMAGE
export CI_BASE_PUSH_STABLE_TAG
BASE_IMAGE="$(metadata_get rocm-base-image)"
CI_BASE_PUSH_STABLE_TAG="$(metadata_get rocm-base-push-stable-tag)"
CI_BASE_PUSH_STABLE_TAG="${CI_BASE_PUSH_STABLE_TAG:-0}"
echo "Using refreshed ROCm base image for ci_base: ${BASE_IMAGE}"
echo "Push stable ci_base tag: ${CI_BASE_PUSH_STABLE_TAG}"
fi
bash .buildkite/scripts/ci-bake-rocm.sh ci-base-rocm-ci-with-deps
}
main "$@"
@@ -1,68 +0,0 @@
#!/usr/bin/env bash
# Build the ROCm CI test image or wheel artifact.
#
# When Dockerfile.rocm_base changes, always build the full image so downstream
# ROCm tests can validate the freshly rebuilt base -> ci_base -> ci image chain.
set -euo pipefail
metadata_get() {
local key="$1"
if command -v buildkite-agent >/dev/null 2>&1; then
buildkite-agent meta-data get "${key}" 2>/dev/null || true
fi
}
use_ci_base_if_present() {
local ci_base_image=""
ci_base_image="$(metadata_get rocm-ci-base-image)"
if [[ -z "${ci_base_image}" ]]; then
return 1
fi
export CI_BASE_IMAGE="${ci_base_image}"
echo "Using ROCm ci_base image selected by the preceding build step: ${CI_BASE_IMAGE}"
}
use_refreshed_base_if_present() {
local base_refreshed=""
base_refreshed="$(metadata_get rocm-base-refresh)"
if [[ "${base_refreshed}" != "1" ]]; then
return 1
fi
export BASE_IMAGE
export IMAGE_TAG_LATEST
BASE_IMAGE="$(metadata_get rocm-base-image)"
IMAGE_TAG_LATEST="$(metadata_get rocm-ci-image-descriptive)"
echo "Using refreshed ROCm base image for test image: ${BASE_IMAGE}"
if [[ -n "${IMAGE_TAG_LATEST}" ]]; then
echo "Also tagging full ROCm CI image as: ${IMAGE_TAG_LATEST}"
fi
return 0
}
main() {
local base_refreshed=0
use_ci_base_if_present || true
if use_refreshed_base_if_present; then
base_refreshed=1
fi
if [[ "${ROCM_CI_ARTIFACT_ONLY:-0}" == "1" && "${base_refreshed}" != "1" ]]; then
echo "ROCM_CI_ARTIFACT_ONLY=1; building ROCm wheel artifact only"
IMAGE_TAG="" bash .buildkite/scripts/ci-bake-rocm.sh test-rocm-ci-with-artifacts
return
fi
bash .buildkite/scripts/ci-bake-rocm.sh test-rocm-ci-with-wheel
}
main "$@"
@@ -1,513 +0,0 @@
#!/usr/bin/env bash
# Build and publish a fresh ROCm base image when Dockerfile.rocm_base changes.
#
# Normal AMD CI builds should not pay for this path. The script no-ops unless
# docker/Dockerfile.rocm_base changed relative to the branch base, the previous
# main commit, or ROCM_BASE_REFRESH_FORCE=1 is set.
set -euo pipefail
DOCKERFILE="${ROCM_BASE_DOCKERFILE:-docker/Dockerfile.rocm_base}"
BASE_REPO="${ROCM_BASE_IMAGE_REPO:-rocm/vllm-dev}"
CI_IMAGE_REPO="${ROCM_CI_IMAGE_REPO:-rocm/vllm-ci}"
BUILDER_NAME="${ROCM_BASE_BUILDER_NAME:-vllm-rocm-base-builder}"
DEFAULT_ROCM_BASE_METADATA_VERSION="1"
DEFAULT_ROCM_BASE_CONTENT_FILES="${DOCKERFILE}"
DEFAULT_ROCM_BASE_CONTENT_ARGS="BASE_IMAGE TRITON_BRANCH TRITON_REPO PYTORCH_BRANCH PYTORCH_REPO PYTORCH_VISION_BRANCH PYTORCH_VISION_REPO PYTORCH_AUDIO_BRANCH PYTORCH_AUDIO_REPO FA_BRANCH FA_REPO AITER_BRANCH AITER_REPO MORI_BRANCH MORI_REPO PYTORCH_ROCM_ARCH PYTHON_VERSION USE_SCCACHE"
metadata_set() {
local key="$1"
local value="$2"
[[ -n "${value}" ]] || return 0
if command -v buildkite-agent >/dev/null 2>&1; then
buildkite-agent meta-data set "${key}" "${value}" || true
fi
}
compute_content_hash() {
local path=""
local file=""
for path in "$@"; do
if [[ -d "${path}" ]]; then
while IFS= read -r -d '' file; do
printf 'file:%s\n' "${file}"
sha256sum "${file}"
done < <(find "${path}" -type f -print0 | sort -z)
elif [[ -f "${path}" ]]; then
printf 'file:%s\n' "${path}"
sha256sum "${path}"
else
printf 'missing:%s\n' "${path}"
fi
done | sha256sum | cut -d' ' -f1
}
clean_docker_tag() {
local input="$1"
echo "${input}" | sed 's/[^a-zA-Z0-9._-]/_/g' | cut -c1-128
}
tag_component() {
local input="$1"
local max_chars="${2:-24}"
clean_docker_tag "${input:-unknown}" | cut -c1-"${max_chars}"
}
extract_arg_default() {
local arg_name="$1"
sed -n -E "s/^[[:space:]]*ARG[[:space:]]+${arg_name}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \
"${DOCKERFILE}" | head -1
}
resolve_image_digest() {
local image_ref="$1"
docker buildx imagetools inspect "${image_ref}" 2>/dev/null \
| sed -n -E 's/^Digest:[[:space:]]+//p' \
| head -1 || true
}
resolve_rocm_base_arg_value() {
local arg_name="$1"
local use_sccache="$2"
case "${arg_name}" in
USE_SCCACHE)
printf '%s\n' "${use_sccache}"
;;
*)
extract_arg_default "${arg_name}"
;;
esac
}
hash_rocm_base_arg_values() {
local use_sccache="$1"
local base_image_digest="$2"
local arg_name=""
local arg_value=""
shift 2 || true
for arg_name in "$@"; do
[[ -n "${arg_name}" ]] || continue
arg_value=$(resolve_rocm_base_arg_value "${arg_name}" "${use_sccache}")
printf 'arg:%s=%s\n' "${arg_name}" "${arg_value:-<empty>}"
if [[ "${arg_name}" == "BASE_IMAGE" && -n "${arg_value}" ]]; then
printf 'arg:%s.digest=%s\n' "${arg_name}" "${base_image_digest:-unknown}"
fi
done
}
rocm_version_from_base_image() {
local base_image="$1"
local version=""
version="$(sed -n -E 's/.*:([0-9]+\.[0-9]+(\.[0-9]+)?)-.*/\1/p' <<<"${base_image}")"
tag_component "${version:-${base_image}}" 16
}
git_diff_changed_base() {
local range="$1"
[[ -n "$(git diff --name-only "${range}" -- "${DOCKERFILE}" 2>/dev/null)" ]]
}
short_git_ref() {
local ref="$1"
git rev-parse --short "${ref}" 2>/dev/null || printf '%s\n' "${ref}"
}
extract_arg_default_from_ref() {
local ref="$1"
local arg_name="$2"
local content=""
content="$(git show "${ref}:${DOCKERFILE}" 2>/dev/null || true)"
sed -n -E "s/^[[:space:]]*ARG[[:space:]]+${arg_name}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \
<<<"${content}" | head -1
}
log_arg_default_changes() {
local old_ref="$1"
local new_ref="$2"
local content_args="${ROCM_BASE_CONTENT_ARGS:-${DEFAULT_ROCM_BASE_CONTENT_ARGS}}"
local arg_name=""
local old_value=""
local new_value=""
local changed=0
echo "Changed ROCm base ARG defaults:"
for arg_name in ${content_args}; do
old_value="$(extract_arg_default_from_ref "${old_ref}" "${arg_name}")"
new_value="$(extract_arg_default_from_ref "${new_ref}" "${arg_name}")"
if [[ "${old_value}" != "${new_value}" ]]; then
echo " - ${arg_name}: ${old_value:-<unset>} -> ${new_value:-<unset>}"
changed=1
fi
done
if [[ "${changed}" == "0" ]]; then
echo " - none detected; Dockerfile instructions changed outside tracked ARG defaults"
fi
}
log_arg_line_diff() {
local range="$1"
local arg_diff=""
arg_diff="$(
git diff --unified=0 "${range}" -- "${DOCKERFILE}" 2>/dev/null \
| awk '/^[+-][[:space:]]*ARG[[:space:]]/ && $0 !~ /^(---|\+\+\+)/ { print " " $0 }' \
|| true
)"
if [[ -n "${arg_diff}" ]]; then
echo "Changed Dockerfile ARG lines:"
printf '%s\n' "${arg_diff}"
fi
}
log_rocm_base_change_check() {
local context="$1"
local range="$2"
local old_ref="$3"
local old_short=""
local head_short=""
old_short="$(short_git_ref "${old_ref}")"
head_short="$(short_git_ref HEAD)"
echo "--- :mag: ROCm base refresh check"
echo "Context: ${context}"
echo "Dockerfile: ${DOCKERFILE}"
echo "Base revision: ${old_short}"
echo "Head revision: ${head_short}"
echo "Git diff range: ${range}"
}
log_rocm_base_rebuild_reason() {
local context="$1"
local range="$2"
local old_ref="$3"
local changed_files=""
log_rocm_base_change_check "${context}" "${range}" "${old_ref}"
changed_files="$(git diff --name-only "${range}" -- "${DOCKERFILE}" 2>/dev/null || true)"
echo "Changed files:"
if [[ -n "${changed_files}" ]]; then
sed 's/^/ - /' <<<"${changed_files}"
else
echo " - ${DOCKERFILE}"
fi
log_arg_default_changes "${old_ref}" HEAD
log_arg_line_diff "${range}"
echo "Decision: rebuilding ROCm base image because ${DOCKERFILE} changed."
}
rocm_base_changed_in_range() {
local context="$1"
local range="$2"
local old_ref="$3"
if git_diff_changed_base "${range}"; then
log_rocm_base_rebuild_reason "${context}" "${range}" "${old_ref}"
return 0
fi
log_rocm_base_change_check "${context}" "${range}" "${old_ref}"
echo "Decision: ROCm base refresh not required; ${DOCKERFILE} is unchanged."
return 1
}
rocm_base_changed() {
local base_branch="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}"
local base_ref="refs/remotes/origin/${base_branch}"
local merge_base=""
if [[ "${ROCM_BASE_REFRESH_SKIP:-0}" == "1" ]]; then
echo "ROCM_BASE_REFRESH_SKIP=1 set; skipping ROCm base refresh"
return 1
fi
if [[ "${ROCM_BASE_REFRESH_FORCE:-0}" == "1" ]]; then
echo "ROCM_BASE_REFRESH_FORCE=1 set; refreshing ROCm base image"
return 0
fi
if ! git rev-parse --is-inside-work-tree >/dev/null 2>&1; then
echo "Not in a git checkout; skipping ROCm base refresh unless forced"
return 1
fi
if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" ]]; then
git fetch --no-tags --depth=200 origin \
"+refs/heads/${base_branch}:${base_ref}" >/dev/null 2>&1 || true
merge_base=$(git merge-base HEAD "${base_ref}" 2>/dev/null || true)
if [[ -z "${merge_base}" ]]; then
echo "Unable to determine merge base with PR base ${base_ref}; skipping ROCm base refresh unless forced"
return 1
fi
if rocm_base_changed_in_range \
"pull request build against ${base_ref}" \
"${merge_base}...HEAD" \
"${merge_base}"; then
return 0
fi
elif [[ "${BUILDKITE_BRANCH:-}" == "${ROCM_BASE_STABLE_BRANCH:-main}" ]] \
&& git rev-parse --verify HEAD~1 >/dev/null 2>&1; then
if rocm_base_changed_in_range \
"stable branch build; comparing against previous ${ROCM_BASE_STABLE_BRANCH:-main} commit" \
"HEAD~1..HEAD" \
"HEAD~1"; then
return 0
fi
else
git fetch --no-tags --depth=200 origin \
"+refs/heads/${base_branch}:${base_ref}" >/dev/null 2>&1 || true
merge_base=$(git merge-base HEAD "${base_ref}" 2>/dev/null || true)
if [[ -z "${merge_base}" ]]; then
echo "Unable to determine merge base with branch base ${base_ref}; skipping ROCm base refresh unless forced"
return 1
fi
if rocm_base_changed_in_range \
"branch build against ${base_ref}" \
"${merge_base}...HEAD" \
"${merge_base}"; then
return 0
fi
fi
return 1
}
should_push_stable_tag() {
if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" ]]; then
return 1
fi
if [[ "${ROCM_BASE_PUSH_STABLE_TAG:-}" == "1" ]]; then
return 0
fi
if [[ "${ROCM_BASE_PUSH_STABLE_TAG:-}" == "0" ]]; then
return 1
fi
[[ "${BUILDKITE_PULL_REQUEST:-false}" == "false" \
&& "${BUILDKITE_BRANCH:-}" == "${ROCM_BASE_STABLE_BRANCH:-main}" ]]
}
setup_builder() {
echo "--- :buildkite: Setting up buildx builder for ROCm base"
if docker buildx inspect "${BUILDER_NAME}" >/dev/null 2>&1; then
docker buildx use "${BUILDER_NAME}"
else
docker buildx create --name "${BUILDER_NAME}" --driver docker-container --use
fi
docker buildx inspect --bootstrap
}
compute_base_content_hash() {
local use_sccache="$1"
local base_image_digest="$2"
local content_files="${ROCM_BASE_CONTENT_FILES:-${DEFAULT_ROCM_BASE_CONTENT_FILES}}"
local content_args="${ROCM_BASE_CONTENT_ARGS:-${DEFAULT_ROCM_BASE_CONTENT_ARGS}}"
local -a content_paths=()
local -a content_arg_names=()
read -r -a content_paths <<< "${content_files}"
read -r -a content_arg_names <<< "${content_args}"
{
printf 'content-files-hash:%s\n' "$(compute_content_hash "${content_paths[@]}")"
printf 'dockerfile:%s\n' "${DOCKERFILE}"
printf 'resolved-build-args:\n'
hash_rocm_base_arg_values \
"${use_sccache}" "${base_image_digest}" "${content_arg_names[@]}"
} | sha256sum | cut -d' ' -f1
}
build_base_image() {
local use_sccache="${ROCM_BASE_USE_SCCACHE:-${USE_SCCACHE:-0}}"
local base_hash=""
local build_date=""
local build_suffix=""
local base_image_arg=""
local base_image_digest=""
local rocm_version=""
local triton_arg=""
local pytorch_arg=""
local pytorch_vision_arg=""
local pytorch_audio_arg=""
local fa_arg=""
local aiter_arg=""
local mori_arg=""
local python_version_arg=""
local pytorch_rocm_arch_arg=""
local pytorch_branch=""
local aiter_branch=""
local dependency_summary=""
local descriptor=""
local ci_descriptor=""
local descriptive_tag=""
local stable_tag="${BASE_REPO}:base"
local ci_descriptive_tag=""
local content_files="${ROCM_BASE_CONTENT_FILES:-${DEFAULT_ROCM_BASE_CONTENT_FILES}}"
local content_args="${ROCM_BASE_CONTENT_ARGS:-${DEFAULT_ROCM_BASE_CONTENT_ARGS}}"
local content_files_hash=""
local metadata_version="${ROCM_BASE_METADATA_VERSION:-${DEFAULT_ROCM_BASE_METADATA_VERSION}}"
local -a tags=()
local -a no_cache_args=()
local -a sccache_args=()
local -a content_paths=()
if [[ ! -f "${DOCKERFILE}" ]]; then
echo "Error: ROCm base Dockerfile not found: ${DOCKERFILE}" >&2
exit 1
fi
build_date="${ROCM_BASE_TAG_DATE:-$(date -u +%Y%m%d)}"
if [[ -n "${BUILDKITE_BUILD_NUMBER:-}" ]]; then
build_suffix="_bk_${BUILDKITE_BUILD_NUMBER}"
fi
base_image_arg="$(extract_arg_default BASE_IMAGE)"
base_image_digest="$(resolve_image_digest "${base_image_arg}")"
read -r -a content_paths <<< "${content_files}"
content_files_hash="$(compute_content_hash "${content_paths[@]}")"
base_hash=$(compute_base_content_hash "${use_sccache}" "${base_image_digest}")
rocm_version="$(rocm_version_from_base_image "${base_image_arg}")"
triton_arg="$(extract_arg_default TRITON_BRANCH)"
pytorch_arg="$(extract_arg_default PYTORCH_BRANCH)"
pytorch_vision_arg="$(extract_arg_default PYTORCH_VISION_BRANCH)"
pytorch_audio_arg="$(extract_arg_default PYTORCH_AUDIO_BRANCH)"
fa_arg="$(extract_arg_default FA_BRANCH)"
aiter_arg="$(extract_arg_default AITER_BRANCH)"
mori_arg="$(extract_arg_default MORI_BRANCH)"
python_version_arg="$(extract_arg_default PYTHON_VERSION)"
pytorch_rocm_arch_arg="$(extract_arg_default PYTORCH_ROCM_ARCH)"
pytorch_branch="$(tag_component "${pytorch_arg}" 16)"
aiter_branch="$(tag_component "${aiter_arg}" 24)"
dependency_summary="base=${base_image_arg},rocm=${rocm_version},python=${python_version_arg},pytorch=${pytorch_arg},torchvision=${pytorch_vision_arg},torchaudio=${pytorch_audio_arg},triton=${triton_arg},flash-attn=${fa_arg},aiter=${aiter_arg},mori=${mori_arg},pytorch-rocm-arch=${pytorch_rocm_arch_arg}"
descriptor="$(clean_docker_tag "base_custom_aiter_${aiter_branch}_torch_${pytorch_branch}_${build_date}${build_suffix}")"
ci_descriptor="$(clean_docker_tag "ci_custom_aiter_${aiter_branch}_torch_${pytorch_branch}_${build_date}${build_suffix}")"
descriptive_tag="${BASE_REPO}:${descriptor}"
ci_descriptive_tag="${CI_IMAGE_REPO}:${ci_descriptor}"
tags=(-t "${descriptive_tag}")
if should_push_stable_tag; then
tags+=(-t "${stable_tag}")
metadata_set "rocm-base-push-stable-tag" "1"
else
metadata_set "rocm-base-push-stable-tag" "0"
fi
if [[ "${ROCM_BASE_NO_CACHE:-1}" == "1" ]]; then
no_cache_args=(--no-cache)
fi
for env_name in \
SCCACHE_DOWNLOAD_URL \
SCCACHE_ENDPOINT \
SCCACHE_BUCKET_NAME \
SCCACHE_REGION_NAME \
SCCACHE_S3_NO_CREDENTIALS; do
if [[ -n "${!env_name:-}" ]]; then
sccache_args+=(--build-arg "${env_name}=${!env_name}")
fi
done
echo "--- :docker: Building ROCm base image"
echo "Dockerfile: ${DOCKERFILE}"
echo "Descriptive tag: ${descriptive_tag}"
echo "Stable tag: ${stable_tag} ($(should_push_stable_tag && echo enabled || echo disabled))"
echo "Content hash: ${base_hash}"
echo "Dependency summary: ${dependency_summary}"
echo "USE_SCCACHE: ${use_sccache}"
docker buildx build \
"${no_cache_args[@]}" \
--pull \
--progress "${BUILDKIT_PROGRESS:-plain}" \
--file "${DOCKERFILE}" \
--build-arg "USE_SCCACHE=${use_sccache}" \
"${sccache_args[@]}" \
--label "org.opencontainers.image.source=https://github.com/vllm-project/vllm" \
--label "org.opencontainers.image.vendor=vLLM" \
--label "org.opencontainers.image.title=vLLM ROCm base" \
--label "org.opencontainers.image.revision=${BUILDKITE_COMMIT:-}" \
--label "vllm.rocm_base.metadata_version=${metadata_version}" \
--label "vllm.rocm_base.content_hash=${base_hash}" \
--label "vllm.rocm_base.content_files_hash=${content_files_hash}" \
--label "vllm.rocm_base.dockerfile=${DOCKERFILE}" \
--label "vllm.rocm_base.image.descriptive=${descriptive_tag}" \
--label "vllm.rocm_base.image.stable=${stable_tag}" \
--label "vllm.rocm_base.git_commit=${BUILDKITE_COMMIT:-}" \
--label "vllm.rocm_base.stable_branch=${ROCM_BASE_STABLE_BRANCH:-main}" \
--label "vllm.rocm_base.descriptor=${descriptor}" \
--label "vllm.rocm_base.dependency_summary=${dependency_summary}" \
--label "vllm.rocm_base.base_image=${base_image_arg}" \
--label "vllm.rocm_base.base_image_digest=${base_image_digest}" \
--label "vllm.rocm_base.dependency.rocm=${rocm_version}" \
--label "vllm.rocm_base.dependency.python=${python_version_arg}" \
--label "vllm.rocm_base.dependency.pytorch=${pytorch_arg}" \
--label "vllm.rocm_base.dependency.torchvision=${pytorch_vision_arg}" \
--label "vllm.rocm_base.dependency.torchaudio=${pytorch_audio_arg}" \
--label "vllm.rocm_base.dependency.triton=${triton_arg}" \
--label "vllm.rocm_base.dependency.flash_attention=${fa_arg}" \
--label "vllm.rocm_base.dependency.aiter=${aiter_arg}" \
--label "vllm.rocm_base.dependency.mori=${mori_arg}" \
--label "vllm.rocm_base.pytorch_rocm_arch=${pytorch_rocm_arch_arg}" \
"${tags[@]}" \
--push \
.
docker buildx imagetools inspect "${descriptive_tag}" >/dev/null
metadata_set "rocm-base-refresh" "1"
metadata_set "rocm-base-image" "${descriptive_tag}"
metadata_set "rocm-base-image-descriptive" "${descriptive_tag}"
metadata_set "rocm-base-image-stable" "${stable_tag}"
metadata_set "rocm-base-image-ci-descriptive" "${ci_descriptive_tag}"
metadata_set "rocm-base-metadata-version" "${metadata_version}"
metadata_set "rocm-base-content-hash" "${base_hash}"
metadata_set "rocm-base-content-files-hash" "${content_files_hash}"
metadata_set "rocm-base-content-files" "${content_files}"
metadata_set "rocm-base-content-args" "${content_args}"
metadata_set "rocm-base-base-image-digest" "${base_image_digest}"
metadata_set "rocm-base-dockerfile" "${DOCKERFILE}"
metadata_set "rocm-base-descriptor" "${descriptor}"
metadata_set "rocm-base-dependency-summary" "${dependency_summary}"
metadata_set "rocm-base-dependency-rocm" "${rocm_version}"
metadata_set "rocm-base-dependency-python" "${python_version_arg}"
metadata_set "rocm-base-dependency-pytorch" "${pytorch_arg}"
metadata_set "rocm-base-dependency-torchvision" "${pytorch_vision_arg}"
metadata_set "rocm-base-dependency-torchaudio" "${pytorch_audio_arg}"
metadata_set "rocm-base-dependency-triton" "${triton_arg}"
metadata_set "rocm-base-dependency-flash-attention" "${fa_arg}"
metadata_set "rocm-base-dependency-aiter" "${aiter_arg}"
metadata_set "rocm-base-dependency-mori" "${mori_arg}"
metadata_set "rocm-base-pytorch-rocm-arch" "${pytorch_rocm_arch_arg}"
metadata_set "rocm-ci-image-descriptive" "${ci_descriptive_tag}"
echo "--- :white_check_mark: ROCm base image published"
echo "Use BASE_IMAGE=${descriptive_tag} for downstream ROCm CI builds"
}
main() {
metadata_set "rocm-base-refresh" "0"
if ! rocm_base_changed; then
echo "ROCm base Dockerfile did not change; skipping base image refresh"
return 0
fi
setup_builder
build_base_image
}
main "$@"
@@ -1,32 +0,0 @@
#!/usr/bin/env bash
# Fast structural smoke test for the full ROCm CI image.
set -euo pipefail
image_ref="${VLLM_CI_SMOKE_IMAGE:-rocm/vllm-ci:${BUILDKITE_COMMIT:?BUILDKITE_COMMIT is required}}"
docker run --rm --network=none --entrypoint /bin/bash "${image_ref}" -ec '
if [ ! -d /vllm-workspace ]; then echo Missing directory: /vllm-workspace >&2; exit 1; fi
if [ ! -d /vllm-workspace/tests ]; then echo Missing directory: /vllm-workspace/tests >&2; exit 1; fi
if [ ! -d /vllm-workspace/src/vllm ]; then echo Missing directory: /vllm-workspace/src/vllm >&2; exit 1; fi
if [ ! -x /vllm-workspace/src/vllm/vllm-rs ]; then echo Missing executable: /vllm-workspace/src/vllm/vllm-rs >&2; exit 1; fi
command -v python3
command -v uv
command -v pytest
if ! command -v amd-smi >/dev/null 2>&1 && ! command -v rocminfo >/dev/null 2>&1; then
echo No ROCm CLI found in image >&2
exit 1
fi
python3 - <<PY
import torch
import vllm
print(torch.__version__)
print(vllm.__version__)
PY
echo AMD image smoke OK
'
+1 -3
View File
@@ -109,9 +109,7 @@ run_nodes() {
if [ "$node" -ne 0 ]; then
docker exec -d "node$node" /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}"
else
# Allocate a TTY (-t -i) for the foreground head node so its output
# keeps ANSI color in the Buildkite log (see run-amd-test.sh).
docker exec -t -i "node$node" /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}"
docker exec "node$node" /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}"
fi
done
}
+36 -119
View File
@@ -8,12 +8,7 @@ if [[ "$MODE" != "style-clippy" && "$MODE" != "test" ]]; then
exit 2
fi
if ROOT_DIR="$(git rev-parse --show-toplevel 2>/dev/null)"; then
:
else
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
ROOT_DIR="$(cd -- "${SCRIPT_DIR}/../.." && pwd -P)"
fi
ROOT_DIR="$(git rev-parse --show-toplevel)"
cd "$ROOT_DIR"
export CARGO_TERM_COLOR="${CARGO_TERM_COLOR:-always}"
@@ -21,21 +16,16 @@ export CARGO_HOME="${CARGO_HOME:-$HOME/.cargo}"
export RUSTUP_HOME="${RUSTUP_HOME:-$HOME/.rustup}"
export PATH="$CARGO_HOME/bin:$PATH"
PROTOC_VERSION="${PROTOC_VERSION:-31.1}"
CARGO_BINSTALL_VERSION="${CARGO_BINSTALL_VERSION:-1.20.1}"
UV_VERSION="${UV_VERSION:-0.11.28}"
PYO3_PYTHON_VERSION="${PYO3_PYTHON_VERSION:-3.12}"
CARGO_SORT_VERSION_REQ="${CARGO_SORT_VERSION_REQ:-2}"
CARGO_DENY_VERSION_REQ="${CARGO_DENY_VERSION_REQ:-0.20}"
CARGO_NEXTEST_VERSION_REQ="${CARGO_NEXTEST_VERSION_REQ:-0.9}"
CARGO_LLVM_COV_VERSION="${CARGO_LLVM_COV_VERSION:-0.8.7}"
log_section() {
echo "--- $*"
}
install_protoc() {
if command -v protoc >/dev/null 2>&1; then
return
fi
local version="${PROTOC_VERSION:-31.1}"
local arch
case "$(uname -m)" in
x86_64)
@@ -50,17 +40,16 @@ install_protoc() {
;;
esac
local url="https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-${arch}.zip"
local url="https://github.com/protocolbuffers/protobuf/releases/download/v${version}/protoc-${version}-linux-${arch}.zip"
local tmp_dir
tmp_dir="$(mktemp -d)"
log_section "Installing protoc ${PROTOC_VERSION}"
log_section "Installing protoc ${version}"
curl -L --proto '=https' --tlsv1.2 -sSf "$url" -o "$tmp_dir/protoc.zip"
mkdir -p "$CARGO_HOME/bin"
unzip -q "$tmp_dir/protoc.zip" bin/protoc 'include/*' -d "$CARGO_HOME"
chmod +x "$CARGO_HOME/bin/protoc"
rm -rf "$tmp_dir"
protoc --version
}
rust_toolchain() {
@@ -81,84 +70,48 @@ install_rust_toolchain() {
}
install_cargo_binstall() {
log_section "Installing cargo-binstall ${CARGO_BINSTALL_VERSION}"
if command -v cargo-binstall >/dev/null 2>&1; then
return
fi
log_section "Installing cargo-binstall"
curl -L --proto '=https' --tlsv1.2 -sSf \
"https://raw.githubusercontent.com/cargo-bins/cargo-binstall/v${CARGO_BINSTALL_VERSION}/install-from-binstall-release.sh" \
| env BINSTALL_VERSION="$CARGO_BINSTALL_VERSION" bash
cargo-binstall -V
https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh \
| bash
}
install_cargo_sort() {
log_section "Installing cargo-sort ${CARGO_SORT_VERSION_REQ}"
cargo binstall --no-confirm --force "cargo-sort@${CARGO_SORT_VERSION_REQ}"
}
if command -v cargo-sort >/dev/null 2>&1; then
return
fi
install_cargo_deny() {
log_section "Installing cargo-deny ${CARGO_DENY_VERSION_REQ}"
cargo binstall --no-confirm --force "cargo-deny@${CARGO_DENY_VERSION_REQ}"
log_section "Installing cargo-sort"
install_cargo_binstall
cargo binstall --no-confirm cargo-sort
}
install_cargo_nextest() {
log_section "Installing cargo-nextest ${CARGO_NEXTEST_VERSION_REQ}"
cargo binstall \
--no-confirm \
--force \
--secure \
"cargo-nextest@${CARGO_NEXTEST_VERSION_REQ}"
}
if command -v cargo-nextest >/dev/null 2>&1; then
return
fi
install_cargo_llvm_cov() {
log_section "Installing cargo-llvm-cov ${CARGO_LLVM_COV_VERSION}"
local toolchain
toolchain="$(rust_toolchain)"
rustup component add --toolchain "$toolchain" llvm-tools-preview
cargo binstall \
--no-confirm \
--force \
--secure \
"cargo-llvm-cov@${CARGO_LLVM_COV_VERSION}"
log_section "Installing cargo-nextest"
install_cargo_binstall
cargo binstall --no-confirm --secure cargo-nextest
}
install_uv() {
log_section "Installing uv ${UV_VERSION}"
curl -L --proto '=https' --tlsv1.2 -sSf \
"https://github.com/astral-sh/uv/releases/download/${UV_VERSION}/uv-installer.sh" \
if command -v uv >/dev/null 2>&1; then
return
fi
log_section "Installing uv"
curl -LsSf --proto '=https' --tlsv1.2 https://astral.sh/uv/install.sh \
| env UV_INSTALL_DIR="$CARGO_HOME/bin" sh
uv --version
}
setup_pyo3_python() {
log_section "Installing Python ${PYO3_PYTHON_VERSION} for PyO3 tests"
uv python install "$PYO3_PYTHON_VERSION"
PYO3_PYTHON="$(uv python find \
--managed-python \
--no-project \
--resolve-links \
"$PYO3_PYTHON_VERSION")"
export PYO3_PYTHON
local python_libdir
python_libdir="$("$PYO3_PYTHON" - <<'PY'
import pathlib
import sysconfig
libdir = pathlib.Path(sysconfig.get_config_var("LIBDIR"))
ldlibrary = sysconfig.get_config_var("LDLIBRARY")
assert sysconfig.get_config_var("Py_ENABLE_SHARED") == 1
assert ldlibrary
assert (libdir / ldlibrary).exists(), libdir / ldlibrary
print(libdir)
PY
)"
export LD_LIBRARY_PATH="${python_libdir}:${LD_LIBRARY_PATH:-}"
export LIBRARY_PATH="${python_libdir}:${LIBRARY_PATH:-}"
}
run_style_clippy() {
install_cargo_binstall
install_cargo_sort
install_cargo_deny
log_section "Checking Rust formatting"
cargo fmt --manifest-path rust/Cargo.toml --all -- --check
@@ -166,13 +119,6 @@ run_style_clippy() {
log_section "Checking Cargo.toml ordering"
cargo sort --workspace --check rust
log_section "Checking Rust dependency bans"
cargo deny \
--manifest-path rust/Cargo.toml \
--config rust/deny.toml \
check \
bans
log_section "Running clippy"
cargo clippy \
--manifest-path rust/Cargo.toml \
@@ -186,44 +132,15 @@ run_style_clippy() {
run_tests() {
install_uv
setup_pyo3_python
install_cargo_binstall
install_cargo_nextest
install_cargo_llvm_cov
log_section "Running cargo nextest with Rust coverage"
mkdir -p artifacts
export LLVM_PROFILE_FILE_NAME="vllm-rust-unit-%4m.profraw"
cargo llvm-cov clean \
--manifest-path rust/Cargo.toml \
--profraw-only
set +e
cargo llvm-cov nextest \
log_section "Running cargo nextest"
cargo nextest run \
--manifest-path rust/Cargo.toml \
--workspace \
--all-features \
--locked \
--no-fail-fast \
--no-clean \
--lcov \
--output-path artifacts/rust-unit.lcov \
--ignore-filename-regex='/\.cargo/(registry|git)/|/rustc/|/target/'
local coverage_rc=$?
local upload_rc=0
if [[ $coverage_rc -eq 0 ]]; then
# shellcheck source=.buildkite/scripts/rust-coverage.sh
source .buildkite/scripts/rust-coverage.sh
rust_coverage_upload artifacts/rust-unit.lcov rust-unit
upload_rc=$?
fi
set -e
if [[ $coverage_rc -ne 0 ]]; then
return "$coverage_rc"
fi
return "$upload_rc"
--no-fail-fast
}
install_protoc
-182
View File
@@ -1,182 +0,0 @@
#!/bin/sh
RUST_CODECOV_VERSION="v11.3.1"
RUST_CODECOV_SHA256="ca1d64196d2d34771084afe76ea657d581bf628e31d993ff8e52ea09cc88a56d"
rust_coverage_repo_root() {
if [ -f /vllm-workspace/.buildkite/scripts/rust-coverage.sh ]; then
printf '%s\n' /vllm-workspace
elif [ -n "${BUILDKITE_BUILD_CHECKOUT_PATH:-}" ] \
&& [ -d "$BUILDKITE_BUILD_CHECKOUT_PATH" ]; then
printf '%s\n' "$BUILDKITE_BUILD_CHECKOUT_PATH"
else
git rev-parse --show-toplevel
fi
}
rust_coverage_start() {
RUST_COVERAGE_FLAG=${1:?coverage flag is required}
RUST_COVERAGE_DIR="/tmp/vllm-rust-coverage/${BUILDKITE_JOB_ID:-local}"
export RUST_COVERAGE_FLAG RUST_COVERAGE_DIR
mkdir -p "$RUST_COVERAGE_DIR"
LLVM_PROFILE_FILE="$RUST_COVERAGE_DIR/rust-%4m.profraw"
export LLVM_PROFILE_FILE
trap rust_coverage_finalize 0
}
rust_coverage_objects() {
rust_cov_objects_manifest="$(dirname "$(command -v llvm-cov)")/../objects"
python3 - "$rust_cov_objects_manifest" <<'PY'
from pathlib import Path
import sys
for relative in Path(sys.argv[1]).read_text().splitlines():
for entry in sys.path:
path = Path(entry or ".").resolve() / relative
if path.is_file():
print(path)
break
else:
raise RuntimeError(f"installed Rust coverage object was not found: {relative}")
PY
}
rust_coverage_collect() {
rust_cov_collect_flag=${1:?coverage flag is required}
rust_cov_collect_lcov="$RUST_COVERAGE_DIR/$rust_cov_collect_flag.lcov"
rust_cov_collect_objects=$(rust_coverage_objects) || return 1
rust_cov_collect_primary=
set --
while IFS= read -r rust_cov_collect_object; do
if [ -z "$rust_cov_collect_primary" ]; then
rust_cov_collect_primary=$rust_cov_collect_object
else
set -- "$@" "--object=$rust_cov_collect_object"
fi
done <<EOF
$rust_cov_collect_objects
EOF
llvm-profdata merge \
-sparse \
"$RUST_COVERAGE_DIR"/*.profraw \
-o "$RUST_COVERAGE_DIR/merged.profdata" || return 1
llvm-cov export \
"$rust_cov_collect_primary" \
"$@" \
--format=lcov \
--instr-profile="$RUST_COVERAGE_DIR/merged.profdata" \
--ignore-filename-regex='/\.cargo/(registry|git)/|/rustc/|/target/' \
> "$rust_cov_collect_lcov" || return 1
RUST_COVERAGE_LCOV=$rust_cov_collect_lcov
export RUST_COVERAGE_LCOV
}
rust_coverage_upload() {
rust_cov_upload_lcov=${1:?LCOV path is required}
rust_cov_upload_flag=${2:?coverage flag is required}
rust_cov_upload_repo_root=$(rust_coverage_repo_root) || return 1
if [ "$(uname -m)" != "x86_64" ]; then
echo "Rust coverage upload currently supports x86_64 CI agents" >&2
return 1
fi
rust_cov_upload_codecov_dir=$(mktemp -d /tmp/codecov-bin.XXXXXX) \
|| return 1
curl -fsSL \
"https://github.com/codecov/codecov-cli/releases/download/${RUST_CODECOV_VERSION}/codecovcli_linux" \
-o "$rust_cov_upload_codecov_dir/codecov" || return 1
echo "$RUST_CODECOV_SHA256 $rust_cov_upload_codecov_dir/codecov" \
| sha256sum -c - || return 1
chmod +x "$rust_cov_upload_codecov_dir/codecov" || return 1
rust_cov_upload_slug="vllm-project/vllm"
if [ -n "${BUILDKITE_PULL_REQUEST:-}" ] \
&& [ "${BUILDKITE_PULL_REQUEST}" != "false" ] \
&& [ -n "${BUILDKITE_PULL_REQUEST_REPO:-}" ]; then
rust_cov_upload_slug=$(echo "$BUILDKITE_PULL_REQUEST_REPO" \
| sed -E 's#(git@|https?://)([^/:]+)[:/]([^/]+/[^/.]+)(\.git)?$#\3#')
case "$rust_cov_upload_slug" in
*/*) ;;
*) rust_cov_upload_slug="vllm-project/vllm" ;;
esac
fi
rust_cov_upload_branch=${BUILDKITE_BRANCH:?BUILDKITE_BRANCH is required}
if [ -z "${CODECOV_TOKEN:-}" ]; then
# Codecov accepts tokenless public uploads on unprotected branch names.
# A colon-separated prefix keeps feature-branch and fork uploads from
# requiring a repository secret.
if [ -n "${BUILDKITE_PULL_REQUEST:-}" ] \
&& [ "${BUILDKITE_PULL_REQUEST}" != "false" ]; then
rust_cov_upload_branch="pr${BUILDKITE_PULL_REQUEST}:$rust_cov_upload_branch"
else
rust_cov_upload_branch="buildkite:$rust_cov_upload_branch"
fi
fi
set --
set -- "$@" upload-process
set -- "$@" --file "$rust_cov_upload_lcov"
# LCOV paths are mapped server-side by codecov.yml. Skip the CLI's local
# source-line fix scanning, which is unrelated to path mapping.
set -- "$@" --disable-search --disable-file-fixes
set -- "$@" --fail-on-error --git-service github
set -- "$@" --build "${BUILDKITE_BUILD_NUMBER:?BUILDKITE_BUILD_NUMBER is required}"
set -- "$@" --branch "$rust_cov_upload_branch"
set -- "$@" --sha "${BUILDKITE_COMMIT:?BUILDKITE_COMMIT is required}"
set -- "$@" --slug "$rust_cov_upload_slug"
set -- "$@" --flag "$rust_cov_upload_flag"
set -- "$@" --name "${rust_cov_upload_flag}-${BUILDKITE_JOB_ID:?BUILDKITE_JOB_ID is required}"
set -- "$@" --dir "$rust_cov_upload_repo_root"
set -- "$@" --network-root-folder "$rust_cov_upload_repo_root"
if [ -n "${BUILDKITE_PULL_REQUEST:-}" ] \
&& [ "${BUILDKITE_PULL_REQUEST}" != "false" ]; then
set -- "$@" --pr "$BUILDKITE_PULL_REQUEST"
fi
rust_cov_upload_log="$rust_cov_upload_codecov_dir/codecov.log"
# E2E steps run from tests/, so execute from the repository root to resolve
# codecov.yml and repository paths consistently.
(
cd "$rust_cov_upload_repo_root" || exit 1
"$rust_cov_upload_codecov_dir/codecov" "$@"
) >"$rust_cov_upload_log" 2>&1
rust_cov_upload_rc=$?
cat "$rust_cov_upload_log"
# v11.3.1 can log API failures while returning zero even with
# --fail-on-error. Preserve the strict CI contract explicitly.
if grep -aEq 'error.* -- ' "$rust_cov_upload_log"; then
echo "Codecov CLI reported an upload error" >&2
rust_cov_upload_rc=1
fi
rm -rf "$rust_cov_upload_codecov_dir"
return "$rust_cov_upload_rc"
}
rust_coverage_finalize() {
rust_cov_finalize_test_rc=$?
trap - 0
set +e
rust_coverage_collect "$RUST_COVERAGE_FLAG"
rust_cov_finalize_collect_rc=$?
rust_cov_finalize_upload_rc=0
if [ "$rust_cov_finalize_collect_rc" -eq 0 ]; then
rust_coverage_upload "$RUST_COVERAGE_LCOV" "$RUST_COVERAGE_FLAG"
rust_cov_finalize_upload_rc=$?
fi
find "$RUST_COVERAGE_DIR" -type f -name '*.profraw' -delete
if [ "$rust_cov_finalize_test_rc" -ne 0 ]; then
exit "$rust_cov_finalize_test_rc"
fi
if [ "$rust_cov_finalize_collect_rc" -ne 0 ]; then
exit "$rust_cov_finalize_collect_rc"
fi
exit "$rust_cov_finalize_upload_rc"
}
@@ -33,14 +33,6 @@ if [[ -n "${ATTENTION_BACKEND:-}" ]]; then
EXTRA_ARGS+=(--attention-backend "${ATTENTION_BACKEND}")
fi
# ROCm: run eager to avoid intermittent HIP-graph decode corruption.
# See https://github.com/ROCm/clr/issues/279
# TODO(aarushjain29): Revert after TheRock 7.14
if command -v rocm-smi &> /dev/null || command -v amd-smi &> /dev/null || [[ -d /opt/rocm ]] || [[ -n "${ROCM_PATH:-}" ]]; then
echo "ROCm platform detected: adding --enforce-eager to avoid HIP-graph decode corruption"
EXTRA_ARGS+=(--enforce-eager)
fi
cleanup() {
if [[ -n "${SERVER_PID:-}" ]] && kill -0 "${SERVER_PID}" 2>/dev/null; then
kill "${SERVER_PID}" 2>/dev/null || true
@@ -18,10 +18,6 @@ wait_for_server() {
MODEL="Qwen/Qwen3-30B-A3B-FP8"
BACK="allgather_reducescatter"
if command -v rocm-smi &> /dev/null || [[ -d /opt/rocm ]] || [[ -n "${ROCM_PATH:-}" ]]; then
# Disable MOE padding for ROCm since it is causing eplb to fail.
export VLLM_ROCM_MOE_PADDING=0
fi
cleanup() {
if [[ -n "${SERVER_PID:-}" ]] && kill -0 "${SERVER_PID}" 2>/dev/null; then
+11 -32
View File
@@ -23,7 +23,6 @@ NC='\033[0m' # No Color
# Default configuration
PIPELINE="ci"
DRY_RUN=true
TORCH_NIGHTLY=false
usage() {
cat <<EOF
@@ -35,14 +34,12 @@ Sets RUN_ALL=1 and NIGHTLY=1 environment variables.
SAFETY: Dry-run by default. Use --execute to actually trigger a build.
Options:
--execute Actually trigger the build (default: dry-run)
--pipeline Buildkite pipeline slug (default: ${PIPELINE})
--commit Override commit SHA (default: current HEAD)
--branch Override branch name (default: current branch)
--message Custom build message (default: auto-generated)
--torch-nightly Also build and run the full suite against torch nightly
(sets TORCH_NIGHTLY=1)
--help Show this help message
--execute Actually trigger the build (default: dry-run)
--pipeline Buildkite pipeline slug (default: ${PIPELINE})
--commit Override commit SHA (default: current HEAD)
--branch Override branch name (default: current branch)
--message Custom build message (default: auto-generated)
--help Show this help message
Prerequisites:
- bk CLI installed: brew tap buildkite/buildkite && brew install buildkite/buildkite/bk
@@ -52,7 +49,6 @@ Examples:
$(basename "$0") # Dry-run, show what would happen
$(basename "$0") --execute # Actually trigger the build
$(basename "$0") --pipeline ci-shadow # Dry-run with different pipeline
$(basename "$0") --torch-nightly # Dry-run a full torch-nightly run
EOF
exit 1
}
@@ -100,10 +96,6 @@ while [[ $# -gt 0 ]]; do
MESSAGE="$2"
shift 2
;;
--torch-nightly)
TORCH_NIGHTLY=true
shift
;;
--help|-h)
usage
;;
@@ -179,17 +171,11 @@ if [[ $(echo "$REMOTE_BRANCHES" | wc -l) -gt 5 ]]; then
fi
echo ""
# Environment variables passed to the build.
BUILD_ENV=("RUN_ALL=1" "NIGHTLY=1")
if [[ "$TORCH_NIGHTLY" == true ]]; then
BUILD_ENV+=("TORCH_NIGHTLY=1")
fi
log_info "Pipeline: ${PIPELINE}"
log_info "Branch: ${BRANCH}"
log_info "Commit: ${COMMIT}"
log_info "Message: ${MESSAGE}"
log_info "Environment: ${BUILD_ENV[*]}"
log_info "Environment: RUN_ALL=1, NIGHTLY=1"
echo ""
# Build the command
@@ -201,10 +187,9 @@ CMD=(bk build create
--commit "${COMMIT}"
--branch "${BRANCH}"
--message "${MESSAGE}"
--env "RUN_ALL=1"
--env "NIGHTLY=1"
)
for env_var in "${BUILD_ENV[@]}"; do
CMD+=(--env "${env_var}")
done
if [[ "$DRY_RUN" == true ]]; then
echo "=========================================="
@@ -225,14 +210,8 @@ if [[ "$DRY_RUN" == true ]]; then
echo " --commit '$(escape_for_shell "${COMMIT}")' \\"
echo " --branch '$(escape_for_shell "${BRANCH}")' \\"
echo " --message '$(escape_for_shell "${MESSAGE}")' \\"
last_idx=$(( ${#BUILD_ENV[@]} - 1 ))
for i in "${!BUILD_ENV[@]}"; do
if [[ $i -eq $last_idx ]]; then
echo " --env '$(escape_for_shell "${BUILD_ENV[$i]}")'"
else
echo " --env '$(escape_for_shell "${BUILD_ENV[$i]}")' \\"
fi
done
echo " --env 'RUN_ALL=1' \\"
echo " --env 'NIGHTLY=1'"
echo ""
echo "=========================================="
echo -e "${YELLOW}To actually trigger this build, run:${NC}"
+4 -12
View File
@@ -6,14 +6,8 @@ set -ex
# manylinux platform tag with auditwheel.
# Index generation is handled separately by generate-and-upload-nightly-index.sh.
# auditwheel is Linux-only; macOS wheels already carry a valid tag, so skip the
# manylinux retag for them.
WHEEL_PLATFORM="${VLLM_WHEEL_PLATFORM:-linux}"
if [[ "$WHEEL_PLATFORM" == "linux" ]]; then
# shellcheck source=lib/manylinux.sh
source .buildkite/scripts/lib/manylinux.sh
fi
# shellcheck source=lib/manylinux.sh
source .buildkite/scripts/lib/manylinux.sh
BUCKET="vllm-wheels"
SUBPATH=$BUILDKITE_COMMIT
@@ -33,10 +27,8 @@ wheel="${wheel_files[0]}"
# ========= detect manylinux tag and rename ==========
if [[ "$WHEEL_PLATFORM" == "linux" ]]; then
wheel="$(apply_manylinux_tag "$wheel")"
echo "Renamed wheel to: $wheel"
fi
wheel="$(apply_manylinux_tag "$wheel")"
echo "Renamed wheel to: $wheel"
# Extract the version from the wheel
version=$(unzip -p "$wheel" '**/METADATA' | grep '^Version: ' | cut -d' ' -f2)
+3 -3
View File
@@ -113,8 +113,8 @@ $PYTHON .buildkite/scripts/generate-nightly-index.py \
echo "Uploading indices to $S3_COMMIT_PREFIX"
aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "$S3_COMMIT_PREFIX"
# Only scheduled nightly builds should update the moving nightly index.
if [[ "${NIGHTLY:-0}" == "1" ]]; then
# Update rocm/nightly/ if on main branch and not a PR
if [[ "$BUILDKITE_BRANCH" == "main" && "$BUILDKITE_PULL_REQUEST" == "false" ]] || [[ "$NIGHTLY" == "1" ]]; then
echo "Updating rocm/nightly/ index..."
aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "s3://$BUCKET/rocm/nightly/"
fi
@@ -147,7 +147,7 @@ echo ""
echo "Install command (by commit):"
echo " pip install vllm --extra-index-url https://${BUCKET}.s3.amazonaws.com/$ROCM_SUBPATH/"
echo ""
if [[ "${NIGHTLY:-0}" == "1" ]]; then
if [[ "$BUILDKITE_BRANCH" == "main" ]] || [[ "$NIGHTLY" == "1" ]]; then
echo "Install command (nightly):"
echo " pip install vllm --extra-index-url https://${BUCKET}.s3.amazonaws.com/rocm/nightly/"
fi
@@ -1,13 +0,0 @@
#!/bin/bash
set -euo pipefail
REGISTRY="public.ecr.aws/q9t5s3a7"
REPO="vllm-release-repo"
ARCH_TAG="${BUILDKITE_COMMIT}-$(uname -m)-xpu"
PLATFORM_TAG="${BUILDKITE_COMMIT}-xpu"
aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin ${REGISTRY}
docker manifest rm ${REGISTRY}/${REPO}:${PLATFORM_TAG} || true
docker manifest create ${REGISTRY}/${REPO}:${PLATFORM_TAG} ${REGISTRY}/${REPO}:${ARCH_TAG} --amend
docker manifest push ${REGISTRY}/${REPO}:${PLATFORM_TAG}
@@ -1,21 +0,0 @@
#!/bin/bash
set -ex
ORIG_TAG_NAME="$BUILDKITE_COMMIT"
REPO="vllm/vllm-openai-xpu"
echo "Pushing original XPU tag ${ORIG_TAG_NAME}-xpu to nightly tags in ${REPO}"
aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:"$ORIG_TAG_NAME"-x86_64-xpu
docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:"$ORIG_TAG_NAME"-x86_64-xpu ${REPO}:nightly-x86_64
docker push ${REPO}:nightly-x86_64
docker manifest rm ${REPO}:nightly || true
docker manifest rm ${REPO}:nightly-"$BUILDKITE_COMMIT" || true
docker manifest create ${REPO}:nightly ${REPO}:nightly-x86_64 --amend
docker manifest create ${REPO}:nightly-"$BUILDKITE_COMMIT" ${REPO}:nightly-x86_64 --amend
docker manifest push ${REPO}:nightly
docker manifest push ${REPO}:nightly-"$BUILDKITE_COMMIT"
+668 -899
View File
File diff suppressed because it is too large Load Diff
+6 -8
View File
@@ -4,7 +4,7 @@ depends_on:
steps:
- label: V1 attention (H100-MI300)
key: v1-attention-h100-mi300
timeout_in_minutes: 85
timeout_in_minutes: 30
device: h100
source_file_dependencies:
- vllm/config/attention.py
@@ -12,12 +12,11 @@ steps:
- vllm/v1/attention
- tests/v1/attention
commands:
- pytest -v -s v1/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
parallelism: 2
- pytest -v -s v1/attention
mirror:
amd:
device: mi325_1
timeout_in_minutes: 95
device: mi300_1
timeout_in_minutes: 70
depends_on:
- image-build-amd
source_file_dependencies:
@@ -31,7 +30,7 @@ steps:
- label: V1 attention (B200)
key: v1-attention-b200
timeout_in_minutes: 80
timeout_in_minutes: 30
device: b200-k8s
source_file_dependencies:
- vllm/config/attention.py
@@ -39,5 +38,4 @@ steps:
- vllm/v1/attention
- tests/v1/attention
commands:
- pytest -v -s v1/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
parallelism: 2
- pytest -v -s v1/attention
+1 -7
View File
@@ -4,7 +4,7 @@ depends_on:
steps:
- label: Basic Correctness
key: basic-correctness
timeout_in_minutes: 45
timeout_in_minutes: 30
device: h200_18gb
source_file_dependencies:
- vllm/
@@ -16,9 +16,3 @@ steps:
- pytest -v -s basic_correctness/test_mem.py
- pytest -v -s basic_correctness/test_basic_correctness.py
- pytest -v -s basic_correctness/test_cpu_offload.py
mirror:
amd:
device: mi325_1
timeout_in_minutes: 70
depends_on:
- image-build-amd
+3 -9
View File
@@ -4,19 +4,13 @@ depends_on:
steps:
- label: Benchmarks CLI Test
key: benchmarks-cli-test
timeout_in_minutes: 30
timeout_in_minutes: 20
device: h200_18gb
source_file_dependencies:
- vllm/
- tests/benchmarks/
commands:
- pytest -v -s benchmarks/
mirror:
amd:
dind: false
device: mi300_1
depends_on:
- image-build-amd
- label: Attention Benchmarks Smoke Test (B200)
key: attention-benchmarks-smoke-test-b200
@@ -24,9 +18,9 @@ steps:
num_gpus: 2
optional: true
working_dir: "/vllm-workspace/"
timeout_in_minutes: 20
timeout_in_minutes: 10
source_file_dependencies:
- benchmarks/attention_benchmarks/
- vllm/v1/attention/
commands:
- python3 benchmarks/attention_benchmarks/benchmark.py --backends flash flashinfer --batch-specs "8q1s1k"
- python3 benchmarks/attention_benchmarks/benchmark.py --backends flash flashinfer --batch-specs "8q1s1k" --repeats 1 --warmup-iters 1
+11 -11
View File
@@ -4,7 +4,7 @@ depends_on:
steps:
- label: Sequence Parallel Correctness Tests (2 GPUs)
key: sequence-parallel-correctness-tests-2-gpus
timeout_in_minutes: 80
timeout_in_minutes: 50
working_dir: "/vllm-workspace/"
num_devices: 2
source_file_dependencies:
@@ -19,7 +19,7 @@ steps:
- label: Sequence Parallel Correctness Tests (2xH100)
key: sequence-parallel-correctness-tests-2xh100
timeout_in_minutes: 75
timeout_in_minutes: 50
working_dir: "/vllm-workspace/"
device: h100
optional: true
@@ -30,7 +30,7 @@ steps:
- label: AsyncTP Correctness Tests (2xH100)
key: asynctp-correctness-tests-2xh100
timeout_in_minutes: 30
timeout_in_minutes: 50
working_dir: "/vllm-workspace/"
device: h100
optional: true
@@ -41,7 +41,7 @@ steps:
- label: AsyncTP Correctness Tests (B200)
key: asynctp-correctness-tests-b200
timeout_in_minutes: 30
timeout_in_minutes: 50
working_dir: "/vllm-workspace/"
device: b200-k8s
optional: true
@@ -52,7 +52,7 @@ steps:
- label: Distributed Compile Unit Tests (2xH100)
key: distributed-compile-unit-tests-2xh100
timeout_in_minutes: 45
timeout_in_minutes: 20
working_dir: "/vllm-workspace/"
device: h100
num_devices: 2
@@ -66,7 +66,7 @@ steps:
- label: Fusion and Compile Unit Tests (2xB200)
key: fusion-and-compile-unit-tests-2xb200
timeout_in_minutes: 30
timeout_in_minutes: 20
working_dir: "/vllm-workspace/"
device: b200-k8s
source_file_dependencies:
@@ -96,7 +96,7 @@ steps:
- label: Fusion E2E Quick (H100)
key: fusion-e2e-quick-h100
timeout_in_minutes: 25
timeout_in_minutes: 15
working_dir: "/vllm-workspace/"
device: h100
num_devices: 1
@@ -115,7 +115,7 @@ steps:
- label: Fusion E2E Config Sweep (H100)
key: fusion-e2e-config-sweep-h100
timeout_in_minutes: 25
timeout_in_minutes: 30
working_dir: "/vllm-workspace/"
device: h100
num_devices: 1
@@ -149,7 +149,7 @@ steps:
- label: Fusion E2E TP2 Quick (H100)
key: fusion-e2e-tp2-quick-h100
timeout_in_minutes: 35
timeout_in_minutes: 20
working_dir: "/vllm-workspace/"
device: h100
num_devices: 2
@@ -167,7 +167,7 @@ steps:
- label: Fusion E2E TP2 AR-RMS Config Sweep (H100)
key: fusion-e2e-tp2-ar-rms-config-sweep-h100
timeout_in_minutes: 30
timeout_in_minutes: 40
working_dir: "/vllm-workspace/"
device: h100
num_devices: 2
@@ -207,7 +207,7 @@ steps:
- label: Fusion E2E TP2 (B200)
key: fusion-e2e-tp2-b200
timeout_in_minutes: 45
timeout_in_minutes: 20
working_dir: "/vllm-workspace/"
device: b200-k8s
num_devices: 2
+5 -6
View File
@@ -2,9 +2,9 @@ group: CUDA
depends_on:
- image-build
steps:
- label: Platform Tests
key: platform-tests
timeout_in_minutes: 20
- label: Platform Tests (CUDA)
key: platform-tests-cuda
timeout_in_minutes: 15
device: h200_18gb
source_file_dependencies:
- vllm/envs.py
@@ -18,9 +18,8 @@ steps:
- pytest -v -s cuda/test_platform_no_cuda_init.py
- label: Cudagraph
device: h200_35gb
key: cudagraph
timeout_in_minutes: 30
timeout_in_minutes: 20
source_file_dependencies:
- tests/v1/cudagraph
- vllm/v1/cudagraph_dispatcher.py
@@ -29,4 +28,4 @@ steps:
commands:
- pytest -v -s v1/cudagraph/test_cudagraph_dispatch.py
- pytest -v -s v1/cudagraph/test_cudagraph_mode.py
- pytest -v -s v1/cudagraph/test_breakable_cudagraph.py
- pytest -v -s v1/cudagraph/test_breakable_cudagraph.py
+7 -128
View File
@@ -4,7 +4,7 @@ depends_on:
steps:
- label: Distributed NixlConnector PD accuracy (4 GPUs)
key: distributed-nixlconnector-pd-accuracy-4-gpus
timeout_in_minutes: 55
timeout_in_minutes: 30
working_dir: "/vllm-workspace/tests"
num_devices: 4
source_file_dependencies:
@@ -13,24 +13,9 @@ steps:
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
mirror:
amd:
dind: false
device: mi300_4
timeout_in_minutes: 85
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- tests/v1/kv_connector/nixl_integration/
- vllm/platforms/rocm.py
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
- label: Distributed FlashInfer NixlConnector PD accuracy (4 GPUs)
key: distributed-flashinfer-nixlconnector-pd-accuracy-4-gpus
timeout_in_minutes: 55
timeout_in_minutes: 30
working_dir: "/vllm-workspace/tests"
num_devices: 4
source_file_dependencies:
@@ -40,19 +25,6 @@ steps:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- FLASHINFER=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
- label: Push NixlConnector PP prefill PD accuracy (4 GPUs)
key: push-nixlconnector-pp-prefill-pd-accuracy-4-gpus
timeout_in_minutes: 30
working_dir: "/vllm-workspace/tests"
num_devices: 4
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- tests/v1/kv_connector/nixl_integration/
- tests/v1/kv_connector/nixl_push_integration/
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- bash v1/kv_connector/nixl_push_integration/config_sweep_accuracy_test.sh
- label: DP EP Distributed NixlConnector PD accuracy tests (4 GPUs)
key: dp-ep-distributed-nixlconnector-pd-accuracy-tests-4-gpus
timeout_in_minutes: 30
@@ -64,24 +36,10 @@ steps:
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- DP_EP=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
mirror:
amd:
dind: false
device: mi300_4
timeout_in_minutes: 60
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- tests/v1/kv_connector/nixl_integration/
- vllm/platforms/rocm.py
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- DP_EP=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
- label: CrossLayer KV layout Distributed NixlConnector PD accuracy tests (4 GPUs)
key: crosslayer-kv-layout-distributed-nixlconnector-pd-accuracy-tests-4-gpus
timeout_in_minutes: 55
timeout_in_minutes: 30
working_dir: "/vllm-workspace/tests"
num_devices: 4
source_file_dependencies:
@@ -90,24 +48,10 @@ steps:
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- CROSS_LAYERS_BLOCKS=True bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
mirror:
amd:
dind: false
device: mi300_4
timeout_in_minutes: 85
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- tests/v1/kv_connector/nixl_integration/
- vllm/platforms/rocm.py
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- CROSS_LAYERS_BLOCKS=True ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
- label: Hybrid SSM NixlConnector PD accuracy tests (4 GPUs)
key: hybrid-ssm-nixlconnector-pd-accuracy-tests-4-gpus
timeout_in_minutes: 60
timeout_in_minutes: 25
working_dir: "/vllm-workspace/tests"
num_devices: 4
source_file_dependencies:
@@ -116,38 +60,10 @@ steps:
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- HYBRID_SSM=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
mirror:
amd:
dind: false
device: mi300_4
timeout_in_minutes: 80
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- tests/v1/kv_connector/nixl_integration/
- vllm/platforms/rocm.py
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- HYBRID_SSM=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
- label: Hybrid SSM NixlConnector PD prefix cache test (2 GPUs)
key: hybrid-ssm-nixlconnector-pd-prefix-cache-2-gpus
timeout_in_minutes: 25
working_dir: "/vllm-workspace/tests"
num_devices: 2
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- vllm/v1/core/sched/
- vllm/v1/core/kv_cache_coordinator.py
- tests/v1/kv_connector/nixl_integration/
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- bash v1/kv_connector/nixl_integration/run_mamba_prefix_cache_test.sh
- label: MultiConnector (Nixl+Offloading) PD accuracy (2 GPUs)
key: multiconnector-nixl-offloading-pd-accuracy-2-gpus
timeout_in_minutes: 40
timeout_in_minutes: 30
working_dir: "/vllm-workspace/tests"
num_devices: 2
source_file_dependencies:
@@ -162,7 +78,7 @@ steps:
- label: NixlConnector PD + Spec Decode acceptance (2 GPUs)
key: nixlconnector-pd-spec-decode-acceptance-2-gpus
timeout_in_minutes: 45
timeout_in_minutes: 30
device: a100
working_dir: "/vllm-workspace/tests"
num_devices: 2
@@ -173,25 +89,10 @@ steps:
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh
mirror:
amd:
dind: false
device: mi300_2
timeout_in_minutes: 70
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
- vllm/v1/worker/kv_connector_model_runner_mixin.py
- tests/v1/kv_connector/nixl_integration/
- vllm/platforms/rocm.py
commands:
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
- KV_CACHE_MEMORY_BYTES=8G ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh
- label: MultiConnector (Nixl+Offloading) PD edge cases (2 GPUs)
key: multiconnector-nixl-offloading-pd-edge-cases-2-gpus
timeout_in_minutes: 25
timeout_in_minutes: 30
working_dir: "/vllm-workspace/tests"
num_devices: 2
source_file_dependencies:
@@ -203,25 +104,3 @@ steps:
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- bash v1/kv_connector/nixl_integration/run_multi_connector_edge_case_test.sh
# P TP 4 - D DPEP 4 test case for DSv4-Flash
- label: DSv4-Flash Disaggregated DP EP
key: dsv4-flash-disaggregated
timeout_in_minutes: 60
device: h200
optional: true
working_dir: "/vllm-workspace/tests"
num_devices: 8
env:
ENABLE_HMA_FLAG: "1"
DP_EP: "1"
GPU_MEMORY_UTILIZATION: "0.85"
PREFILLER_TP_SIZE: "4"
DECODER_TP_SIZE: "4"
PREFILL_BLOCK_SIZE: "256"
DECODE_BLOCK_SIZE: "256"
MODEL_NAMES: "deepseek-ai/DeepSeek-V4-Flash"
VLLM_SERVE_EXTRA_ARGS: "--trust-remote-code,--kv-cache-dtype,fp8"
commands:
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
- bash v1/kv_connector/nixl_integration/run_accuracy_test.sh
+19 -35
View File
@@ -4,7 +4,7 @@ depends_on:
steps:
- label: Distributed Comm Ops
key: distributed-comm-ops
timeout_in_minutes: 25
timeout_in_minutes: 20
working_dir: "/vllm-workspace/tests"
num_devices: 2
source_file_dependencies:
@@ -18,7 +18,7 @@ steps:
- label: Distributed DP Tests (2 GPUs)
key: distributed-dp-tests-2-gpus
timeout_in_minutes: 35
timeout_in_minutes: 20
working_dir: "/vllm-workspace/tests"
num_devices: 2
source_file_dependencies:
@@ -37,26 +37,10 @@ steps:
- TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
- TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py
- DP_SIZE=2 pytest -v -s entrypoints/openai/test_multi_api_servers.py
mirror:
amd:
dind: false
device: mi300_2
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/
- vllm/engine/
- vllm/executor/
- vllm/worker/worker_base.py
- vllm/v1/engine/
- vllm/v1/worker/
- tests/v1/distributed
- tests/entrypoints/openai/test_multi_api_servers.py
- vllm/platforms/rocm.py
- label: Distributed Compile + RPC Tests (2 GPUs)
key: distributed-compile-rpc-tests-2-gpus
timeout_in_minutes: 65
timeout_in_minutes: 20
working_dir: "/vllm-workspace/tests"
num_devices: 2
source_file_dependencies:
@@ -79,7 +63,7 @@ steps:
- label: Distributed Torchrun + Shutdown Tests (2 GPUs)
key: distributed-torchrun-shutdown-tests-2-gpus
timeout_in_minutes: 30
timeout_in_minutes: 20
working_dir: "/vllm-workspace/tests"
num_devices: 2
source_file_dependencies:
@@ -134,7 +118,7 @@ steps:
- label: Distributed DP Tests (4 GPUs)
key: distributed-dp-tests-4-gpus
timeout_in_minutes: 45
timeout_in_minutes: 30
working_dir: "/vllm-workspace/tests"
num_devices: 4
source_file_dependencies:
@@ -155,7 +139,7 @@ steps:
- label: Distributed Compile + Comm (4 GPUs)
key: distributed-compile-comm-4-gpus
timeout_in_minutes: 70
timeout_in_minutes: 30
working_dir: "/vllm-workspace/tests"
num_devices: 4
source_file_dependencies:
@@ -175,9 +159,9 @@ steps:
# test multi-node TP with multiproc executor (simulated on single node)
- pytest -v -s distributed/test_multiproc_executor.py::test_multiproc_executor_multi_node
- label: Distributed Tests (8xH100)
key: distributed-tests-8xh100
timeout_in_minutes: 20
- label: Distributed Tests (8 GPUs)(H100)
key: distributed-tests-8-gpus-h100
timeout_in_minutes: 10
device: h100
num_devices: 8
working_dir: "/vllm-workspace/tests"
@@ -196,8 +180,8 @@ steps:
# test with torchrun tp=2 and dp=4 with ep
- torchrun --nproc-per-node=8 ../examples/features/torchrun/torchrun_dp_example_offline.py --tp-size=2 --pp-size=1 --dp-size=4 --enable-ep
- label: Distributed Tests (4xA100)
key: distributed-tests-4xa100
- label: Distributed Tests (4 GPUs)(A100)
key: distributed-tests-4-gpus-a100
device: a100
optional: true
num_devices: 4
@@ -211,9 +195,9 @@ steps:
- TARGET_TEST_SUITE=A100 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
- pytest -v -s -x lora/test_mixtral.py
- label: Distributed Tests (2xH100-2xMI300)
key: distributed-tests-2xh100-2xmi300
timeout_in_minutes: 30
- label: Distributed Tests (2 GPUs)(H100)
key: distributed-tests-2-gpus-h100
timeout_in_minutes: 15
device: h100
optional: true
working_dir: "/vllm-workspace/"
@@ -226,15 +210,15 @@ steps:
- VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer.py
- pytest -v -s tests/distributed/test_packed_tensor.py
- label: Distributed Tests (2xB200)
key: distributed-tests-2xb200
- label: Distributed Tests (2 GPUs)(B200)
key: distributed-tests-2-gpus-b200
device: b200-k8s
optional: true
working_dir: "/vllm-workspace/"
num_devices: 2
commands:
- pytest -v -s tests/distributed/test_context_parallel.py
- pytest -v -s tests/distributed/test_nccl_symm_mem.py
- pytest -v -s tests/distributed/test_nccl_symm_mem_allreduce.py
- pytest -v -s tests/v1/distributed/test_dbo.py
- pytest -v -s tests/distributed/test_mnnvl_alltoall.py
@@ -260,7 +244,7 @@ steps:
- label: Pipeline + Context Parallelism (4 GPUs)
key: pipeline-context-parallelism-4-gpus
timeout_in_minutes: 55
timeout_in_minutes: 60
working_dir: "/vllm-workspace/tests"
num_devices: 4
source_file_dependencies:
@@ -275,7 +259,7 @@ steps:
- label: RayExecutorV2 (4 GPUs)
key: rayexecutorv2-4-gpus
timeout_in_minutes: 45
timeout_in_minutes: 60
working_dir: "/vllm-workspace/tests"
num_devices: 4
source_file_dependencies:
+1 -1
View File
@@ -3,7 +3,7 @@ depends_on:
- image-build-cpu
steps:
- label: Docker Build Metadata
timeout_in_minutes: 20
timeout_in_minutes: 10
device: cpu-small
source_file_dependencies:
- .buildkite/release-pipeline.yaml
+11 -11
View File
@@ -2,9 +2,9 @@ group: E2E Integration
depends_on:
- image-build
steps:
- label: DeepSeek V2-Lite Sync EPLB Accuracy (4xH100)
key: deepseek-v2-lite-sync-eplb-accuracy-4xh100
timeout_in_minutes: 25
- label: DeepSeek V2-Lite Sync EPLB Accuracy
key: deepseek-v2-lite-sync-eplb-accuracy
timeout_in_minutes: 60
device: h100
optional: true
num_devices: 4
@@ -12,9 +12,9 @@ steps:
commands:
- bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh 0.25 200 8010
- label: Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy (4xH100)
key: qwen3-30b-a3b-fp8-block-sync-eplb-accuracy-4xh100
timeout_in_minutes: 25
- label: Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy
key: qwen3-30b-a3b-fp8-block-sync-eplb-accuracy
timeout_in_minutes: 60
device: h100
optional: true
num_devices: 4
@@ -22,9 +22,9 @@ steps:
commands:
- bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020
- label: Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy (2xB200)
key: qwen3-30b-a3b-fp8-block-sync-eplb-accuracy-2xb200
timeout_in_minutes: 20
- label: Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy (B200)
key: qwen3-30b-a3b-fp8-block-sync-eplb-accuracy-b200
timeout_in_minutes: 60
device: b200-k8s
optional: true
num_devices: 2
@@ -34,7 +34,7 @@ steps:
- label: Qwen3-30B-A3B-FP8 DP4 Async EPLB Accuracy
key: qwen3-30b-a3b-fp8-dp4-async-eplb-accuracy
timeout_in_minutes: 25
timeout_in_minutes: 60
device: h100
optional: true
num_devices: 4
@@ -44,7 +44,7 @@ steps:
- label: DeepSeek V2-Lite Prefetch Offload Accuracy (H100)
key: deepseek-v2-lite-prefetch-offload-accuracy-h100
timeout_in_minutes: 20
timeout_in_minutes: 60
device: h100
optional: true
num_devices: 1
+13 -29
View File
@@ -4,7 +4,7 @@ depends_on:
steps:
- label: Engine
key: engine
timeout_in_minutes: 30
timeout_in_minutes: 15
device: h200_18gb
source_file_dependencies:
- vllm/compilation/
@@ -28,14 +28,14 @@ steps:
- pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py test_jit_monitor.py
mirror:
amd:
device: mi325_1
timeout_in_minutes: 50
device: mi300_1
timeout_in_minutes: 60
depends_on:
- image-build-amd
- label: Engine (1 GPU)
key: engine-1-gpu
timeout_in_minutes: 45
timeout_in_minutes: 30
source_file_dependencies:
- vllm/v1/engine/
- tests/v1/engine/
@@ -44,14 +44,14 @@ steps:
- pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py
mirror:
amd:
device: mi325_1
timeout_in_minutes: 55
device: mi300_1
timeout_in_minutes: 40
depends_on:
- image-build-amd
- label: e2e Scheduling (1 GPU)
key: e2e-scheduling-1-gpu
timeout_in_minutes: 35
timeout_in_minutes: 30
device: h200_18gb
source_file_dependencies:
- vllm/v1/
@@ -60,34 +60,24 @@ steps:
- pytest -v -s v1/e2e/general/test_async_scheduling.py
mirror:
amd:
device: mi325_1
timeout_in_minutes: 70
device: mi250_1
timeout_in_minutes: 60
depends_on:
- image-build-amd
- label: e2e Core (1 GPU)
device: h200_35gb
key: e2e-core-1-gpu
timeout_in_minutes: 40
timeout_in_minutes: 30
source_file_dependencies:
- vllm/v1/
- tests/v1/e2e/general/
commands:
- pytest -v -s v1/e2e/general --ignore v1/e2e/general/test_async_scheduling.py
mirror:
amd:
device: mi325_1
timeout_in_minutes: 60
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/v1/
- tests/v1/e2e/general/
- vllm/platforms/rocm.py
- label: V1 e2e (2 GPUs)
key: v1-e2e-2-gpus
timeout_in_minutes: 25 # TODO: Fix timeout after we have more confidence in the test stability
timeout_in_minutes: 60 # TODO: Fix timeout after we have more confidence in the test stability
optional: true
num_devices: 2
source_file_dependencies:
@@ -112,16 +102,10 @@ steps:
commands:
# Only run tests that need exactly 2 GPUs
- pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "tensor_parallelism"
mirror:
amd:
dind: false
device: mi300_2
depends_on:
- image-build-amd
- label: V1 e2e (4 GPUs)
key: v1-e2e-4-gpus
timeout_in_minutes: 20 # TODO: Fix timeout after we have more confidence in the test stability
timeout_in_minutes: 60 # TODO: Fix timeout after we have more confidence in the test stability
optional: true
num_devices: 4
source_file_dependencies:
@@ -149,7 +133,7 @@ steps:
- label: V1 e2e (4xH100)
key: v1-e2e-4xh100
timeout_in_minutes: 35
timeout_in_minutes: 60
device: h100
num_devices: 4
optional: true
+15 -26
View File
@@ -3,9 +3,8 @@ depends_on:
- image-build
steps:
- label: Entrypoints Unit Tests
device: h200_35gb
key: entrypoints-unit-tests
timeout_in_minutes: 25
timeout_in_minutes: 10
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/entrypoints
@@ -16,9 +15,8 @@ steps:
- pytest -v -s entrypoints/weight_transfer
- label: Entrypoints Integration (LLM)
device: h200_35gb
key: entrypoints-integration-llm
timeout_in_minutes: 60
timeout_in_minutes: 40
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/
@@ -31,35 +29,30 @@ steps:
mirror:
amd:
device: mi325_1
# TODO(akaratza): Test after Torch >= 2.12 bump
soft_fail: true
depends_on:
- image-build-amd
- label: Entrypoints Integration (API Server)
key: entrypoints-integration-api-server
device: h200_35gb
timeout_in_minutes: 50
timeout_in_minutes: 130
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/
- tests/entrypoints/serve
- tests/entrypoints/scale_out
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc
- PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc
- pytest -v -s entrypoints/scale_out
mirror:
amd:
device: mi325_1
device: mi300_1
depends_on:
- image-build-amd
- label: Entrypoints Integration (API Server OpenAI - Part 1)
device: h200_35gb
key: entrypoints-integration-api-server-openai-part-1
timeout_in_minutes: 45
timeout_in_minutes: 50
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/
@@ -70,15 +63,14 @@ steps:
- pytest -v -s entrypoints/openai --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/correctness
mirror:
amd:
device: mi325_1
timeout_in_minutes: 65
device: mi300_1
timeout_in_minutes: 80
depends_on:
- image-build-amd
- label: Entrypoints Integration (API Server OpenAI - Part 2)
device: h200_35gb
key: entrypoints-integration-api-server-openai-part-2
timeout_in_minutes: 45
timeout_in_minutes: 50
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/
@@ -90,13 +82,12 @@ steps:
- pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py
mirror:
amd:
device: mi325_1
device: mi300_1
timeout_in_minutes: 80
depends_on:
- image-build-amd
- label: Entrypoints Integration (API Server Generate)
device: h200_35gb
key: entrypoints-integration-api-server-generate
timeout_in_minutes: 50
working_dir: "/vllm-workspace/tests"
@@ -113,13 +104,12 @@ steps:
- pytest -v -s entrypoints/anthropic
mirror:
amd:
device: mi325_1
timeout_in_minutes: 65
device: mi300_1
timeout_in_minutes: 60
depends_on:
- image-build-amd
- label: Entrypoints Integration (Responses API)
device: h200_35gb
key: entrypoints-integration-responses-api
timeout_in_minutes: 50
working_dir: "/vllm-workspace/tests"
@@ -132,7 +122,7 @@ steps:
- label: Entrypoints Integration (Speech to Text)
device: h200_35gb
key: entrypoints-integration-speech_to_text
timeout_in_minutes: 45
timeout_in_minutes: 50
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/
@@ -144,7 +134,7 @@ steps:
- label: Entrypoints Integration (Multimodal)
device: h200_35gb
key: entrypoints-integration-multimodal
timeout_in_minutes: 45
timeout_in_minutes: 50
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/
@@ -154,7 +144,6 @@ steps:
- pytest -v -s entrypoints/multimodal
- label: Entrypoints Integration (Pooling)
device: h200_35gb
key: entrypoints-integration-pooling
timeout_in_minutes: 50
working_dir: "/vllm-workspace/tests"
@@ -167,7 +156,7 @@ steps:
- label: OpenAI API Correctness
key: openai-api-correctness
timeout_in_minutes: 20
timeout_in_minutes: 30
device: h200_18gb
source_file_dependencies:
- csrc/
@@ -176,7 +165,7 @@ steps:
- pytest -s entrypoints/openai/correctness/
mirror:
amd:
device: mi325_1
device: mi300_1
depends_on:
- image-build-amd
source_file_dependencies:
+3 -14
View File
@@ -4,7 +4,7 @@ depends_on:
steps:
- label: EPLB Algorithm
key: eplb-algorithm
timeout_in_minutes: 20
timeout_in_minutes: 15
device: h200_18gb
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
@@ -14,21 +14,10 @@ steps:
commands:
- pytest -v -s distributed/test_eplb_algo.py
- pytest -v -s distributed/test_eplb_utils.py
mirror:
amd:
dind: false
device: mi300_1
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/distributed/eplb
- tests/distributed/test_eplb_algo.py
- tests/distributed/test_eplb_utils.py
- vllm/platforms/rocm.py
- label: EPLB Execution # 17min
key: eplb-execution
timeout_in_minutes: 25
timeout_in_minutes: 27
working_dir: "/vllm-workspace/tests"
num_devices: 4
source_file_dependencies:
@@ -40,7 +29,7 @@ steps:
- label: Elastic EP Scaling Test
key: elastic-ep-scaling-test
timeout_in_minutes: 30
timeout_in_minutes: 20
device: h100
working_dir: "/vllm-workspace/tests"
num_devices: 4
+28 -90
View File
@@ -4,7 +4,7 @@ depends_on:
steps:
- label: vLLM IR Tests
key: vllm-ir-tests
timeout_in_minutes: 35
timeout_in_minutes: 10
device: h200_18gb
working_dir: "/vllm-workspace/"
source_file_dependencies:
@@ -15,21 +15,19 @@ steps:
- pytest -v -s tests/kernels/ir
- label: Kernels Core Operation Test
device: h200_35gb
key: kernels-core-operation-test
timeout_in_minutes: 120
timeout_in_minutes: 75
source_file_dependencies:
- csrc/
- tests/kernels/core
- tests/kernels/test_concat_mla_q.py
- tests/kernels/test_fused_qk_norm_rope_gate.py
commands:
- pytest -v -s kernels/core --ignore=kernels/core/test_minimax_reduce_rms.py kernels/test_concat_mla_q.py kernels/test_fused_qk_norm_rope_gate.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
parallelism: 3
- pytest -v -s kernels/core --ignore=kernels/core/test_minimax_reduce_rms.py kernels/test_concat_mla_q.py kernels/test_fused_qk_norm_rope_gate.py
- label: Kernels MiniMax Reduce RMS Test (2 GPUs)
key: kernels-minimax-reduce-rms-test-2-gpus
timeout_in_minutes: 20
timeout_in_minutes: 15
num_devices: 2
device: h100
source_file_dependencies:
@@ -43,20 +41,18 @@ steps:
- label: Deepseek V4 Kernel Test (H100)
key: deepseek-v4-kernel-test-h100
timeout_in_minutes: 30
timeout_in_minutes: 15
device: h100
source_file_dependencies:
- csrc/fused_deepseek_v4_qnorm_rope_kv_insert_kernel.cu
- vllm/models/deepseek_v4/common/ops/
- tests/kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py
- tests/kernels/test_top_k_per_row.py # it runs on Blackwell too - some kernels have arch-specific optimizations
commands:
- pytest -v -s kernels/test_fused_deepseek_v4_*.py
- pytest -v -s kernels/test_top_k_per_row.py
- label: Deepseek V4 Kernel Test (B200)
key: deepseek-v4-kernel-test-b200
timeout_in_minutes: 20
timeout_in_minutes: 15
device: b200-k8s
source_file_dependencies:
- csrc/fused_deepseek_v4_qnorm_rope_kv_insert_kernel.cu
@@ -67,7 +63,7 @@ steps:
- label: Kernels Attention Test %N
key: kernels-attention-test
timeout_in_minutes: 65
timeout_in_minutes: 35
source_file_dependencies:
- csrc/attention/
- vllm/v1/attention
@@ -78,37 +74,10 @@ steps:
commands:
- pytest -v -s kernels/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
parallelism: 2
mirror:
amd:
device: mi325_1
timeout_in_minutes: 90
depends_on:
- image-build-amd
source_file_dependencies:
- csrc/attention/
- vllm/v1/attention
- vllm/model_executor/layers/attention
- tests/kernels/attention
- vllm/_aiter_ops.py
- vllm/envs.py
- vllm/platforms/rocm.py
- label: Kernels Attention DiffKV Test (H100)
key: kernels-attention-diffkv-test-h100
timeout_in_minutes: 20
device: h100
num_devices: 1
source_file_dependencies:
- vllm/v1/attention/ops/triton_unified_attention_diffkv.py
- vllm/v1/attention/backends/triton_attn_diffkv.py
- vllm/v1/attention/backends/flash_attn_diffkv.py
- tests/kernels/attention/test_triton_unified_attention_diffkv.py
commands:
- pytest -v -s kernels/attention/test_triton_unified_attention_diffkv.py
- label: Kernels Quantization Test %N
key: kernels-quantization-test
timeout_in_minutes: 60
timeout_in_minutes: 90
source_file_dependencies:
- csrc/quantization/
- vllm/model_executor/layers/quantization
@@ -118,11 +87,10 @@ steps:
parallelism: 2
mirror:
amd:
device: mi325_1
device: mi300_1
source_file_dependencies:
- csrc/quantization/
- vllm/model_executor/layers/quantization
- vllm/config/
- tests/kernels/quantization
- tests/kernels/quantization/test_rocm_skinny_gemms.py
- vllm/_aiter_ops.py
@@ -133,7 +101,7 @@ steps:
- label: Kernels MoE Test %N
key: kernels-moe-test
timeout_in_minutes: 50
timeout_in_minutes: 25
source_file_dependencies:
- csrc/quantization/cutlass_w8a8/moe/
- csrc/moe/
@@ -146,27 +114,10 @@ steps:
- pytest -v -s kernels/moe --ignore=kernels/moe/test_modular_oai_triton_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
- pytest -v -s kernels/moe/test_modular_oai_triton_moe.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
parallelism: 5
mirror:
amd:
device: mi325_1
timeout_in_minutes: 65
source_file_dependencies:
- csrc/quantization/cutlass_w8a8/moe/
- csrc/moe/
- tests/kernels/moe
- vllm/model_executor/layers/fused_moe/
- vllm/distributed/device_communicators/
- vllm/envs.py
- vllm/config
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
depends_on:
- image-build-amd
- label: Kernels Mamba Test
device: h200_35gb
key: kernels-mamba-test
timeout_in_minutes: 40
timeout_in_minutes: 45
source_file_dependencies:
- csrc/mamba/
- tests/kernels/mamba
@@ -175,19 +126,19 @@ steps:
- pytest -v -s kernels/mamba
- label: Kernels KDA Test
timeout_in_minutes: 25
timeout_in_minutes: 20
device: h200_18gb
source_file_dependencies:
- vllm/third_party/flash_linear_attention/ops/kda.py
- vllm/third_party/flash_linear_attention/ops/chunk_delta_h.py
- vllm/third_party/flash_linear_attention/ops/l2norm.py
- vllm/model_executor/layers/fla/ops/kda.py
- vllm/model_executor/layers/fla/ops/chunk_delta_h.py
- vllm/model_executor/layers/fla/ops/l2norm.py
- tests/kernels/test_kda.py
commands:
- pytest -v -s kernels/test_kda.py
- label: Kernels DeepGEMM Test (H100)
key: kernels-deepgemm-test-h100
timeout_in_minutes: 35
timeout_in_minutes: 45
device: h100
num_devices: 1
source_file_dependencies:
@@ -214,7 +165,7 @@ steps:
- label: Kernels (B200)
key: kernels-b200
timeout_in_minutes: 80
timeout_in_minutes: 30
working_dir: "/vllm-workspace/"
device: b200-k8s
# optional: true
@@ -234,15 +185,6 @@ steps:
- vllm/v1/attention/backends/mla/flashinfer_mla.py
- vllm/v1/attention/selector.py
- vllm/platforms/cuda.py
- vllm/model_executor/kernels/linear/cute_dsl/ll_bf16.py
- vllm/model_executor/kernels/linear/cute_dsl/_ll_bf16_dotprod.py
- vllm/model_executor/kernels/linear/cute_dsl/_ll_bf16_splitk.py
- vllm/cute_utils/
- vllm/model_executor/layers/mamba/ops/gdn_chunk_cutedsl/
- vllm/model_executor/layers/fused_moe/router/bf16x3_router_gemm_cutedsl.py
- tests/kernels/mamba/test_gdn_prefill_cutedsl.py
- tests/kernels/test_bf16x3_router_gemm_cutedsl.py
- tests/kernels/test_ll_bf16_gemm.py
- tests/kernels/test_top_k_per_row.py
commands:
- nvidia-smi
@@ -271,28 +213,24 @@ steps:
- pytest -v -s tests/kernels/moe/test_flashinfer_moe.py
- pytest -v -s tests/kernels/moe/test_trtllm_nvfp4_moe.py
- pytest -v -s tests/kernels/moe/test_cutedsl_moe.py
- pytest -v -s tests/kernels/mamba/test_gdn_prefill_cutedsl.py
- pytest -v -s tests/kernels/test_bf16x3_router_gemm_cutedsl.py
- pytest -v -s tests/kernels/test_ll_bf16_gemm.py
# e2e
- pytest -v -s tests/models/quantization/test_nvfp4.py
- label: Kernels Helion Test
key: kernels-helion-test
timeout_in_minutes: 115
timeout_in_minutes: 30
device: h100
source_file_dependencies:
- vllm/utils/import_utils.py
- tests/kernels/helion/
commands:
- pip install helion==1.1.0
- pytest -v -s kernels/helion/ --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
parallelism: 2
- pip install helion==1.0.0
- pytest -v -s kernels/helion/
- label: Kernels FP8 MoE Test (1xH100)
key: kernels-fp8-moe-test-1xh100
timeout_in_minutes: 40
- label: Kernels FP8 MoE Test (1 H100)
key: kernels-fp8-moe-test-1-h100
timeout_in_minutes: 90
device: h100
num_devices: 1
optional: true
@@ -307,9 +245,9 @@ steps:
- pytest -v -s kernels/moe/test_triton_moe_no_act_mul.py
- pytest -v -s kernels/moe/test_triton_moe_ptpc_fp8.py
- label: Kernels FP8 MoE Test (2xH100)
key: kernels-fp8-moe-test-2xh100
timeout_in_minutes: 45
- label: Kernels FP8 MoE Test (2 H100s)
key: kernels-fp8-moe-test-2-h100s
timeout_in_minutes: 90
device: h100
num_devices: 2
optional: true
@@ -319,7 +257,7 @@ steps:
- label: Kernels Fp4 MoE Test (B200)
key: kernels-fp4-moe-test-b200
timeout_in_minutes: 25
timeout_in_minutes: 60
device: b200-k8s
num_devices: 1
optional: true
@@ -332,7 +270,7 @@ steps:
- label: Kernels FusedMoE Layer Test (2 H100s)
key: kernels-fusedmoe-layer-test-2-h100s
timeout_in_minutes: 30
timeout_in_minutes: 90
device: h100
num_devices: 2
source_file_dependencies:
+25 -220
View File
@@ -5,7 +5,7 @@ steps:
- label: LM Eval Small Models
device: h200_35gb
key: lm-eval-small-models
timeout_in_minutes: 45
timeout_in_minutes: 75
source_file_dependencies:
- csrc/
- vllm/model_executor/layers/quantization
@@ -14,7 +14,7 @@ steps:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small.txt
mirror:
amd:
device: mi325_1
device: mi300_1
timeout_in_minutes: 55
depends_on:
- image-build-amd
@@ -28,8 +28,7 @@ steps:
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
# - label: LM Eval Large Models (4xA100)
# key: lm-eval-large-models-4xa100
# - label: LM Eval Large Models (4 GPUs)(A100)
# device: a100
# optional: true
# num_devices: 4
@@ -41,8 +40,8 @@ steps:
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
# - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large.txt --tp-size=4
- label: LM Eval Large Models (4xH100)
key: lm-eval-large-models-4xh100
- label: LM Eval Large Models (4 GPUs)(H100)
key: lm-eval-large-models-4-gpus-h100
device: h100
optional: true
num_devices: 4
@@ -54,9 +53,9 @@ steps:
- export VLLM_USE_DEEP_GEMM=0 # We found Triton is faster than DeepGEMM for H100
- pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-hopper.txt --tp-size=4
- label: LM Eval Small Models (1xB200)
key: lm-eval-small-models-1xb200
timeout_in_minutes: 50
- label: LM Eval Small Models (B200)
key: lm-eval-small-models-b200
timeout_in_minutes: 120
device: b200-k8s
optional: true
source_file_dependencies:
@@ -65,45 +64,10 @@ steps:
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell.txt
- label: LM Eval Small Models Distributed (2xB200)
key: lm-eval-small-models-distributed-2xb200
- label: LM Eval Large Models (B200, EP)
key: lm-eval-large-models-b200-ep
timeout_in_minutes: 120
device: b200-k8s
num_devices: 2
optional: true
source_file_dependencies:
- csrc/
- vllm/model_executor/layers/quantization
autorun_on_main: true
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small-tp.txt
- label: LM Eval PCP (4xB200)
key: lm-eval-pcp-4xb200
timeout_in_minutes: 360
device: b200-k8s
num_devices: 4
optional: true
source_file_dependencies:
- csrc/
- tests/evals/gsm8k/configs/GLM-5.2-NVFP4-TP2-PCP2-EP.yaml
- tests/evals/gsm8k/configs/GLM-5.2-NVFP4-TP1-PCP4-EP.yaml
- tests/evals/gsm8k/configs/models-pcp.txt
- vllm/model_executor/layers/quantization
- vllm/config/parallel.py
- vllm/distributed/parallel_state.py
- vllm/model_executor/layers/attention/mla_attention.py
- vllm/model_executor/layers/attention/pcp.py
- vllm/v1/worker/gpu/model_runner.py
- vllm/v1/worker/gpu/pcp_manager.py
autorun_on_main: true
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-pcp.txt
- label: LM Eval Large Models EP (2xB200)
key: lm-eval-large-models-ep-2xb200
timeout_in_minutes: 60
device: b200-k8s
optional: true
num_devices: 2
source_file_dependencies:
@@ -112,9 +76,9 @@ steps:
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell-ep.txt
- label: LM Eval Qwen3.5 Models (2xB200)
key: lm-eval-qwen3-5-models-2xb200
timeout_in_minutes: 45
- label: LM Eval Qwen3.5 Models (B200)
key: lm-eval-qwen3-5-models-b200
timeout_in_minutes: 120
device: b200-k8s
optional: true
num_devices: 2
@@ -125,29 +89,18 @@ steps:
- vllm/transformers_utils/configs/qwen3_5_moe.py
- vllm/model_executor/models/qwen3_next.py
- vllm/model_executor/models/qwen3_next_mtp.py
- vllm/third_party/flash_linear_attention/ops/
- vllm/model_executor/layers/fla/ops/
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-qwen35-blackwell.txt
- label: LM Eval Large Models (8xH200)
key: lm-eval-large-models-8xh200
timeout_in_minutes: 50
- label: LM Eval Large Models (H200)
key: lm-eval-large-models-h200
timeout_in_minutes: 60
device: h200
optional: true
num_devices: 8
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200.txt
mirror:
amd:
dind: false
device: mi300_8
timeout_in_minutes: 60
depends_on:
- image-build-amd
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- export PYTORCH_ROCM_ARCH=gfx942 # Limit Quark compilation to save time
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx.txt
- label: MoE Refactor Integration Test (H100 - TEMPORARY)
key: moe-refactor-integration-test-h100-temporary
@@ -173,101 +126,10 @@ steps:
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor-dp-ep/config-b200.txt
- label: LM Eval Humming f16 (A100 - TEMPORARY)
key: lm-eval-humming-f16-a100
timeout_in_minutes: 75
device: a100
optional: true
num_devices: 1
source_file_dependencies:
- vllm/model_executor/layers/quantization/humming.py
- vllm/model_executor/layers/quantization/utils/humming_utils.py
- vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
- vllm/model_executor/layers/fused_moe/oracle/
- vllm/model_executor/kernels/linear/
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config.txt
- label: LM Eval Humming Act int8 (A100 - TEMPORARY)
key: lm-eval-humming-act-a100
timeout_in_minutes: 45
device: a100
optional: true
num_devices: 1
source_file_dependencies:
- vllm/model_executor/layers/quantization/humming.py
- vllm/model_executor/layers/quantization/utils/humming_utils.py
- vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
- vllm/model_executor/layers/fused_moe/oracle/
- vllm/model_executor/kernels/linear/
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt
- label: LM Eval Humming f16 (H100 - TEMPORARY)
key: lm-eval-humming-f16-h100
timeout_in_minutes: 70
device: h100
optional: true
num_devices: 1
source_file_dependencies:
- vllm/model_executor/layers/quantization/humming.py
- vllm/model_executor/layers/quantization/utils/humming_utils.py
- vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
- vllm/model_executor/layers/fused_moe/oracle/
- vllm/model_executor/kernels/linear/
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config.txt
- label: LM Eval Humming Act fp8/int8 (H100 - TEMPORARY)
key: lm-eval-humming-act-h100
timeout_in_minutes: 70
device: h100
optional: true
num_devices: 1
source_file_dependencies:
- vllm/model_executor/layers/quantization/humming.py
- vllm/model_executor/layers/quantization/utils/humming_utils.py
- vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
- vllm/model_executor/layers/fused_moe/oracle/
- vllm/model_executor/kernels/linear/
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-fp8.txt
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt
- label: LM Eval Humming f16 (B200 - TEMPORARY)
key: lm-eval-humming-f16-b200
timeout_in_minutes: 50
device: b200-k8s
optional: true
num_devices: 1
source_file_dependencies:
- vllm/model_executor/layers/quantization/humming.py
- vllm/model_executor/layers/quantization/utils/humming_utils.py
- vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
- vllm/model_executor/layers/fused_moe/oracle/
- vllm/model_executor/kernels/linear/
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config.txt
- label: LM Eval Humming Act fp8/int8 (B200 - TEMPORARY)
key: lm-eval-humming-act-b200
timeout_in_minutes: 50
device: b200-k8s
optional: true
num_devices: 1
source_file_dependencies:
- vllm/model_executor/layers/quantization/humming.py
- vllm/model_executor/layers/quantization/utils/humming_utils.py
- vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
- vllm/model_executor/layers/fused_moe/oracle/
- vllm/model_executor/kernels/linear/
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-fp8.txt
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt
- label: LM Eval TurboQuant KV Cache
key: lm-eval-turboquant-kv-cache
timeout_in_minutes: 55
timeout_in_minutes: 75
device: h200_18gb
source_file_dependencies:
- vllm/model_executor/layers/quantization/turboquant/
@@ -277,9 +139,9 @@ steps:
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant.txt
- label: GPQA Eval (GPT-OSS) (2xH100)
key: gpqa-eval-gpt-oss-2xh100
timeout_in_minutes: 35
- label: GPQA Eval (GPT-OSS) (H100)
key: gpqa-eval-gpt-oss-h100
timeout_in_minutes: 120
device: h100
optional: true
num_devices: 2
@@ -291,9 +153,9 @@ steps:
- uv pip install --system 'gpt-oss[eval]==0.0.5'
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-h100.txt
- label: GPQA Eval (GPT-OSS) (2xB200)
key: gpqa-eval-gpt-oss-2xb200
timeout_in_minutes: 30
- label: GPQA Eval (GPT-OSS) (B200)
key: gpqa-eval-gpt-oss-b200
timeout_in_minutes: 120
device: b200-k8s
optional: true
num_devices: 2
@@ -305,66 +167,9 @@ steps:
- uv pip install --system 'gpt-oss[eval]==0.0.5'
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-b200.txt
- label: GPQA Eval (GPT-OSS) (DGX Spark)
key: gpqa-eval-gpt-oss-spark
timeout_in_minutes: 35
device: dgx-spark
optional: true
num_devices: 1
depends_on:
- arm64-image-build
source_file_dependencies:
- csrc/
- vllm/model_executor/layers/quantization
- tests/evals/gpt_oss/
commands:
- uv pip install --system 'gpt-oss[eval]==0.0.5'
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-spark.txt
- label: LM Eval KV-Offload (1xH200)
key: kv-offload-small
timeout_in_minutes: 30
device: h200_35gb
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/offloading/
- vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py
- vllm/v1/kv_offload/
- vllm/v1/simple_kv_offload/
- tests/evals/gsm8k/test_gsm8k_offloading.py
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_offloading.py -k "nemotron-h-8b or gemma-4-e4b-it"
- label: LM Eval KV-Offload (2xH100)
key: kv-offload-medium
timeout_in_minutes: 30
device: h100
num_devices: 2
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/offloading/
- vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py
- vllm/v1/kv_offload/
- vllm/v1/simple_kv_offload/
- tests/evals/gsm8k/test_gsm8k_offloading.py
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_offloading.py -k "qwen3.5-35b"
- label: LM Eval KV-Offload (4xH100)
key: kv-offload-large
timeout_in_minutes: 40
device: h100
num_devices: 4
source_file_dependencies:
- vllm/distributed/kv_transfer/kv_connector/v1/offloading/
- vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py
- vllm/v1/kv_offload/
- vllm/v1/simple_kv_offload/
- tests/evals/gsm8k/test_gsm8k_offloading.py
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_offloading.py -k "deepseek-v4-flash"
- label: MRCR Eval Small Models
device: h200_35gb
timeout_in_minutes: 25
timeout_in_minutes: 30
source_file_dependencies:
- tests/evals/mrcr/
commands:
+2 -13
View File
@@ -5,29 +5,18 @@ steps:
- label: LoRA %N
device: h200_35gb
key: lora
timeout_in_minutes: 40
timeout_in_minutes: 30
source_file_dependencies:
- vllm/lora
- tests/lora
commands:
- pytest -v -s lora --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --ignore=lora/test_chatglm3_tp.py --ignore=lora/test_llama_tp.py --ignore=lora/test_qwen3_with_multi_loras.py --ignore=lora/test_olmoe_tp.py --ignore=lora/test_deepseekv2_tp.py --ignore=lora/test_gptoss_tp.py --ignore=lora/test_qwen3moe_tp.py --ignore=lora/test_qwen35_densemodel_lora.py
parallelism: 4
mirror:
amd:
device: mi325_1
working_dir: "/vllm-workspace/tests"
timeout_in_minutes: 65
source_file_dependencies:
- vllm/lora
- tests/lora
- vllm/platforms/rocm.py
depends_on:
- image-build-amd
- label: LoRA TP (Distributed)
key: lora-tp-distributed
timeout_in_minutes: 60
timeout_in_minutes: 30
num_devices: 4
source_file_dependencies:
- vllm/lora
+17 -78
View File
@@ -5,7 +5,7 @@ steps:
- label: V1 Spec Decode
device: h200_35gb
key: v1-spec-decode
timeout_in_minutes: 40
timeout_in_minutes: 30
source_file_dependencies:
- vllm/config/
- vllm/distributed/
@@ -21,17 +21,10 @@ steps:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
# TODO: create another `optional` test group for slow tests
- pytest -v -s -m 'not slow_test' v1/spec_decode
mirror:
amd:
dind: false
device: mi300_1
timeout_in_minutes: 75
depends_on:
- image-build-amd
- label: V1 Sample + Logits
key: v1-sample-logits
timeout_in_minutes: 45
timeout_in_minutes: 30
device: h200_18gb
source_file_dependencies:
- vllm/config/
@@ -64,9 +57,8 @@ steps:
- image-build-amd
- label: V1 Core + KV + Metrics
device: h200_35gb
key: v1-core-kv-metrics
timeout_in_minutes: 80
timeout_in_minutes: 30
source_file_dependencies:
- vllm/config/
- vllm/distributed/
@@ -91,7 +83,6 @@ steps:
- tests/v1/simple_kv_offload
- tests/v1/worker
- tests/v1/kv_connector/unit
- tests/v1/ec_connector/unit
- tests/v1/metrics
- tests/entrypoints/openai/correctness/test_lmeval.py
commands:
@@ -104,17 +95,10 @@ steps:
- pytest -v -s v1/simple_kv_offload
- pytest -v -s v1/worker
- pytest -v -s -m 'not cpu_test' v1/kv_connector/unit
- pytest -v -s -m 'not cpu_test' v1/ec_connector/unit
- pytest -v -s -m 'not cpu_test' v1/metrics
# Integration test for streaming correctness (requires special branch).
- pip install -U git+https://github.com/vllm-project/lm-evaluation-harness.git@streaming-api
- pip install -U git+https://github.com/robertgshaw2-redhat/lm-evaluation-harness.git@streaming-api
- pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
mirror:
amd:
device: mi325_1
timeout_in_minutes: 75
depends_on:
- image-build-amd
- label: V1 Others (CPU)
key: v1-others-cpu
@@ -154,29 +138,14 @@ steps:
- vllm/v1/spec_decode/extract_hidden_states.py
- vllm/model_executor/models/extract_hidden_states.py
- vllm/transformers_utils/configs/extract_hidden_states.py
- vllm/distributed/kv_transfer/kv_connector/v1/example_hidden_states_connector.py
- tests/v1/kv_connector/extract_hidden_states_integration
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -v -s v1/kv_connector/extract_hidden_states_integration
- label: Extract Hidden States Integration (2 GPUs)
key: extract-hidden-states-integration-2-gpus
timeout_in_minutes: 20
num_devices: 2
source_file_dependencies:
- vllm/v1/spec_decode/extract_hidden_states.py
- vllm/model_executor/models/extract_hidden_states.py
- vllm/transformers_utils/configs/extract_hidden_states.py
- vllm/distributed/kv_transfer/kv_connector/v1/example_hidden_states_connector.py
- tests/v1/kv_connector/extract_hidden_states_integration
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -v -s -m 'distributed' v1/kv_connector/extract_hidden_states_integration
- label: Regression
key: regression
timeout_in_minutes: 30
timeout_in_minutes: 20
device: h200_18gb
source_file_dependencies:
- vllm/config/
@@ -192,14 +161,14 @@ steps:
- vllm/v1/
- tests/test_regression
commands:
- pip install 'modelscope<1.38'
- pip install modelscope
- pytest -v -s test_regression.py
working_dir: "/vllm-workspace/tests" # optional
- label: Examples
device: h200_35gb
key: examples
timeout_in_minutes: 40
timeout_in_minutes: 45
working_dir: "/vllm-workspace/examples"
source_file_dependencies:
- vllm/entrypoints
@@ -228,20 +197,10 @@ steps:
- python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048
# https://github.com/vllm-project/vllm/pull/26682 uses slightly more memory in PyTorch 2.9+ causing this test to OOM in 1xL4 GPU
- python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536
mirror:
amd:
device: mi325_1
source_file_dependencies:
- vllm/entrypoints
- vllm/multimodal
- examples/
- vllm/platforms/rocm.py
depends_on:
- image-build-amd
- label: Metrics, Tracing (2 GPUs)
key: metrics-tracing-2-gpus
timeout_in_minutes: 25
timeout_in_minutes: 20
num_devices: 2
source_file_dependencies:
- vllm/config/
@@ -264,12 +223,6 @@ steps:
'opentelemetry-exporter-otlp>=1.26.0' \
'opentelemetry-semantic-conventions-ai>=0.4.1'"
- pytest -v -s v1/tracing
mirror:
amd:
device: mi325_2
depends_on:
- image-build-amd
optional: true
- label: Python-only Installation
key: python-only-installation
@@ -282,21 +235,11 @@ steps:
- setup.py
commands:
- bash standalone_tests/python_only_compile.sh
mirror:
amd:
device: mi325_1
timeout_in_minutes: 45
depends_on:
- image-build-amd
source_file_dependencies:
- tests/standalone_tests/python_only_compile.sh
- setup.py
- vllm/platforms/rocm.py
- label: Async Engine, Inputs, Utils, Worker
device: h200_35gb
key: async-engine-inputs-utils-worker
timeout_in_minutes: 25
timeout_in_minutes: 50
source_file_dependencies:
- vllm/assets/
- vllm/config/
@@ -323,7 +266,7 @@ steps:
key: async-engine-inputs-utils-worker-config-cpu
depends_on:
- image-build-cpu
timeout_in_minutes: 65
timeout_in_minutes: 30
source_file_dependencies:
- vllm/assets/
- vllm/config/
@@ -350,42 +293,38 @@ steps:
- vllm/transformers_utils/
- vllm/utils/
- vllm/v1/
- tests/test_envs.py
- tests/test_inputs.py
- tests/test_outputs.py
- tests/test_pooling_params.py
- tests/test_ray_env.py
- tests/test_sampling_params.py
- tests/multimodal
- tests/renderers
- tests/standalone_tests/lazy_imports.py
- tests/tokenizers_
- tests/reasoning
- tests/tool_parsers
- tests/tokenizers_
- tests/parser
- tests/transformers_utils
- tests/config
device: cpu-small
commands:
- python3 standalone_tests/lazy_imports.py
- pytest -v -s test_envs.py
- pytest -v -s test_inputs.py
- pytest -v -s test_outputs.py
- pytest -v -s test_pooling_params.py
- pytest -v -s test_ray_env.py
- pytest -v -s test_sampling_params.py
- pytest -v -s -m 'cpu_test' multimodal
- pytest -v -s renderers
- pytest -v -s reasoning
- pytest -v -s tool_parsers
- pytest -v -s tokenizers_
- pytest -v -s reasoning --ignore=reasoning/test_seedoss_reasoning_parser.py --ignore=reasoning/test_glm4_moe_reasoning_parser.py
- pytest -v -s tool_parsers
- pytest -v -s parser
- pytest -v -s transformers_utils
- pytest -v -s config
- label: Batch Invariance (A100)
key: batch-invariance-a100
timeout_in_minutes: 40
timeout_in_minutes: 30
device: a100
source_file_dependencies:
- vllm/v1/attention
@@ -399,7 +338,7 @@ steps:
- label: Batch Invariance (H100)
key: batch-invariance-h100
timeout_in_minutes: 40
timeout_in_minutes: 30
device: h100
source_file_dependencies:
- vllm/v1/attention
@@ -415,7 +354,7 @@ steps:
- label: Batch Invariance (B200)
key: batch-invariance-b200
timeout_in_minutes: 35
timeout_in_minutes: 30
device: b200-k8s
source_file_dependencies:
- vllm/v1/attention
@@ -434,7 +373,7 @@ steps:
- label: Acceptance Length Test (Large Models) # optional
device: h200_35gb
key: acceptance-length-test-large-models
timeout_in_minutes: 20
timeout_in_minutes: 25
gpu: h100
optional: true
num_gpus: 1
+1 -20
View File
@@ -3,16 +3,13 @@ depends_on:
- image-build
steps:
- label: Model Executor
device: h200_35gb
key: model-executor
timeout_in_minutes: 45
timeout_in_minutes: 35
source_file_dependencies:
- vllm/engine/arg_utils.py
- vllm/config/model.py
- vllm/model_executor
- vllm/model_executor/warmup
- tests/model_executor
- tests/model_executor/test_jit_warmup.py
- tests/entrypoints/openai/completion/test_tensorizer_entrypoint.py
commands:
- apt-get update && apt-get install -y curl libsodium23
@@ -26,19 +23,3 @@ steps:
# calls that the signal method cannot interrupt.
- pytest -v -s model_executor -m '(not slow_test)' --timeout=900 --timeout-method=thread
- pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py --timeout=900 --timeout-method=thread
mirror:
amd:
dind: false
device: mi300_1
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/engine/arg_utils.py
- vllm/config/model.py
- vllm/model_executor
- vllm/model_executor/warmup
- tests/model_executor
- tests/model_executor/test_jit_warmup.py
- tests/entrypoints/openai/completion/test_tensorizer_entrypoint.py
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
+7 -5
View File
@@ -5,7 +5,7 @@ steps:
- label: Model Runner V2 Core Tests
device: h200_35gb
key: model-runner-v2-core-tests
timeout_in_minutes: 35
timeout_in_minutes: 45
source_file_dependencies:
- vllm/v1/worker/gpu/
- vllm/v1/worker/gpu_worker.py
@@ -18,7 +18,9 @@ steps:
- set -x
- export VLLM_USE_V2_MODEL_RUNNER=1
- pytest -v -s v1/engine/test_llm_engine.py -k "not test_engine_metrics"
- pytest -v -s v1/e2e/general/test_async_scheduling.py -k "not ngram"
# This requires eager until we sort out CG correctness issues.
# TODO: remove ENFORCE_EAGER here after https://github.com/vllm-project/vllm/pull/32936 is merged.
- ENFORCE_EAGER=1 pytest -v -s v1/e2e/general/test_async_scheduling.py -k "not ngram"
- pytest -v -s v1/e2e/general/test_context_length.py
- pytest -v -s v1/e2e/general/test_min_tokens.py
# Temporary hack filter to exclude ngram spec decoding based tests.
@@ -27,7 +29,7 @@ steps:
- label: Model Runner V2 Examples
device: h200_35gb
key: model-runner-v2-examples
timeout_in_minutes: 35
timeout_in_minutes: 45
working_dir: "/vllm-workspace/examples"
source_file_dependencies:
- vllm/v1/worker/gpu/
@@ -63,7 +65,7 @@ steps:
- label: Model Runner V2 Distributed (2 GPUs)
key: model-runner-v2-distributed-2-gpus
timeout_in_minutes: 30
timeout_in_minutes: 45
working_dir: "/vllm-workspace/tests"
num_devices: 2
source_file_dependencies:
@@ -84,7 +86,7 @@ steps:
- label: Model Runner V2 Pipeline Parallelism (4 GPUs)
key: model-runner-v2-pipeline-parallelism-4-gpus
timeout_in_minutes: 50
timeout_in_minutes: 60
working_dir: "/vllm-workspace/tests"
num_devices: 4
source_file_dependencies:
+13 -14
View File
@@ -4,8 +4,9 @@ depends_on:
steps:
- label: Basic Models Tests (Initialization)
key: basic-models-tests-initialization
timeout_in_minutes: 25
timeout_in_minutes: 45
device: h200_18gb
torch_nightly: true
source_file_dependencies:
- vllm/
- tests/models/test_initialization.py
@@ -13,11 +14,13 @@ steps:
commands:
# Run a subset of model initialization tests
- pytest -v -s models/test_initialization.py::test_can_initialize_small_subset
mirror:
torch_nightly: {}
- label: Basic Models Tests (Extra Initialization) %N
device: h200_35gb
key: basic-models-tests-extra-initialization
timeout_in_minutes: 100
timeout_in_minutes: 45
source_file_dependencies:
- vllm/model_executor/models/
- tests/models/test_initialization.py
@@ -27,35 +30,31 @@ steps:
# subset of supported models (the complement of the small subset in the above
# test.) Also run if model initialization test file is modified
- pytest -v -s models/test_initialization.py -k 'not test_can_initialize_small_subset' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
parallelism: 4
parallelism: 2
mirror:
torch_nightly: {}
- label: Basic Models Tests (Other)
device: h200_35gb
key: basic-models-tests-other
timeout_in_minutes: 35
timeout_in_minutes: 45
source_file_dependencies:
- vllm/
- tests/models/test_terratorch.py
- tests/models/transformers/test_backend.py
- tests/models/test_transformers.py
- tests/models/test_registry.py
commands:
- pytest -v -s models/test_terratorch.py models/transformers/test_backend.py models/test_registry.py
mirror:
amd:
device: mi325_1
depends_on:
- image-build-amd
- pytest -v -s models/test_terratorch.py models/test_transformers.py models/test_registry.py
- label: Basic Models Test (Other CPU) # 5min
key: basic-models-test-other-cpu
depends_on:
- image-build-cpu
timeout_in_minutes: 20
timeout_in_minutes: 10
source_file_dependencies:
- vllm/
- tests/models/test_utils.py
- tests/models/test_vision.py
- tests/models/transformers/fusers/
device: cpu-small
commands:
- pytest -v -s models/test_utils.py models/test_vision.py models/transformers/fusers/
- pytest -v -s models/test_utils.py models/test_vision.py
@@ -4,7 +4,7 @@ depends_on:
steps:
- label: Distributed Model Tests (2 GPUs)
key: distributed-model-tests-2-gpus
timeout_in_minutes: 60
timeout_in_minutes: 50
working_dir: "/vllm-workspace/tests"
num_devices: 2
source_file_dependencies:
@@ -17,7 +17,7 @@ steps:
- TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
- CUDA_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)'
# Avoid importing model tests that cause CUDA reinitialization error
- pytest models/transformers/test_backend.py -v -s -m 'distributed(num_gpus=2)'
- pytest models/test_transformers.py -v -s -m 'distributed(num_gpus=2)'
- pytest models/language -v -s -m 'distributed(num_gpus=2)'
- pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)'
- pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py
+16 -49
View File
@@ -4,7 +4,7 @@ depends_on:
steps:
- label: Language Models Tests (Standard)
key: language-models-tests-standard
timeout_in_minutes: 30
timeout_in_minutes: 25
device: h200_18gb
source_file_dependencies:
- vllm/
@@ -14,16 +14,11 @@ steps:
- pip freeze | grep -E 'torch'
- pytest -v -s models/language -m 'core_model and (not slow_test)'
mirror:
amd:
dind: false
device: mi300_1
depends_on:
- image-build-amd
torch_nightly: {}
- label: Language Models Tests (Extra Standard) %N
device: h200_35gb
key: language-models-tests-extra-standard
timeout_in_minutes: 40
timeout_in_minutes: 45
source_file_dependencies:
- vllm/model_executor/models/
- tests/models/language/pooling/test_embedding.py
@@ -36,26 +31,11 @@ steps:
- pytest -v -s models/language -m 'core_model and slow_test' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
parallelism: 2
mirror:
amd:
dind: false
device: mi300_1
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/model_executor/models/
- vllm/model_executor/model_loader/
- vllm/model_executor/layers/
- vllm/v1/attention/backends/
- vllm/v1/attention/selector.py
- tests/models/language/pooling/test_embedding.py
- tests/models/language/generation/test_common.py
- tests/models/language/pooling/test_classification.py
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
torch_nightly: {}
- label: Language Models Tests (Hybrid) %N
device: h200_35gb
key: language-models-tests-hybrid
timeout_in_minutes: 65
timeout_in_minutes: 75
source_file_dependencies:
- vllm/
- tests/models/language/generation
@@ -64,13 +44,14 @@ steps:
# Note: also needed to run plamo2 model in vLLM
- uv pip install --system --no-build-isolation 'git+https://github.com/state-spaces/mamba@v2.3.0'
- uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0'
# Shard the hybrid language model tests that are numerically stable on Hopper.
- pytest -v -s models/language/generation -m hybrid_model -k 'not granite-4.0-tiny-preview' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
# Shard hybrid language model tests
- pytest -v -s models/language/generation -m hybrid_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
parallelism: 2
mirror:
torch_nightly: {}
amd:
device: mi325_1
timeout_in_minutes: 70
timeout_in_minutes: 90
depends_on:
- image-build-amd
commands:
@@ -78,24 +59,10 @@ steps:
- uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0'
- pytest -v -s models/language/generation -m hybrid_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
# Granite 4 hybrid generation is sensitive to hardware-specific Triton SSD
# autotuning (https://github.com/vllm-project/vllm/issues/25194). Keep this one
# correctness test on L4 until its H200 output matches the Transformers reference.
- label: Language Models Tests (Granite L4 Compatibility)
key: language-models-tests-granite-l4-compatibility
timeout_in_minutes: 65
source_file_dependencies:
- vllm/
- tests/models/language/generation
commands:
- uv pip install --system --no-build-isolation 'git+https://github.com/state-spaces/mamba@v2.3.0'
- uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0'
- pytest -v -s models/language/generation -m hybrid_model -k 'granite-4.0-tiny-preview'
- label: Language Models Test (Extended Generation) # 80min
device: h200_35gb
key: language-models-test-extended-generation
timeout_in_minutes: 65
timeout_in_minutes: 110
optional: true
source_file_dependencies:
- vllm/
@@ -109,7 +76,7 @@ steps:
- label: Language Models Test (PPL)
key: language-models-test-ppl
timeout_in_minutes: 30
timeout_in_minutes: 110
device: h200_18gb
optional: true
source_file_dependencies:
@@ -118,10 +85,10 @@ steps:
commands:
- pytest -v -s models/language/generation_ppl_test
- label: Language Models Test (Extended Pooling)
- label: Language Models Test (Extended Pooling) # 36min
device: h200_35gb
key: language-models-test-extended-pooling
timeout_in_minutes: 120
timeout_in_minutes: 50
optional: true
source_file_dependencies:
- vllm/
@@ -131,13 +98,13 @@ steps:
mirror:
amd:
device: mi325_1
timeout_in_minutes: 120
timeout_in_minutes: 100
depends_on:
- image-build-amd
- label: Language Models Test (MTEB)
key: language-models-test-mteb
timeout_in_minutes: 45
timeout_in_minutes: 110
device: h200_18gb
optional: true
source_file_dependencies:
+24 -35
View File
@@ -10,94 +10,93 @@ steps:
- vllm/
- tests/models/multimodal
commands:
- pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2"
- pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model
mirror:
amd:
device: mi325_1
device: mi300_1
depends_on:
- image-build-amd
- label: "Multi-Modal Models (Standard) 2: qwen3 + gemma"
key: multi-modal-models-standard-2-qwen3-gemma
timeout_in_minutes: 50
timeout_in_minutes: 45
device: h200_18gb
source_file_dependencies:
- vllm/
- tests/models/multimodal
commands:
- pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen3 or gemma"
- pytest -v -s models/multimodal/generation/test_mm_prefix_lm.py -m core_model
- pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model
- pytest -v -s models/multimodal/generation/test_vit_cudagraph.py -m core_model
mirror:
amd:
device: mi325_1
device: mi300_1
depends_on:
- image-build-amd
- label: "Multi-Modal Models (Standard) 3: llava + qwen2_vl"
device: h200_35gb
key: multi-modal-models-standard-3-llava-qwen2-vl
timeout_in_minutes: 40
timeout_in_minutes: 45
source_file_dependencies:
- vllm/
- tests/models/multimodal
commands:
- pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma"
- pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model
mirror:
amd:
device: mi325_1
device: mi300_1
depends_on:
- image-build-amd
- label: "Multi-Modal Models (Standard) 4: other + whisper"
device: h200_35gb
key: multi-modal-models-standard-4-other-whisper
timeout_in_minutes: 50
timeout_in_minutes: 45
source_file_dependencies:
- vllm/
- tests/models/multimodal
commands:
- pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_mm_prefix_lm.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/generation/test_vit_cudagraph.py --ignore models/multimodal/processing
- pytest -v -s models/multimodal/generation/test_vit_cudagraph.py -m core_model
- pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/processing
- pytest models/multimodal/generation/test_memory_leak.py -m core_model
- cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model # Otherwise, mp_method="spawn" doesn't work
mirror:
amd:
device: mi325_1
depends_on:
- image-build-amd
- label: Multi-Modal Processor (CPU) %N
- label: Multi-Modal Processor (CPU)
key: multi-modal-processor-cpu
depends_on:
- image-build-cpu
timeout_in_minutes: 125
timeout_in_minutes: 60
source_file_dependencies:
- vllm/
- tests/models/multimodal
- tests/models/registry.py
device: cpu-medium
commands:
- pytest -v -s models/multimodal/processing --ignore models/multimodal/processing/test_tensor_schema.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
parallelism: 4
- pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- pytest -v -s models/multimodal/processing --ignore models/multimodal/processing/test_tensor_schema.py
- label: Multi-Modal Processor # 44min
key: multi-modal-processor
timeout_in_minutes: 65
timeout_in_minutes: 60
device: h200_18gb
source_file_dependencies:
- vllm/
- tests/models/multimodal
- tests/models/registry.py
commands:
- pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- pytest -v -s models/multimodal/processing/test_tensor_schema.py
- label: Multi-Modal Accuracy Eval (Small Models) # 50min
device: h200_35gb
key: multi-modal-accuracy-eval-small-models
timeout_in_minutes: 30
timeout_in_minutes: 70
working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
source_file_dependencies:
- vllm/multimodal/
@@ -105,21 +104,8 @@ steps:
- vllm/v1/core/
commands:
- pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-mm-small.txt --tp-size=1
mirror:
amd:
dind: false
device: mi300_1
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/multimodal/
- vllm/inputs/
- vllm/v1/core/
- vllm/platforms/rocm.py
- vllm/model_executor/model_loader/
- label: Multi-Modal Models (Extended Generation 1)
device: h200_35gb
key: multi-modal-models-extended-generation-1
optional: true
source_file_dependencies:
@@ -127,6 +113,7 @@ steps:
- tests/models/multimodal/generation
- tests/models/multimodal/test_mapping.py
commands:
- pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- pytest -v -s models/multimodal/generation -m 'not core_model' --ignore models/multimodal/generation/test_common.py
- pytest -v -s models/multimodal/test_mapping.py
mirror:
@@ -143,6 +130,7 @@ steps:
- vllm/
- tests/models/multimodal/generation
commands:
- pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=0) and not core_model'
- label: Multi-Modal Models (Extended Generation 3)
@@ -153,6 +141,7 @@ steps:
- vllm/
- tests/models/multimodal/generation
commands:
- pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
- pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model'
- label: Multi-Modal Models (Extended Pooling)
@@ -166,8 +155,8 @@ steps:
- pytest -v -s models/multimodal/pooling -m 'not core_model'
mirror:
amd:
device: mi325_1
timeout_in_minutes: 75
device: mi300_1
timeout_in_minutes: 60
depends_on:
- image-build-amd
source_file_dependencies:
+1 -24
View File
@@ -4,7 +4,7 @@ depends_on:
steps:
- label: Plugin Tests (2 GPUs)
key: plugin-tests-2-gpus
timeout_in_minutes: 35
timeout_in_minutes: 60
working_dir: "/vllm-workspace/tests"
num_devices: 2
source_file_dependencies:
@@ -27,21 +27,12 @@ steps:
- pip install -e ./plugins/bge_m3_sparse_plugin
- pytest -v -s plugins_tests/test_bge_m3_sparse_io_processor_plugins.py
- pip uninstall bge_m3_sparse_plugin -y
# test colbert_query io_processor plugin
- pip install -e ./plugins/colbert_query_plugin
- pytest -v -s plugins_tests/test_colbert_query_io_processor_plugins.py
- pip uninstall colbert_query_plugin -y
# end io_processor plugins test
# begin stat_logger plugins test
- pip install -e ./plugins/vllm_add_dummy_stat_logger
- pytest -v -s plugins_tests/test_stats_logger_plugins.py
- pip uninstall dummy_stat_logger -y
# end stat_logger plugins test
# begin endpoint plugins test
- pip install -e ./plugins/vllm_add_dummy_endpoint_plugin
- pytest -v -s plugins_tests/test_endpoint_plugins.py
- pip uninstall vllm_add_dummy_endpoint_plugin -y
# end endpoint plugins test
# other tests continue here:
- pytest -v -s plugins_tests/test_scheduler_plugins.py
- pip install -e ./plugins/vllm_add_dummy_model
@@ -49,17 +40,3 @@ steps:
- pytest -v -s plugins_tests/test_oot_registration_online.py # it needs a clean process
- pytest -v -s plugins_tests/test_oot_registration_offline.py # it needs a clean process
- pytest -v -s plugins_tests/lora_resolvers # unit tests for in-tree lora resolver plugins
- label: GGUF Plugin
key: gguf-plugin
device: h200_18gb
timeout_in_minutes: 30
soft_fail: true
optional: true
source_file_dependencies:
- vllm/model_executor/layers/quantization
- tests/plugins_tests/test_gguf_plugin.py
commands:
- pip install "vllm-gguf-plugin >= 0.0.2"
- pytest -v -s plugins_tests/gguf
+5 -57
View File
@@ -5,7 +5,7 @@ steps:
- label: PyTorch Compilation Unit Tests
device: h200_35gb
key: pytorch-compilation-unit-tests
timeout_in_minutes: 90
timeout_in_minutes: 10
source_file_dependencies:
- vllm/__init__.py
- vllm/_aiter_ops.py
@@ -78,7 +78,7 @@ steps:
- label: PyTorch Compilation Passes Unit Tests
key: pytorch-compilation-passes-unit-tests
timeout_in_minutes: 45
timeout_in_minutes: 20
source_file_dependencies:
- vllm/__init__.py
- vllm/_aiter_ops.py
@@ -107,18 +107,10 @@ steps:
- tests/compile/passes
commands:
- pytest -s -v compile/passes --ignore compile/passes/distributed
mirror:
amd:
dind: false
device: mi300_1
timeout_in_minutes: 65
depends_on:
- image-build-amd
- label: PyTorch Fullgraph Smoke Test
device: h200_35gb
key: pytorch-fullgraph-smoke-test
timeout_in_minutes: 90
timeout_in_minutes: 35
source_file_dependencies:
- vllm/__init__.py
- vllm/_aiter_ops.py
@@ -150,46 +142,11 @@ steps:
# as it is a heavy test that is covered in other steps.
# Use `find` to launch multiple instances of pytest so that
# they do not suffer from https://github.com/vllm-project/vllm/issues/28965
- "find compile/fullgraph/ -name 'test_*.py' -not -name 'test_full_cudagraph.py' -not -name 'test_full_graph.py' -print0 | xargs -0 -n1 -I{} pytest -s -v '{}'"
# Hopper-only DeepSeek-V2-Lite cases in this file require two 29.3-GiB model
# instances and cannot fit a 35GB MIG slice. L4 retains the original coverage:
# those SM90 cases skip while the architecture-compatible cases still run.
- label: PyTorch Fullgraph CUDAGraph (L4 Compatibility)
key: pytorch-fullgraph-cudagraph-l4-compatibility
timeout_in_minutes: 60
source_file_dependencies:
- vllm/__init__.py
- vllm/_aiter_ops.py
- vllm/_custom_ops.py
- vllm/compilation/
- vllm/config/
- vllm/distributed/
- vllm/engine/
- vllm/env_override.py
- vllm/envs.py
- vllm/forward_context.py
- vllm/inputs/
- vllm/ir/
- vllm/kernels/
- vllm/logger.py
- vllm/model_executor/
- vllm/multimodal/
- vllm/platforms/
- vllm/plugins/
- vllm/sampling_params.py
- vllm/sequence.py
- vllm/transformers_utils/
- vllm/triton_utils/
- vllm/utils/
- vllm/v1/
- tests/compile
commands:
- pytest -s -v compile/fullgraph/test_full_cudagraph.py
- "find compile/fullgraph/ -name 'test_*.py' -not -name 'test_full_graph.py' -print0 | xargs -0 -n1 -I{} pytest -s -v '{}'"
- label: PyTorch Fullgraph
key: pytorch-fullgraph
timeout_in_minutes: 40
timeout_in_minutes: 30
device: h200_18gb
source_file_dependencies:
- vllm/__init__.py
@@ -232,12 +189,3 @@ steps:
- requirements/test/nightly-torch.txt
commands:
- bash standalone_tests/pytorch_nightly_dependency.sh
mirror:
amd:
dind: false
device: mi300_1
depends_on:
- image-build-amd
source_file_dependencies:
- requirements/test/nightly-torch.txt
- vllm/platforms/rocm.py
+5 -28
View File
@@ -3,11 +3,8 @@ depends_on:
- image-build
steps:
- label: Quantization
device: h200_35gb
key: quantization
timeout_in_minutes: 75
env:
VLLM_USE_V2_MODEL_RUNNER: "0"
timeout_in_minutes: 90
source_file_dependencies:
- csrc/
- vllm/model_executor/layers/quantization
@@ -22,27 +19,11 @@ steps:
# TODO(jerryzh168): resolve the above comment
- uv pip install --system torchao==0.17.0 --index-url https://download.pytorch.org/whl/cu130
- uv pip install --system conch-triton-kernels
# The SM90-only checkpoint currently contains a removed weight_chan_scale
# parameter. It was not exercised by the previous L4 job.
- VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py -k 'not test_compressed_tensors_w4a8_fp8' --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
parallelism: 8
- label: Quantized Fusions
device: h200_35gb
key: quantized-fusions
timeout_in_minutes: 20
source_file_dependencies:
- tests/fusion
- vllm/model_executor/layers/fusion
- vllm/model_executor/kernels/linear
- vllm/model_executor/layers/quantization/compressed_tensors
- vllm/model_executor/layers/quantization/modelopt.py
commands:
- pytest -v -s fusion/
- VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py
- label: Quantized MoE Test (B200)
key: quantized-moe-test-b200
timeout_in_minutes: 120
timeout_in_minutes: 60
working_dir: "/vllm-workspace/"
device: b200-k8s
source_file_dependencies:
@@ -59,14 +40,10 @@ steps:
- pytest -s -v tests/quantization/test_blackwell_moe.py
- label: Quantized Models Test
device: h200_35gb
key: quantized-models-test
timeout_in_minutes: 65
env:
VLLM_USE_V2_MODEL_RUNNER: "0"
timeout_in_minutes: 60
source_file_dependencies:
- vllm/model_executor/layers/quantization
- tests/models/quantization
commands:
- pytest -v -s models/quantization --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
parallelism: 3
- pytest -v -s models/quantization
+16 -58
View File
@@ -3,16 +3,11 @@ depends_on:
- image-build
steps:
- label: Rust Frontend OpenAI Coverage
timeout_in_minutes: 30
timeout_in_minutes: 90
device: h200_18gb
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- rust/
- build_rust.sh
- tools/build_rust.py
- rust-toolchain.toml
- .buildkite/scripts/rust-coverage.sh
- codecov.yml
- vllm/benchmarks/
- vllm/entrypoints/openai/
- vllm/entrypoints/serve/
@@ -20,130 +15,93 @@ steps:
- tests/utils.py
- tests/benchmarks/test_serve_cli.py
- tests/entrypoints/openai/chat_completion/test_chat_completion.py
- tests/entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py
# - tests/entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py
# - tests/entrypoints/openai/completion/test_prompt_validation.py
- tests/entrypoints/openai/completion/test_shutdown.py
- tests/entrypoints/openai/test_return_token_ids.py
- tests/entrypoints/openai/test_uds.py
# - tests/entrypoints/openai/test_return_token_ids.py
# - tests/entrypoints/openai/test_uds.py
- tests/v1/sample/test_logprobs_e2e.py
commands:
- . /vllm-workspace/.buildkite/scripts/rust-coverage.sh && rust_coverage_start rust-e2e
- export VLLM_USE_RUST_FRONTEND=1
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -v -s benchmarks/test_serve_cli.py -k "not insecure and not (test_bench_serve and not test_bench_serve_chat)"
- pytest -v -s entrypoints/openai/chat_completion/test_chat_completion.py -k "not test_invalid_json_schema and not test_invalid_regex"
- pytest -v -s entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py -k "not multiple"
- pytest -v -s entrypoints/openai/chat_completion/test_chat_completion.py
# - pytest -v -s entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py -k "not invalid"
# - pytest -v -s entrypoints/openai/completion/test_prompt_validation.py -k "not prompt_embeds"
- pytest -v -s entrypoints/openai/completion/test_shutdown.py -k "not engine_failure and not test_abort_timeout_exits_quickly"
# test_comparison streams differently: Rust emits a separate first (prompt_token_ids) chunk and
# finish chunk without logprobs, while the test reads `logprobs.tokens` on every chunk.
- pytest -v -s entrypoints/openai/test_return_token_ids.py -k "not test_comparison"
- pytest -v -s entrypoints/openai/test_uds.py
# - pytest -v -s entrypoints/openai/test_return_token_ids.py
# - pytest -v -s entrypoints/openai/test_uds.py
- pytest -v -s v1/sample/test_logprobs_e2e.py -k "test_prompt_logprobs_e2e_server"
- label: Rust Frontend Serve/Admin Coverage
timeout_in_minutes: 25
timeout_in_minutes: 60
device: h200_18gb
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- rust/
- build_rust.sh
- tools/build_rust.py
- rust-toolchain.toml
- .buildkite/scripts/rust-coverage.sh
- codecov.yml
- vllm/entrypoints/openai/
- vllm/entrypoints/serve/
- vllm/v1/engine/
- tests/utils.py
- tests/entrypoints/serve/dev/rpc/test_collective_rpc.py
- tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py
# - tests/entrypoints/serve/dev/rpc/test_collective_rpc.py
- tests/entrypoints/serve/disagg/test_serving_tokens.py
- tests/entrypoints/serve/instrumentator/test_basic.py
- tests/entrypoints/serve/instrumentator/test_metrics.py
# - tests/entrypoints/serve/dev/test_sleep.py
- tests/entrypoints/serve/tokenize/test_tokenization.py
commands:
- . /vllm-workspace/.buildkite/scripts/rust-coverage.sh && rust_coverage_start rust-e2e
- export VLLM_USE_RUST_FRONTEND=1
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc/test_collective_rpc.py
# server_load can be flaky under the Rust frontend; keep it excluded for now.
# - pytest -v -s entrypoints/serve/dev/rpc/test_collective_rpc.py
- pytest -v -s entrypoints/serve/instrumentator/test_basic.py -k "not show_version and not server_load"
# test_generate_logprobs expects Python-style top_logprobs truncation (dedup sampled + cap at max(k, 1)).
- pytest -v -s entrypoints/scale_out/token_in_token_out/test_serving_tokens.py -k "not stream and not lora and not test_generate_logprobs and not stop_string_workflow"
- pytest -v -s entrypoints/serve/disagg/test_serving_tokens.py -k "not stream and not lora and not test_generate_logprobs and not stop_string_workflow"
- pytest -v -s entrypoints/serve/instrumentator/test_metrics.py -k "text and not show and not run_batch and not test_metrics_counts and not test_metrics_exist"
# - pytest -v -s entrypoints/serve/dev/test_sleep.py
# /tokenizer_info is not implemented in the Rust frontend (the CLI flag is accepted as a no-op).
- pytest -v -s entrypoints/serve/tokenize/test_tokenization.py -k "not tokenizer_info"
- label: Rust Frontend Core Correctness
timeout_in_minutes: 20
timeout_in_minutes: 30
device: h200_18gb
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- rust/
- build_rust.sh
- tools/build_rust.py
- rust-toolchain.toml
- .buildkite/scripts/rust-coverage.sh
- codecov.yml
- vllm/entrypoints/openai/
- tests/utils.py
- tests/entrypoints/openai/correctness/test_lmeval.py
commands:
- . /vllm-workspace/.buildkite/scripts/rust-coverage.sh && rust_coverage_start rust-e2e
- export VLLM_USE_RUST_FRONTEND=1
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
- label: Rust Frontend Tool Use
device: h200_35gb
timeout_in_minutes: 25
timeout_in_minutes: 60
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- rust/
- build_rust.sh
- tools/build_rust.py
- rust-toolchain.toml
- .buildkite/scripts/rust-coverage.sh
- codecov.yml
- vllm/entrypoints/openai/
- vllm/tool_parsers/
- tests/utils.py
- tests/tool_use/
commands:
- . /vllm-workspace/.buildkite/scripts/rust-coverage.sh && rust_coverage_start rust-e2e
- export VLLM_USE_RUST_FRONTEND=1
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -v -s tool_use --ignore=tool_use/mistral --models llama3.2 -k "not test_response_format_with_tool_choice_required and not test_parallel_tool_calls_false and not test_tool_call_and_choice"
- label: Rust Frontend Distributed
timeout_in_minutes: 25
timeout_in_minutes: 30
num_devices: 4
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- rust/
- build_rust.sh
- tools/build_rust.py
- rust-toolchain.toml
- .buildkite/scripts/rust-coverage.sh
- codecov.yml
- vllm/distributed/
- vllm/engine/
- vllm/executor/
- vllm/v1/engine/
- vllm/v1/worker/
- tests/utils.py
- tests/v1/distributed/test_external_lb_dp.py
- tests/v1/distributed/test_hybrid_lb_dp.py
- tests/v1/distributed/test_internal_lb_dp.py
commands:
- . /vllm-workspace/.buildkite/scripts/rust-coverage.sh && rust_coverage_start rust-e2e
- export VLLM_USE_RUST_FRONTEND=1
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- export NCCL_CUMEM_HOST_ENABLE=0
- TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_internal_lb_dp.py -k "not 4 and not server_info"
- TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py -k "not 4 and not server_info"
- TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_hybrid_lb_dp.py -k "not 4 and not server_info"
@@ -4,7 +4,7 @@ steps:
- label: Rust Frontend Cargo Style + Clippy
key: rust-frontend-cargo-style-clippy
depends_on: []
timeout_in_minutes: 20
timeout_in_minutes: 30
device: cpu-medium
no_plugin: true
source_file_dependencies:
@@ -18,7 +18,7 @@ steps:
- label: Rust Frontend Cargo Tests
key: rust-frontend-cargo-tests
depends_on: []
timeout_in_minutes: 20
timeout_in_minutes: 30
device: cpu-medium
no_plugin: true
source_file_dependencies:
@@ -26,7 +26,5 @@ steps:
- rust-toolchain.toml
- .buildkite/test_areas/rust_frontend_cargo.yaml
- .buildkite/scripts/run-rust-frontend-cargo-ci.sh
- .buildkite/scripts/rust-coverage.sh
- codecov.yml
commands:
- .buildkite/scripts/run-rust-frontend-cargo-ci.sh test
+2 -2
View File
@@ -5,7 +5,7 @@ steps:
- label: Samplers Test
device: h200_35gb
key: samplers-test
timeout_in_minutes: 40
timeout_in_minutes: 75
source_file_dependencies:
- vllm/model_executor/layers
- vllm/sampling_metadata.py
@@ -19,7 +19,7 @@ steps:
- VLLM_USE_FLASHINFER_SAMPLER=1 pytest -v -s samplers
mirror:
amd:
device: mi325_1
device: mi250_1
depends_on:
- image-build-amd
commands:
+13 -29
View File
@@ -4,7 +4,7 @@ depends_on:
steps:
- label: Spec Decode Eagle
key: spec-decode-eagle
timeout_in_minutes: 25
timeout_in_minutes: 30
device: h200_18gb
source_file_dependencies:
- vllm/v1/spec_decode/
@@ -12,24 +12,10 @@ steps:
- tests/v1/e2e/spec_decode/
commands:
- pytest -v -s v1/e2e/spec_decode -k "eagle_correctness"
mirror:
amd:
device: mi325_1
timeout_in_minutes: 60
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/v1/spec_decode/
- vllm/v1/worker/gpu/spec_decode/
- vllm/model_executor/model_loader/
- vllm/v1/sample/
- vllm/model_executor/layers/
- tests/v1/e2e/spec_decode/
- vllm/platforms/rocm.py
- label: Spec Decode Eagle Nightly B200
key: spec-decode-eagle-nightly-b200
timeout_in_minutes: 25
timeout_in_minutes: 30
device: b200-k8s
optional: true
source_file_dependencies:
@@ -41,7 +27,7 @@ steps:
- label: Spec Decode Speculators + MTP
key: spec-decode-speculators-mtp
timeout_in_minutes: 20
timeout_in_minutes: 30
device: h200_18gb
source_file_dependencies:
- vllm/v1/spec_decode/
@@ -53,7 +39,7 @@ steps:
- pytest -v -s v1/e2e/spec_decode -k "speculators or mtp_correctness"
mirror:
amd:
device: mi325_1
device: mi300_1
timeout_in_minutes: 65
depends_on:
- image-build-amd
@@ -82,7 +68,7 @@ steps:
- label: Spec Decode Ngram + Suffix
key: spec-decode-ngram-suffix
timeout_in_minutes: 20
timeout_in_minutes: 30
device: h200_18gb
source_file_dependencies:
- vllm/v1/spec_decode/
@@ -92,10 +78,8 @@ steps:
- pytest -v -s v1/e2e/spec_decode -k "ngram or suffix"
mirror:
amd:
device: mi325_1
timeout_in_minutes: 55
# TODO(akaratza): Test after Torch >= 2.12 bump
soft_fail: true
device: mi300_1
timeout_in_minutes: 65
depends_on:
- image-build-amd
source_file_dependencies:
@@ -109,7 +93,7 @@ steps:
- label: Spec Decode Draft Model
key: spec-decode-draft-model
timeout_in_minutes: 45
timeout_in_minutes: 30
device: h200_18gb
source_file_dependencies:
- vllm/v1/spec_decode/
@@ -119,8 +103,8 @@ steps:
- pytest -v -s v1/e2e/spec_decode -k "draft_model or no_sync or batch_inference"
mirror:
amd:
device: mi325_1
timeout_in_minutes: 55
device: mi300_1
timeout_in_minutes: 50
depends_on:
- image-build-amd
source_file_dependencies:
@@ -134,7 +118,7 @@ steps:
- label: Spec Decode Draft Model Nightly B200
key: spec-decode-draft-model-nightly-b200
timeout_in_minutes: 40
timeout_in_minutes: 30
device: b200-k8s
optional: true
source_file_dependencies:
@@ -146,7 +130,7 @@ steps:
- label: Speculators Correctness
key: speculators-correctness
timeout_in_minutes: 30
timeout_in_minutes: 60
device: h100
optional: true
num_devices: 1
@@ -159,7 +143,7 @@ steps:
- pytest -v -s v1/spec_decode/test_speculators_correctness.py -m slow_test
- label: Spec Decode MTP hybrid (B200)
timeout_in_minutes: 20
timeout_in_minutes: 30
device: b200-k8s
optional: true
source_file_dependencies:
+1 -9
View File
@@ -4,7 +4,7 @@ depends_on:
steps:
- label: Weight Loading Multiple GPU # 33min
key: weight-loading-multiple-gpu
timeout_in_minutes: 50
timeout_in_minutes: 45
working_dir: "/vllm-workspace/tests"
num_devices: 2
optional: true
@@ -13,14 +13,6 @@ steps:
- tests/weight_loading
commands:
- bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models.txt
mirror:
amd:
dind: false
device: mi300_2
depends_on:
- image-build-amd
commands:
- bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-amd.txt
# - label: Weight Loading Multiple GPU - Large Models # optional
# working_dir: "/vllm-workspace/tests"
@@ -1,35 +0,0 @@
---
name: ci-fails-buildkite
description: Fetch and diagnose vLLM Buildkite CI failure logs. Use when investigating failing CI jobs on a PR or build, when the user pastes a buildkite.com URL, or asks to fetch/diagnose CI logs.
---
# Diagnosing vLLM Buildkite CI Failures
Buildkite logs are public; no login needed.
`.buildkite/scripts/ci-fetch-log.sh` saves each log as `ci-<build>-<job-name>.log`, stripped of timestamps and ANSI codes. Existing files are kept; set `CI_FETCH_LOG_FORCE=1` to refetch.
## Fetching logs
```bash
# All failed jobs in a PR's latest build (current branch's PR if omitted):
.buildkite/scripts/ci-fetch-log.sh --pr <PR>
# All failed jobs in a build (--soft also includes soft-failed jobs;
# --all fetches every finished job):
.buildkite/scripts/ci-fetch-log.sh "https://buildkite.com/vllm/ci/builds/<N>"
# One job — `gh pr checks` URLs (#<job_uuid>) and web UI URLs (?sid=) both
# work; pass "-" as a second argument to stream to stdout:
.buildkite/scripts/ci-fetch-log.sh "https://buildkite.com/vllm/ci/builds/<N>#<job_uuid>"
```
To clean an already-downloaded log with `.buildkite/scripts/ci-clean-log.sh`:
```bash
./ci-clean-log.sh ci.log
```
## Reference
See [docs/contributing/ci/failures.md](../../../docs/contributing/ci/failures.md) for the full guide: filing CI failure issues, investigating/bisecting, reproducing flaky tests, and daily triage.
+25 -22
View File
@@ -2,16 +2,17 @@
# for more info about CODEOWNERS file
# This lists cover the "core" components of vLLM that require careful review
/vllm/compilation @zou3519 @youkaichao @ProExpertProg @BoyuanFeng
/vllm/distributed/kv_transfer @NickLucche @ApostaC @orozery @xuechendi @ivanium
/vllm/compilation @zou3519 @youkaichao @ProExpertProg @BoyuanFeng @vadiklyutiy
/vllm/distributed/kv_transfer @NickLucche @ApostaC @orozery @xuechendi
/vllm/lora @jeejeelee
/vllm/model_executor/layers/attention @LucasWilkinson @MatthewBonanni
/vllm/model_executor/layers/fused_moe @mgoin @pavanimajety @zyongye
/vllm/model_executor/layers/quantization @mgoin @robertgshaw2-redhat @tlrmchlsmth @yewentao256 @pavanimajety @zyongye
/vllm/model_executor/layers/mamba @tdoublep @tomeras91
/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py @tdoublep @ZJY0516 @vadiklyutiy
/vllm/model_executor/layers/mamba/gdn_linear_attn.py @tdoublep @ZJY0516 @vadiklyutiy
/vllm/model_executor/layers/rotary_embedding.py @vadiklyutiy
/vllm/model_executor/model_loader @22quinn
/vllm/model_executor/layers/batch_invariant.py @yewentao256
/vllm/model_executor/layers/batch_invariant.py @yewentao256
/vllm/ir @ProExpertProg
/vllm/kernels/ @ProExpertProg @tjtanaa
/vllm/kernels/helion @ProExpertProg @zou3519
@@ -22,13 +23,8 @@
# Any change to the VllmConfig changes can have a large user-facing impact,
# so spam a lot of people
/vllm/config @WoosukKwon @youkaichao @robertgshaw2-redhat @mgoin @tlrmchlsmth @houseroad @yewentao256 @ProExpertProg
/vllm/config/cache.py @heheda12345 @ivanium
# Config utils
/vllm/config/utils.py @hmellor
/vllm/engine/arg_utils.py @hmellor
/vllm/utils/argparse_utils.py
/vllm/config @WoosukKwon @youkaichao @robertgshaw2-redhat @mgoin @tlrmchlsmth @houseroad @hmellor @yewentao256 @ProExpertProg
/vllm/config/cache.py @heheda12345
# Entrypoints
/vllm/entrypoints/anthropic @mgoin @DarkLight1337
@@ -47,7 +43,6 @@
# Rust Frontend
/rust/ @BugenZhao @njhill
/rust/src/bench @esmeetu
/build_rust.sh @BugenZhao @njhill
/rust-toolchain.toml @BugenZhao @njhill
/.buildkite/test_areas/rust* @BugenZhao @njhill
@@ -68,20 +63,19 @@
/vllm/v1/attention/backends/flashinfer.py @mgoin @pavanimajety @vadiklyutiy
/vllm/v1/attention/backends/triton_attn.py @tdoublep
/vllm/v1/attention/backends/gdn_attn.py @ZJY0516 @vadiklyutiy
/vllm/v1/core @WoosukKwon @robertgshaw2-redhat @njhill @ywang96 @alexm-redhat @heheda12345 @ApostaC @orozery @ivanium
/vllm/v1/core @WoosukKwon @robertgshaw2-redhat @njhill @ywang96 @alexm-redhat @heheda12345 @ApostaC @orozery
/vllm/v1/sample @22quinn @houseroad @njhill
/vllm/v1/spec_decode @benchislett @luccafong @MatthewBonanni
/vllm/v1/structured_output @mgoin @russellb @aarnphm @benchislett
/vllm/v1/kv_cache_interface.py @heheda12345 @ivanium
/vllm/v1/kv_cache_interface.py @heheda12345
/vllm/v1/kv_offload @ApostaC @orozery
/vllm/v1/simple_kv_offload @ivanium
/vllm/v1/engine @njhill
/vllm/v1/executor @njhill
/vllm/v1/worker @njhill
/vllm/v1/worker/kv_connector_model_runner_mixin.py @orozery @NickLucche @ivanium
/vllm/v1/worker/kv_connector_model_runner_mixin.py @orozery @NickLucche
# Model runner V2
/vllm/v1/worker/gpu @WoosukKwon @njhill @yewentao256
/vllm/v1/worker/gpu @WoosukKwon @njhill
/vllm/v1/worker/gpu/kv_connector.py @orozery
# CI & building
@@ -105,14 +99,13 @@
/tests/test_inputs.py @DarkLight1337 @ywang96
/tests/entrypoints/llm/test_struct_output_generate.py @mgoin @russellb @aarnphm
/tests/v1/structured_output @mgoin @russellb @aarnphm
/tests/v1/core @WoosukKwon @robertgshaw2-redhat @njhill @ywang96 @alexm-redhat @heheda12345 @ApostaC @orozery @ivanium
/tests/v1/core @WoosukKwon @robertgshaw2-redhat @njhill @ywang96 @alexm-redhat @heheda12345 @ApostaC @orozery
/tests/weight_loading @mgoin @youkaichao @yewentao256
/tests/lora @jeejeelee
/tests/models/language/generation/test_hybrid.py @tdoublep @tomeras91
/tests/v1/kv_connector/nixl_integration @NickLucche
/tests/v1/kv_connector @ApostaC @orozery @ivanium
/tests/v1/kv_connector @ApostaC @orozery
/tests/v1/kv_offload @ApostaC @orozery
/tests/v1/simple_kv_offload @ivanium
/tests/v1/determinism @yewentao256
/tests/reasoning @aarnphm @chaunceyjiang @sfeng33 @bbrowning
/tests/tool_parsers @aarnphm @chaunceyjiang @sfeng33 @bbrowning
@@ -120,7 +113,17 @@
# Transformers modeling backend
/vllm/model_executor/models/transformers @hmellor
/tests/models/transformers @hmellor
/tests/models/test_transformers.py @hmellor
# Observability
/vllm/config/observability.py @markmc
/vllm/v1/metrics @markmc
/tests/v1/metrics @markmc
/vllm/tracing.py @markmc
/tests/v1/tracing/test_tracing.py @markmc
/vllm/config/kv_events.py @markmc
/vllm/distributed/kv_events.py @markmc
/tests/distributed/test_events.py @markmc
# Docs
/docs/mkdocs @hmellor
@@ -173,7 +176,7 @@ mkdocs.yaml @hmellor
# Kernels
/vllm/v1/attention/ops/chunked_prefill_paged_decode.py @tdoublep
/vllm/v1/attention/ops/triton_unified_attention.py @tdoublep
/vllm/third_party/flash_linear_attention @ZJY0516 @vadiklyutiy
/vllm/model_executor/layers/fla @ZJY0516 @vadiklyutiy
# ROCm related: specify owner with write access to notify AMD folks for careful code review
/vllm/**/*rocm* @tjtanaa @dllehr-amd
-7
View File
@@ -1,7 +0,0 @@
# Custom self-hosted runner labels (e.g. the autoscaling vllm-runners pool) so
# actionlint doesn't flag them as unknown in `runs-on`.
self-hosted-runner:
labels:
- vllm-runners
# Not yet in actionlint's known-label set.
- macos-26
+1
View File
@@ -21,6 +21,7 @@ updates:
- dependency-name: "torchvision"
- dependency-name: "xformers"
- dependency-name: "lm-format-enforcer"
- dependency-name: "gguf"
- dependency-name: "compressed-tensors"
- dependency-name: "ray[cgraph]" # Ray Compiled Graph
- dependency-name: "lm-eval"
+4 -8
View File
@@ -144,12 +144,12 @@ pull_request_rules:
- label != stale
- or:
- files~=^examples/.*mistral.*\.py
- files~=^tests/.*(?:mistral|voxtral|mixtral|pixtral).*\.py
- files~=^vllm/model_executor/models/.*(?:mistral|voxtral|mixtral|pixtral).*\.py
- files~=^tests/.*mistral.*\.py
- files~=^vllm/model_executor/models/.*mistral.*\.py
- files~=^vllm/reasoning/.*mistral.*\.py
- files~=^vllm/tool_parsers/.*mistral.*\.py
- files~=^vllm/transformers_utils/.*(?:mistral|voxtral|pixtral).*\.py
- title~=(?i)(?:mistral|ministral|voxtral|mixtral|pixtral)
- files~=^vllm/transformers_utils/.*mistral.*\.py
- title~=(?i)Mistral
actions:
label:
add:
@@ -388,13 +388,9 @@ pull_request_rules:
- or:
- files~=^tests/tool_use/
- files~=^tests/tool_parsers/
- files~=^tests/parser/
- files~=^tests/reasoning/
- files~=^tests/entrypoints/openai/.*tool.*
- files~=^tests/entrypoints/anthropic/.*tool.*
- files~=^vllm/tool_parsers/
- files~=^vllm/parser/
- files~=^vllm/reasoning/
- files=docs/features/tool_calling.md
- files~=^examples/tool_calling/
actions:
+2 -2
View File
@@ -323,11 +323,11 @@ jobs:
// {users} will be replaced with @mentions
const ccConfig = {
rocm: {
users: ['hongxiayang', 'tjtanaa', 'vllmellm', 'giuseppegrossi'],
users: ['hongxiayang', 'tjtanaa', 'vllmellm'],
message: 'CC {users} for ROCm-related issue',
},
mistral: {
users: ['patrickvonplaten', 'juliendenize', 'andylolu2', 'NickLucche'],
users: ['patrickvonplaten', 'juliendenize', 'andylolu2'],
message: 'CC {users} for Mistral-related issue',
},
// Add more label -> user mappings here
+10 -19
View File
@@ -11,25 +11,13 @@ permissions:
jobs:
macos-m1-smoke-test:
# macos-26 (the supported target) is still a preview runner, so gate on GA
# macos-15 and keep macos-26 non-blocking.
strategy:
fail-fast: false
matrix:
include:
- os: macos-15
required: true
- os: macos-26
required: false
name: macos-m1-smoke-test (${{ matrix.os }})
runs-on: ${{ matrix.os }}
continue-on-error: ${{ !matrix.required }}
runs-on: macos-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- uses: actions/checkout@v6.0.1
- uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
- uses: astral-sh/setup-uv@v7
with:
enable-cache: true
cache-dependency-glob: |
@@ -84,11 +72,14 @@ jobs:
# Test health endpoint
curl -f http://localhost:8000/health
# Long prompt: hits the split-KV path that short prompts skip (#46769).
PAYLOAD=$(python -c "import json; print(json.dumps({'model': 'Qwen/Qwen3-0.6B', 'prompt': 'The quick brown fox jumps over the lazy dog. ' * 24, 'max_tokens': 16}))")
curl -f --max-time 120 http://localhost:8000/v1/completions \
# Test completion
curl -f http://localhost:8000/v1/completions \
-H "Content-Type: application/json" \
-d "$PAYLOAD"
-d '{
"model": "Qwen/Qwen3-0.6B",
"prompt": "Hello",
"max_tokens": 5
}'
# Cleanup
kill "$SERVER_PID"
+4 -9
View File
@@ -28,8 +28,7 @@ jobs:
pull_number: context.payload.pull_request.number,
});
const readyLabels = ['ready', 'ready-run-all-tests'];
const hasReadyLabel = pr.labels.some(l => readyLabels.includes(l.name));
const hasReadyLabel = pr.labels.some(l => l.name === 'ready');
const hasVerifiedLabel = pr.labels.some(l => l.name === 'verified');
const { data: mergedPRs } = await github.rest.search.issuesAndPullRequests({
@@ -41,22 +40,18 @@ jobs:
if (hasReadyLabel || hasVerifiedLabel || mergedCount >= 4) {
core.info(`Check passed: verified label=${hasVerifiedLabel}, ready label=${hasReadyLabel}, 4+ merged PRs=${mergedCount >= 4}`);
} else {
core.setFailed(`PR must have the 'verified', 'ready', or 'ready-run-all-tests' label (the ready labels also trigger tests) or the author must have at least 4 merged PRs (found ${mergedCount}).`);
core.setFailed(`PR must have the 'verified' or 'ready' (which also triggers tests) label or the author must have at least 4 merged PRs (found ${mergedCount}).`);
}
pre-commit:
needs: pre-run-check
if: always() && (needs.pre-run-check.result == 'success' || needs.pre-run-check.result == 'skipped')
runs-on: [self-hosted, linux, x64, vllm-runners]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
- uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
with:
python-version: "3.12"
# Provide shellcheck on PATH so tools/pre_commit/shellcheck.sh skips its
# wget + tar -xJ self-download, which the self-hosted runner image lacks
# (no wget/xz). Pinned to shellcheck 0.10.0 to match the script's "stable".
- run: python -m pip install shellcheck-py==0.10.0.1
- run: echo "::add-matcher::.github/workflows/matchers/actionlint.json"
- run: echo "::add-matcher::.github/workflows/matchers/markdownlint.json"
- run: echo "::add-matcher::.github/workflows/matchers/mypy.json"
+2 -5
View File
@@ -9,7 +9,7 @@ PATH=${cuda_home}/bin:$PATH
LD_LIBRARY_PATH=${cuda_home}/lib64:$LD_LIBRARY_PATH
# Install requirements
if [ "$(echo "$2" | cut -d. -f1)" = "12" ]; then
if [ "$(echo $2 | cut -d. -f1)" = "12" ]; then
sed -i 's/^nvidia-cutlass-dsl\[cu13\]>=/nvidia-cutlass-dsl>=/' requirements/cuda.txt
fi
$python_executable -m pip install -r requirements/build/cuda.txt -r requirements/cuda.txt
@@ -17,10 +17,7 @@ $python_executable -m pip install -r requirements/build/cuda.txt -r requirements
# Limit the number of parallel jobs to avoid OOM
export MAX_JOBS=1
# Make sure release wheels are built for the following architectures
# Do not add +PTX here: vLLM filters torch's top-level PTX flag when it
# converts global gencode flags into per-kernel arch lists. If a specific
# kernel needs PTX, add +PTX to that kernel's CMake arch list instead.
export TORCH_CUDA_ARCH_LIST="7.5 8.0 8.6 8.9 9.0 10.0 12.0"
export TORCH_CUDA_ARCH_LIST="7.5 8.0 8.6 8.9 9.0 10.0 12.0+PTX"
bash tools/check_repo.sh
+2 -11
View File
@@ -15,12 +15,6 @@ vllm/third_party/flashmla/flash_mla_interface.py
# DeepGEMM vendored package built from source
vllm/third_party/deep_gemm/
# fmha_sm100 vendored package built from source
vllm/third_party/fmha_sm100/
# tml-fa4 vendored package built from source
vllm/third_party/tml_fa4/
# triton jit
.triton
@@ -202,9 +196,7 @@ cython_debug/
.vscode/
# Claude
.claude/*
!.claude/skills/
!.claude/skills/**
.claude/
# Codex
.codex/
@@ -241,7 +233,7 @@ actionlint
shellcheck*/
# Ignore moe/marlin_moe gen code
csrc/libtorch_stable/moe/marlin_moe_wna16/kernel_*
csrc/moe/marlin_moe_wna16/kernel_*
# Ignore ep_kernels_workspace folder
ep_kernels_workspace/
@@ -257,4 +249,3 @@ vllm/grpc/vllm_engine_pb2.pyi
# Ignore generated cpu headers
csrc/cpu/cpu_attn_dispatch_generated.h
rust-coverage-tools/
+3 -16
View File
@@ -21,7 +21,7 @@ repos:
rev: v21.1.2
hooks:
- id: clang-format
exclude: 'csrc/libtorch_stable/moe/topk_softmax_kernels.cu|vllm/third_party/.*'
exclude: 'csrc/(moe/topk_softmax_kernels.cu|libtorch_stable/quantization/gguf/(ggml-common.h|dequantize.cuh|vecdotq.cuh|mmq.cuh|mmvq.cuh))|vllm/third_party/.*'
types_or: [c++, cuda]
args: [--style=file, --verbose]
- repo: https://github.com/DavidAnson/markdownlint-cli2
@@ -30,7 +30,7 @@ repos:
- id: markdownlint-cli2
language_version: lts
args: [--fix]
exclude: (^|/)CLAUDE\.md$
exclude: ^CLAUDE\.md$
- repo: https://github.com/rhysd/actionlint
rev: v1.7.7
hooks:
@@ -131,19 +131,6 @@ repos:
--python-version, "3.12",
]
files: ^requirements/(common|xpu|test/xpu)\.(in|txt)$
- id: pip-compile
alias: pip-compile-cpu
name: pip-compile-cpu
args: [
requirements/test/cuda.in,
-o, requirements/test/cpu.txt,
--index-strategy, unsafe-best-match,
--torch-backend, cpu,
--python-platform, x86_64-manylinux_2_28,
--python-version, "3.12",
]
files: ^requirements/(common|cpu|test/(cuda|cpu))\.(in|txt)$
exclude: ^requirements/test/cuda\.txt$
- id: pip-compile
alias: pip-compile-docs
name: pip-compile-docs
@@ -210,7 +197,7 @@ repos:
name: Check SPDX headers
entry: python tools/pre_commit/check_spdx_header.py
language: python
types_or: [python, rust, proto]
types: [python]
- id: check-root-lazy-imports
name: Check root lazy imports
entry: python tools/pre_commit/check_init_lazy_imports.py
+2
View File
@@ -0,0 +1,2 @@
collect_env.py
vllm/model_executor/layers/fla/ops/*.py

Some files were not shown because too many files have changed in this diff Show More