Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ee0da84ab9 | ||
|
|
217c64a976 | ||
|
|
cfe8a4d063 | ||
|
|
6d37570a1c | ||
|
|
f85a9f112a | ||
|
|
836b5acb1b | ||
|
|
b36db10f27 | ||
|
|
b70c13ea47 | ||
|
|
6829a6d55f | ||
|
|
6ed56e04ff | ||
|
|
21089b4762 | ||
|
|
7b3d595eb1 | ||
|
|
e66b766bc4 | ||
|
|
51000a95a1 | ||
|
|
2e0c5f52f7 | ||
|
|
0085058ca8 | ||
|
|
fdd8e4efd3 | ||
|
|
c6561c2fb6 |
@@ -1,103 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Audit vLLM compiled libraries for PyTorch stable ABI compliance."""
|
||||
|
||||
import fnmatch
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from torch_abi_audit import inspect_package
|
||||
from torch_abi_audit.report import ExtensionReport, PackageReport
|
||||
|
||||
# Temporary allowlist of extensions not yet on the stable ABI.
|
||||
# Shrink and remove over time.
|
||||
ALLOWED_UNSTABLE_LIBRARIES: tuple[str, ...] = (
|
||||
"_flashkda_C.abi3.so",
|
||||
"vllm_flash_attn/_vllm_fa2_C.abi3.so",
|
||||
"vllm_flash_attn/_vllm_fa3_C.abi3.so",
|
||||
"third_party/deep_gemm/_C*.so",
|
||||
)
|
||||
|
||||
|
||||
def _relative_path(lib: ExtensionReport, package_root: Path) -> str:
|
||||
try:
|
||||
return lib.path.relative_to(package_root).as_posix()
|
||||
except ValueError:
|
||||
return lib.path.name
|
||||
|
||||
|
||||
def _is_torch_unstable(lib: ExtensionReport) -> bool:
|
||||
return lib.error is None and lib.torch.uses_torch and not lib.torch.stable
|
||||
|
||||
|
||||
def _matches_allowlist(rel_path: str, patterns: tuple[str, ...]) -> bool:
|
||||
return any(fnmatch.fnmatch(rel_path, pattern) for pattern in patterns)
|
||||
|
||||
|
||||
def _iter_libs(report: PackageReport) -> tuple[ExtensionReport, ...]:
|
||||
return (*report.extensions, *report.bundled_libs)
|
||||
|
||||
|
||||
def _collect_unstable(report: PackageReport) -> list[str]:
|
||||
return sorted(
|
||||
_relative_path(lib, report.root)
|
||||
for lib in _iter_libs(report)
|
||||
if _is_torch_unstable(lib)
|
||||
)
|
||||
|
||||
|
||||
def _find_stale_allowlist_entries(
|
||||
report: PackageReport, patterns: tuple[str, ...]
|
||||
) -> list[str]:
|
||||
"""Allowlist patterns that match a built library which is no longer unstable."""
|
||||
stale: list[str] = []
|
||||
for pattern in patterns:
|
||||
for lib in _iter_libs(report):
|
||||
if lib.error is not None:
|
||||
continue
|
||||
if not fnmatch.fnmatch(_relative_path(lib, report.root), pattern):
|
||||
continue
|
||||
if not _is_torch_unstable(lib):
|
||||
stale.append(pattern)
|
||||
break
|
||||
return stale
|
||||
|
||||
|
||||
def check_torch_abi(
|
||||
package: str = "vllm",
|
||||
patterns: tuple[str, ...] = ALLOWED_UNSTABLE_LIBRARIES,
|
||||
) -> int:
|
||||
report = inspect_package(package)
|
||||
if report.error:
|
||||
print(f"error: failed to inspect {package!r}: {report.error}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
unstable = _collect_unstable(report)
|
||||
unexpected = [
|
||||
rel_path for rel_path in unstable if not _matches_allowlist(rel_path, patterns)
|
||||
]
|
||||
stale = _find_stale_allowlist_entries(report, patterns)
|
||||
|
||||
if unexpected or stale:
|
||||
if unexpected:
|
||||
print(
|
||||
"Not allowed: torch-unstable libraries outside "
|
||||
f"ALLOWED_UNSTABLE_LIBRARIES: {', '.join(unexpected)}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
if stale:
|
||||
print(
|
||||
"Not allowed: stale ALLOWED_UNSTABLE_LIBRARIES entries: "
|
||||
f"{', '.join(stale)}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
print("Torch stable ABI check passed.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print(">>> Auditing vLLM extension modules for PyTorch stable ABI compliance")
|
||||
sys.exit(check_torch_abi())
|
||||
@@ -14,7 +14,6 @@ run_all_patterns:
|
||||
- "setup.py"
|
||||
- "csrc/"
|
||||
- "cmake/"
|
||||
- ".buildkite/check-torch-abi.py"
|
||||
run_all_exclude_patterns:
|
||||
- "docker/Dockerfile."
|
||||
- "csrc/cpu/"
|
||||
|
||||
@@ -2,17 +2,17 @@ name: vllm_intel_ci
|
||||
job_dirs:
|
||||
- ".buildkite/intel_jobs"
|
||||
run_all_patterns:
|
||||
- ".buildkite/ci_config_intel.yaml"
|
||||
- ".buildkite/scripts/hardware_ci/run-intel-test.sh"
|
||||
- "docker/Dockerfile"
|
||||
- "docker/Dockerfile.xpu"
|
||||
- "CMakeLists.txt"
|
||||
- "requirements/common.txt"
|
||||
- "requirements/xpu.txt"
|
||||
- "requirements/build/cuda.txt"
|
||||
- "requirements/test/cuda.txt"
|
||||
- "setup.py"
|
||||
- "csrc/"
|
||||
- "cmake/"
|
||||
run_all_exclude_patterns:
|
||||
- "docker/Dockerfile."
|
||||
- "csrc/cpu/"
|
||||
- "csrc/rocm/"
|
||||
- "cmake/hipify.py"
|
||||
|
||||
@@ -8,7 +8,6 @@ run_all_patterns:
|
||||
- "docker/docker-bake-rocm.hcl"
|
||||
- ".buildkite/hardware_tests/amd.yaml"
|
||||
- ".buildkite/scripts/ci-bake-rocm.sh"
|
||||
- ".buildkite/scripts/rocm/"
|
||||
- ".buildkite/scripts/hardware_ci/run-amd-test.py"
|
||||
- ".buildkite/scripts/hardware_ci/run-amd-test.sh"
|
||||
- "CMakeLists.txt"
|
||||
|
||||
@@ -1,55 +1,24 @@
|
||||
group: Hardware - AMD Build
|
||||
|
||||
# ROCm image flow:
|
||||
# 1. Refresh the long-lived ROCm base image only when Dockerfile.rocm_base changes.
|
||||
# 2. Build ci_base from either the stable base or the freshly refreshed base.
|
||||
# 3. Build the per-commit ROCm CI image and smoke-test it before GPU jobs run.
|
||||
steps:
|
||||
- label: "AMD: :docker: refresh ROCm base"
|
||||
key: refresh-rocm-base-amd
|
||||
depends_on: []
|
||||
device: amd_cpu
|
||||
no_plugin: true
|
||||
commands:
|
||||
- bash .buildkite/scripts/rocm/refresh-base-image.sh
|
||||
env:
|
||||
DOCKER_BUILDKIT: "1"
|
||||
BUILDKIT_PROGRESS: "tty"
|
||||
TERM: "xterm-256color"
|
||||
retry:
|
||||
automatic:
|
||||
- exit_status: 1 # Transient Docker/BuildKit failure
|
||||
limit: 1
|
||||
- exit_status: -1 # Agent was lost
|
||||
limit: 1
|
||||
- exit_status: -10 # Agent was lost
|
||||
limit: 1
|
||||
|
||||
# Ensure ci_base is up-to-date before building the test image.
|
||||
# Compares a content hash of ci_base-affecting files against the remote
|
||||
# image label. If hashes match the build is skipped (< 30 s); if they
|
||||
# differ ci_base is rebuilt and pushed automatically.
|
||||
- label: "AMD: :docker: ensure ci_base"
|
||||
key: ensure-ci-base-amd
|
||||
soft_fail: false
|
||||
depends_on:
|
||||
- refresh-rocm-base-amd
|
||||
depends_on: []
|
||||
device: amd_cpu
|
||||
no_plugin: true
|
||||
commands:
|
||||
- bash .buildkite/scripts/rocm/build-ci-base.sh
|
||||
- bash .buildkite/scripts/ci-bake-rocm.sh ci-base-rocm-ci-with-deps
|
||||
env:
|
||||
DOCKER_BUILDKIT: "1"
|
||||
BUILDKIT_PROGRESS: "tty"
|
||||
TERM: "xterm-256color"
|
||||
VLLM_BAKE_FILE: "docker/docker-bake-rocm.hcl"
|
||||
PYTORCH_ROCM_ARCH: "gfx90a;gfx942;gfx950"
|
||||
REMOTE_VLLM: "1"
|
||||
VLLM_BRANCH: "$BUILDKITE_COMMIT"
|
||||
retry:
|
||||
automatic:
|
||||
- exit_status: 1 # Transient Docker/BuildKit failure
|
||||
limit: 1
|
||||
- exit_status: -1 # Agent was lost
|
||||
limit: 1
|
||||
- exit_status: -10 # Agent was lost
|
||||
@@ -57,18 +26,40 @@ steps:
|
||||
|
||||
- label: "AMD: :docker: build test image and artifacts"
|
||||
key: image-build-amd
|
||||
soft_fail: false
|
||||
depends_on:
|
||||
- ensure-ci-base-amd
|
||||
device: amd_cpu
|
||||
no_plugin: true
|
||||
commands:
|
||||
- bash .buildkite/scripts/rocm/build-test-image.sh
|
||||
- bash .buildkite/scripts/rocm/smoke-test-image.sh
|
||||
- |
|
||||
if [[ "${ROCM_CI_ARTIFACT_ONLY:-0}" == "1" ]]; then
|
||||
echo "ROCM_CI_ARTIFACT_ONLY=1; building ROCm wheel artifact only"
|
||||
IMAGE_TAG="" bash .buildkite/scripts/ci-bake-rocm.sh test-rocm-ci-with-artifacts
|
||||
else
|
||||
bash .buildkite/scripts/ci-bake-rocm.sh test-rocm-ci-with-wheel
|
||||
fi
|
||||
- |
|
||||
docker run --rm --network=none --entrypoint /bin/bash "rocm/vllm-ci:${BUILDKITE_COMMIT}" -ec '
|
||||
if [ ! -d /vllm-workspace ]; then echo Missing directory: /vllm-workspace >&2; exit 1; fi
|
||||
if [ ! -d /vllm-workspace/tests ]; then echo Missing directory: /vllm-workspace/tests >&2; exit 1; fi
|
||||
if [ ! -d /vllm-workspace/src/vllm ]; then echo Missing directory: /vllm-workspace/src/vllm >&2; exit 1; fi
|
||||
if [ ! -x /vllm-workspace/src/vllm/vllm-rs ]; then echo Missing executable: /vllm-workspace/src/vllm/vllm-rs >&2; exit 1; fi
|
||||
command -v python3
|
||||
command -v uv
|
||||
command -v pytest
|
||||
if ! command -v amd-smi >/dev/null 2>&1 && ! command -v rocminfo >/dev/null 2>&1; then
|
||||
echo No ROCm CLI found in image >&2
|
||||
exit 1
|
||||
fi
|
||||
python3 - <<PY
|
||||
import torch, vllm
|
||||
print(torch.__version__)
|
||||
print(vllm.__version__)
|
||||
PY
|
||||
echo AMD image smoke OK
|
||||
'
|
||||
env:
|
||||
DOCKER_BUILDKIT: "1"
|
||||
BUILDKIT_PROGRESS: "tty"
|
||||
TERM: "xterm-256color"
|
||||
VLLM_BAKE_FILE: "docker/docker-bake-rocm.hcl"
|
||||
PYTORCH_ROCM_ARCH: "gfx90a;gfx942;gfx950"
|
||||
IMAGE_TAG: "rocm/vllm-ci:$BUILDKITE_COMMIT"
|
||||
@@ -76,8 +67,6 @@ steps:
|
||||
VLLM_BRANCH: "$BUILDKITE_COMMIT"
|
||||
retry:
|
||||
automatic:
|
||||
- exit_status: 1 # Transient Docker/BuildKit failure
|
||||
limit: 1
|
||||
- exit_status: -1 # Agent was lost
|
||||
limit: 1
|
||||
- exit_status: -10 # Agent was lost
|
||||
|
||||
@@ -17,22 +17,16 @@ steps:
|
||||
- tests/kernels/test_awq_int4_to_int8.py
|
||||
- tests/kernels/quantization/test_cpu_fp8_scaled_mm.py
|
||||
- tests/kernels/mamba/cpu/test_cpu_gdn_ops.py
|
||||
- tests/kernels/mamba/test_cpu_short_conv.py
|
||||
- tests/kernels/mamba/test_causal_conv1d.py
|
||||
- tests/kernels/mamba/test_mamba_ssm.py
|
||||
commands:
|
||||
- |
|
||||
bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 30m "
|
||||
pytest -x -v -s tests/kernels/attention/test_cpu_attn.py
|
||||
pytest -x -v -s tests/kernels/moe/test_cpu_fused_moe.py
|
||||
pytest -x -v -s tests/kernels/moe/test_cpu_quant_fused_moe.py
|
||||
pytest -x -v -s tests/kernels/mamba/test_cpu_short_conv.py
|
||||
pytest -x -v -s tests/kernels/test_onednn.py
|
||||
pytest -x -v -s tests/kernels/test_awq_int4_to_int8.py
|
||||
pytest -x -v -s tests/kernels/quantization/test_cpu_fp8_scaled_mm.py
|
||||
pytest -x -v -s tests/kernels/mamba/cpu/test_cpu_gdn_ops.py
|
||||
pytest -x -v -s tests/kernels/mamba/test_causal_conv1d.py
|
||||
pytest -x -v -s tests/kernels/mamba/test_mamba_ssm.py"
|
||||
pytest -x -v -s tests/kernels/mamba/cpu/test_cpu_gdn_ops.py"
|
||||
|
||||
# Note: SDE can't be downloaded from CI host because of AWS WAF
|
||||
# - label: CPU-Compatibility Tests
|
||||
@@ -59,7 +53,7 @@ steps:
|
||||
- tests/models/language/pooling/
|
||||
commands:
|
||||
- |
|
||||
bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 50m "
|
||||
bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 40m "
|
||||
pytest -x -v -s tests/models/language/generation -m cpu_model
|
||||
pytest -x -v -s tests/models/language/pooling -m cpu_model"
|
||||
|
||||
@@ -74,15 +68,13 @@ steps:
|
||||
- vllm/v1/sample/ops/topk_topp_triton.py
|
||||
- vllm/v1/sample/ops/topk_topp_sampler.py
|
||||
- tests/v1/sample/test_topk_topp_sampler.py
|
||||
- tests/v1/e2e/test_cpu_linear_attn_chunked_prefix.py
|
||||
commands:
|
||||
- |
|
||||
bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 45m "
|
||||
uv pip install git+https://github.com/triton-lang/triton-cpu.git@270e696d
|
||||
VLLM_USE_V2_MODEL_RUNNER=1 pytest -x -v -s tests/models/language/generation/test_granite.py -m cpu_model
|
||||
# TODO: move to CPU-Kernel Tests once triton-cpu has a pre-built wheel
|
||||
pytest -x -v -s tests/v1/sample/test_topk_topp_sampler.py::TestTritonTopkTopp
|
||||
pytest -x -v -s tests/v1/e2e/test_cpu_linear_attn_chunked_prefix.py"
|
||||
pytest -x -v -s tests/v1/sample/test_topk_topp_sampler.py::TestTritonTopkTopp"
|
||||
|
||||
- label: CPU-Quantization Model Tests
|
||||
depends_on: []
|
||||
@@ -97,13 +89,11 @@ steps:
|
||||
- vllm/model_executor/layers/fused_moe/experts/cpu_moe.py
|
||||
- tests/quantization/test_compressed_tensors.py
|
||||
- tests/quantization/test_cpu_wna16.py
|
||||
- tests/quantization/test_cpu_w8a8.py
|
||||
commands:
|
||||
- |
|
||||
bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 45m "
|
||||
pytest -x -v -s tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_logprobs
|
||||
pytest -x -v -s tests/quantization/test_cpu_wna16.py
|
||||
pytest -x -v -s tests/quantization/test_cpu_w8a8.py"
|
||||
pytest -x -v -s tests/quantization/test_cpu_wna16.py"
|
||||
|
||||
- label: CPU-Distributed Tests (PP+TP)
|
||||
depends_on: []
|
||||
@@ -145,22 +135,9 @@ steps:
|
||||
commands:
|
||||
- |
|
||||
bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 45m "
|
||||
pytest -x -v -s tests/models/multimodal/generation --ignore=tests/models/multimodal/generation/test_pixtral.py --ignore=tests/models/multimodal/generation/test_qwen2_5_vl.py -m cpu_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
|
||||
pytest -x -v -s tests/models/multimodal/generation --ignore=tests/models/multimodal/generation/test_pixtral.py -m cpu_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB"
|
||||
parallelism: 4
|
||||
|
||||
- label: CPU-Qwen2.5-VL Multimodal Tests
|
||||
depends_on: []
|
||||
device: intel_cpu
|
||||
no_plugin: true
|
||||
source_file_dependencies:
|
||||
# - vllm/
|
||||
- vllm/model_executor/layers/rotary_embedding
|
||||
- tests/models/multimodal/generation/
|
||||
commands:
|
||||
- |
|
||||
bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 40m "
|
||||
VLLM_CI_ENV=0 pytest -x -v -s tests/models/multimodal/generation/test_qwen2_5_vl.py"
|
||||
|
||||
- label: "Arm CPU Test"
|
||||
depends_on: []
|
||||
soft_fail: false
|
||||
|
||||
@@ -18,7 +18,7 @@ steps:
|
||||
- label: "XPU example Test"
|
||||
depends_on:
|
||||
- image-build-xpu
|
||||
timeout_in_minutes: 50
|
||||
timeout_in_minutes: 30
|
||||
optional: true
|
||||
device: intel_gpu
|
||||
agent_tags:
|
||||
@@ -39,13 +39,13 @@ steps:
|
||||
- label: "XPU V1 test"
|
||||
depends_on:
|
||||
- image-build-xpu
|
||||
timeout_in_minutes: 70
|
||||
timeout_in_minutes: 30
|
||||
optional: true
|
||||
device: intel_gpu
|
||||
agent_tags:
|
||||
label: production
|
||||
gpu: 1+
|
||||
mem: 24+
|
||||
mem: 16+
|
||||
no_plugin: true
|
||||
env:
|
||||
REGISTRY: "public.ecr.aws/q9t5s3a7"
|
||||
@@ -60,7 +60,7 @@ steps:
|
||||
- label: "XPU server test"
|
||||
depends_on:
|
||||
- image-build-xpu
|
||||
timeout_in_minutes: 45
|
||||
timeout_in_minutes: 30
|
||||
optional: true
|
||||
device: intel_gpu
|
||||
agent_tags:
|
||||
|
||||
@@ -79,18 +79,12 @@ setup_buildx_builder() {
|
||||
docker buildx ls | grep -E '^\*|^NAME' || docker buildx ls
|
||||
}
|
||||
|
||||
annotate_image_tags() {
|
||||
.buildkite/scripts/annotate-image-build.sh \
|
||||
"${IMAGE_TAG:-}" "${IMAGE_TAG_LATEST:-}"
|
||||
}
|
||||
|
||||
check_and_skip_if_image_exists() {
|
||||
if [[ -n "${IMAGE_TAG:-}" ]]; then
|
||||
echo "--- :mag: Checking if image exists"
|
||||
if docker manifest inspect "${IMAGE_TAG}" >/dev/null 2>&1; then
|
||||
echo "Image already exists: ${IMAGE_TAG}"
|
||||
echo "Skipping build"
|
||||
annotate_image_tags
|
||||
exit 0
|
||||
fi
|
||||
echo "Image not found, proceeding with build"
|
||||
@@ -177,18 +171,6 @@ BRANCH=$4
|
||||
IMAGE_TAG=$5
|
||||
IMAGE_TAG_LATEST=${6:-} # only used for main branch, optional
|
||||
|
||||
# When TORCH_NIGHTLY=1, build the base CI image against PyTorch nightly so the
|
||||
# entire existing pipeline runs on nightly torch (CUDA/GPU lane only). Delegate
|
||||
# to the dedicated nightly build (PYTORCH_NIGHTLY=1, CUDA 13.0) and tag it at the
|
||||
# normal IMAGE_TAG that every test step already pulls -- no separate image tag,
|
||||
# no duplicate "vLLM Against PyTorch Nightly" pipeline section.
|
||||
if [[ "${TORCH_NIGHTLY:-0}" == "1" ]]; then
|
||||
echo "--- :warning: TORCH_NIGHTLY=1 -- building base image on PyTorch nightly"
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
exec "${SCRIPT_DIR}/image_build_torch_nightly.sh" \
|
||||
"${REGISTRY}" "${REPO}" "${BUILDKITE_COMMIT}" "${BRANCH}" "${IMAGE_TAG}"
|
||||
fi
|
||||
|
||||
# build config
|
||||
TARGET="test-ci"
|
||||
VLLM_BAKE_FILE_PATH="${VLLM_BAKE_FILE_PATH:-docker/docker-bake.hcl}"
|
||||
@@ -272,5 +254,3 @@ echo "--- :docker: Building ${TARGET}"
|
||||
docker --debug buildx bake -f "${VLLM_BAKE_FILE_PATH}" -f "${CI_HCL_PATH}" --progress plain "${TARGET}"
|
||||
|
||||
echo "--- :white_check_mark: Build complete"
|
||||
|
||||
annotate_image_tags
|
||||
|
||||
@@ -9,31 +9,30 @@ fi
|
||||
REGISTRY=$1
|
||||
REPO=$2
|
||||
BUILDKITE_COMMIT=$3
|
||||
IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-arm64"
|
||||
|
||||
# authenticate with AWS ECR
|
||||
aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true
|
||||
|
||||
# skip build if image already exists
|
||||
if docker manifest inspect "$IMAGE" >/dev/null 2>&1; then
|
||||
echo "Image found"
|
||||
else
|
||||
if [[ -z $(docker manifest inspect "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64) ]]; then
|
||||
echo "Image not found, proceeding with build..."
|
||||
# build for arm64 GPU targets: Grace/GH200 (sm_90),
|
||||
# Blackwell/Thor (sm_100/sm_103/sm_110), and DGX Spark/GB10
|
||||
# (sm_121, family-covered by 12.0 under CUDA 13)
|
||||
docker build --file docker/Dockerfile \
|
||||
--platform linux/arm64 \
|
||||
--build-arg max_jobs=16 \
|
||||
--build-arg nvcc_threads=4 \
|
||||
--build-arg torch_cuda_arch_list="9.0 10.0 11.0 12.0" \
|
||||
--build-arg USE_SCCACHE=1 \
|
||||
--build-arg buildkite_commit="$BUILDKITE_COMMIT" \
|
||||
--tag "$IMAGE" \
|
||||
--target test \
|
||||
--progress plain .
|
||||
# push
|
||||
docker push "$IMAGE"
|
||||
else
|
||||
echo "Image found"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
.buildkite/scripts/annotate-image-build.sh "$IMAGE"
|
||||
# build for arm64 GPU targets: Grace/GH200 (sm_90) and DGX Spark/GB10
|
||||
# (sm_121, family-covered by 12.0 under CUDA 13)
|
||||
docker build --file docker/Dockerfile \
|
||||
--platform linux/arm64 \
|
||||
--build-arg max_jobs=16 \
|
||||
--build-arg nvcc_threads=4 \
|
||||
--build-arg torch_cuda_arch_list="9.0 12.0" \
|
||||
--build-arg USE_SCCACHE=1 \
|
||||
--build-arg buildkite_commit="$BUILDKITE_COMMIT" \
|
||||
--tag "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64 \
|
||||
--target test \
|
||||
--progress plain .
|
||||
|
||||
# push
|
||||
docker push "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64
|
||||
|
||||
@@ -9,26 +9,26 @@ fi
|
||||
REGISTRY=$1
|
||||
REPO=$2
|
||||
BUILDKITE_COMMIT=$3
|
||||
IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-cpu"
|
||||
|
||||
# authenticate with AWS ECR
|
||||
aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true
|
||||
|
||||
# skip build if image already exists
|
||||
if docker manifest inspect "$IMAGE" >/dev/null 2>&1; then
|
||||
echo "Image found"
|
||||
else
|
||||
if [[ -z $(docker manifest inspect "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-cpu) ]]; then
|
||||
echo "Image not found, proceeding with build..."
|
||||
# build
|
||||
docker build --file docker/Dockerfile.cpu \
|
||||
--build-arg max_jobs=16 \
|
||||
--build-arg buildkite_commit="$BUILDKITE_COMMIT" \
|
||||
--build-arg VLLM_CPU_X86=true \
|
||||
--tag "$IMAGE" \
|
||||
--target vllm-test \
|
||||
--progress plain .
|
||||
# push
|
||||
docker push "$IMAGE"
|
||||
else
|
||||
echo "Image found"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
.buildkite/scripts/annotate-image-build.sh "$IMAGE"
|
||||
# build
|
||||
docker build --file docker/Dockerfile.cpu \
|
||||
--build-arg max_jobs=16 \
|
||||
--build-arg buildkite_commit="$BUILDKITE_COMMIT" \
|
||||
--build-arg VLLM_CPU_X86=true \
|
||||
--tag "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-cpu \
|
||||
--target vllm-test \
|
||||
--progress plain .
|
||||
|
||||
# push
|
||||
docker push "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-cpu
|
||||
|
||||
@@ -9,25 +9,25 @@ fi
|
||||
REGISTRY=$1
|
||||
REPO=$2
|
||||
BUILDKITE_COMMIT=$3
|
||||
IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-arm64-cpu"
|
||||
|
||||
# authenticate with AWS ECR
|
||||
aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true
|
||||
|
||||
# skip build if image already exists
|
||||
if docker manifest inspect "$IMAGE" >/dev/null 2>&1; then
|
||||
echo "Image found"
|
||||
else
|
||||
if [[ -z $(docker manifest inspect "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64-cpu) ]]; then
|
||||
echo "Image not found, proceeding with build..."
|
||||
# build
|
||||
docker build --file docker/Dockerfile.cpu \
|
||||
--build-arg max_jobs=16 \
|
||||
--build-arg buildkite_commit="$BUILDKITE_COMMIT" \
|
||||
--tag "$IMAGE" \
|
||||
--target vllm-test \
|
||||
--progress plain .
|
||||
# push
|
||||
docker push "$IMAGE"
|
||||
else
|
||||
echo "Image found"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
.buildkite/scripts/annotate-image-build.sh "$IMAGE"
|
||||
# build
|
||||
docker build --file docker/Dockerfile.cpu \
|
||||
--build-arg max_jobs=16 \
|
||||
--build-arg buildkite_commit="$BUILDKITE_COMMIT" \
|
||||
--tag "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64-cpu \
|
||||
--target vllm-test \
|
||||
--progress plain .
|
||||
|
||||
# push
|
||||
docker push "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64-cpu
|
||||
|
||||
@@ -9,26 +9,26 @@ fi
|
||||
REGISTRY=$1
|
||||
REPO=$2
|
||||
BUILDKITE_COMMIT=$3
|
||||
IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-hpu"
|
||||
|
||||
# authenticate with AWS ECR
|
||||
aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true
|
||||
|
||||
# skip build if image already exists
|
||||
if docker manifest inspect "$IMAGE" >/dev/null 2>&1; then
|
||||
echo "Image found"
|
||||
else
|
||||
if [[ -z $(docker manifest inspect "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-hpu) ]]; then
|
||||
echo "Image not found, proceeding with build..."
|
||||
# build
|
||||
docker build \
|
||||
--file tests/pytorch_ci_hud_benchmark/Dockerfile.hpu \
|
||||
--build-arg max_jobs=16 \
|
||||
--build-arg buildkite_commit="$BUILDKITE_COMMIT" \
|
||||
--tag "$IMAGE" \
|
||||
--progress plain \
|
||||
https://github.com/vllm-project/vllm-gaudi.git
|
||||
# push
|
||||
docker push "$IMAGE"
|
||||
else
|
||||
echo "Image found"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
.buildkite/scripts/annotate-image-build.sh "$IMAGE"
|
||||
# build
|
||||
docker build \
|
||||
--file tests/pytorch_ci_hud_benchmark/Dockerfile.hpu \
|
||||
--build-arg max_jobs=16 \
|
||||
--build-arg buildkite_commit="$BUILDKITE_COMMIT" \
|
||||
--tag "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-hpu \
|
||||
--progress plain \
|
||||
https://github.com/vllm-project/vllm-gaudi.git
|
||||
|
||||
# push
|
||||
docker push "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-hpu
|
||||
|
||||
@@ -40,7 +40,6 @@ docker buildx ls
|
||||
echo "--- :mag: Checking if image already exists"
|
||||
if docker manifest inspect "$IMAGE_TAG" >/dev/null 2>&1; then
|
||||
echo "Image found: $IMAGE_TAG — skipping build"
|
||||
.buildkite/scripts/annotate-image-build.sh "$IMAGE_TAG"
|
||||
exit 0
|
||||
fi
|
||||
echo "Image not found, proceeding with build..."
|
||||
@@ -67,5 +66,3 @@ docker buildx build --file docker/Dockerfile \
|
||||
--progress plain .
|
||||
|
||||
echo "--- :white_check_mark: Torch nightly image build complete: $IMAGE_TAG"
|
||||
|
||||
.buildkite/scripts/annotate-image-build.sh "$IMAGE_TAG"
|
||||
|
||||
@@ -9,26 +9,26 @@ fi
|
||||
REGISTRY=$1
|
||||
REPO=$2
|
||||
BUILDKITE_COMMIT=$3
|
||||
IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-xpu"
|
||||
|
||||
# authenticate with AWS ECR
|
||||
aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true
|
||||
aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 936637512419.dkr.ecr.us-east-1.amazonaws.com || true
|
||||
|
||||
# skip build if image already exists
|
||||
if docker manifest inspect "$IMAGE" &> /dev/null; then
|
||||
echo "Image found"
|
||||
else
|
||||
if ! docker manifest inspect "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-xpu &> /dev/null; then
|
||||
echo "Image not found, proceeding with build..."
|
||||
# build
|
||||
docker build \
|
||||
--file docker/Dockerfile.xpu \
|
||||
--build-arg max_jobs=16 \
|
||||
--build-arg buildkite_commit="$BUILDKITE_COMMIT" \
|
||||
--tag "$IMAGE" \
|
||||
--progress plain .
|
||||
# push
|
||||
docker push "$IMAGE"
|
||||
else
|
||||
echo "Image found"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
.buildkite/scripts/annotate-image-build.sh "$IMAGE"
|
||||
# build
|
||||
docker build \
|
||||
--file docker/Dockerfile.xpu \
|
||||
--build-arg max_jobs=16 \
|
||||
--build-arg buildkite_commit="$BUILDKITE_COMMIT" \
|
||||
--tag "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-xpu \
|
||||
--progress plain .
|
||||
|
||||
# push
|
||||
docker push "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-xpu
|
||||
|
||||
@@ -3,7 +3,7 @@ depends_on:
|
||||
- image-build-xpu
|
||||
steps:
|
||||
- label: XPU Sleep Mode
|
||||
timeout_in_minutes: 45
|
||||
timeout_in_minutes: 30
|
||||
device: intel_gpu
|
||||
agent_tags:
|
||||
label: production
|
||||
@@ -23,5 +23,4 @@ steps:
|
||||
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
||||
'cd tests &&
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
|
||||
pytest -v -s basic_correctness/test_cpu_offload.py &&
|
||||
pytest -v -s basic_correctness/test_mem.py::test_end_to_end'
|
||||
|
||||
@@ -1,26 +0,0 @@
|
||||
group: Benchmarks
|
||||
depends_on:
|
||||
- image-build-xpu
|
||||
steps:
|
||||
- label: Benchmarks CLI Test
|
||||
key: benchmarks-cli-test
|
||||
timeout_in_minutes: 40
|
||||
device: intel_gpu
|
||||
agent_tags:
|
||||
label: production
|
||||
gpu: 1+
|
||||
mem: 16+
|
||||
no_plugin: true
|
||||
working_dir: "."
|
||||
env:
|
||||
REGISTRY: "public.ecr.aws/q9t5s3a7"
|
||||
REPO: "vllm-ci-test-repo"
|
||||
VLLM_TEST_DEVICE: "xpu"
|
||||
source_file_dependencies:
|
||||
- vllm/
|
||||
- tests/benchmarks/
|
||||
commands:
|
||||
- >-
|
||||
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
||||
'cd tests &&
|
||||
pytest -v -s benchmarks/'
|
||||
@@ -2,44 +2,6 @@ group: Engine Intel
|
||||
depends_on:
|
||||
- image-build-xpu
|
||||
steps:
|
||||
- label: Engine
|
||||
key: engine
|
||||
timeout_in_minutes: 40
|
||||
device: intel_gpu
|
||||
agent_tags:
|
||||
label: production
|
||||
gpu: 1+
|
||||
mem: 16+
|
||||
no_plugin: true
|
||||
working_dir: "."
|
||||
env:
|
||||
REGISTRY: "public.ecr.aws/q9t5s3a7"
|
||||
REPO: "vllm-ci-test-repo"
|
||||
VLLM_TEST_DEVICE: "xpu"
|
||||
source_file_dependencies:
|
||||
- vllm/compilation/
|
||||
- vllm/config/
|
||||
- vllm/engine/
|
||||
- vllm/entrypoints/logger.py
|
||||
- vllm/envs.py
|
||||
- vllm/logger.py
|
||||
- vllm/logging_utils/
|
||||
- vllm/platforms/
|
||||
- vllm/sequence.py
|
||||
- vllm/triton_utils/
|
||||
- vllm/utils/
|
||||
- tests/engine
|
||||
- tests/test_sequence
|
||||
- tests/test_config
|
||||
- tests/test_logger
|
||||
- tests/test_vllm_port
|
||||
- tests/test_jit_monitor.py
|
||||
commands:
|
||||
- >-
|
||||
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
||||
'cd tests &&
|
||||
pytest -v -s engine/test_arg_utils.py test_sequence.py test_logger.py test_vllm_port.py test_jit_monitor.py'
|
||||
|
||||
- label: Engine (1 GPU)
|
||||
timeout_in_minutes: 30
|
||||
device: intel_gpu
|
||||
@@ -61,41 +23,3 @@ steps:
|
||||
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
||||
'cd tests &&
|
||||
pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py'
|
||||
|
||||
- label: V1 e2e (2 GPUs)
|
||||
timeout_in_minutes: 30
|
||||
device: intel_gpu
|
||||
agent_tags:
|
||||
label: production
|
||||
gpu: 2+
|
||||
mem: 16+
|
||||
no_plugin: true
|
||||
working_dir: "."
|
||||
env:
|
||||
REGISTRY: "public.ecr.aws/q9t5s3a7"
|
||||
REPO: "vllm-ci-test-repo"
|
||||
VLLM_TEST_DEVICE: "xpu"
|
||||
source_file_dependencies:
|
||||
- vllm/compilation/
|
||||
- vllm/config/
|
||||
- vllm/distributed/
|
||||
- vllm/engine/
|
||||
- vllm/envs.py
|
||||
- vllm/forward_context.py
|
||||
- vllm/inputs/
|
||||
- vllm/logger.py
|
||||
- vllm/logging_utils/
|
||||
- vllm/model_executor/
|
||||
- vllm/multimodal/
|
||||
- vllm/platforms/
|
||||
- vllm/sampling_params.py
|
||||
- vllm/transformers_utils/
|
||||
- vllm/triton_utils/
|
||||
- vllm/utils/
|
||||
- vllm/v1/
|
||||
- tests/v1/e2e/spec_decode
|
||||
commands:
|
||||
- >-
|
||||
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
||||
'cd tests &&
|
||||
pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "tensor_parallelism"'
|
||||
|
||||
@@ -81,12 +81,10 @@ steps:
|
||||
'cd tests &&
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
|
||||
set -o pipefail &&
|
||||
pytest -v -s lora/test_punica_ops.py::test_kernels &&
|
||||
pytest -v -s lora/test_punica_ops.py::test_kernels_hidden_size &&
|
||||
pytest -v -s lora/test_punica_ops.py::test_add_lora_fused_moe_early_exit'
|
||||
pytest -v -s lora/test_punica_ops.py --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[expand-0-xpu:0-dtype0-3-43264-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype1-1-2049-64-128-16]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-128-1-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-256-1-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-256-8-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels[expand-0-xpu:0-dtype0-3-2049-128-8-16]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-128-8-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels[expand-0-xpu:0-dtype1-1-2049-256-128-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype0-3-64256-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype1-2-29696-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype1-3-49408-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype0-2-16384-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[expand-0-xpu:0-dtype0-2-51328-32-4-4]"'
|
||||
|
||||
- label: LoRA Punica FP8/XPU Ops
|
||||
timeout_in_minutes: 60
|
||||
timeout_in_minutes: 45
|
||||
device: intel_gpu
|
||||
agent_tags:
|
||||
label: production
|
||||
@@ -130,10 +128,10 @@ steps:
|
||||
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
||||
'cd tests &&
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
|
||||
(pytest -v -s lora/test_mixtral.py --deselect="tests/lora/test_mixtral.py::test_mixtral_lora[4]" || true) &&
|
||||
pytest -v -s lora/test_quant_model.py --deselect="tests/lora/test_quant_model.py::test_quant_model_lora[model0]" --deselect="tests/lora/test_quant_model.py::test_quant_model_lora[model1]" --deselect="tests/lora/test_quant_model.py::test_quant_model_tp_equality[model0]" &&
|
||||
pytest -v -s lora/test_transformers_model.py &&
|
||||
pytest -v -s lora/test_chatglm3_tp.py &&
|
||||
pytest -v -s lora/test_llama_tp.py::test_llama_lora &&
|
||||
pytest -s -v lora/test_minicpmv_tp.py'
|
||||
|
||||
- label: LoRA Multimodal
|
||||
|
||||
@@ -3,7 +3,7 @@ depends_on:
|
||||
- image-build-xpu
|
||||
steps:
|
||||
- label: V1 Core + KV + Metrics
|
||||
timeout_in_minutes: 45
|
||||
timeout_in_minutes: 30
|
||||
device: intel_gpu
|
||||
agent_tags:
|
||||
label: production
|
||||
@@ -33,12 +33,12 @@ steps:
|
||||
pytest -v -s v1/executor'
|
||||
|
||||
- label: V1 Sample + Logits
|
||||
timeout_in_minutes: 90
|
||||
timeout_in_minutes: 30
|
||||
device: intel_gpu
|
||||
agent_tags:
|
||||
label: production
|
||||
gpu: 1+
|
||||
mem: 24+
|
||||
mem: 16+
|
||||
no_plugin: true
|
||||
working_dir: "."
|
||||
env:
|
||||
@@ -72,31 +72,9 @@ steps:
|
||||
pytest -v -s v1/test_oracle.py &&
|
||||
pytest -v -s v1/test_request.py &&
|
||||
pytest -v -s v1/test_outputs.py &&
|
||||
pytest -v -s v1/sample'
|
||||
|
||||
- label: Basic Models Tests (Initialization)
|
||||
timeout_in_minutes: 60
|
||||
device: intel_gpu
|
||||
agent_tags:
|
||||
label: production
|
||||
gpu: 1+
|
||||
mem: 16+
|
||||
no_plugin: true
|
||||
working_dir: "."
|
||||
env:
|
||||
REGISTRY: "public.ecr.aws/q9t5s3a7"
|
||||
REPO: "vllm-ci-test-repo"
|
||||
VLLM_TEST_DEVICE: "xpu"
|
||||
source_file_dependencies:
|
||||
- vllm/
|
||||
- tests/models/test_initialization.py
|
||||
- tests/models/registry.py
|
||||
commands:
|
||||
- >-
|
||||
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
||||
'export VLLM_XPU_FUSED_MOE_USE_REF=1 &&
|
||||
cd tests &&
|
||||
pytest -v -s models/test_initialization.py::test_can_initialize_large_subset[Eagle3MiniMaxM2ForCausalLM]'
|
||||
pytest -v -s v1/sample/test_topk_topp_sampler.py &&
|
||||
pytest -v -s v1/sample/test_logprobs.py &&
|
||||
pytest -v -s v1/sample/test_logprobs_e2e.py'
|
||||
|
||||
- label: XPU CPU Offload
|
||||
timeout_in_minutes: 60
|
||||
@@ -125,37 +103,9 @@ steps:
|
||||
pytest -v -s v1/kv_offload &&
|
||||
pytest -v -s v1/kv_connector/unit/test_offloading_connector.py'
|
||||
|
||||
- label: NixlConnector PD accuracy (4 GPUs)
|
||||
timeout_in_minutes: 60
|
||||
num_devices: 4
|
||||
device: intel_gpu
|
||||
agent_tags:
|
||||
label: production
|
||||
gpu: 4+
|
||||
mem: 16+
|
||||
no_plugin: true
|
||||
working_dir: "."
|
||||
env:
|
||||
REGISTRY: "public.ecr.aws/q9t5s3a7"
|
||||
REPO: "vllm-ci-test-repo"
|
||||
VLLM_TEST_DEVICE: "xpu"
|
||||
source_file_dependencies:
|
||||
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
|
||||
- vllm/v1/worker/kv_connector_model_runner_mixin.py
|
||||
- tests/v1/kv_connector/nixl_integration/
|
||||
- vllm/platforms/xpu.py
|
||||
commands:
|
||||
- >-
|
||||
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
||||
'cd tests &&
|
||||
bash v1/kv_connector/nixl_integration/run_xpu_disagg_accuracy_test.sh &&
|
||||
PREFILLER_TP_SIZE=2 DECODER_TP_SIZE=1 bash v1/kv_connector/nixl_integration/run_xpu_disagg_accuracy_test.sh &&
|
||||
PREFILLER_TP_SIZE=1 DECODER_TP_SIZE=2 bash v1/kv_connector/nixl_integration/run_xpu_disagg_accuracy_test.sh &&
|
||||
PREFILLER_TP_SIZE=2 DECODER_TP_SIZE=2 bash v1/kv_connector/nixl_integration/run_xpu_disagg_accuracy_test.sh'
|
||||
|
||||
- label: Regression
|
||||
key: regression
|
||||
timeout_in_minutes: 50
|
||||
timeout_in_minutes: 30
|
||||
device: intel_gpu
|
||||
agent_tags:
|
||||
label: production
|
||||
@@ -183,13 +133,13 @@ steps:
|
||||
commands:
|
||||
- >-
|
||||
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
||||
'pip install modelscope\<1.38 &&
|
||||
'pip install modelscope &&
|
||||
cd tests &&
|
||||
pytest -v -s test_regression.py'
|
||||
|
||||
- label: Metrics, Tracing (2 GPUs)
|
||||
key: metrics-tracing-2-gpus
|
||||
timeout_in_minutes: 45
|
||||
timeout_in_minutes: 30
|
||||
num_devices: 2
|
||||
device: intel_gpu
|
||||
agent_tags:
|
||||
@@ -225,7 +175,7 @@ steps:
|
||||
|
||||
- label: Async Engine, Inputs, Utils, Worker
|
||||
key: async-engine-inputs-utils-worker
|
||||
timeout_in_minutes: 55
|
||||
timeout_in_minutes: 30
|
||||
device: intel_gpu
|
||||
agent_tags:
|
||||
label: production
|
||||
@@ -262,25 +212,3 @@ steps:
|
||||
pytest -v -s detokenizer &&
|
||||
pytest -v -s -m "not cpu_test" ./multimodal &&
|
||||
pytest -v -s utils_ --ignore=utils_/test_mem_utils.py'
|
||||
|
||||
- label: Fusion Unit Tests
|
||||
timeout_in_minutes: 30
|
||||
device: intel_gpu
|
||||
agent_tags:
|
||||
label: production
|
||||
gpu: 1+
|
||||
mem: 16+
|
||||
no_plugin: true
|
||||
working_dir: "."
|
||||
env:
|
||||
REGISTRY: "public.ecr.aws/q9t5s3a7"
|
||||
REPO: "vllm-ci-test-repo"
|
||||
VLLM_TEST_DEVICE: "xpu"
|
||||
source_file_dependencies:
|
||||
- vllm/compilation/
|
||||
- tests/compile/passes/test_qk_norm_rope_fusion.py
|
||||
commands:
|
||||
- >-
|
||||
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
||||
'cd tests &&
|
||||
pytest -v -s compile/passes/test_qk_norm_rope_fusion.py'
|
||||
@@ -1,33 +0,0 @@
|
||||
group: Model Executor Intel
|
||||
depends_on:
|
||||
- image-build-xpu
|
||||
steps:
|
||||
- label: Model Executor (Intel)
|
||||
key: model-executor-intel
|
||||
timeout_in_minutes: 45
|
||||
device: intel_gpu
|
||||
agent_tags:
|
||||
label: production
|
||||
gpu: 1+
|
||||
mem: 24+
|
||||
no_plugin: true
|
||||
working_dir: "."
|
||||
env:
|
||||
REGISTRY: "public.ecr.aws/q9t5s3a7"
|
||||
REPO: "vllm-ci-test-repo"
|
||||
VLLM_TEST_DEVICE: "xpu"
|
||||
source_file_dependencies:
|
||||
- vllm/engine/arg_utils.py
|
||||
- vllm/config/model.py
|
||||
- vllm/model_executor
|
||||
- tests/model_executor
|
||||
commands:
|
||||
- >-
|
||||
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
||||
'apt-get update && apt-get install -y curl libsodium23 &&
|
||||
pip3 install tensorizer==2.10.1 &&
|
||||
pip3 install runai-model-streamer[s3,gcs,azure]\>=0.15.7 &&
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
|
||||
export PYTHONFAULTHANDLER=1 &&
|
||||
cd tests &&
|
||||
pytest -v -s model_executor -m "not slow_test" --ignore="model_executor/layers/test_rocm_unquantized_gemm.py" --deselect="tests/model_executor/model_loader/test_reload.py::test_kv_scale_reload"'
|
||||
@@ -8,7 +8,7 @@ steps:
|
||||
agent_tags:
|
||||
label: production
|
||||
gpu: 2+
|
||||
mem: 24+
|
||||
mem: 16+
|
||||
no_plugin: true
|
||||
working_dir: "."
|
||||
env:
|
||||
@@ -28,9 +28,7 @@ steps:
|
||||
'export VLLM_USE_V2_MODEL_RUNNER=1 &&
|
||||
cd tests &&
|
||||
pytest -v -s v1/engine/test_llm_engine.py -k "not test_engine_metrics" &&
|
||||
pytest -v -s v1/e2e/general/test_context_length.py &&
|
||||
ENFORCE_EAGER=1 pytest -v -s v1/e2e/general/test_async_scheduling.py -k "not ngram" &&
|
||||
pytest -v -s entrypoints/llm/test_struct_output_generate.py -k "xgrammar and not speculative_config6 and not speculative_config7 and not speculative_config8 and not speculative_config0" &&
|
||||
pytest -v -s v1/e2e/general/test_min_tokens.py'
|
||||
|
||||
- label: Model Runner V2 Examples (Intel)
|
||||
@@ -62,55 +60,3 @@ steps:
|
||||
python3 basic/offline_inference/generate.py --model facebook/opt-125m &&
|
||||
python3 generate/multimodal/vision_language_offline.py --seed 0 &&
|
||||
python3 features/automatic_prefix_caching/prefix_caching_offline.py'
|
||||
|
||||
- label: Model Runner V2 Distributed (2 GPUs)
|
||||
timeout_in_minutes: 50
|
||||
device: intel_gpu
|
||||
agent_tags:
|
||||
label: production
|
||||
gpu: 2+
|
||||
mem: 16+
|
||||
no_plugin: true
|
||||
working_dir: "."
|
||||
env:
|
||||
REGISTRY: "public.ecr.aws/q9t5s3a7"
|
||||
REPO: "vllm-ci-test-repo"
|
||||
VLLM_TEST_DEVICE: "xpu"
|
||||
source_file_dependencies:
|
||||
- vllm/v1/worker/gpu/
|
||||
- vllm/v1/worker/gpu_worker.py
|
||||
- tests/basic_correctness/test_basic_correctness.py
|
||||
- tests/v1/distributed/test_async_llm_dp.py
|
||||
- tests/v1/distributed/test_eagle_dp.py
|
||||
commands:
|
||||
- >-
|
||||
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
||||
'export VLLM_USE_V2_MODEL_RUNNER=1 &&
|
||||
cd tests &&
|
||||
TARGET_TEST_SUITE=L4 pytest -v -s basic_correctness/test_basic_correctness.py -m "distributed\(num_gpus=2\)" -k "not ray and not True"'
|
||||
|
||||
- label: Model Runner V2 Spec Decode
|
||||
timeout_in_minutes: 50
|
||||
device: intel_gpu
|
||||
agent_tags:
|
||||
label: production
|
||||
gpu: 1+
|
||||
mem: 24+
|
||||
no_plugin: true
|
||||
working_dir: "."
|
||||
env:
|
||||
REGISTRY: "public.ecr.aws/q9t5s3a7"
|
||||
REPO: "vllm-ci-test-repo"
|
||||
VLLM_TEST_DEVICE: "xpu"
|
||||
source_file_dependencies:
|
||||
- vllm/v1/worker/gpu/
|
||||
- vllm/v1/worker/gpu_worker.py
|
||||
- tests/v1/spec_decode/test_max_len.py
|
||||
- tests/v1/spec_decode/test_rejection_sampler_utils.py
|
||||
- tests/v1/e2e/spec_decode/test_spec_decode.py
|
||||
commands:
|
||||
- >-
|
||||
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
||||
'export VLLM_USE_V2_MODEL_RUNNER=1 &&
|
||||
cd tests &&
|
||||
pytest -v -s v1/spec_decode/test_synthetic_rejection_sampler_utils.py'
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
group: Models - Distributed
|
||||
depends_on:
|
||||
- image-build-xpu
|
||||
steps:
|
||||
- label: Distributed Model Tests (2 GPUs)
|
||||
key: distributed-model-tests-2-gpus
|
||||
timeout_in_minutes: 65
|
||||
device: intel_gpu
|
||||
agent_tags:
|
||||
label: production
|
||||
gpu: 2+
|
||||
mem: 16+
|
||||
no_plugin: true
|
||||
working_dir: "."
|
||||
env:
|
||||
REGISTRY: "public.ecr.aws/q9t5s3a7"
|
||||
REPO: "vllm-ci-test-repo"
|
||||
VLLM_TEST_DEVICE: "xpu"
|
||||
source_file_dependencies:
|
||||
- vllm/model_executor/model_loader/sharded_state_loader.py
|
||||
- vllm/model_executor/models/
|
||||
- tests/model_executor/model_loader/test_sharded_state_loader.py
|
||||
commands:
|
||||
- >-
|
||||
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
||||
'cd tests &&
|
||||
pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m "not slow_test"'
|
||||
@@ -4,32 +4,7 @@ depends_on:
|
||||
steps:
|
||||
- label: "Multi-Modal Models (Standard) 1: qwen2"
|
||||
key: multi-modal-models-standard-1-qwen2
|
||||
timeout_in_minutes: 70
|
||||
device: intel_gpu
|
||||
agent_tags:
|
||||
label: production
|
||||
gpu: 1+
|
||||
mem: 24+
|
||||
no_plugin: true
|
||||
working_dir: "."
|
||||
env:
|
||||
REGISTRY: "public.ecr.aws/q9t5s3a7"
|
||||
REPO: "vllm-ci-test-repo"
|
||||
VLLM_TEST_DEVICE: "xpu"
|
||||
source_file_dependencies:
|
||||
- vllm/
|
||||
- tests/models/multimodal
|
||||
commands:
|
||||
- >-
|
||||
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
||||
'pip install av &&
|
||||
cd tests &&
|
||||
pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2" &&
|
||||
pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model'
|
||||
|
||||
- label: "Multi-Modal Models (Standard) 2: qwen3 + gemma"
|
||||
key: multi-modal-models-standard-2-qwen3-gemma
|
||||
timeout_in_minutes: 70
|
||||
timeout_in_minutes: 45
|
||||
device: intel_gpu
|
||||
agent_tags:
|
||||
label: production
|
||||
@@ -47,17 +22,19 @@ steps:
|
||||
commands:
|
||||
- >-
|
||||
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
||||
'cd tests &&
|
||||
pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model'
|
||||
'pip install av git+https://github.com/TIGER-AI-Lab/Mantis.git &&
|
||||
cd tests &&
|
||||
pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2" &&
|
||||
pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model'
|
||||
|
||||
- label: "Multi-Modal Models (Standard) 3: llava + qwen2_vl"
|
||||
key: multi-modal-models-standard-3-llava-qwen2-vl
|
||||
timeout_in_minutes: 65
|
||||
- label: "Multi-Modal Models (Standard) 2: qwen3 + gemma"
|
||||
key: multi-modal-models-standard-2-qwen3-gemma
|
||||
timeout_in_minutes: 45
|
||||
device: intel_gpu
|
||||
agent_tags:
|
||||
label: production
|
||||
gpu: 1+
|
||||
mem: 24+
|
||||
mem: 16+
|
||||
no_plugin: true
|
||||
working_dir: "."
|
||||
env:
|
||||
@@ -70,12 +47,12 @@ steps:
|
||||
commands:
|
||||
- >-
|
||||
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
||||
'cd tests &&
|
||||
pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma" &&
|
||||
pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model'
|
||||
'pip install git+https://github.com/TIGER-AI-Lab/Mantis.git &&
|
||||
cd tests &&
|
||||
pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model'
|
||||
|
||||
- label: "Multi-Modal Models (Standard) 4: other + whisper"
|
||||
key: multi-modal-models-standard-4-other-whisper
|
||||
- label: "Multi-Modal Models (Standard) 3: llava + qwen2_vl"
|
||||
key: multi-modal-models-standard-3-llava-qwen2-vl
|
||||
timeout_in_minutes: 45
|
||||
device: intel_gpu
|
||||
agent_tags:
|
||||
@@ -94,18 +71,43 @@ steps:
|
||||
commands:
|
||||
- >-
|
||||
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
||||
'pip install av &&
|
||||
'pip install git+https://github.com/TIGER-AI-Lab/Mantis.git &&
|
||||
cd tests &&
|
||||
pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma" &&
|
||||
pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model'
|
||||
|
||||
- label: "Multi-Modal Models (Standard) 4: other + whisper"
|
||||
key: multi-modal-models-standard-4-other-whisper
|
||||
timeout_in_minutes: 45
|
||||
device: intel_gpu
|
||||
agent_tags:
|
||||
label: production
|
||||
gpu: 1+
|
||||
mem: 16+
|
||||
no_plugin: true
|
||||
working_dir: "."
|
||||
env:
|
||||
REGISTRY: "public.ecr.aws/q9t5s3a7"
|
||||
REPO: "vllm-ci-test-repo"
|
||||
VLLM_TEST_DEVICE: "xpu"
|
||||
source_file_dependencies:
|
||||
- vllm/
|
||||
- tests/models/multimodal
|
||||
commands:
|
||||
- >-
|
||||
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
||||
'pip install av git+https://github.com/TIGER-AI-Lab/Mantis.git &&
|
||||
cd tests &&
|
||||
pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/processing'
|
||||
|
||||
- label: Multi-Modal Processor # 44min
|
||||
key: multi-modal-processor
|
||||
timeout_in_minutes: 60
|
||||
timeout_in_minutes: 45
|
||||
device: intel_gpu
|
||||
agent_tags:
|
||||
label: production
|
||||
gpu: 1+
|
||||
mem: 24+
|
||||
mem: 16+
|
||||
no_plugin: true
|
||||
working_dir: "."
|
||||
env:
|
||||
@@ -119,9 +121,11 @@ steps:
|
||||
commands:
|
||||
- >-
|
||||
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
||||
'pip install av matplotlib ftfy &&
|
||||
'pip install av matplotlib ftfy git+https://github.com/TIGER-AI-Lab/Mantis.git &&
|
||||
pip install open-clip-torch --no-deps &&
|
||||
cd tests &&
|
||||
pytest -v -s models/multimodal/processing/test_tensor_schema.py
|
||||
--deselect "tests/models/multimodal/processing/test_tensor_schema.py::test_model_tensor_schema[mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4]"
|
||||
--deselect "tests/models/multimodal/processing/test_tensor_schema.py::test_model_tensor_schema[Qwen/Qwen2.5-Omni-7B-AWQ]"
|
||||
--num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB'
|
||||
parallelism: 4
|
||||
|
||||
@@ -1,28 +0,0 @@
|
||||
group: Quantization
|
||||
depends_on:
|
||||
- image-build-xpu
|
||||
steps:
|
||||
- label: Quantization
|
||||
key: quantization
|
||||
timeout_in_minutes: 30
|
||||
env:
|
||||
REGISTRY: "public.ecr.aws/q9t5s3a7"
|
||||
REPO: "vllm-ci-test-repo"
|
||||
VLLM_TEST_DEVICE: "xpu"
|
||||
no_plugin: true
|
||||
working_dir: "."
|
||||
device: intel_gpu
|
||||
agent_tags:
|
||||
label: production
|
||||
gpu: 1+
|
||||
mem: 16+
|
||||
source_file_dependencies:
|
||||
- csrc/
|
||||
- vllm/model_executor/layers/quantization
|
||||
- tests/quantization
|
||||
commands:
|
||||
# - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py
|
||||
- >-
|
||||
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
||||
'VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s tests/quantization/test_per_token_kv_cache.py --deselect="tests/quantization/test_per_token_kv_cache.py::test_triton_unified_attention_per_token_head_scale[int4-16-128-num_heads0-seq_lens1]"'
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
group: Samplers Intel
|
||||
depends_on:
|
||||
- image-build-xpu
|
||||
steps:
|
||||
- label: Samplers Test (FlashInfer)
|
||||
key: samplers-test-flashinfer-intel
|
||||
timeout_in_minutes: 40
|
||||
device: intel_gpu
|
||||
agent_tags:
|
||||
label: production
|
||||
gpu: 1+
|
||||
mem: 24+
|
||||
no_plugin: true
|
||||
working_dir: "."
|
||||
env:
|
||||
REGISTRY: "public.ecr.aws/q9t5s3a7"
|
||||
REPO: "vllm-ci-test-repo"
|
||||
VLLM_TEST_DEVICE: "xpu"
|
||||
source_file_dependencies:
|
||||
- vllm/model_executor/layers
|
||||
- vllm/sampling_metadata.py
|
||||
- tests/samplers
|
||||
- tests/conftest.py
|
||||
- vllm/entrypoints/generate/beam_search
|
||||
commands:
|
||||
- >-
|
||||
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
||||
'cd tests &&
|
||||
VLLM_USE_FLASHINFER_SAMPLER=1 pytest -v -s samplers'
|
||||
@@ -17,7 +17,7 @@ steps:
|
||||
- label: "XPU example Test"
|
||||
depends_on:
|
||||
- image-build-xpu
|
||||
timeout_in_minutes: 50
|
||||
timeout_in_minutes: 30
|
||||
device: intel_gpu
|
||||
agent_tags:
|
||||
label: production
|
||||
@@ -42,46 +42,21 @@ steps:
|
||||
python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --kv-cache-dtype fp8 &&
|
||||
python3 examples/basic/offline_inference/generate.py --model nvidia/Llama-3.1-8B-Instruct-FP8 --block-size 64 --enforce-eager --quantization modelopt --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --max-model-len 4096 &&
|
||||
python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --block-size 64 --enforce-eager --max-model-len 8192 &&
|
||||
python3 examples/basic/offline_inference/generate.py --model TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ --block-size 64 --enforce-eager &&
|
||||
python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 &&
|
||||
python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 --enable-expert-parallel &&
|
||||
python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --max-model-len 8192 &&
|
||||
VLLM_XPU_FUSED_MOE_USE_REF=1 python3 examples/basic/offline_inference/generate.py --model Qwen/Qwen3-30B-A3B-Instruct-2507-FP8 --enforce-eager -tp 2 --max-model-len 8192 &&
|
||||
python3 examples/basic/offline_inference/generate.py --model INCModel/Qwen3-30B-A3B-Instruct-2507-MXFP4-LLMC --enforce-eager -tp 2 --max-model-len 8192
|
||||
'
|
||||
- label: "XPU W8A8 FP8 Linear Examples"
|
||||
depends_on:
|
||||
- image-build-xpu
|
||||
timeout_in_minutes: 60
|
||||
device: intel_gpu
|
||||
agent_tags:
|
||||
label: production
|
||||
gpu: 1+
|
||||
mem: 24+
|
||||
no_plugin: true
|
||||
env:
|
||||
REGISTRY: "public.ecr.aws/q9t5s3a7"
|
||||
REPO: "vllm-ci-test-repo"
|
||||
VLLM_TEST_DEVICE: "xpu"
|
||||
source_file_dependencies:
|
||||
- vllm/
|
||||
- .buildkite/intel_jobs/test-intel.yaml
|
||||
commands:
|
||||
- >-
|
||||
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
||||
'python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8 --enforce-eager --max-model-len 4096 &&
|
||||
python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model neuralmagic/Llama-3.2-1B-Instruct-FP8-dynamic --enforce-eager --max-model-len 4096 &&
|
||||
python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8 --enforce-eager --max-model-len 4096
|
||||
'
|
||||
- label: "XPU V1 test"
|
||||
depends_on:
|
||||
- image-build-xpu
|
||||
timeout_in_minutes: 70
|
||||
timeout_in_minutes: 30
|
||||
device: intel_gpu
|
||||
agent_tags:
|
||||
label: production
|
||||
gpu: 1+
|
||||
mem: 24+
|
||||
mem: 16+
|
||||
no_plugin: true
|
||||
env:
|
||||
REGISTRY: "public.ecr.aws/q9t5s3a7"
|
||||
@@ -93,19 +68,19 @@ steps:
|
||||
- >-
|
||||
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
||||
'cd tests &&
|
||||
bash v1/kv_connector/nixl_integration/run_xpu_disagg_accuracy_test.sh &&
|
||||
pytest -v -s v1/core --ignore=v1/core/test_reset_prefix_cache_e2e.py --ignore=v1/core/test_scheduler_e2e.py &&
|
||||
pytest -v -s v1/engine --ignore=v1/engine/test_output_processor.py &&
|
||||
pytest -v -s v1/sample --ignore=v1/sample/test_logprobs.py --ignore=v1/sample/test_logprobs_e2e.py -k "not test_topk_only and not test_topp_only and not test_topk_and_topp" &&
|
||||
pytest -v -s v1/worker --ignore=v1/worker/test_gpu_model_runner.py --ignore=v1/worker/test_worker_memory_snapshot.py &&
|
||||
pytest -v -s v1/structured_output &&
|
||||
pytest -v -s v1/test_serial_utils.py &&
|
||||
pytest -v -s v1/e2e/general/test_correctness_sliding_window.py --deselect="tests/v1/e2e/general/test_correctness_sliding_window.py::test_sliding_window_retrieval[True-1-5-google/gemma-3-1b-it]" &&
|
||||
pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py --ignore=v1/spec_decode/test_speculators_correctness.py &&
|
||||
pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_multi_connector.py --ignore=v1/kv_connector/unit/test_example_connector.py --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py --ignore=v1/kv_connector/unit/test_offloading_connector.py'
|
||||
- label: "XPU server test"
|
||||
depends_on:
|
||||
- image-build-xpu
|
||||
timeout_in_minutes: 45
|
||||
timeout_in_minutes: 30
|
||||
device: intel_gpu
|
||||
agent_tags:
|
||||
label: production
|
||||
@@ -145,52 +120,4 @@ steps:
|
||||
- >-
|
||||
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
||||
'cd tests &&
|
||||
pytest -v -s quantization/test_auto_round.py &&
|
||||
pytest -v -s quantization/test_online.py'
|
||||
- label: "XPU GPQA Eval (GPT-OSS)"
|
||||
depends_on:
|
||||
- image-build-xpu
|
||||
timeout_in_minutes: 60
|
||||
device: intel_gpu
|
||||
agent_tags:
|
||||
label: production
|
||||
gpu: 1+
|
||||
mem: 24+
|
||||
no_plugin: true
|
||||
env:
|
||||
REGISTRY: "public.ecr.aws/q9t5s3a7"
|
||||
REPO: "vllm-ci-test-repo"
|
||||
VLLM_TEST_DEVICE: "xpu"
|
||||
source_file_dependencies:
|
||||
- vllm/
|
||||
- tests/evals/gpt_oss/
|
||||
- .buildkite/intel_jobs/test-intel.yaml
|
||||
commands:
|
||||
- >-
|
||||
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
||||
'pip install "gpt-oss[eval]==0.0.5" &&
|
||||
cd tests &&
|
||||
pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-xpu.txt'
|
||||
- label: "XPU compressed tensors FP8 test"
|
||||
depends_on:
|
||||
- image-build-xpu
|
||||
timeout_in_minutes: 60
|
||||
device: intel_gpu
|
||||
agent_tags:
|
||||
label: production
|
||||
gpu: 1+
|
||||
mem: 16+
|
||||
no_plugin: true
|
||||
env:
|
||||
REGISTRY: "public.ecr.aws/q9t5s3a7"
|
||||
REPO: "vllm-ci-test-repo"
|
||||
VLLM_TEST_DEVICE: "xpu"
|
||||
source_file_dependencies:
|
||||
- vllm/
|
||||
- tests/quantization/test_compressed_tensors.py
|
||||
- .buildkite/intel_jobs/test-intel.yaml
|
||||
commands:
|
||||
- >-
|
||||
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
||||
'cd tests &&
|
||||
pytest -v -s quantization/test_compressed_tensors.py::test_compressed_tensors_fp8'
|
||||
pytest -v -s quantization/test_auto_round.py'
|
||||
@@ -1,7 +1,6 @@
|
||||
# For hf script, without -t option (tensor parallel size).
|
||||
# bash .buildkite/lm-eval-harness/run-lm-eval-mmlupro-vllm-baseline.sh -m meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 -l 250 -t 8 -f 5
|
||||
model_name: "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8"
|
||||
rocm_safetensors_load_strategy: lazy
|
||||
required_gpu_arch:
|
||||
- gfx942
|
||||
- gfx950
|
||||
|
||||
@@ -72,11 +72,6 @@ def launch_lm_eval(eval_config, tp_size):
|
||||
if moe_backend is not None:
|
||||
model_args += f"moe_backend={moe_backend},"
|
||||
|
||||
if current_platform.is_rocm():
|
||||
rocm_load_strategy = eval_config.get("rocm_safetensors_load_strategy")
|
||||
if rocm_load_strategy is not None:
|
||||
model_args += f"safetensors_load_strategy={rocm_load_strategy},"
|
||||
|
||||
env_vars = eval_config.get("env_vars", None)
|
||||
with scoped_env_vars(env_vars):
|
||||
results = lm_eval.simple_evaluate(
|
||||
|
||||
@@ -28,6 +28,11 @@
|
||||
"dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json"
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "sharegpt",
|
||||
"dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json"
|
||||
}
|
||||
},
|
||||
{
|
||||
"test_name": "serving_llama8B_tp1_random_128_128",
|
||||
"server_parameters": {
|
||||
|
||||
+368
-488
File diff suppressed because it is too large
Load Diff
@@ -3,8 +3,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
#
|
||||
# Append a build artifact line to the Buildkite annotation.
|
||||
# Usage: annotate-build-artifact.sh <label> <value> <context>
|
||||
# Usage: annotate-build-artifact.sh <label> <value>
|
||||
set -e
|
||||
echo "- **${1}**: \`${2}\`" | \
|
||||
buildkite-agent annotate --append --style 'info' \
|
||||
--context "${3:?context is required}"
|
||||
buildkite-agent annotate --append --style 'info' --context 'release-artifacts'
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
#!/bin/bash
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
#
|
||||
# Append the Docker image tag(s) an image-build step pushed to a Buildkite
|
||||
# annotation, so the built image tags show up on the build page instead of
|
||||
# being buried in the job logs.
|
||||
#
|
||||
# Usage: annotate-image-build.sh <image_tag> [<image_tag> ...]
|
||||
set -euo pipefail
|
||||
|
||||
# buildkite-agent only exists on Buildkite agents; no-op elsewhere so the
|
||||
# image build scripts stay runnable locally.
|
||||
if ! command -v buildkite-agent >/dev/null 2>&1; then
|
||||
echo "buildkite-agent not found; skipping image tag annotation"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
label="${BUILDKITE_LABEL:-Image build}"
|
||||
content=""
|
||||
for image in "$@"; do
|
||||
[[ -n "$image" ]] || continue
|
||||
content+="- **${label}**: \`${image}\`"$'\n'
|
||||
done
|
||||
|
||||
if [[ -z "$content" ]]; then
|
||||
echo "No image tags provided; nothing to annotate"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Best-effort: a flaky annotation must never fail an otherwise successful
|
||||
# (and expensive) image build.
|
||||
if ! printf '%s' "$content" | \
|
||||
buildkite-agent annotate --append --style 'info' --context 'docker-images'; then
|
||||
echo "warning: failed to annotate build with image tags"
|
||||
fi
|
||||
@@ -1,35 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
#
|
||||
# Build the macOS arm64 CPU wheel natively on a macOS agent (the `macmini`
|
||||
# queue) into artifacts/dist/ for upload-nightly-wheels.sh.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# The macmini queue uses persistent checkouts, so refresh tags for setuptools-scm.
|
||||
git fetch --tags --force origin
|
||||
|
||||
# The Rust frontend build needs protoc.
|
||||
if ! command -v protoc >/dev/null 2>&1; then
|
||||
brew install protobuf
|
||||
fi
|
||||
|
||||
# upload-nightly-wheels.sh expects exactly one wheel.
|
||||
rm -rf artifacts/dist
|
||||
mkdir -p artifacts/dist
|
||||
|
||||
export VLLM_TARGET_DEVICE=cpu
|
||||
export VLLM_REQUIRE_RUST_FRONTEND=1
|
||||
export MACOSX_DEPLOYMENT_TARGET=11.0
|
||||
# uv's CPython is universal2; force an arm64-only build and tag so the wheel
|
||||
# isn't mislabelled universal2 and installed on Intel Macs where import fails.
|
||||
export ARCHFLAGS="-arch arm64"
|
||||
export _PYTHON_HOST_PLATFORM="macosx-11.0-arm64"
|
||||
export CMAKE_BUILD_PARALLEL_LEVEL="${CMAKE_BUILD_PARALLEL_LEVEL:-4}"
|
||||
|
||||
uv venv --python 3.12
|
||||
uv pip install -r requirements/build/cpu.txt --index-strategy unsafe-best-match
|
||||
uv build --wheel --no-build-isolation -o artifacts/dist
|
||||
|
||||
ls -l artifacts/dist/*.whl
|
||||
@@ -29,11 +29,7 @@ if python3 -c "import torch; assert torch.version.hip" 2>/dev/null; then
|
||||
TORCH_INDEX_URL=""
|
||||
fi
|
||||
else
|
||||
if [ "${TORCH_NIGHTLY:-0}" = "1" ]; then
|
||||
TORCH_INDEX_URL="https://download.pytorch.org/whl/nightly/cu130"
|
||||
else
|
||||
TORCH_INDEX_URL="https://download.pytorch.org/whl/cu130"
|
||||
fi
|
||||
TORCH_INDEX_URL="https://download.pytorch.org/whl/cu130"
|
||||
fi
|
||||
echo ">>> Using PyTorch index: ${TORCH_INDEX_URL:-PyPI default}"
|
||||
|
||||
|
||||
@@ -15,10 +15,9 @@ set -euo pipefail
|
||||
|
||||
DEFAULT_REPO_SLUG="vllm-project/vllm"
|
||||
DEFAULT_CI_HCL_SOURCE="docker/ci-rocm.hcl"
|
||||
DEFAULT_CI_BASE_CONTENT_FILES="requirements/common.txt requirements/rocm.txt requirements/test/rocm.txt docker/Dockerfile.rocm_base docker/ci-rocm.hcl docker/docker-bake-rocm.hcl tools/install_torchcodec_rocm.sh tools/install_protoc.sh rust-toolchain.toml tests/vllm_test_utils .buildkite/scripts/ci-bake-rocm.sh .buildkite/scripts/rocm/build-ci-base.sh"
|
||||
DEFAULT_CI_BASE_CONTENT_FILES="requirements/common.txt requirements/rocm.txt requirements/test/rocm.txt docker/Dockerfile.rocm_base tools/install_torchcodec_rocm.sh tests/vllm_test_utils"
|
||||
DEFAULT_CI_BASE_DOCKERFILE="docker/Dockerfile.rocm"
|
||||
DEFAULT_CI_BASE_DOCKERFILE_STAGES="base rust_toolchain_input_0 rust_toolchain_input_1 rust-toolchain-input rust-toolchain build_nixl build_rocshmem build_deepep mori_base ci_base"
|
||||
DEFAULT_CI_BASE_METADATA_VERSION="1"
|
||||
DEFAULT_CI_BASE_DOCKERFILE_STAGES="base build_rixl build_rocshmem build_deepep mori_base ci_base"
|
||||
IMAGE_EXISTED_BEFORE_BUILD=0
|
||||
|
||||
TARGET=""
|
||||
@@ -285,7 +284,7 @@ get_content_arg_names() {
|
||||
fi | awk 'NF && !seen[$0]++'
|
||||
}
|
||||
|
||||
compute_ci_base_content_hash_once() {
|
||||
compute_ci_base_content_hash() {
|
||||
local -a content_paths=()
|
||||
local -a content_args=()
|
||||
local dockerfile="${CI_BASE_DOCKERFILE:-}"
|
||||
@@ -301,8 +300,7 @@ compute_ci_base_content_hash_once() {
|
||||
if [[ -n "${dockerfile}" ]]; then
|
||||
printf 'dockerfile:%s\n' "${dockerfile}"
|
||||
printf 'resolved-build-args:\n'
|
||||
hash_dockerfile_arg_values "${dockerfile}" "${content_args[@]}" \
|
||||
|| return 1
|
||||
hash_dockerfile_arg_values "${dockerfile}" "${content_args[@]}"
|
||||
if [[ -n "${stages}" ]]; then
|
||||
printf 'dockerfile-stages:%s\n' "${stages}"
|
||||
if [[ -f "${dockerfile}" ]]; then
|
||||
@@ -315,53 +313,6 @@ compute_ci_base_content_hash_once() {
|
||||
} | sha256sum | cut -d' ' -f1
|
||||
}
|
||||
|
||||
compute_ci_base_content_hash() {
|
||||
local attempts="${CI_BASE_HASH_ATTEMPTS:-3}"
|
||||
local delay_secs="${CI_BASE_HASH_RETRY_DELAY:-5}"
|
||||
local attempt=0
|
||||
local hash=""
|
||||
local failed=0
|
||||
local -a hashes=()
|
||||
|
||||
if [[ ! "${attempts}" =~ ^[1-9][0-9]*$ ]]; then
|
||||
echo "Invalid CI_BASE_HASH_ATTEMPTS: ${attempts}" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ ! "${delay_secs}" =~ ^[0-9]+$ ]]; then
|
||||
echo "Invalid CI_BASE_HASH_RETRY_DELAY: ${delay_secs}" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
for ((attempt = 1; attempt <= attempts; attempt++)); do
|
||||
if ! hash=$(compute_ci_base_content_hash_once); then
|
||||
echo "ci_base content hash calculation ${attempt}/${attempts} failed" >&2
|
||||
failed=1
|
||||
else
|
||||
hashes+=("${hash}")
|
||||
echo "ci_base content hash calculation ${attempt}/${attempts}: ${hash}" >&2
|
||||
fi
|
||||
|
||||
if ((attempt < attempts)); then
|
||||
sleep "${delay_secs}"
|
||||
fi
|
||||
done
|
||||
|
||||
if ((failed)) || ((${#hashes[@]} != attempts)); then
|
||||
echo "Could not calculate a reliable ci_base content hash" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
for hash in "${hashes[@]:1}"; do
|
||||
if [[ "${hash}" != "${hashes[0]}" ]]; then
|
||||
echo "ci_base content hash changed between calculations" >&2
|
||||
printf ' observed: %s\n' "${hashes[@]}" >&2
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
|
||||
printf '%s\n' "${hashes[0]}"
|
||||
}
|
||||
|
||||
extract_dockerfile_arg_default() {
|
||||
local dockerfile="$1"
|
||||
local arg_name="$2"
|
||||
@@ -414,11 +365,7 @@ hash_dockerfile_arg_values() {
|
||||
printf 'arg:%s=%s\n' "${arg_name}" "${arg_value:-<empty>}"
|
||||
if [[ "${arg_name}" == "BASE_IMAGE" && -n "${arg_value}" ]]; then
|
||||
digest=$(resolve_image_digest "${arg_value}")
|
||||
if [[ -z "${digest}" ]]; then
|
||||
echo "Failed to resolve digest for BASE_IMAGE=${arg_value}" >&2
|
||||
return 1
|
||||
fi
|
||||
printf 'arg:%s.digest=%s\n' "${arg_name}" "${digest}"
|
||||
printf 'arg:%s.digest=%s\n' "${arg_name}" "${digest:-unknown}"
|
||||
fi
|
||||
done
|
||||
}
|
||||
@@ -445,16 +392,6 @@ should_upload_wheel_artifacts() {
|
||||
|| "${TARGET}" == *"artifact"* ]]
|
||||
}
|
||||
|
||||
set_buildkite_metadata() {
|
||||
local key="$1"
|
||||
local value="$2"
|
||||
|
||||
[[ -n "${value}" ]] || return 0
|
||||
if command -v buildkite-agent >/dev/null 2>&1; then
|
||||
buildkite-agent meta-data set "${key}" "${value}" || true
|
||||
fi
|
||||
}
|
||||
|
||||
get_remote_image_label() {
|
||||
local image_ref="$1"
|
||||
local label_key="$2"
|
||||
@@ -588,22 +525,6 @@ get_remote_image_label_with_retry() {
|
||||
return 0
|
||||
}
|
||||
|
||||
remote_ci_base_metadata_is_current() {
|
||||
local image_ref="$1"
|
||||
local metadata_version=""
|
||||
|
||||
metadata_version=$(get_remote_image_label "${image_ref}" "vllm.ci_base.metadata_version")
|
||||
[[ "${metadata_version}" == "${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}" ]]
|
||||
}
|
||||
|
||||
remote_ci_base_metadata_is_current_with_retry() {
|
||||
local image_ref="$1"
|
||||
local metadata_version=""
|
||||
|
||||
metadata_version=$(get_remote_image_label_with_retry "${image_ref}" "vllm.ci_base.metadata_version")
|
||||
[[ "${metadata_version}" == "${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}" ]]
|
||||
}
|
||||
|
||||
remote_image_exists() {
|
||||
local image_ref="$1"
|
||||
docker manifest inspect "${image_ref}" >/dev/null 2>&1
|
||||
@@ -660,7 +581,6 @@ init_config() {
|
||||
CI_BASE_CONTENT_FILES="${CI_BASE_CONTENT_FILES:-${DEFAULT_CI_BASE_CONTENT_FILES}}"
|
||||
CI_BASE_DOCKERFILE="${CI_BASE_DOCKERFILE:-${DEFAULT_CI_BASE_DOCKERFILE}}"
|
||||
CI_BASE_DOCKERFILE_STAGES="${CI_BASE_DOCKERFILE_STAGES:-${DEFAULT_CI_BASE_DOCKERFILE_STAGES}}"
|
||||
CI_BASE_METADATA_VERSION="${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}"
|
||||
CI_BASE_IMAGE_TAG="${CI_BASE_IMAGE_TAG:-rocm/vllm-dev:ci_base}"
|
||||
export PYTORCH_ROCM_ARCH
|
||||
|
||||
@@ -715,10 +635,6 @@ load_ci_hcl() {
|
||||
echo "Copied ${CI_HCL_SOURCE} to ${CI_HCL_PATH}"
|
||||
}
|
||||
|
||||
init_bake_files() {
|
||||
BAKE_FILES=(-f "${VLLM_BAKE_FILE}" -f "${CI_HCL_PATH}")
|
||||
}
|
||||
|
||||
compute_ci_base_hash_if_needed() {
|
||||
if [[ -z "${CI_BASE_CONTENT_FILES:-}" ]]; then
|
||||
return 0
|
||||
@@ -760,14 +676,12 @@ configure_ci_base_image_refs() {
|
||||
fi
|
||||
|
||||
content_tag=$(ci_base_tag_with_suffix "${stable_tag}" "${CI_BASE_CONTENT_HASH}")
|
||||
CI_BASE_IMAGE_TAG_CONTENT_REF="${content_tag}"
|
||||
if [[ -n "${BUILDKITE_COMMIT:-}" ]]; then
|
||||
commit_tag=$(ci_base_tag_with_suffix "${stable_tag}" "${BUILDKITE_COMMIT}")
|
||||
CI_BASE_IMAGE_TAG_COMMIT="${commit_tag}"
|
||||
export CI_BASE_IMAGE_TAG_COMMIT
|
||||
fi
|
||||
CI_BASE_IMAGE_TAG_COMMIT_REF="${commit_tag}"
|
||||
|
||||
# *_REF is the logical tag recorded in metadata. *_EXTRA is only passed to
|
||||
# bake when that tag is not already the primary tag, avoiding duplicates.
|
||||
if should_push_stable_ci_base_tag; then
|
||||
primary_tag="${content_tag}"
|
||||
CI_BASE_IMAGE_TAG_STABLE="${stable_tag}"
|
||||
@@ -777,35 +691,19 @@ configure_ci_base_image_refs() {
|
||||
fi
|
||||
CI_BASE_IMAGE_TAG="${primary_tag}"
|
||||
if [[ "${primary_tag}" == "${content_tag}" ]]; then
|
||||
CI_BASE_IMAGE_TAG_CONTENT_EXTRA=""
|
||||
CI_BASE_IMAGE_TAG_CONTENT=""
|
||||
else
|
||||
CI_BASE_IMAGE_TAG_CONTENT_EXTRA="${content_tag}"
|
||||
CI_BASE_IMAGE_TAG_CONTENT="${content_tag}"
|
||||
fi
|
||||
if [[ -n "${commit_tag}" && "${commit_tag}" != "${primary_tag}" ]]; then
|
||||
CI_BASE_IMAGE_TAG_COMMIT_EXTRA="${commit_tag}"
|
||||
else
|
||||
CI_BASE_IMAGE_TAG_COMMIT_EXTRA=""
|
||||
fi
|
||||
export CI_BASE_IMAGE_TAG
|
||||
export CI_BASE_IMAGE_TAG_COMMIT_EXTRA
|
||||
export CI_BASE_IMAGE_TAG_CONTENT_EXTRA
|
||||
export CI_BASE_IMAGE_TAG_CONTENT_REF
|
||||
export CI_BASE_IMAGE_TAG_COMMIT_REF
|
||||
export CI_BASE_IMAGE_TAG_STABLE
|
||||
export CI_BASE_IMAGE_TAG CI_BASE_IMAGE_TAG_CONTENT CI_BASE_IMAGE_TAG_STABLE
|
||||
|
||||
if is_ci_base_target; then
|
||||
IMAGE_TAG="${primary_tag}"
|
||||
CI_BASE_IMAGE="${primary_tag}"
|
||||
export CI_BASE_IMAGE
|
||||
export IMAGE_TAG
|
||||
|
||||
echo "ci_base primary image tag: ${CI_BASE_IMAGE_TAG}"
|
||||
if [[ -n "${commit_tag}" ]]; then
|
||||
if [[ "${commit_tag}" == "${primary_tag}" ]]; then
|
||||
echo "ci_base commit image tag: ${commit_tag} (primary)"
|
||||
else
|
||||
echo "ci_base commit image tag: ${commit_tag}"
|
||||
fi
|
||||
if [[ -n "${CI_BASE_IMAGE_TAG_COMMIT:-}" ]]; then
|
||||
echo "ci_base commit image tag: ${CI_BASE_IMAGE_TAG_COMMIT}"
|
||||
fi
|
||||
echo "ci_base content image tag: ${content_tag}"
|
||||
if [[ -n "${CI_BASE_IMAGE_TAG_STABLE}" ]]; then
|
||||
@@ -814,10 +712,6 @@ configure_ci_base_image_refs() {
|
||||
echo "ci_base stable alias will not be pushed for this build"
|
||||
echo "Set NIGHTLY=1 on ${CI_BASE_STABLE_BRANCH:-main} to refresh ${stable_tag}"
|
||||
fi
|
||||
set_buildkite_metadata "rocm-ci-base-image" "${CI_BASE_IMAGE_TAG}"
|
||||
set_buildkite_metadata "rocm-ci-base-image-content" "${content_tag}"
|
||||
set_buildkite_metadata "rocm-ci-base-image-commit" "${CI_BASE_IMAGE_TAG_COMMIT_REF:-}"
|
||||
set_buildkite_metadata "rocm-ci-base-image-stable" "${CI_BASE_IMAGE_TAG_STABLE:-}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
@@ -834,8 +728,8 @@ ci_base_candidate_refs() {
|
||||
printf '%s\n' \
|
||||
"${IMAGE_TAG:-}" \
|
||||
"${CI_BASE_IMAGE_TAG:-}" \
|
||||
"${CI_BASE_IMAGE_TAG_COMMIT_EXTRA:-}" \
|
||||
"${CI_BASE_IMAGE_TAG_CONTENT_EXTRA:-}" \
|
||||
"${CI_BASE_IMAGE_TAG_COMMIT:-}" \
|
||||
"${CI_BASE_IMAGE_TAG_CONTENT:-}" \
|
||||
"${CI_BASE_IMAGE_TAG_STABLE:-}" \
|
||||
| awk 'NF && !seen[$0]++'
|
||||
}
|
||||
@@ -849,10 +743,6 @@ find_matching_ci_base_ref() {
|
||||
remote_image_exists "${candidate}" || continue
|
||||
candidate_hash=$(get_remote_image_label "${candidate}" "vllm.ci_base.content_hash")
|
||||
if [[ "${candidate_hash}" == "${CI_BASE_CONTENT_HASH}" ]]; then
|
||||
if ! remote_ci_base_metadata_is_current "${candidate}"; then
|
||||
echo "Found matching ci_base content hash but stale metadata: ${candidate}" >&2
|
||||
continue
|
||||
fi
|
||||
printf '%s\n' "${candidate}"
|
||||
return 0
|
||||
fi
|
||||
@@ -927,10 +817,6 @@ maybe_skip_existing_image() {
|
||||
if [[ -n "${remote_hash}" ]]; then
|
||||
echo "Remote ci_base content hash: ${remote_hash:0:16}..."
|
||||
if [[ "${remote_hash}" == "${CI_BASE_CONTENT_HASH}" ]]; then
|
||||
if ! remote_ci_base_metadata_is_current "${IMAGE_TAG}"; then
|
||||
echo "Content hashes match but ci_base metadata is stale; rebuilding to refresh metadata"
|
||||
return 0
|
||||
fi
|
||||
if ! refresh_ci_base_tags_from_ref "${IMAGE_TAG}"; then
|
||||
echo "ci_base tag refresh failed; rebuilding to push expected tags"
|
||||
return 0
|
||||
@@ -1112,104 +998,12 @@ prepare_git_cache_metadata() {
|
||||
fi
|
||||
}
|
||||
|
||||
ci_base_metadata_pairs() {
|
||||
local dockerfile="${CI_BASE_DOCKERFILE:-${DEFAULT_CI_BASE_DOCKERFILE}}"
|
||||
local stages="${CI_BASE_DOCKERFILE_STAGES:-${DEFAULT_CI_BASE_DOCKERFILE_STAGES}}"
|
||||
local content_files="${CI_BASE_CONTENT_FILES:-${DEFAULT_CI_BASE_CONTENT_FILES}}"
|
||||
local content_files_hash=""
|
||||
local base_image=""
|
||||
local base_image_digest=""
|
||||
local git_branch=""
|
||||
local -a content_paths=()
|
||||
local -a content_args=()
|
||||
|
||||
read -r -a content_paths <<< "${content_files}"
|
||||
if [[ ${#content_paths[@]} -gt 0 ]]; then
|
||||
content_files_hash=$(compute_content_hash "${content_paths[@]}")
|
||||
fi
|
||||
mapfile -t content_args < <(
|
||||
get_content_arg_names "${dockerfile}" "${stages}" "${CI_BASE_CONTENT_ARGS:-}"
|
||||
)
|
||||
|
||||
base_image=$(resolve_dockerfile_arg_value "${dockerfile}" "BASE_IMAGE")
|
||||
if [[ -n "${base_image}" ]]; then
|
||||
base_image_digest=$(resolve_image_digest "${base_image}")
|
||||
fi
|
||||
git_branch="${BUILDKITE_BRANCH:-${VLLM_BRANCH:-}}"
|
||||
|
||||
metadata_pair "vllm.ci_base.metadata_version" "${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}"
|
||||
metadata_pair "vllm.ci_base.content_hash" "${CI_BASE_CONTENT_HASH:-}"
|
||||
metadata_pair "vllm.ci_base.content_files_hash" "${content_files_hash}"
|
||||
metadata_pair "vllm.ci_base.content_files" "${content_files}"
|
||||
metadata_pair "vllm.ci_base.content_args" "$(join_words "${content_args[@]}")"
|
||||
metadata_pair "vllm.ci_base.dockerfile" "${dockerfile}"
|
||||
metadata_pair "vllm.ci_base.dockerfile_stages" "${stages}"
|
||||
metadata_pair "vllm.ci_base.image.primary" "${CI_BASE_IMAGE_TAG:-}"
|
||||
metadata_pair "vllm.ci_base.image.content" "${CI_BASE_IMAGE_TAG_CONTENT_REF:-${CI_BASE_IMAGE_TAG_CONTENT_EXTRA:-}}"
|
||||
metadata_pair "vllm.ci_base.image.commit" "${CI_BASE_IMAGE_TAG_COMMIT_REF:-${CI_BASE_IMAGE_TAG_COMMIT_EXTRA:-}}"
|
||||
metadata_pair "vllm.ci_base.image.stable" "${CI_BASE_IMAGE_TAG_STABLE:-}"
|
||||
metadata_pair "vllm.ci_base.git_commit" "${BUILDKITE_COMMIT:-}"
|
||||
metadata_pair "vllm.ci_base.git_branch" "${git_branch}"
|
||||
metadata_pair "vllm.ci_base.vllm_branch" "${VLLM_BRANCH:-}"
|
||||
metadata_pair "vllm.ci_base.stable_branch" "${CI_BASE_STABLE_BRANCH:-main}"
|
||||
|
||||
metadata_pair "vllm.rocm.base_image" "${base_image}"
|
||||
metadata_pair "vllm.rocm.base_image_digest" "${base_image_digest}"
|
||||
metadata_pair "vllm.rocm.pytorch_rocm_arch" "${PYTORCH_ROCM_ARCH:-}"
|
||||
metadata_pair "vllm.rocm.nic_backend" "$(resolve_dockerfile_arg_value "${dockerfile}" "NIC_BACKEND")"
|
||||
metadata_pair "vllm.rocm.ainic_version" "$(resolve_dockerfile_arg_value "${dockerfile}" "AINIC_VERSION")"
|
||||
metadata_pair "vllm.rocm.ubuntu_codename" "$(resolve_dockerfile_arg_value "${dockerfile}" "UBUNTU_CODENAME")"
|
||||
metadata_pair "vllm.rocm.nixl_repo" "$(resolve_dockerfile_arg_value "${dockerfile}" "NIXL_REPO")"
|
||||
metadata_pair "vllm.rocm.nixl_commit" "${NIXL_BRANCH:-$(resolve_dockerfile_arg_value "${dockerfile}" "NIXL_BRANCH")}"
|
||||
metadata_pair "vllm.rocm.ucx_repo" "$(resolve_dockerfile_arg_value "${dockerfile}" "UCX_REPO")"
|
||||
metadata_pair "vllm.rocm.ucx_commit" "${UCX_BRANCH:-$(resolve_dockerfile_arg_value "${dockerfile}" "UCX_BRANCH")}"
|
||||
metadata_pair "vllm.rocm.rocshmem_repo" "$(resolve_dockerfile_arg_value "${dockerfile}" "ROCSHMEM_REPO")"
|
||||
metadata_pair "vllm.rocm.rocshmem_commit" "${ROCSHMEM_BRANCH:-$(resolve_dockerfile_arg_value "${dockerfile}" "ROCSHMEM_BRANCH")}"
|
||||
metadata_pair "vllm.rocm.deepep_repo" "$(resolve_dockerfile_arg_value "${dockerfile}" "DEEPEP_REPO")"
|
||||
metadata_pair "vllm.rocm.deepep_commit" "${DEEPEP_BRANCH:-$(resolve_dockerfile_arg_value "${dockerfile}" "DEEPEP_BRANCH")}"
|
||||
metadata_pair "vllm.rocm.deepep_nic" "$(resolve_dockerfile_arg_value "${dockerfile}" "DEEPEP_NIC")"
|
||||
metadata_pair "vllm.rocm.deepep_rocm_arch" "$(resolve_dockerfile_arg_value "${dockerfile}" "DEEPEP_ROCM_ARCH")"
|
||||
metadata_pair "vllm.rocm.nixl_cache_key" "${NIXL_CACHE_KEY:-}"
|
||||
metadata_pair "vllm.rocm.rocshmem_cache_key" "${ROCSHMEM_CACHE_KEY:-}"
|
||||
metadata_pair "vllm.rocm.deepep_cache_key" "${DEEPEP_CACHE_KEY:-}"
|
||||
|
||||
metadata_pair "vllm.buildkite.build_number" "${BUILDKITE_BUILD_NUMBER:-}"
|
||||
metadata_pair "vllm.buildkite.build_id" "${BUILDKITE_BUILD_ID:-}"
|
||||
}
|
||||
|
||||
write_ci_base_metadata_annotations() {
|
||||
local metadata="$1"
|
||||
local key=""
|
||||
local value=""
|
||||
local annotation=""
|
||||
|
||||
[[ -n "${metadata}" ]] || return 0
|
||||
while IFS=$'\t' read -r key value; do
|
||||
[[ -n "${key}" && -n "${value}" ]] || continue
|
||||
annotation="manifest:${key}=${value}"
|
||||
printf ' "%s",\n' "$(hcl_escape_string "${annotation}")"
|
||||
done <<< "${metadata}"
|
||||
}
|
||||
|
||||
write_ci_base_metadata_labels() {
|
||||
local metadata="$1"
|
||||
local key=""
|
||||
local value=""
|
||||
|
||||
[[ -n "${metadata}" ]] || return 0
|
||||
while IFS=$'\t' read -r key value; do
|
||||
[[ -n "${key}" && -n "${value}" ]] || continue
|
||||
printf ' "%s" = "%s"\n' \
|
||||
"$(hcl_escape_string "${key}")" \
|
||||
"$(hcl_escape_string "${value}")"
|
||||
done <<< "${metadata}"
|
||||
}
|
||||
|
||||
write_ci_base_label_override() {
|
||||
local target_name=""
|
||||
local metadata=""
|
||||
local -a ci_base_targets=()
|
||||
|
||||
BAKE_FILES=(-f "${VLLM_BAKE_FILE}" -f "${CI_HCL_PATH}")
|
||||
|
||||
if [[ -z "${CI_BASE_CONTENT_HASH:-}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
@@ -1225,23 +1019,16 @@ write_ci_base_label_override() {
|
||||
return 0
|
||||
fi
|
||||
|
||||
metadata=$(ci_base_metadata_pairs)
|
||||
|
||||
: > "${CI_BASE_LABEL_OVERRIDE_PATH}"
|
||||
for target_name in "${ci_base_targets[@]}"; do
|
||||
cat >> "${CI_BASE_LABEL_OVERRIDE_PATH}" <<EOF
|
||||
target "${target_name}" {
|
||||
annotations = [
|
||||
"manifest:org.opencontainers.image.revision=",
|
||||
EOF
|
||||
write_ci_base_metadata_annotations "${metadata}" >> "${CI_BASE_LABEL_OVERRIDE_PATH}"
|
||||
cat >> "${CI_BASE_LABEL_OVERRIDE_PATH}" <<EOF
|
||||
]
|
||||
labels = {
|
||||
"org.opencontainers.image.revision" = ""
|
||||
EOF
|
||||
write_ci_base_metadata_labels "${metadata}" >> "${CI_BASE_LABEL_OVERRIDE_PATH}"
|
||||
cat >> "${CI_BASE_LABEL_OVERRIDE_PATH}" <<EOF
|
||||
"vllm.ci_base.content_hash" = "${CI_BASE_CONTENT_HASH}"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1249,7 +1036,7 @@ EOF
|
||||
done
|
||||
|
||||
BAKE_FILES+=(-f "${CI_BASE_LABEL_OVERRIDE_PATH}")
|
||||
echo "Appended ci_base metadata label override for targets: ${ci_base_targets[*]}"
|
||||
echo "Appended ci_base content-hash label override for targets: ${ci_base_targets[*]}"
|
||||
}
|
||||
|
||||
uses_rocm_csrc_cache() {
|
||||
@@ -1263,24 +1050,12 @@ uses_rocm_csrc_cache() {
|
||||
esac
|
||||
}
|
||||
|
||||
uses_rocm_rust_cache() {
|
||||
case "${TARGET}" in
|
||||
rust-rocm-ci|test-rocm-ci|test-rocm-ci-with-wheel|test-rocm-ci-with-artifacts|export-wheel-rocm)
|
||||
return 0
|
||||
;;
|
||||
*)
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
compute_rocm_csrc_content_hash() {
|
||||
local bake_dir=""
|
||||
local dockerfile_rocm=""
|
||||
local -a content_paths=(
|
||||
"requirements/common.txt"
|
||||
"requirements/rocm.txt"
|
||||
"pyproject.toml"
|
||||
"setup.py"
|
||||
"CMakeLists.txt"
|
||||
"cmake"
|
||||
@@ -1324,56 +1099,6 @@ compute_rocm_csrc_content_hash_if_needed() {
|
||||
echo "ROCm csrc content cache ref: ${ROCM_CSRC_CONTENT_CACHE_REF}"
|
||||
}
|
||||
|
||||
compute_rocm_rust_content_hash() {
|
||||
local bake_dir=""
|
||||
local dockerfile_rocm=""
|
||||
local -a content_paths=(
|
||||
"requirements/build/rust.txt"
|
||||
"rust/Cargo.lock"
|
||||
"rust/Cargo.toml"
|
||||
"rust/proto"
|
||||
"rust/src"
|
||||
"rust-toolchain.toml"
|
||||
"tools/build_rust.py"
|
||||
"tools/install_protoc.sh"
|
||||
"build_rust.sh"
|
||||
)
|
||||
local -a content_args=()
|
||||
|
||||
bake_dir=$(dirname "${VLLM_BAKE_FILE}")
|
||||
dockerfile_rocm="${bake_dir}/Dockerfile.rocm"
|
||||
mapfile -t content_args < <(
|
||||
get_content_arg_names "${dockerfile_rocm}" "base rust_toolchain_input_0 rust_toolchain_input_1 rust-toolchain-input rust_input_0 rust_input_1 rust-input rust-toolchain rust-build" "${ROCM_RUST_CONTENT_ARGS:-}"
|
||||
)
|
||||
|
||||
{
|
||||
printf 'rust-input-files-hash:%s\n' "$(compute_content_hash "${content_paths[@]}")"
|
||||
printf 'dockerfile:%s\n' "${dockerfile_rocm}"
|
||||
printf 'resolved-build-args:\n'
|
||||
hash_dockerfile_arg_values "${dockerfile_rocm}" "${content_args[@]}"
|
||||
printf 'dockerfile-stages:base rust_toolchain_input_0 rust_toolchain_input_1 rust-toolchain-input rust_input_0 rust_input_1 rust-input rust-toolchain rust-build\n'
|
||||
if [[ -f "${dockerfile_rocm}" ]]; then
|
||||
hash_dockerfile_stages "${dockerfile_rocm}" "base rust_toolchain_input_0 rust_toolchain_input_1 rust-toolchain-input rust_input_0 rust_input_1 rust-input rust-toolchain rust-build"
|
||||
else
|
||||
printf 'missing:%s\n' "${dockerfile_rocm}"
|
||||
fi
|
||||
} | sha256sum | cut -d' ' -f1
|
||||
}
|
||||
|
||||
compute_rocm_rust_content_hash_if_needed() {
|
||||
local cache_repo="${DOCKERHUB_CACHE_REPO:-rocm/vllm-ci-cache}"
|
||||
|
||||
if [[ "${ROCM_RUST_CONTENT_CACHE:-1}" == "0" ]] || ! uses_rocm_rust_cache; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
ROCM_RUST_CONTENT_HASH=$(compute_rocm_rust_content_hash)
|
||||
ROCM_RUST_CONTENT_CACHE_REF="${cache_repo}:rust-rocm-input-${ROCM_RUST_CONTENT_HASH}"
|
||||
export ROCM_RUST_CONTENT_HASH
|
||||
export ROCM_RUST_CONTENT_CACHE_REF
|
||||
echo "ROCm Rust content cache ref: ${ROCM_RUST_CONTENT_CACHE_REF}"
|
||||
}
|
||||
|
||||
write_hcl_string_list_entries() {
|
||||
local indent="$1"
|
||||
local value=""
|
||||
@@ -1394,18 +1119,6 @@ hcl_escape_string() {
|
||||
printf '%s' "${value}"
|
||||
}
|
||||
|
||||
join_words() {
|
||||
local IFS=" "
|
||||
printf '%s' "$*"
|
||||
}
|
||||
|
||||
metadata_pair() {
|
||||
local key="$1"
|
||||
local value="${2:-}"
|
||||
|
||||
printf '%s\t%s\n' "${key}" "${value}"
|
||||
}
|
||||
|
||||
write_hcl_string_list() {
|
||||
local indent="$1"
|
||||
shift
|
||||
@@ -1431,7 +1144,6 @@ write_rocm_build_arg_override() {
|
||||
"${CI_BASE_DOCKERFILE_STAGES:-${DEFAULT_CI_BASE_DOCKERFILE_STAGES}}" \
|
||||
"${CI_BASE_CONTENT_ARGS:-}"
|
||||
get_content_arg_names "${dockerfile_rocm}" "base csrc-build" "${ROCM_CSRC_CONTENT_ARGS:-}"
|
||||
get_content_arg_names "${dockerfile_rocm}" "base rust_toolchain_input_0 rust_toolchain_input_1 rust-toolchain-input rust_input_0 rust_input_1 rust-input rust-toolchain rust-build" "${ROCM_RUST_CONTENT_ARGS:-}"
|
||||
} | awk 'NF && !seen[$0]++'
|
||||
)
|
||||
|
||||
@@ -1480,133 +1192,46 @@ validate_cache_export_mode() {
|
||||
esac
|
||||
}
|
||||
|
||||
validate_content_cache_export_mode() {
|
||||
local mode="$1"
|
||||
local env_name="$2"
|
||||
|
||||
case "${mode}" in
|
||||
missing|always|never)
|
||||
;;
|
||||
*)
|
||||
echo "Error: ${env_name} must be one of: missing, always, never"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
should_export_content_cache_ref() {
|
||||
local cache_ref="$1"
|
||||
local cache_name="$2"
|
||||
local mode="${ROCM_CONTENT_CACHE_EXPORT_MODE:-missing}"
|
||||
|
||||
case "${mode}" in
|
||||
always)
|
||||
echo "${cache_name} content cache export mode is always; exporting ${cache_ref}"
|
||||
return 0
|
||||
;;
|
||||
never)
|
||||
echo "${cache_name} content cache export mode is never; not exporting ${cache_ref}"
|
||||
return 1
|
||||
;;
|
||||
missing|"")
|
||||
if docker buildx imagetools inspect "${cache_ref}" >/dev/null 2>&1; then
|
||||
echo "${cache_name} content cache exists; not re-exporting ${cache_ref}"
|
||||
return 1
|
||||
fi
|
||||
echo "${cache_name} content cache missing; will export ${cache_ref}"
|
||||
return 0
|
||||
;;
|
||||
*)
|
||||
echo "Error: ROCM_CONTENT_CACHE_EXPORT_MODE must be one of: missing, always, never"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
write_rocm_cache_override() {
|
||||
local cache_repo="${DOCKERHUB_CACHE_REPO:-rocm/vllm-ci-cache}"
|
||||
local content_cache_export_mode="${ROCM_CONTENT_CACHE_EXPORT_MODE:-missing}"
|
||||
local csrc_cache_to_mode="${ROCM_CSRC_CACHE_TO_MODE:-max}"
|
||||
local rust_cache_to_mode="${ROCM_RUST_CACHE_TO_MODE:-max}"
|
||||
local rocm_cache_to_mode="${ROCM_FINAL_CACHE_TO_MODE:-min}"
|
||||
local -a csrc_content_cache_from=()
|
||||
local -a rust_content_cache_from=()
|
||||
local -a combined_content_cache_from=()
|
||||
local -a content_cache_from=()
|
||||
local -a csrc_cache_to=()
|
||||
local -a rust_cache_to=()
|
||||
local -a rocm_cache_to=()
|
||||
local -a export_wheel_cache_to=()
|
||||
local export_csrc_cache=1
|
||||
local export_rust_cache=1
|
||||
|
||||
if ! uses_rocm_csrc_cache && ! uses_rocm_rust_cache; then
|
||||
if ! uses_rocm_csrc_cache; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
validate_content_cache_export_mode \
|
||||
"${content_cache_export_mode}" \
|
||||
"ROCM_CONTENT_CACHE_EXPORT_MODE"
|
||||
validate_cache_export_mode "${csrc_cache_to_mode}" "ROCM_CSRC_CACHE_TO_MODE"
|
||||
validate_cache_export_mode "${rust_cache_to_mode}" "ROCM_RUST_CACHE_TO_MODE"
|
||||
validate_cache_export_mode "${rocm_cache_to_mode}" "ROCM_FINAL_CACHE_TO_MODE"
|
||||
echo "ROCm content cache export mode: ${content_cache_export_mode}"
|
||||
echo "ROCm csrc cache export mode: ${csrc_cache_to_mode}"
|
||||
echo "ROCm Rust cache export mode: ${rust_cache_to_mode}"
|
||||
echo "ROCm final image cache export mode: ${rocm_cache_to_mode}"
|
||||
|
||||
if [[ -n "${ROCM_CSRC_CONTENT_CACHE_REF:-}" ]]; then
|
||||
csrc_content_cache_from+=("type=registry,ref=${ROCM_CSRC_CONTENT_CACHE_REF}")
|
||||
if should_export_content_cache_ref "${ROCM_CSRC_CONTENT_CACHE_REF}" "ROCm csrc"; then
|
||||
csrc_cache_to+=(
|
||||
"type=registry,ref=${ROCM_CSRC_CONTENT_CACHE_REF},mode=${csrc_cache_to_mode},ignore-error=true"
|
||||
)
|
||||
else
|
||||
export_csrc_cache=0
|
||||
fi
|
||||
content_cache_from+=("type=registry,ref=${ROCM_CSRC_CONTENT_CACHE_REF}")
|
||||
csrc_cache_to+=(
|
||||
"type=registry,ref=${ROCM_CSRC_CONTENT_CACHE_REF},mode=${csrc_cache_to_mode},ignore-error=true"
|
||||
)
|
||||
fi
|
||||
|
||||
if [[ -n "${ROCM_RUST_CONTENT_CACHE_REF:-}" ]]; then
|
||||
rust_content_cache_from+=("type=registry,ref=${ROCM_RUST_CONTENT_CACHE_REF}")
|
||||
if should_export_content_cache_ref "${ROCM_RUST_CONTENT_CACHE_REF}" "ROCm Rust"; then
|
||||
rust_cache_to+=(
|
||||
"type=registry,ref=${ROCM_RUST_CONTENT_CACHE_REF},mode=${rust_cache_to_mode},ignore-error=true"
|
||||
)
|
||||
else
|
||||
export_rust_cache=0
|
||||
fi
|
||||
fi
|
||||
|
||||
combined_content_cache_from=("${csrc_content_cache_from[@]}" "${rust_content_cache_from[@]}")
|
||||
|
||||
# Docker Hub cache exports are best-effort. A cache-only target failure can
|
||||
# otherwise cancel the sibling image target before its manifest is pushed.
|
||||
if [[ -n "${BUILDKITE_COMMIT:-}" ]]; then
|
||||
if [[ ${export_csrc_cache} -eq 1 ]]; then
|
||||
csrc_cache_to+=(
|
||||
"type=registry,ref=${cache_repo}:csrc-rocm-${BUILDKITE_COMMIT},mode=${csrc_cache_to_mode},ignore-error=true"
|
||||
)
|
||||
fi
|
||||
if [[ ${export_rust_cache} -eq 1 ]]; then
|
||||
rust_cache_to+=(
|
||||
"type=registry,ref=${cache_repo}:rust-rocm-${BUILDKITE_COMMIT},mode=${rust_cache_to_mode},ignore-error=true"
|
||||
)
|
||||
fi
|
||||
csrc_cache_to+=(
|
||||
"type=registry,ref=${cache_repo}:csrc-rocm-${BUILDKITE_COMMIT},mode=${csrc_cache_to_mode},ignore-error=true"
|
||||
)
|
||||
rocm_cache_to+=(
|
||||
"type=registry,ref=${cache_repo}:rocm-${BUILDKITE_COMMIT},mode=${rocm_cache_to_mode},ignore-error=true"
|
||||
)
|
||||
fi
|
||||
|
||||
if [[ -n "${ROCM_CACHE_BRANCH_TAG:-}" ]]; then
|
||||
if [[ ${export_csrc_cache} -eq 1 ]]; then
|
||||
csrc_cache_to+=(
|
||||
"type=registry,ref=${cache_repo}:csrc-rocm-branch-${ROCM_CACHE_BRANCH_TAG},mode=${csrc_cache_to_mode},ignore-error=true"
|
||||
)
|
||||
fi
|
||||
if [[ ${export_rust_cache} -eq 1 ]]; then
|
||||
rust_cache_to+=(
|
||||
"type=registry,ref=${cache_repo}:rust-rocm-branch-${ROCM_CACHE_BRANCH_TAG},mode=${rust_cache_to_mode},ignore-error=true"
|
||||
)
|
||||
fi
|
||||
csrc_cache_to+=(
|
||||
"type=registry,ref=${cache_repo}:csrc-rocm-branch-${ROCM_CACHE_BRANCH_TAG},mode=${csrc_cache_to_mode},ignore-error=true"
|
||||
)
|
||||
rocm_cache_to+=(
|
||||
"type=registry,ref=${cache_repo}:rocm-branch-${ROCM_CACHE_BRANCH_TAG},mode=${rocm_cache_to_mode},ignore-error=true"
|
||||
)
|
||||
@@ -1624,7 +1249,7 @@ target "csrc-rocm-ci" {
|
||||
cache-from = concat(
|
||||
get_cache_from_rocm_csrc(),
|
||||
EOF
|
||||
write_hcl_string_list " " "${csrc_content_cache_from[@]}"
|
||||
write_hcl_string_list " " "${content_cache_from[@]}"
|
||||
cat <<EOF
|
||||
)
|
||||
EOF
|
||||
@@ -1632,23 +1257,11 @@ EOF
|
||||
cat <<EOF
|
||||
}
|
||||
|
||||
target "rust-rocm-ci" {
|
||||
cache-from = concat(
|
||||
get_cache_from_rocm_rust(),
|
||||
EOF
|
||||
write_hcl_string_list " " "${rust_content_cache_from[@]}"
|
||||
cat <<EOF
|
||||
)
|
||||
EOF
|
||||
write_hcl_string_list_attr " " "cache-to" "${rust_cache_to[@]}"
|
||||
cat <<EOF
|
||||
}
|
||||
|
||||
target "test-rocm-ci" {
|
||||
cache-from = concat(
|
||||
get_cache_from_rocm(),
|
||||
EOF
|
||||
write_hcl_string_list " " "${combined_content_cache_from[@]}"
|
||||
write_hcl_string_list " " "${content_cache_from[@]}"
|
||||
cat <<EOF
|
||||
)
|
||||
EOF
|
||||
@@ -1660,7 +1273,7 @@ target "export-wheel-rocm" {
|
||||
cache-from = concat(
|
||||
get_cache_from_rocm(),
|
||||
EOF
|
||||
write_hcl_string_list " " "${combined_content_cache_from[@]}"
|
||||
write_hcl_string_list " " "${content_cache_from[@]}"
|
||||
cat <<EOF
|
||||
)
|
||||
EOF
|
||||
@@ -1686,7 +1299,7 @@ extract_dependency_pins() {
|
||||
return 0
|
||||
fi
|
||||
|
||||
for var in NIXL_BRANCH UCX_BRANCH ROCSHMEM_BRANCH DEEPEP_BRANCH; do
|
||||
for var in RIXL_BRANCH UCX_BRANCH ROCSHMEM_BRANCH DEEPEP_BRANCH; do
|
||||
if [[ -n "${!var:-}" ]]; then
|
||||
echo "Using provided ${var}: ${!var}"
|
||||
continue
|
||||
@@ -1706,30 +1319,30 @@ extract_dependency_pins() {
|
||||
compute_dependency_cache_keys() {
|
||||
local bake_dir=""
|
||||
local dockerfile_rocm=""
|
||||
local nixl_branch=""
|
||||
local rixl_branch=""
|
||||
local ucx_branch=""
|
||||
local rocshmem_branch=""
|
||||
local deepep_branch=""
|
||||
local nixl_material=""
|
||||
local rixl_material=""
|
||||
local rocshmem_material=""
|
||||
local deepep_material=""
|
||||
|
||||
bake_dir=$(dirname "${VLLM_BAKE_FILE}")
|
||||
dockerfile_rocm="${bake_dir}/Dockerfile.rocm"
|
||||
nixl_branch=$(resolve_dockerfile_arg_value "${dockerfile_rocm}" "NIXL_BRANCH")
|
||||
rixl_branch=$(resolve_dockerfile_arg_value "${dockerfile_rocm}" "RIXL_BRANCH")
|
||||
ucx_branch=$(resolve_dockerfile_arg_value "${dockerfile_rocm}" "UCX_BRANCH")
|
||||
rocshmem_branch=$(resolve_dockerfile_arg_value "${dockerfile_rocm}" "ROCSHMEM_BRANCH")
|
||||
deepep_branch=$(resolve_dockerfile_arg_value "${dockerfile_rocm}" "DEEPEP_BRANCH")
|
||||
|
||||
if [[ -n "${nixl_branch}" && -n "${ucx_branch}" ]]; then
|
||||
nixl_material=$(compose_stage_cache_material "${dockerfile_rocm}" "base build_nixl")
|
||||
NIXL_CACHE_KEY=$(
|
||||
if [[ -n "${rixl_branch}" && -n "${ucx_branch}" ]]; then
|
||||
rixl_material=$(compose_stage_cache_material "${dockerfile_rocm}" "base build_rixl")
|
||||
RIXL_CACHE_KEY=$(
|
||||
compose_dependency_cache_key \
|
||||
"${nixl_branch}-ucx-${ucx_branch}" \
|
||||
"${nixl_material}"
|
||||
"${rixl_branch}-ucx-${ucx_branch}" \
|
||||
"${rixl_material}"
|
||||
)
|
||||
export NIXL_CACHE_KEY
|
||||
echo "NIXL dependency cache key: ${NIXL_CACHE_KEY}"
|
||||
export RIXL_CACHE_KEY
|
||||
echo "RIXL dependency cache key: ${RIXL_CACHE_KEY}"
|
||||
fi
|
||||
|
||||
if [[ -n "${rocshmem_branch}" ]]; then
|
||||
@@ -1780,11 +1393,11 @@ dependency_cache_ref_for_target() {
|
||||
local cache_repo="${DOCKERHUB_CACHE_REPO:-rocm/vllm-ci-cache}"
|
||||
|
||||
case "${target}" in
|
||||
nixl-rocm-ci)
|
||||
if [[ -n "${NIXL_CACHE_KEY:-}" ]]; then
|
||||
printf '%s\n' "${cache_repo}:nixl-rocm-${NIXL_CACHE_KEY}"
|
||||
elif [[ -n "${NIXL_BRANCH:-}" ]]; then
|
||||
printf '%s\n' "${cache_repo}:nixl-rocm-${NIXL_BRANCH}-ucx-${UCX_BRANCH:-}"
|
||||
rixl-rocm-ci)
|
||||
if [[ -n "${RIXL_CACHE_KEY:-}" ]]; then
|
||||
printf '%s\n' "${cache_repo}:rixl-rocm-${RIXL_CACHE_KEY}"
|
||||
elif [[ -n "${RIXL_BRANCH:-}" ]]; then
|
||||
printf '%s\n' "${cache_repo}:rixl-rocm-${RIXL_BRANCH}-ucx-${UCX_BRANCH:-}"
|
||||
fi
|
||||
;;
|
||||
rocshmem-rocm-ci)
|
||||
@@ -1815,7 +1428,7 @@ add_dependency_cache_target() {
|
||||
|
||||
resolve_ci_base_dependency_targets() {
|
||||
local mode="${ROCM_DEP_CACHE_EXPORT_MODE:-missing}"
|
||||
local nixl_ref=""
|
||||
local rixl_ref=""
|
||||
local rocshmem_ref=""
|
||||
local deepep_ref=""
|
||||
|
||||
@@ -1824,7 +1437,7 @@ resolve_ci_base_dependency_targets() {
|
||||
case "${mode}" in
|
||||
always)
|
||||
echo "ROCM_DEP_CACHE_EXPORT_MODE=always; exporting all dependency caches serially"
|
||||
for target in nixl-rocm-ci rocshmem-rocm-ci deepep-rocm-ci; do
|
||||
for target in rixl-rocm-ci rocshmem-rocm-ci deepep-rocm-ci; do
|
||||
if [[ -n "$(dependency_cache_ref_for_target "${target}")" ]]; then
|
||||
add_dependency_cache_target "${target}"
|
||||
fi
|
||||
@@ -1844,13 +1457,13 @@ resolve_ci_base_dependency_targets() {
|
||||
;;
|
||||
esac
|
||||
|
||||
if [[ "${mode}" != "always" && -n "${NIXL_CACHE_KEY:-}" ]]; then
|
||||
nixl_ref=$(dependency_cache_ref_for_target "nixl-rocm-ci")
|
||||
if dependency_cache_ref_exists "${nixl_ref}"; then
|
||||
echo "NIXL dependency cache exists: ${nixl_ref}"
|
||||
if [[ "${mode}" != "always" && -n "${RIXL_CACHE_KEY:-}" ]]; then
|
||||
rixl_ref=$(dependency_cache_ref_for_target "rixl-rocm-ci")
|
||||
if dependency_cache_ref_exists "${rixl_ref}"; then
|
||||
echo "RIXL dependency cache exists: ${rixl_ref}"
|
||||
else
|
||||
echo "NIXL dependency cache missing; will seed: ${nixl_ref}"
|
||||
add_dependency_cache_target "nixl-rocm-ci"
|
||||
echo "RIXL dependency cache missing; will seed: ${rixl_ref}"
|
||||
add_dependency_cache_target "rixl-rocm-ci"
|
||||
fi
|
||||
fi
|
||||
|
||||
@@ -1928,13 +1541,7 @@ confirm_remote_image_push() {
|
||||
|
||||
remote_hash=$(get_remote_image_label_with_retry "${image_ref}" "vllm.ci_base.content_hash")
|
||||
if [[ -n "${remote_hash}" && "${remote_hash}" == "${CI_BASE_CONTENT_HASH}" ]]; then
|
||||
if remote_ci_base_metadata_is_current_with_retry "${image_ref}"; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "Remote image exists with the expected ci_base content hash but stale metadata."
|
||||
echo " expected metadata version: ${CI_BASE_METADATA_VERSION:-${DEFAULT_CI_BASE_METADATA_VERSION}}"
|
||||
return 1
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "Remote image exists but does not have the expected ci_base content hash."
|
||||
@@ -1950,8 +1557,8 @@ confirm_remote_image_push() {
|
||||
fi
|
||||
|
||||
if [[ -z "${remote_revision}" \
|
||||
&& ${IMAGE_EXISTED_BEFORE_BUILD} -eq 0 ]] \
|
||||
&& image_tag_is_commit_scoped; then
|
||||
&& ${IMAGE_EXISTED_BEFORE_BUILD} -eq 0 \
|
||||
&& image_tag_is_commit_scoped ]]; then
|
||||
echo "Remote image exists under a commit-scoped tag; accepting push despite missing revision label."
|
||||
return 0
|
||||
fi
|
||||
@@ -2009,10 +1616,7 @@ seed_dependency_caches_if_needed() {
|
||||
|
||||
echo "--- :docker: Seeding ${target}"
|
||||
echo "Expected cache ref: ${cache_ref}"
|
||||
docker buildx bake \
|
||||
"${BAKE_FILES[@]}" \
|
||||
--progress "${BUILDKIT_PROGRESS:-plain}" \
|
||||
"${target}"
|
||||
docker buildx bake "${BAKE_FILES[@]}" --progress plain "${target}"
|
||||
verify_dependency_cache_ref "${cache_ref}"
|
||||
done
|
||||
}
|
||||
@@ -2040,10 +1644,7 @@ run_bake() {
|
||||
local build_rc=0
|
||||
|
||||
echo "--- :docker: Building ${TARGET}"
|
||||
docker buildx bake \
|
||||
"${BAKE_FILES[@]}" \
|
||||
--progress "${BUILDKIT_PROGRESS:-plain}" \
|
||||
"${BAKE_TARGETS[@]}" || build_rc=$?
|
||||
docker buildx bake "${BAKE_FILES[@]}" --progress plain "${BAKE_TARGETS[@]}" || build_rc=$?
|
||||
|
||||
if [[ ${build_rc} -eq 0 ]]; then
|
||||
echo "--- :white_check_mark: Build complete"
|
||||
@@ -2081,57 +1682,36 @@ upload_wheel_artifacts_if_present() {
|
||||
local wheel_dir="./wheel-export"
|
||||
local artifact_dir="artifacts/vllm-rocm-install"
|
||||
local archive_name="vllm-rocm-install.tar.gz"
|
||||
local metadata_dir="${wheel_dir}/.vllm-ci-artifact"
|
||||
local native_base_image=""
|
||||
local whl=""
|
||||
local whl_name=""
|
||||
local -a wheels=()
|
||||
|
||||
if ! should_upload_wheel_artifacts; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [[ -d "${wheel_dir}" ]]; then
|
||||
mapfile -t wheels < <(find "${wheel_dir}" -maxdepth 1 -type f -name '*.whl' -print)
|
||||
fi
|
||||
if [[ ${#wheels[@]} -ne 1 ]]; then
|
||||
echo "Expected exactly one ROCm wheel in ${wheel_dir}; found ${#wheels[@]}" >&2
|
||||
return 1
|
||||
fi
|
||||
whl="${wheels[0]}"
|
||||
whl_name=$(basename "${whl}")
|
||||
native_base_image="${CI_BASE_IMAGE_TAG_COMMIT_REF:-${CI_BASE_IMAGE:-}}"
|
||||
if [[ -z "${native_base_image}" ]]; then
|
||||
echo "Native ROCm artifact requires a ci_base image reference" >&2
|
||||
return 1
|
||||
if [[ ! -d "${wheel_dir}" ]] || ! ls "${wheel_dir}"/*.whl >/dev/null 2>&1; then
|
||||
echo "No ROCm wheel artifacts found in ${wheel_dir}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "--- :package: Uploading ROCm vLLM install artifact"
|
||||
rm -rf "${artifact_dir}" "${metadata_dir}"
|
||||
mkdir -p "${artifact_dir}" "${metadata_dir}"
|
||||
|
||||
printf '%s\n' "${BUILDKITE_COMMIT:-local}" > "${metadata_dir}/commit.txt"
|
||||
printf '%s\n' "${native_base_image}" > "${metadata_dir}/native-base-image.txt"
|
||||
printf '%s\n' "${CI_BASE_IMAGE:-}" > "${metadata_dir}/ci-base-image.txt"
|
||||
printf '%s\n' "${IMAGE_TAG:-}" > "${metadata_dir}/fallback-image.txt"
|
||||
printf '%s\n' "${whl_name}" > "${metadata_dir}/wheel-filename.txt"
|
||||
mkdir -p "${artifact_dir}"
|
||||
|
||||
tar -C "${wheel_dir}" -czf "${artifact_dir}/${archive_name}" .
|
||||
(
|
||||
cd "${artifact_dir}"
|
||||
sha256sum "${archive_name}" > "${archive_name}.sha256"
|
||||
)
|
||||
echo "Created ${archive_name}: $(du -sh "${artifact_dir}/${archive_name}" | cut -f1)"
|
||||
cp "${metadata_dir}"/*.txt "${artifact_dir}/"
|
||||
cp "${whl}" "${artifact_dir}/${whl_name}"
|
||||
echo "Copied ${whl_name}: $(du -sh "${artifact_dir}/${whl_name}" | cut -f1)"
|
||||
printf '%s\n' "${CI_BASE_IMAGE:-}" > "${artifact_dir}/ci-base-image.txt"
|
||||
printf '%s\n' "${IMAGE_TAG:-}" > "${artifact_dir}/fallback-image.txt"
|
||||
|
||||
for whl in "${wheel_dir}"/*.whl; do
|
||||
[[ -f "${whl}" ]] || continue
|
||||
whl_name=$(basename "${whl}")
|
||||
cp "${whl}" "${artifact_dir}/${whl_name}"
|
||||
echo "Copied ${whl_name}: $(du -sh "${artifact_dir}/${whl_name}" | cut -f1)"
|
||||
done
|
||||
|
||||
if command -v buildkite-agent >/dev/null 2>&1; then
|
||||
buildkite-agent artifact upload "${artifact_dir}/*" || return 1
|
||||
buildkite-agent artifact upload "${artifact_dir}/*"
|
||||
echo "ROCm vLLM install artifacts uploaded to ${artifact_dir}/"
|
||||
elif [[ "${BUILDKITE:-false}" == "true" ]]; then
|
||||
echo "buildkite-agent not found; cannot upload required ROCm artifacts" >&2
|
||||
return 1
|
||||
else
|
||||
echo "Not in Buildkite, skipping artifact upload"
|
||||
fi
|
||||
@@ -2144,18 +1724,16 @@ main() {
|
||||
print_header
|
||||
validate_inputs
|
||||
load_ci_hcl
|
||||
init_bake_files
|
||||
compute_ci_base_hash_if_needed
|
||||
configure_ci_base_image_refs
|
||||
maybe_skip_existing_image
|
||||
setup_builder
|
||||
prepare_git_cache_metadata
|
||||
write_ci_base_label_override
|
||||
extract_dependency_pins
|
||||
write_rocm_build_arg_override
|
||||
compute_dependency_cache_keys
|
||||
write_ci_base_label_override
|
||||
compute_rocm_csrc_content_hash_if_needed
|
||||
compute_rocm_rust_content_hash_if_needed
|
||||
write_rocm_cache_override
|
||||
resolve_ci_base_dependency_targets
|
||||
print_bake_config
|
||||
@@ -2163,11 +1741,6 @@ main() {
|
||||
echo "BAKE_PRINT_ONLY=1 set; skipping build"
|
||||
return 0
|
||||
fi
|
||||
if should_upload_wheel_artifacts; then
|
||||
# wheel-export is an output directory, not a BuildKit cache. Starting
|
||||
# clean prevents a failed/retried export from packaging a stale wheel.
|
||||
rm -rf ./wheel-export
|
||||
fi
|
||||
seed_dependency_caches_if_needed
|
||||
run_bake
|
||||
upload_wheel_artifacts_if_present
|
||||
|
||||
@@ -45,10 +45,8 @@ $PYTHON .buildkite/scripts/generate-nightly-index.py --version "$SUBPATH" --curr
|
||||
echo "Uploading indices to $S3_COMMIT_PREFIX"
|
||||
aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "$S3_COMMIT_PREFIX"
|
||||
|
||||
# copy to /nightly/ only when enabled for a main branch build that is not a PR
|
||||
if [[ "${UPDATE_NIGHTLY_INDEX:-1}" == "1" && \
|
||||
"$BUILDKITE_BRANCH" == "main" && \
|
||||
"$BUILDKITE_PULL_REQUEST" == "false" ]]; then
|
||||
# copy to /nightly/ only if it is on the main branch and not a PR
|
||||
if [[ "$BUILDKITE_BRANCH" == "main" && "$BUILDKITE_PULL_REQUEST" == "false" ]]; then
|
||||
echo "Uploading indices to overwrite /nightly/"
|
||||
aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "s3://$BUCKET/nightly/"
|
||||
fi
|
||||
@@ -69,7 +67,7 @@ pure_version="${version%%+*}"
|
||||
echo "Pure version (without variant): $pure_version"
|
||||
|
||||
# re-generate and copy to /<pure_version>/ only if it does not have "dev" in the version
|
||||
if [[ "${UPDATE_VERSION_INDEX:-1}" == "1" && "$version" != *"dev"* ]]; then
|
||||
if [[ "$version" != *"dev"* ]]; then
|
||||
echo "Re-generating indices for /$pure_version/"
|
||||
rm -rf "${INDICES_OUTPUT_DIR:?}"
|
||||
mkdir -p "$INDICES_OUTPUT_DIR"
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
#!/bin/bash
|
||||
|
||||
# This script runs ROCm tests either directly in a native CI pod or inside the
|
||||
# corresponding Docker container. Multi-node tests continue to use Docker.
|
||||
# This script runs tests inside the corresponding ROCm docker container.
|
||||
# It handles both single-node and multi-node test configurations.
|
||||
#
|
||||
# Multi-node detection: Instead of matching on fragile group names, we detect
|
||||
# multi-node jobs structurally by looking for the bracket command syntax
|
||||
@@ -28,34 +28,6 @@
|
||||
###############################################################################
|
||||
set -o pipefail
|
||||
|
||||
: "${BUILDKIT_PROGRESS:=plain}"
|
||||
: "${TERM:=xterm-256color}"
|
||||
: "${FORCE_COLOR:=1}"
|
||||
: "${CLICOLOR_FORCE:=1}"
|
||||
: "${PY_COLORS:=1}"
|
||||
: "${ROCM_DOCKER_TTY:=1}"
|
||||
: "${PYTHONFAULTHANDLER:=1}"
|
||||
: "${PYTEST_TIMEOUT:=2400}"
|
||||
if [[ " ${PYTEST_ADDOPTS:-} " != *" --color"* ]]; then
|
||||
PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--color=yes"
|
||||
fi
|
||||
if [[ " ${PYTEST_ADDOPTS:-} " != *" --durations="* ]]; then
|
||||
PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--durations=25"
|
||||
fi
|
||||
if [[ " ${PYTEST_ADDOPTS:-} " != *" --durations-min="* ]]; then
|
||||
PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--durations-min=1.0"
|
||||
fi
|
||||
# Dump stacks after 25 minutes, then stop an individual test after 40 minutes.
|
||||
if [[ " ${PYTEST_ADDOPTS:-} " != *" faulthandler_timeout="* ]]; then
|
||||
PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }-o faulthandler_timeout=1500"
|
||||
fi
|
||||
if [[ " ${PYTEST_ADDOPTS:-} " != *" --timeout-method="* &&
|
||||
" ${PYTEST_ADDOPTS:-} " != *" --timeout-method "* ]]; then
|
||||
PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--timeout-method=thread"
|
||||
fi
|
||||
export BUILDKIT_PROGRESS TERM FORCE_COLOR CLICOLOR_FORCE PY_COLORS PYTEST_ADDOPTS PYTEST_TIMEOUT ROCM_DOCKER_TTY
|
||||
export PYTHONFAULTHANDLER
|
||||
|
||||
# Export Python path for commands that run directly on the host. Containerized
|
||||
# tests set this to /vllm-workspace below so spawned Python processes do not
|
||||
# depend on their current working directory.
|
||||
@@ -70,28 +42,6 @@ report_docker_usage() {
|
||||
docker system df || true
|
||||
}
|
||||
|
||||
clear_ci_orchestration_env() {
|
||||
unset -v \
|
||||
VLLM_TEST_GROUP_NAME \
|
||||
VLLM_CI_REQUIRE_PERSISTENT_HF_CACHE \
|
||||
VLLM_CI_ARTIFACT_STEP \
|
||||
VLLM_TEST_CACHE \
|
||||
VLLM_CI_EXECUTION_MODE \
|
||||
VLLM_CI_WORKSPACE \
|
||||
VLLM_CI_REQUIRE_WORKSPACE_MOUNT \
|
||||
VLLM_TEST_COMMANDS \
|
||||
VLLM_CI_BRANCH \
|
||||
VLLM_CI_BASE_IMAGE \
|
||||
VLLM_CI_FALLBACK_IMAGE \
|
||||
VLLM_CI_DOCKER_DISABLED \
|
||||
VLLM_CI_ARTIFACT_GLOB \
|
||||
VLLM_CI_ARTIFACT_CHECKSUM_GLOB \
|
||||
VLLM_CI_EXPECTED_GPU_COUNT \
|
||||
VLLM_CI_USE_ARTIFACTS \
|
||||
VLLM_CI_RESULTS_ROOT \
|
||||
VLLM_ALLOW_DEPRECATED_BEAM_SEARCH
|
||||
}
|
||||
|
||||
cleanup_network() {
|
||||
local max_nodes=${NUM_NODES:-2}
|
||||
for node in $(seq 0 $((max_nodes - 1))); do
|
||||
@@ -184,11 +134,7 @@ prepare_artifact_image() {
|
||||
fi
|
||||
|
||||
cp "${wheel_dir}"/*.whl "${context_dir}/wheels/" || return 1
|
||||
tar -C "${wheel_dir}" \
|
||||
--exclude='*.whl' \
|
||||
--exclude='.vllm-ci-artifact' \
|
||||
--exclude='./.vllm-ci-artifact' \
|
||||
-cf - . \
|
||||
tar -C "${wheel_dir}" --exclude='*.whl' -cf - . \
|
||||
| tar -C "${workspace_dir}" -xf - || return 1
|
||||
cat > "${context_dir}/Dockerfile" <<'EOF'
|
||||
ARG BASE_IMAGE
|
||||
@@ -203,7 +149,6 @@ EOF
|
||||
echo "--- Building local ROCm test image"
|
||||
docker build \
|
||||
--pull=false \
|
||||
--progress "${BUILDKIT_PROGRESS}" \
|
||||
--build-arg "BASE_IMAGE=${base_image}" \
|
||||
-t "${artifact_image}" \
|
||||
"${context_dir}" || return 1
|
||||
@@ -211,276 +156,6 @@ EOF
|
||||
return 0
|
||||
}
|
||||
|
||||
is_native_runtime() {
|
||||
[[ "${AMD_CI_RUNTIME:-}" == "native" || "${NATIVE_CI:-}" == "true" ]]
|
||||
}
|
||||
|
||||
validate_native_workspace() {
|
||||
local workspace_dir="${VLLM_CI_WORKSPACE:-/vllm-workspace}"
|
||||
local workspace_real=""
|
||||
local checkout_real=""
|
||||
local workspace_mount=""
|
||||
|
||||
mkdir -p "${workspace_dir}" || return 1
|
||||
workspace_real=$(readlink -m "${workspace_dir}") || return 1
|
||||
if [[ -n "${BUILDKITE_BUILD_CHECKOUT_PATH:-}" ]]; then
|
||||
checkout_real=$(readlink -m "${BUILDKITE_BUILD_CHECKOUT_PATH}") || return 1
|
||||
if [[ "${checkout_real}" == "${workspace_real}" \
|
||||
|| "${checkout_real}" == "${workspace_real}/"* \
|
||||
|| "${workspace_real}" == "${checkout_real}/"* ]]; then
|
||||
echo "Refusing to replace ${workspace_real}; it overlaps the Buildkite checkout ${checkout_real}" >&2
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
if [[ "${VLLM_CI_REQUIRE_WORKSPACE_MOUNT:-1}" == "1" ]]; then
|
||||
if ! command -v findmnt >/dev/null 2>&1; then
|
||||
echo "findmnt is required to verify the native workspace mount" >&2
|
||||
return 1
|
||||
fi
|
||||
workspace_mount=$(findmnt -n -T "${workspace_real}" -o TARGET 2>/dev/null || true)
|
||||
if [[ "$(readlink -m "${workspace_mount:-/}")" != "${workspace_real}" ]]; then
|
||||
echo "Native CI requires a dedicated volume mounted at ${workspace_real}" >&2
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
prepare_native_workspace() {
|
||||
if [[ "${VLLM_CI_USE_ARTIFACTS:-0}" != "1" ]]; then
|
||||
echo "Native CI requires VLLM_CI_USE_ARTIFACTS=1"
|
||||
return 1
|
||||
fi
|
||||
if ! command -v buildkite-agent >/dev/null 2>&1; then
|
||||
echo "buildkite-agent not found; cannot download ROCm wheel artifact"
|
||||
return 1
|
||||
fi
|
||||
validate_native_workspace || return 1
|
||||
|
||||
local artifact_glob="${VLLM_CI_ARTIFACT_GLOB:-artifacts/vllm-rocm-install/vllm-rocm-install.tar.gz}"
|
||||
local artifact_checksum_glob="${VLLM_CI_ARTIFACT_CHECKSUM_GLOB:-${artifact_glob}.sha256}"
|
||||
local artifact_step="${VLLM_CI_ARTIFACT_STEP:-image-build-amd}"
|
||||
local archive=""
|
||||
local checksum=""
|
||||
local download_dir=""
|
||||
local metadata_dir=""
|
||||
local recorded_base=""
|
||||
local recorded_commit=""
|
||||
local recorded_wheel=""
|
||||
local workspace_dir="${VLLM_CI_WORKSPACE:-/vllm-workspace}"
|
||||
local wheel_dir=""
|
||||
local attempt=0
|
||||
local attempt_dir=""
|
||||
local -a archives=()
|
||||
local -a checksums=()
|
||||
local -a wheels=()
|
||||
|
||||
artifact_work_dir=$(mktemp -d -t vllm-rocm-artifact.XXXXXX) || return 1
|
||||
wheel_dir="${artifact_work_dir}/wheels"
|
||||
mkdir -p "${wheel_dir}" || return 1
|
||||
|
||||
echo "--- Downloading ROCm wheel artifact from ${artifact_step} (native in-pod)"
|
||||
for attempt in 1 2 3; do
|
||||
attempt_dir="${artifact_work_dir}/download-${attempt}"
|
||||
rm -rf "${attempt_dir}" || return 1
|
||||
mkdir -p "${attempt_dir}" || return 1
|
||||
if buildkite-agent artifact download \
|
||||
"${artifact_glob}" "${attempt_dir}" --step "${artifact_step}" \
|
||||
&& buildkite-agent artifact download \
|
||||
"${artifact_checksum_glob}" "${attempt_dir}" --step "${artifact_step}"; then
|
||||
download_dir="${attempt_dir}"
|
||||
break
|
||||
fi
|
||||
echo "Artifact download attempt ${attempt}/3 failed"
|
||||
if [[ "${attempt}" -lt 3 ]]; then
|
||||
sleep $((attempt * 2))
|
||||
fi
|
||||
done
|
||||
if [[ -z "${download_dir}" ]]; then
|
||||
echo "Failed to download ${artifact_glob} and ${artifact_checksum_glob} from ${artifact_step}"
|
||||
return 1
|
||||
fi
|
||||
|
||||
mapfile -t archives < <(
|
||||
find "${download_dir}" -name "vllm-rocm-install.tar.gz" -type f -print
|
||||
)
|
||||
mapfile -t checksums < <(
|
||||
find "${download_dir}" -name "vllm-rocm-install.tar.gz.sha256" -type f -print
|
||||
)
|
||||
if [[ ${#archives[@]} -ne 1 || ${#checksums[@]} -ne 1 ]]; then
|
||||
echo "Expected exactly one ROCm archive and checksum; found ${#archives[@]} archive(s) and ${#checksums[@]} checksum(s)" >&2
|
||||
return 1
|
||||
fi
|
||||
archive="${archives[0]}"
|
||||
checksum="${checksums[0]}"
|
||||
if [[ "$(dirname "${archive}")" != "$(dirname "${checksum}")" ]]; then
|
||||
echo "ROCm archive and checksum were downloaded to different directories" >&2
|
||||
return 1
|
||||
fi
|
||||
(
|
||||
cd "$(dirname "${archive}")"
|
||||
sha256sum -c "$(basename "${checksum}")"
|
||||
) || return 1
|
||||
|
||||
tar --no-same-owner -xzf "${archive}" -C "${wheel_dir}" || return 1
|
||||
mapfile -t wheels < <(
|
||||
find "${wheel_dir}" -maxdepth 1 -type f -name '*.whl' -print
|
||||
)
|
||||
if [[ ${#wheels[@]} -ne 1 ]]; then
|
||||
echo "ROCm artifact must contain exactly one top-level wheel; found ${#wheels[@]}" >&2
|
||||
return 1
|
||||
fi
|
||||
metadata_dir="${wheel_dir}/.vllm-ci-artifact"
|
||||
for metadata_file in commit.txt native-base-image.txt wheel-filename.txt; do
|
||||
if [[ ! -s "${metadata_dir}/${metadata_file}" ]]; then
|
||||
echo "ROCm artifact metadata is missing ${metadata_file}" >&2
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
for metadata_file in ci-base-image.txt fallback-image.txt; do
|
||||
if [[ ! -f "${metadata_dir}/${metadata_file}" ]]; then
|
||||
echo "ROCm artifact metadata is missing ${metadata_file}" >&2
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
|
||||
recorded_commit=$(tr -d '\r\n' < "${metadata_dir}/commit.txt")
|
||||
recorded_base=$(tr -d '\r\n' < "${metadata_dir}/native-base-image.txt")
|
||||
recorded_wheel=$(tr -d '\r\n' < "${metadata_dir}/wheel-filename.txt")
|
||||
if [[ -z "${BUILDKITE_COMMIT:-}" || "${recorded_commit}" != "${BUILDKITE_COMMIT}" ]]; then
|
||||
echo "ROCm artifact commit ${recorded_commit} does not match ${BUILDKITE_COMMIT:-unset}" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ -z "${VLLM_CI_BASE_IMAGE:-}" || "${recorded_base}" != "${VLLM_CI_BASE_IMAGE}" ]]; then
|
||||
echo "ROCm artifact base ${recorded_base} does not match ${VLLM_CI_BASE_IMAGE:-unset}" >&2
|
||||
return 1
|
||||
fi
|
||||
if [[ "${recorded_wheel}" != "$(basename "${wheels[0]}")" ]]; then
|
||||
echo "ROCm artifact wheel manifest ${recorded_wheel} does not match $(basename "${wheels[0]}")" >&2
|
||||
return 1
|
||||
fi
|
||||
for required_dir in tests .buildkite requirements; do
|
||||
if [[ ! -d "${wheel_dir}/${required_dir}" ]]; then
|
||||
echo "ROCm wheel artifact did not contain ${required_dir}/" >&2
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo "--- Installing ROCm wheel into pod environment"
|
||||
python3 -m pip install --no-deps --force-reinstall "${wheels[0]}" || return 1
|
||||
|
||||
echo "--- Preparing ${workspace_dir} from artifact"
|
||||
find "${workspace_dir}" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + || return 1
|
||||
tar -C "${wheel_dir}" \
|
||||
--exclude='*.whl' \
|
||||
--exclude='.vllm-ci-artifact' \
|
||||
--exclude='./.vllm-ci-artifact' \
|
||||
-cf - . | tar --no-same-owner -C "${workspace_dir}" -xf - || return 1
|
||||
if [[ ! -d "${workspace_dir}/tests" ]]; then
|
||||
echo "Failed to stage the native test workspace" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
initialize_native_environment() {
|
||||
local job_id="${BUILDKITE_JOB_ID:-${BUILDKITE_PARALLEL_JOB:-local}}"
|
||||
local job_id_suffix=""
|
||||
local native_root=""
|
||||
local hf_fstype=""
|
||||
local hf_mount=""
|
||||
|
||||
if [[ "$(id -u)" -ne 0 ]]; then
|
||||
echo "Native ROCm CI currently requires the ci_base container to run as root" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
job_id="${job_id//[^A-Za-z0-9_.-]/_}"
|
||||
job_id_suffix="${job_id##*-}"
|
||||
job_id_suffix="${job_id_suffix:0:12}"
|
||||
native_root="/tmp/vllm-native-${job_id}"
|
||||
TMPDIR="/tmp/vllm-${job_id_suffix}/tmp"
|
||||
VLLM_RPC_BASE_PATH="/tmp"
|
||||
TORCHINDUCTOR_CACHE_DIR="${native_root}/cache/torchinductor"
|
||||
TRITON_CACHE_DIR="${native_root}/cache/triton"
|
||||
VLLM_CACHE_ROOT="${native_root}/cache/vllm"
|
||||
XDG_CACHE_HOME="${native_root}/cache/xdg"
|
||||
: "${HF_HOME:=/home/buildkite-agent/huggingface}"
|
||||
# datasets uses POSIX locks that are unsupported by the shared HF NFS cache.
|
||||
# Keep processed datasets job-local while retaining the persistent Hub cache.
|
||||
HF_DATASETS_CACHE="${native_root}/cache/huggingface/datasets"
|
||||
: "${HF_HUB_DOWNLOAD_TIMEOUT:=300}"
|
||||
: "${HF_HUB_ETAG_TIMEOUT:=60}"
|
||||
export TMPDIR VLLM_RPC_BASE_PATH
|
||||
export TORCHINDUCTOR_CACHE_DIR TRITON_CACHE_DIR VLLM_CACHE_ROOT XDG_CACHE_HOME
|
||||
export HF_HOME HF_DATASETS_CACHE HF_HUB_DOWNLOAD_TIMEOUT HF_HUB_ETAG_TIMEOUT
|
||||
export PYTORCH_ROCM_ARCH=""
|
||||
|
||||
mkdir -p "${TMPDIR}" \
|
||||
"${TORCHINDUCTOR_CACHE_DIR}" \
|
||||
"${TRITON_CACHE_DIR}" \
|
||||
"${VLLM_CACHE_ROOT}" \
|
||||
"${XDG_CACHE_HOME}" \
|
||||
"${HF_HOME}" \
|
||||
"${HF_DATASETS_CACHE}" || return 1
|
||||
|
||||
echo "Native compile caches: VLLM_CACHE_ROOT=${VLLM_CACHE_ROOT} TORCHINDUCTOR_CACHE_DIR=${TORCHINDUCTOR_CACHE_DIR}"
|
||||
|
||||
if [[ "${VLLM_CI_REQUIRE_PERSISTENT_HF_CACHE:-0}" == "1" ]]; then
|
||||
if ! command -v findmnt >/dev/null 2>&1; then
|
||||
echo "findmnt is required to verify the native Hugging Face cache mount" >&2
|
||||
return 1
|
||||
fi
|
||||
hf_mount=$(findmnt -n -T "${HF_HOME}" -o TARGET 2>/dev/null || true)
|
||||
if [[ -z "${hf_mount}" || "${hf_mount}" == "/" ]]; then
|
||||
echo "Native CI requires a persistent volume mounted at or above ${HF_HOME}" >&2
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if command -v findmnt >/dev/null 2>&1; then
|
||||
hf_fstype=$(findmnt -n -T "${HF_HOME}" -o FSTYPE 2>/dev/null || true)
|
||||
fi
|
||||
if [[ "${hf_fstype}" == nfs || "${hf_fstype}" == nfs4 ]]; then
|
||||
# Keep hf-xet state local and avoid vectored writes on shared NFS.
|
||||
export HF_XET_CACHE="${native_root}/cache/hf-xet"
|
||||
export HF_XET_HIGH_PERFORMANCE=0
|
||||
export HF_XET_RECONSTRUCTION_USE_VECTORED_WRITE=0
|
||||
mkdir -p "${HF_XET_CACHE}" || return 1
|
||||
echo "Configured hf-xet for shared ${hf_fstype} cache at ${HF_HOME}"
|
||||
fi
|
||||
}
|
||||
|
||||
run_native_preflight() {
|
||||
local expected_gpus="${VLLM_CI_EXPECTED_GPU_COUNT:-1}"
|
||||
|
||||
if [[ ! "${expected_gpus}" =~ ^[0-9]+$ ]]; then
|
||||
echo "Invalid VLLM_CI_EXPECTED_GPU_COUNT=${expected_gpus}" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
python3 -c "import encodings, importlib.metadata as im, importlib.util as iu; [im.version(d) for d in ('transformers', 'torch', 'ray', 'sympy', 'markupsafe', 'vllm')]; missing=[m for m in ('torch.utils.model_zoo', 'transformers.models.nomic_bert', 'ray.dag', 'sympy.physics', 'markupsafe._speedups') if iu.find_spec(m) is None]; assert not missing, missing" || return 1
|
||||
|
||||
if [[ "${expected_gpus}" == "0" ]]; then
|
||||
echo "Native CPU-only AMD job: skipping ROCm device validation"
|
||||
return 0
|
||||
fi
|
||||
|
||||
echo "--- ROCm info"
|
||||
rocminfo || return 1
|
||||
VLLM_CI_EXPECTED_GPU_COUNT="${expected_gpus}" python3 - <<'PY'
|
||||
import os
|
||||
|
||||
import torch
|
||||
|
||||
expected = int(os.environ["VLLM_CI_EXPECTED_GPU_COUNT"])
|
||||
assert torch.version.hip, "PyTorch is not a ROCm build"
|
||||
assert torch.cuda.is_available(), "ROCm GPU is not available to PyTorch"
|
||||
actual = torch.cuda.device_count()
|
||||
assert actual == expected, f"Expected {expected} ROCm GPU(s), found {actual}"
|
||||
PY
|
||||
}
|
||||
|
||||
is_multi_node() {
|
||||
local cmds="$1"
|
||||
# Primary signal: NUM_NODES environment variable set by the pipeline
|
||||
@@ -663,58 +338,7 @@ re_quote_pytest_markers() {
|
||||
# Main
|
||||
###############################################################################
|
||||
|
||||
if is_native_runtime; then
|
||||
echo "--- Native in-pod ROCm CI (AMD_CI_RUNTIME=${AMD_CI_RUNTIME:-unset}, NATIVE_CI=${NATIVE_CI:-unset})"
|
||||
artifact_work_dir=""
|
||||
|
||||
cleanup_native_workspace() {
|
||||
if [[ -n "${artifact_work_dir}" ]]; then
|
||||
rm -rf "${artifact_work_dir}"
|
||||
fi
|
||||
}
|
||||
trap cleanup_native_workspace EXIT
|
||||
|
||||
if [[ -n "${VLLM_TEST_COMMANDS:-}" ]]; then
|
||||
commands="${VLLM_TEST_COMMANDS}"
|
||||
commands_source="env"
|
||||
else
|
||||
commands="$*"
|
||||
commands_source="argv"
|
||||
if [[ -z "$commands" ]]; then
|
||||
echo "Error: No test commands provided for native CI." >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$commands_source" == "argv" ]]; then
|
||||
commands=$(re_quote_pytest_markers "$commands")
|
||||
fi
|
||||
|
||||
if is_multi_node "$commands"; then
|
||||
echo "Native CI does not support multi-node jobs yet."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! initialize_native_environment; then
|
||||
echo "Failed to initialize the native test environment"
|
||||
exit 1
|
||||
fi
|
||||
if ! prepare_native_workspace; then
|
||||
echo "Failed to prepare native test workspace"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
export PYTHONPATH="${VLLM_CI_WORKSPACE:-/vllm-workspace}"
|
||||
|
||||
echo "Native test commands: $commands"
|
||||
run_native_preflight || exit 1
|
||||
# Keep AMD CI orchestration variables out of vLLM's runtime environment.
|
||||
clear_ci_orchestration_env
|
||||
/bin/bash -o pipefail -c "${commands}"
|
||||
handle_pytest_exit "$?"
|
||||
fi
|
||||
|
||||
# --- GPU initialization for legacy Docker execution ---
|
||||
# --- GPU initialization ---
|
||||
echo "--- ROCm info"
|
||||
rocminfo
|
||||
|
||||
@@ -743,20 +367,6 @@ remove_docker_container() {
|
||||
}
|
||||
trap remove_docker_container EXIT
|
||||
|
||||
# python_only_compile.sh runs `python setup.py develop` and needs the full repo tree
|
||||
# under /vllm-workspace (Dockerfile.rocm test stage: mkdir src && mv vllm).
|
||||
# The ROCm wheel artifact tarball only ships a thin tree (tests, etc.), so
|
||||
# artifact images cannot satisfy that test — use the full rocm/vllm-ci image.
|
||||
_cmd_probe="${VLLM_TEST_COMMANDS:-}"
|
||||
if [[ -z "${_cmd_probe}" ]]; then
|
||||
_cmd_probe="$*"
|
||||
fi
|
||||
if [[ "${VLLM_CI_USE_ARTIFACTS:-0}" == "1" && "${_cmd_probe}" == *python_only_compile.sh* ]]; then
|
||||
echo "INFO: disabling VLLM_CI_USE_ARTIFACTS for python_only_compile (requires full /vllm-workspace tree)"
|
||||
export VLLM_CI_USE_ARTIFACTS=0
|
||||
fi
|
||||
unset -v _cmd_probe
|
||||
|
||||
if ! prepare_artifact_image; then
|
||||
echo "Using full ROCm CI image: ${image_name}"
|
||||
docker pull "${image_name}" || exit 1
|
||||
@@ -816,28 +426,6 @@ fi
|
||||
|
||||
echo "Final commands: $commands"
|
||||
|
||||
standalone_merge_base_env=()
|
||||
if [[ "$commands" == *python_only_compile.sh* ]]; then
|
||||
# The ROCm test image often ships /vllm-workspace without .git. Resolve the
|
||||
# wheels.vllm.ai commit from the agent checkout for this test only.
|
||||
vllm_standalone_merge_base=""
|
||||
checkout="${BUILDKITE_BUILD_CHECKOUT_PATH:-}"
|
||||
if [[ -z "${checkout}" || ! -d "${checkout}" ]]; then
|
||||
checkout="."
|
||||
fi
|
||||
# Pass safe.directory per-command because Buildkite uses mixed user IDs.
|
||||
if git -c "safe.directory=${checkout}" -C "${checkout}" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
|
||||
vllm_standalone_merge_base="$(
|
||||
git -c "safe.directory=${checkout}" -C "${checkout}" merge-base HEAD origin/main 2>/dev/null || true
|
||||
)"
|
||||
fi
|
||||
if [[ -z "${vllm_standalone_merge_base}" ]]; then
|
||||
vllm_standalone_merge_base="${BUILDKITE_COMMIT:-}"
|
||||
fi
|
||||
echo "INFO: passing CI_STANDALONE_MERGE_BASE into container: ${vllm_standalone_merge_base}"
|
||||
standalone_merge_base_env=(-e "CI_STANDALONE_MERGE_BASE=${vllm_standalone_merge_base}")
|
||||
fi
|
||||
|
||||
MYPYTHONPATH="/vllm-workspace"
|
||||
|
||||
container_job_id="${BUILDKITE_JOB_ID:-${BUILDKITE_PARALLEL_JOB:-0}}"
|
||||
@@ -867,7 +455,6 @@ else
|
||||
fi
|
||||
|
||||
# --- Route: multi-node vs single-node ---
|
||||
clear_ci_orchestration_env
|
||||
if is_multi_node "$commands"; then
|
||||
echo "--- Multi-node job detected"
|
||||
export DCKR_VER=$(docker --version | sed 's/Docker version \(.*\), build .*/\1/')
|
||||
@@ -914,37 +501,14 @@ if is_multi_node "$commands"; then
|
||||
else
|
||||
echo "--- Single-node job"
|
||||
echo "Render devices: $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES"
|
||||
docker_run_terminal_args=(-i)
|
||||
if [[ "${ROCM_DOCKER_TTY}" == "1" ]]; then
|
||||
docker_run_terminal_args+=(-t)
|
||||
echo "Docker interactive stdin: enabled; TTY allocation: enabled"
|
||||
else
|
||||
echo "Docker interactive stdin: enabled; TTY allocation: disabled"
|
||||
fi
|
||||
|
||||
ulimit_core_hard=$(ulimit -H -c)
|
||||
if [[ "$ulimit_core_hard" == "unlimited" ]]; then
|
||||
# docker run can't pass "unlimited" to --ulimit
|
||||
ulimit_core_hard="-1"
|
||||
fi
|
||||
# Disable core dumps in the ROCm test container unless the ROCm debug agent is enabled
|
||||
coredump_flags="--ulimit core=0:$ulimit_core_hard"
|
||||
if [[ "$commands" == *"ROCm debug agent enabled"* ]]; then
|
||||
# Works around https://github.com/rocm/rocm-systems/issues/6206
|
||||
coredump_flags='-e HSA_COREDUMP_PATTERN="/tmp/gpucore.%p"'
|
||||
else
|
||||
echo "ROCm debug agent not enabled, coredumps are disabled in the test container."
|
||||
fi
|
||||
|
||||
docker run \
|
||||
"${docker_run_terminal_args[@]}" \
|
||||
--device /dev/kfd $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES \
|
||||
$RDMA_FLAGS \
|
||||
--network=host \
|
||||
--shm-size=16gb \
|
||||
--group-add "$render_gid" \
|
||||
--rm \
|
||||
$coredump_flags \
|
||||
-e HF_TOKEN \
|
||||
-e "HF_HUB_DOWNLOAD_TIMEOUT=${HF_HUB_DOWNLOAD_TIMEOUT}" \
|
||||
-e "HF_HUB_ETAG_TIMEOUT=${HF_HUB_ETAG_TIMEOUT}" \
|
||||
@@ -952,13 +516,6 @@ else
|
||||
-e AWS_SECRET_ACCESS_KEY \
|
||||
-e BUILDKITE_PARALLEL_JOB \
|
||||
-e BUILDKITE_PARALLEL_JOB_COUNT \
|
||||
-e TERM \
|
||||
-e FORCE_COLOR \
|
||||
-e CLICOLOR_FORCE \
|
||||
-e PY_COLORS \
|
||||
-e PYTHONFAULTHANDLER \
|
||||
-e PYTEST_ADDOPTS \
|
||||
-e PYTEST_TIMEOUT \
|
||||
-v "${HF_CACHE}:${HF_MOUNT}" \
|
||||
-e "HF_HOME=${HF_MOUNT}" \
|
||||
-e "PYTHONPATH=${MYPYTHONPATH}" \
|
||||
@@ -968,7 +525,6 @@ else
|
||||
-e "VLLM_CACHE_ROOT=${CONTAINER_CACHE_ROOT}/vllm" \
|
||||
-e "XDG_CACHE_HOME=${CONTAINER_CACHE_ROOT}/xdg" \
|
||||
-e "PYTORCH_ROCM_ARCH=" \
|
||||
"${standalone_merge_base_env[@]}" \
|
||||
--name "${container_name}" \
|
||||
"${image_name}" \
|
||||
/bin/bash -c "${CONTAINER_PREFLIGHT} && ${commands}"
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
#!/bin/bash
|
||||
set -euox pipefail
|
||||
|
||||
export VLLM_CPU_KVCACHE_SPACE=1
|
||||
export VLLM_CPU_KVCACHE_SPACE=1
|
||||
export VLLM_CPU_CI_ENV=1
|
||||
# Skip torch.compile via vLLM's --enforce-eager flag (passed below) instead of
|
||||
# TORCH_COMPILE_DISABLE=1, which torch 2.12 no longer treats as a silent no-op
|
||||
# when callers specify fullgraph=True.
|
||||
# Reduce sub-processes for acceleration
|
||||
export TORCH_COMPILE_DISABLE=1
|
||||
export VLLM_ENABLE_V1_MULTIPROCESSING=0
|
||||
|
||||
SDE_ARCHIVE="sde-external-10.7.0-2026-02-18-lin.tar.xz"
|
||||
@@ -50,15 +49,15 @@ wait_for_pid_and_check_log() {
|
||||
}
|
||||
|
||||
# Test Sky Lake (AVX512F)
|
||||
./sde/sde64 -skl -- python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --dtype bfloat16 --enforce-eager > test_0.log 2>&1 &
|
||||
./sde/sde64 -skl -- python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --dtype bfloat16 > test_0.log 2>&1 &
|
||||
PID_TEST_0=$!
|
||||
|
||||
# Test Cascade Lake (AVX512F + VNNI)
|
||||
./sde/sde64 -clx -- python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --dtype bfloat16 --enforce-eager > test_1.log 2>&1 &
|
||||
./sde/sde64 -clx -- python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --dtype bfloat16 > test_1.log 2>&1 &
|
||||
PID_TEST_1=$!
|
||||
|
||||
# Test Cooper Lake (AVX512F + VNNI + BF16)
|
||||
./sde/sde64 -cpx -- python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --dtype bfloat16 --enforce-eager > test_2.log 2>&1 &
|
||||
./sde/sde64 -cpx -- python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --dtype bfloat16 > test_2.log 2>&1 &
|
||||
PID_TEST_2=$!
|
||||
|
||||
wait_for_pid_and_check_log $PID_TEST_0 test_0.log
|
||||
|
||||
@@ -8,7 +8,7 @@ set -ex
|
||||
CORE_RANGE=${CORE_RANGE:-0-31}
|
||||
OMP_CORE_RANGE=${OMP_CORE_RANGE:-0-31}
|
||||
|
||||
export CMAKE_BUILD_PARALLEL_LEVEL=32
|
||||
export CMAKE_BUILD_PARALLEL_LEVEL=16
|
||||
|
||||
# Setup cleanup
|
||||
remove_docker_container() {
|
||||
@@ -37,12 +37,8 @@ function cpu_tests() {
|
||||
pytest -x -v -s tests/kernels/test_onednn.py
|
||||
pytest -x -v -s tests/kernels/attention/test_cpu_attn.py
|
||||
pytest -x -v -s tests/kernels/core/test_cpu_activation.py
|
||||
pytest -x -v -s tests/kernels/moe/test_cpu_fused_moe.py
|
||||
pytest -x -v -s tests/kernels/mamba/cpu/test_cpu_gdn_ops.py
|
||||
pytest -x -v -s tests/kernels/moe/test_cpu_int4_moe.py
|
||||
pytest -x -v -s tests/kernels/mamba/test_cpu_short_conv.py
|
||||
pytest -x -v -s tests/kernels/mamba/test_causal_conv1d.py
|
||||
pytest -x -v -s tests/kernels/mamba/test_mamba_ssm.py"
|
||||
pytest -x -v -s tests/kernels/moe/test_moe.py -k test_cpu_fused_moe_basic
|
||||
pytest -x -v -s tests/kernels/mamba/cpu/test_cpu_gdn_ops.py"
|
||||
|
||||
# skip tests requiring model downloads if HF_TOKEN is not set
|
||||
# due to rate-limits
|
||||
@@ -66,6 +62,7 @@ function cpu_tests() {
|
||||
set -e
|
||||
pytest -x -v -s tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_logprobs"
|
||||
|
||||
|
||||
# basic online serving
|
||||
docker exec cpu-test bash -c '
|
||||
set -e
|
||||
@@ -99,4 +96,3 @@ function cpu_tests() {
|
||||
# All of CPU tests are expected to be finished less than 40 mins.
|
||||
export -f cpu_tests
|
||||
timeout 2h bash -c cpu_tests
|
||||
|
||||
|
||||
@@ -7,49 +7,10 @@ set -euox pipefail
|
||||
# allow to bind to different cores
|
||||
CORE_RANGE=${CORE_RANGE:-48-95}
|
||||
NUMA_NODE=${NUMA_NODE:-1}
|
||||
AGENT_SLOT=${AGENT_SLOT:-}
|
||||
IMAGE_NAME="cpu-test-${NUMA_NODE}${AGENT_SLOT:+-${AGENT_SLOT}}"
|
||||
IMAGE_NAME="cpu-test-$NUMA_NODE"
|
||||
TIMEOUT_VAL=$1
|
||||
TEST_COMMAND=$2
|
||||
|
||||
# Disk hygiene knobs. Reclaim space only once the Docker root filesystem crosses
|
||||
# DISK_USAGE_THRESHOLD percent, and cap the shared BuildKit cache at
|
||||
# BUILDKIT_CACHE_MAX so subsequent builds keep reusing the hottest layers.
|
||||
DISK_USAGE_THRESHOLD=${DISK_USAGE_THRESHOLD:-70}
|
||||
BUILDKIT_CACHE_MAX=${BUILDKIT_CACHE_MAX:-80GB}
|
||||
|
||||
# Reclaim disk only when the host is under pressure. We trim (not purge) the
|
||||
# shared BuildKit cache so cross-job/cross-agent reuse stays intact, and only
|
||||
# touch dangling images; other agents' uniquely tagged images are left alone.
|
||||
prune_if_disk_pressure() {
|
||||
local docker_root disk_usage
|
||||
docker_root=$(docker info -f '{{.DockerRootDir}}' 2>/dev/null || true)
|
||||
if [ -z "$docker_root" ]; then
|
||||
return 0
|
||||
fi
|
||||
disk_usage=$(df "$docker_root" 2>/dev/null | tail -1 | awk '{print $5}' | tr -d '%')
|
||||
if [ "${disk_usage:-0}" -gt "$DISK_USAGE_THRESHOLD" ]; then
|
||||
echo "--- :broom: Disk usage ${disk_usage}% exceeds ${DISK_USAGE_THRESHOLD}%, reclaiming space"
|
||||
docker image prune -f || true
|
||||
docker builder prune -f --keep-storage="$BUILDKIT_CACHE_MAX" || true
|
||||
else
|
||||
echo "Disk usage ${disk_usage:-unknown}% within ${DISK_USAGE_THRESHOLD}% threshold; skipping prune"
|
||||
fi
|
||||
}
|
||||
|
||||
# Always drop this agent's image once the job ends (the default builder never
|
||||
# uses it as a cache source, so removing it costs no rebuild speed), then
|
||||
# reclaim space if needed. Guard every docker call with `|| true` so the trap
|
||||
# never overrides the test's exit code.
|
||||
cleanup() {
|
||||
docker image rm -f "$IMAGE_NAME" || true
|
||||
prune_if_disk_pressure
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# Free space up front so a nearly-full host doesn't fail the build.
|
||||
prune_if_disk_pressure
|
||||
|
||||
# building the docker image
|
||||
echo "--- :docker: Building Docker image"
|
||||
docker build --progress plain --tag "$IMAGE_NAME" --target vllm-test -f docker/Dockerfile.cpu .
|
||||
|
||||
@@ -21,7 +21,6 @@ case "${test_suite}" in
|
||||
python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --kv-cache-dtype fp8
|
||||
python3 examples/basic/offline_inference/generate.py --model nvidia/Llama-3.1-8B-Instruct-FP8 --block-size 64 --enforce-eager --quantization modelopt --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --max-model-len 4096
|
||||
python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --block-size 64 --enforce-eager --max-model-len 8192
|
||||
python3 examples/basic/offline_inference/generate.py --model TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ --block-size 64 --enforce-eager
|
||||
python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2
|
||||
python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 --enable-expert-parallel
|
||||
python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --max-model-len 8192
|
||||
@@ -35,7 +34,6 @@ case "${test_suite}" in
|
||||
pytest -v -s v1/worker --ignore=v1/worker/test_gpu_model_runner.py --ignore=v1/worker/test_worker_memory_snapshot.py
|
||||
pytest -v -s v1/structured_output
|
||||
pytest -v -s v1/test_serial_utils.py
|
||||
pytest -v -s v1/e2e/general/test_correctness_sliding_window.py --deselect="tests/v1/e2e/general/test_correctness_sliding_window.py::test_sliding_window_retrieval[True-1-5-google/gemma-3-1b-it]"
|
||||
pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py --ignore=v1/spec_decode/test_speculators_correctness.py
|
||||
pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_multi_connector.py --ignore=v1/kv_connector/unit/test_example_connector.py --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py --ignore=v1/kv_connector/unit/test_offloading_connector.py
|
||||
;;
|
||||
|
||||
@@ -360,7 +360,7 @@ export HF_TOKEN ZE_AFFINITY_MASK
|
||||
--ipc=host \
|
||||
--privileged \
|
||||
-v /dev/dri/by-path:/dev/dri/by-path \
|
||||
-v "/data/huggingface:/root/.cache/huggingface" \
|
||||
-v "${HOME}/.cache/huggingface:/root/.cache/huggingface" \
|
||||
--entrypoint='' \
|
||||
-e HF_TOKEN \
|
||||
-e ZE_AFFINITY_MASK \
|
||||
|
||||
@@ -85,7 +85,7 @@ RUN pip config set global.index-url http://cache-service-vllm.nginx-pypi-cache.s
|
||||
|
||||
# Install for pytest to make the docker build cache layer always valid
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
pip install pytest>=6.0 'modelscope<1.38'
|
||||
pip install pytest>=6.0 modelscope
|
||||
|
||||
WORKDIR /workspace/vllm
|
||||
|
||||
|
||||
@@ -130,22 +130,6 @@ docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${ROCM_BASE_CACHE_KEY}-rocm
|
||||
docker push vllm/vllm-openai-rocm:latest-base
|
||||
docker push vllm/vllm-openai-rocm:v${RELEASE_VERSION}-base
|
||||
|
||||
# ---- XPU ----
|
||||
|
||||
docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-xpu
|
||||
|
||||
docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-xpu vllm/vllm-openai-xpu:latest-x86_64
|
||||
docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-x86_64-xpu vllm/vllm-openai-xpu:v${RELEASE_VERSION}-x86_64
|
||||
docker push vllm/vllm-openai-xpu:latest-x86_64
|
||||
docker push vllm/vllm-openai-xpu:v${RELEASE_VERSION}-x86_64
|
||||
|
||||
docker manifest rm vllm/vllm-openai-xpu:latest || true
|
||||
docker manifest rm vllm/vllm-openai-xpu:v${RELEASE_VERSION} || true
|
||||
docker manifest create vllm/vllm-openai-xpu:latest vllm/vllm-openai-xpu:latest-x86_64 --amend
|
||||
docker manifest create vllm/vllm-openai-xpu:v${RELEASE_VERSION} vllm/vllm-openai-xpu:v${RELEASE_VERSION}-x86_64 --amend
|
||||
docker manifest push vllm/vllm-openai-xpu:latest
|
||||
docker manifest push vllm/vllm-openai-xpu:v${RELEASE_VERSION}
|
||||
|
||||
# ---- CPU ----
|
||||
# CPU images are behind separate block steps and may not have been built.
|
||||
# All-or-nothing: inspect both arches first, then either publish everything
|
||||
|
||||
@@ -1,32 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build the ROCm ci_base image, optionally from a freshly rebuilt ROCm base.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
metadata_get() {
|
||||
local key="$1"
|
||||
if command -v buildkite-agent >/dev/null 2>&1; then
|
||||
buildkite-agent meta-data get "${key}" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
main() {
|
||||
local base_refreshed=""
|
||||
|
||||
base_refreshed="$(metadata_get rocm-base-refresh)"
|
||||
if [[ "${base_refreshed}" == "1" ]]; then
|
||||
export BASE_IMAGE
|
||||
export CI_BASE_PUSH_STABLE_TAG
|
||||
|
||||
BASE_IMAGE="$(metadata_get rocm-base-image)"
|
||||
CI_BASE_PUSH_STABLE_TAG="$(metadata_get rocm-base-push-stable-tag)"
|
||||
CI_BASE_PUSH_STABLE_TAG="${CI_BASE_PUSH_STABLE_TAG:-0}"
|
||||
|
||||
echo "Using refreshed ROCm base image for ci_base: ${BASE_IMAGE}"
|
||||
echo "Push stable ci_base tag: ${CI_BASE_PUSH_STABLE_TAG}"
|
||||
fi
|
||||
|
||||
bash .buildkite/scripts/ci-bake-rocm.sh ci-base-rocm-ci-with-deps
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -1,68 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build the ROCm CI test image or wheel artifact.
|
||||
#
|
||||
# When Dockerfile.rocm_base changes, always build the full image so downstream
|
||||
# ROCm tests can validate the freshly rebuilt base -> ci_base -> ci image chain.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
metadata_get() {
|
||||
local key="$1"
|
||||
if command -v buildkite-agent >/dev/null 2>&1; then
|
||||
buildkite-agent meta-data get "${key}" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
use_ci_base_if_present() {
|
||||
local ci_base_image=""
|
||||
|
||||
ci_base_image="$(metadata_get rocm-ci-base-image)"
|
||||
if [[ -z "${ci_base_image}" ]]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
export CI_BASE_IMAGE="${ci_base_image}"
|
||||
echo "Using ROCm ci_base image selected by the preceding build step: ${CI_BASE_IMAGE}"
|
||||
}
|
||||
|
||||
use_refreshed_base_if_present() {
|
||||
local base_refreshed=""
|
||||
|
||||
base_refreshed="$(metadata_get rocm-base-refresh)"
|
||||
if [[ "${base_refreshed}" != "1" ]]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
export BASE_IMAGE
|
||||
export IMAGE_TAG_LATEST
|
||||
|
||||
BASE_IMAGE="$(metadata_get rocm-base-image)"
|
||||
IMAGE_TAG_LATEST="$(metadata_get rocm-ci-image-descriptive)"
|
||||
|
||||
echo "Using refreshed ROCm base image for test image: ${BASE_IMAGE}"
|
||||
if [[ -n "${IMAGE_TAG_LATEST}" ]]; then
|
||||
echo "Also tagging full ROCm CI image as: ${IMAGE_TAG_LATEST}"
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
main() {
|
||||
local base_refreshed=0
|
||||
|
||||
use_ci_base_if_present || true
|
||||
|
||||
if use_refreshed_base_if_present; then
|
||||
base_refreshed=1
|
||||
fi
|
||||
|
||||
if [[ "${ROCM_CI_ARTIFACT_ONLY:-0}" == "1" && "${base_refreshed}" != "1" ]]; then
|
||||
echo "ROCM_CI_ARTIFACT_ONLY=1; building ROCm wheel artifact only"
|
||||
IMAGE_TAG="" bash .buildkite/scripts/ci-bake-rocm.sh test-rocm-ci-with-artifacts
|
||||
return
|
||||
fi
|
||||
|
||||
bash .buildkite/scripts/ci-bake-rocm.sh test-rocm-ci-with-wheel
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -1,513 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build and publish a fresh ROCm base image when Dockerfile.rocm_base changes.
|
||||
#
|
||||
# Normal AMD CI builds should not pay for this path. The script no-ops unless
|
||||
# docker/Dockerfile.rocm_base changed relative to the branch base, the previous
|
||||
# main commit, or ROCM_BASE_REFRESH_FORCE=1 is set.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
DOCKERFILE="${ROCM_BASE_DOCKERFILE:-docker/Dockerfile.rocm_base}"
|
||||
BASE_REPO="${ROCM_BASE_IMAGE_REPO:-rocm/vllm-dev}"
|
||||
CI_IMAGE_REPO="${ROCM_CI_IMAGE_REPO:-rocm/vllm-ci}"
|
||||
BUILDER_NAME="${ROCM_BASE_BUILDER_NAME:-vllm-rocm-base-builder}"
|
||||
DEFAULT_ROCM_BASE_METADATA_VERSION="1"
|
||||
DEFAULT_ROCM_BASE_CONTENT_FILES="${DOCKERFILE}"
|
||||
DEFAULT_ROCM_BASE_CONTENT_ARGS="BASE_IMAGE TRITON_BRANCH TRITON_REPO PYTORCH_BRANCH PYTORCH_REPO PYTORCH_VISION_BRANCH PYTORCH_VISION_REPO PYTORCH_AUDIO_BRANCH PYTORCH_AUDIO_REPO FA_BRANCH FA_REPO AITER_BRANCH AITER_REPO MORI_BRANCH MORI_REPO PYTORCH_ROCM_ARCH PYTHON_VERSION USE_SCCACHE"
|
||||
|
||||
metadata_set() {
|
||||
local key="$1"
|
||||
local value="$2"
|
||||
|
||||
[[ -n "${value}" ]] || return 0
|
||||
if command -v buildkite-agent >/dev/null 2>&1; then
|
||||
buildkite-agent meta-data set "${key}" "${value}" || true
|
||||
fi
|
||||
}
|
||||
|
||||
compute_content_hash() {
|
||||
local path=""
|
||||
local file=""
|
||||
|
||||
for path in "$@"; do
|
||||
if [[ -d "${path}" ]]; then
|
||||
while IFS= read -r -d '' file; do
|
||||
printf 'file:%s\n' "${file}"
|
||||
sha256sum "${file}"
|
||||
done < <(find "${path}" -type f -print0 | sort -z)
|
||||
elif [[ -f "${path}" ]]; then
|
||||
printf 'file:%s\n' "${path}"
|
||||
sha256sum "${path}"
|
||||
else
|
||||
printf 'missing:%s\n' "${path}"
|
||||
fi
|
||||
done | sha256sum | cut -d' ' -f1
|
||||
}
|
||||
|
||||
clean_docker_tag() {
|
||||
local input="$1"
|
||||
echo "${input}" | sed 's/[^a-zA-Z0-9._-]/_/g' | cut -c1-128
|
||||
}
|
||||
|
||||
tag_component() {
|
||||
local input="$1"
|
||||
local max_chars="${2:-24}"
|
||||
|
||||
clean_docker_tag "${input:-unknown}" | cut -c1-"${max_chars}"
|
||||
}
|
||||
|
||||
extract_arg_default() {
|
||||
local arg_name="$1"
|
||||
|
||||
sed -n -E "s/^[[:space:]]*ARG[[:space:]]+${arg_name}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \
|
||||
"${DOCKERFILE}" | head -1
|
||||
}
|
||||
|
||||
resolve_image_digest() {
|
||||
local image_ref="$1"
|
||||
|
||||
docker buildx imagetools inspect "${image_ref}" 2>/dev/null \
|
||||
| sed -n -E 's/^Digest:[[:space:]]+//p' \
|
||||
| head -1 || true
|
||||
}
|
||||
|
||||
resolve_rocm_base_arg_value() {
|
||||
local arg_name="$1"
|
||||
local use_sccache="$2"
|
||||
|
||||
case "${arg_name}" in
|
||||
USE_SCCACHE)
|
||||
printf '%s\n' "${use_sccache}"
|
||||
;;
|
||||
*)
|
||||
extract_arg_default "${arg_name}"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
hash_rocm_base_arg_values() {
|
||||
local use_sccache="$1"
|
||||
local base_image_digest="$2"
|
||||
local arg_name=""
|
||||
local arg_value=""
|
||||
shift 2 || true
|
||||
|
||||
for arg_name in "$@"; do
|
||||
[[ -n "${arg_name}" ]] || continue
|
||||
arg_value=$(resolve_rocm_base_arg_value "${arg_name}" "${use_sccache}")
|
||||
printf 'arg:%s=%s\n' "${arg_name}" "${arg_value:-<empty>}"
|
||||
if [[ "${arg_name}" == "BASE_IMAGE" && -n "${arg_value}" ]]; then
|
||||
printf 'arg:%s.digest=%s\n' "${arg_name}" "${base_image_digest:-unknown}"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
rocm_version_from_base_image() {
|
||||
local base_image="$1"
|
||||
local version=""
|
||||
|
||||
version="$(sed -n -E 's/.*:([0-9]+\.[0-9]+(\.[0-9]+)?)-.*/\1/p' <<<"${base_image}")"
|
||||
tag_component "${version:-${base_image}}" 16
|
||||
}
|
||||
|
||||
git_diff_changed_base() {
|
||||
local range="$1"
|
||||
[[ -n "$(git diff --name-only "${range}" -- "${DOCKERFILE}" 2>/dev/null)" ]]
|
||||
}
|
||||
|
||||
short_git_ref() {
|
||||
local ref="$1"
|
||||
|
||||
git rev-parse --short "${ref}" 2>/dev/null || printf '%s\n' "${ref}"
|
||||
}
|
||||
|
||||
extract_arg_default_from_ref() {
|
||||
local ref="$1"
|
||||
local arg_name="$2"
|
||||
local content=""
|
||||
|
||||
content="$(git show "${ref}:${DOCKERFILE}" 2>/dev/null || true)"
|
||||
sed -n -E "s/^[[:space:]]*ARG[[:space:]]+${arg_name}=\"?([^\"[:space:]]+)\"?.*/\\1/p" \
|
||||
<<<"${content}" | head -1
|
||||
}
|
||||
|
||||
log_arg_default_changes() {
|
||||
local old_ref="$1"
|
||||
local new_ref="$2"
|
||||
local content_args="${ROCM_BASE_CONTENT_ARGS:-${DEFAULT_ROCM_BASE_CONTENT_ARGS}}"
|
||||
local arg_name=""
|
||||
local old_value=""
|
||||
local new_value=""
|
||||
local changed=0
|
||||
|
||||
echo "Changed ROCm base ARG defaults:"
|
||||
for arg_name in ${content_args}; do
|
||||
old_value="$(extract_arg_default_from_ref "${old_ref}" "${arg_name}")"
|
||||
new_value="$(extract_arg_default_from_ref "${new_ref}" "${arg_name}")"
|
||||
if [[ "${old_value}" != "${new_value}" ]]; then
|
||||
echo " - ${arg_name}: ${old_value:-<unset>} -> ${new_value:-<unset>}"
|
||||
changed=1
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "${changed}" == "0" ]]; then
|
||||
echo " - none detected; Dockerfile instructions changed outside tracked ARG defaults"
|
||||
fi
|
||||
}
|
||||
|
||||
log_arg_line_diff() {
|
||||
local range="$1"
|
||||
local arg_diff=""
|
||||
|
||||
arg_diff="$(
|
||||
git diff --unified=0 "${range}" -- "${DOCKERFILE}" 2>/dev/null \
|
||||
| awk '/^[+-][[:space:]]*ARG[[:space:]]/ && $0 !~ /^(---|\+\+\+)/ { print " " $0 }' \
|
||||
|| true
|
||||
)"
|
||||
|
||||
if [[ -n "${arg_diff}" ]]; then
|
||||
echo "Changed Dockerfile ARG lines:"
|
||||
printf '%s\n' "${arg_diff}"
|
||||
fi
|
||||
}
|
||||
|
||||
log_rocm_base_change_check() {
|
||||
local context="$1"
|
||||
local range="$2"
|
||||
local old_ref="$3"
|
||||
local old_short=""
|
||||
local head_short=""
|
||||
|
||||
old_short="$(short_git_ref "${old_ref}")"
|
||||
head_short="$(short_git_ref HEAD)"
|
||||
|
||||
echo "--- :mag: ROCm base refresh check"
|
||||
echo "Context: ${context}"
|
||||
echo "Dockerfile: ${DOCKERFILE}"
|
||||
echo "Base revision: ${old_short}"
|
||||
echo "Head revision: ${head_short}"
|
||||
echo "Git diff range: ${range}"
|
||||
}
|
||||
|
||||
log_rocm_base_rebuild_reason() {
|
||||
local context="$1"
|
||||
local range="$2"
|
||||
local old_ref="$3"
|
||||
local changed_files=""
|
||||
|
||||
log_rocm_base_change_check "${context}" "${range}" "${old_ref}"
|
||||
|
||||
changed_files="$(git diff --name-only "${range}" -- "${DOCKERFILE}" 2>/dev/null || true)"
|
||||
echo "Changed files:"
|
||||
if [[ -n "${changed_files}" ]]; then
|
||||
sed 's/^/ - /' <<<"${changed_files}"
|
||||
else
|
||||
echo " - ${DOCKERFILE}"
|
||||
fi
|
||||
log_arg_default_changes "${old_ref}" HEAD
|
||||
log_arg_line_diff "${range}"
|
||||
echo "Decision: rebuilding ROCm base image because ${DOCKERFILE} changed."
|
||||
}
|
||||
|
||||
rocm_base_changed_in_range() {
|
||||
local context="$1"
|
||||
local range="$2"
|
||||
local old_ref="$3"
|
||||
|
||||
if git_diff_changed_base "${range}"; then
|
||||
log_rocm_base_rebuild_reason "${context}" "${range}" "${old_ref}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
log_rocm_base_change_check "${context}" "${range}" "${old_ref}"
|
||||
echo "Decision: ROCm base refresh not required; ${DOCKERFILE} is unchanged."
|
||||
return 1
|
||||
}
|
||||
|
||||
rocm_base_changed() {
|
||||
local base_branch="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}"
|
||||
local base_ref="refs/remotes/origin/${base_branch}"
|
||||
local merge_base=""
|
||||
|
||||
if [[ "${ROCM_BASE_REFRESH_SKIP:-0}" == "1" ]]; then
|
||||
echo "ROCM_BASE_REFRESH_SKIP=1 set; skipping ROCm base refresh"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ "${ROCM_BASE_REFRESH_FORCE:-0}" == "1" ]]; then
|
||||
echo "ROCM_BASE_REFRESH_FORCE=1 set; refreshing ROCm base image"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if ! git rev-parse --is-inside-work-tree >/dev/null 2>&1; then
|
||||
echo "Not in a git checkout; skipping ROCm base refresh unless forced"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" ]]; then
|
||||
git fetch --no-tags --depth=200 origin \
|
||||
"+refs/heads/${base_branch}:${base_ref}" >/dev/null 2>&1 || true
|
||||
merge_base=$(git merge-base HEAD "${base_ref}" 2>/dev/null || true)
|
||||
if [[ -z "${merge_base}" ]]; then
|
||||
echo "Unable to determine merge base with PR base ${base_ref}; skipping ROCm base refresh unless forced"
|
||||
return 1
|
||||
fi
|
||||
if rocm_base_changed_in_range \
|
||||
"pull request build against ${base_ref}" \
|
||||
"${merge_base}...HEAD" \
|
||||
"${merge_base}"; then
|
||||
return 0
|
||||
fi
|
||||
elif [[ "${BUILDKITE_BRANCH:-}" == "${ROCM_BASE_STABLE_BRANCH:-main}" ]] \
|
||||
&& git rev-parse --verify HEAD~1 >/dev/null 2>&1; then
|
||||
if rocm_base_changed_in_range \
|
||||
"stable branch build; comparing against previous ${ROCM_BASE_STABLE_BRANCH:-main} commit" \
|
||||
"HEAD~1..HEAD" \
|
||||
"HEAD~1"; then
|
||||
return 0
|
||||
fi
|
||||
else
|
||||
git fetch --no-tags --depth=200 origin \
|
||||
"+refs/heads/${base_branch}:${base_ref}" >/dev/null 2>&1 || true
|
||||
merge_base=$(git merge-base HEAD "${base_ref}" 2>/dev/null || true)
|
||||
if [[ -z "${merge_base}" ]]; then
|
||||
echo "Unable to determine merge base with branch base ${base_ref}; skipping ROCm base refresh unless forced"
|
||||
return 1
|
||||
fi
|
||||
if rocm_base_changed_in_range \
|
||||
"branch build against ${base_ref}" \
|
||||
"${merge_base}...HEAD" \
|
||||
"${merge_base}"; then
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
should_push_stable_tag() {
|
||||
if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" ]]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ "${ROCM_BASE_PUSH_STABLE_TAG:-}" == "1" ]]; then
|
||||
return 0
|
||||
fi
|
||||
if [[ "${ROCM_BASE_PUSH_STABLE_TAG:-}" == "0" ]]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
[[ "${BUILDKITE_PULL_REQUEST:-false}" == "false" \
|
||||
&& "${BUILDKITE_BRANCH:-}" == "${ROCM_BASE_STABLE_BRANCH:-main}" ]]
|
||||
}
|
||||
|
||||
setup_builder() {
|
||||
echo "--- :buildkite: Setting up buildx builder for ROCm base"
|
||||
if docker buildx inspect "${BUILDER_NAME}" >/dev/null 2>&1; then
|
||||
docker buildx use "${BUILDER_NAME}"
|
||||
else
|
||||
docker buildx create --name "${BUILDER_NAME}" --driver docker-container --use
|
||||
fi
|
||||
docker buildx inspect --bootstrap
|
||||
}
|
||||
|
||||
compute_base_content_hash() {
|
||||
local use_sccache="$1"
|
||||
local base_image_digest="$2"
|
||||
local content_files="${ROCM_BASE_CONTENT_FILES:-${DEFAULT_ROCM_BASE_CONTENT_FILES}}"
|
||||
local content_args="${ROCM_BASE_CONTENT_ARGS:-${DEFAULT_ROCM_BASE_CONTENT_ARGS}}"
|
||||
local -a content_paths=()
|
||||
local -a content_arg_names=()
|
||||
|
||||
read -r -a content_paths <<< "${content_files}"
|
||||
read -r -a content_arg_names <<< "${content_args}"
|
||||
|
||||
{
|
||||
printf 'content-files-hash:%s\n' "$(compute_content_hash "${content_paths[@]}")"
|
||||
printf 'dockerfile:%s\n' "${DOCKERFILE}"
|
||||
printf 'resolved-build-args:\n'
|
||||
hash_rocm_base_arg_values \
|
||||
"${use_sccache}" "${base_image_digest}" "${content_arg_names[@]}"
|
||||
} | sha256sum | cut -d' ' -f1
|
||||
}
|
||||
|
||||
build_base_image() {
|
||||
local use_sccache="${ROCM_BASE_USE_SCCACHE:-${USE_SCCACHE:-0}}"
|
||||
local base_hash=""
|
||||
local build_date=""
|
||||
local build_suffix=""
|
||||
local base_image_arg=""
|
||||
local base_image_digest=""
|
||||
local rocm_version=""
|
||||
local triton_arg=""
|
||||
local pytorch_arg=""
|
||||
local pytorch_vision_arg=""
|
||||
local pytorch_audio_arg=""
|
||||
local fa_arg=""
|
||||
local aiter_arg=""
|
||||
local mori_arg=""
|
||||
local python_version_arg=""
|
||||
local pytorch_rocm_arch_arg=""
|
||||
local pytorch_branch=""
|
||||
local aiter_branch=""
|
||||
local dependency_summary=""
|
||||
local descriptor=""
|
||||
local ci_descriptor=""
|
||||
local descriptive_tag=""
|
||||
local stable_tag="${BASE_REPO}:base"
|
||||
local ci_descriptive_tag=""
|
||||
local content_files="${ROCM_BASE_CONTENT_FILES:-${DEFAULT_ROCM_BASE_CONTENT_FILES}}"
|
||||
local content_args="${ROCM_BASE_CONTENT_ARGS:-${DEFAULT_ROCM_BASE_CONTENT_ARGS}}"
|
||||
local content_files_hash=""
|
||||
local metadata_version="${ROCM_BASE_METADATA_VERSION:-${DEFAULT_ROCM_BASE_METADATA_VERSION}}"
|
||||
local -a tags=()
|
||||
local -a no_cache_args=()
|
||||
local -a sccache_args=()
|
||||
local -a content_paths=()
|
||||
|
||||
if [[ ! -f "${DOCKERFILE}" ]]; then
|
||||
echo "Error: ROCm base Dockerfile not found: ${DOCKERFILE}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
build_date="${ROCM_BASE_TAG_DATE:-$(date -u +%Y%m%d)}"
|
||||
if [[ -n "${BUILDKITE_BUILD_NUMBER:-}" ]]; then
|
||||
build_suffix="_bk_${BUILDKITE_BUILD_NUMBER}"
|
||||
fi
|
||||
base_image_arg="$(extract_arg_default BASE_IMAGE)"
|
||||
base_image_digest="$(resolve_image_digest "${base_image_arg}")"
|
||||
read -r -a content_paths <<< "${content_files}"
|
||||
content_files_hash="$(compute_content_hash "${content_paths[@]}")"
|
||||
base_hash=$(compute_base_content_hash "${use_sccache}" "${base_image_digest}")
|
||||
rocm_version="$(rocm_version_from_base_image "${base_image_arg}")"
|
||||
triton_arg="$(extract_arg_default TRITON_BRANCH)"
|
||||
pytorch_arg="$(extract_arg_default PYTORCH_BRANCH)"
|
||||
pytorch_vision_arg="$(extract_arg_default PYTORCH_VISION_BRANCH)"
|
||||
pytorch_audio_arg="$(extract_arg_default PYTORCH_AUDIO_BRANCH)"
|
||||
fa_arg="$(extract_arg_default FA_BRANCH)"
|
||||
aiter_arg="$(extract_arg_default AITER_BRANCH)"
|
||||
mori_arg="$(extract_arg_default MORI_BRANCH)"
|
||||
python_version_arg="$(extract_arg_default PYTHON_VERSION)"
|
||||
pytorch_rocm_arch_arg="$(extract_arg_default PYTORCH_ROCM_ARCH)"
|
||||
pytorch_branch="$(tag_component "${pytorch_arg}" 16)"
|
||||
aiter_branch="$(tag_component "${aiter_arg}" 24)"
|
||||
dependency_summary="base=${base_image_arg},rocm=${rocm_version},python=${python_version_arg},pytorch=${pytorch_arg},torchvision=${pytorch_vision_arg},torchaudio=${pytorch_audio_arg},triton=${triton_arg},flash-attn=${fa_arg},aiter=${aiter_arg},mori=${mori_arg},pytorch-rocm-arch=${pytorch_rocm_arch_arg}"
|
||||
descriptor="$(clean_docker_tag "base_custom_aiter_${aiter_branch}_torch_${pytorch_branch}_${build_date}${build_suffix}")"
|
||||
ci_descriptor="$(clean_docker_tag "ci_custom_aiter_${aiter_branch}_torch_${pytorch_branch}_${build_date}${build_suffix}")"
|
||||
|
||||
descriptive_tag="${BASE_REPO}:${descriptor}"
|
||||
ci_descriptive_tag="${CI_IMAGE_REPO}:${ci_descriptor}"
|
||||
|
||||
tags=(-t "${descriptive_tag}")
|
||||
if should_push_stable_tag; then
|
||||
tags+=(-t "${stable_tag}")
|
||||
metadata_set "rocm-base-push-stable-tag" "1"
|
||||
else
|
||||
metadata_set "rocm-base-push-stable-tag" "0"
|
||||
fi
|
||||
|
||||
if [[ "${ROCM_BASE_NO_CACHE:-1}" == "1" ]]; then
|
||||
no_cache_args=(--no-cache)
|
||||
fi
|
||||
|
||||
for env_name in \
|
||||
SCCACHE_DOWNLOAD_URL \
|
||||
SCCACHE_ENDPOINT \
|
||||
SCCACHE_BUCKET_NAME \
|
||||
SCCACHE_REGION_NAME \
|
||||
SCCACHE_S3_NO_CREDENTIALS; do
|
||||
if [[ -n "${!env_name:-}" ]]; then
|
||||
sccache_args+=(--build-arg "${env_name}=${!env_name}")
|
||||
fi
|
||||
done
|
||||
|
||||
echo "--- :docker: Building ROCm base image"
|
||||
echo "Dockerfile: ${DOCKERFILE}"
|
||||
echo "Descriptive tag: ${descriptive_tag}"
|
||||
echo "Stable tag: ${stable_tag} ($(should_push_stable_tag && echo enabled || echo disabled))"
|
||||
echo "Content hash: ${base_hash}"
|
||||
echo "Dependency summary: ${dependency_summary}"
|
||||
echo "USE_SCCACHE: ${use_sccache}"
|
||||
|
||||
docker buildx build \
|
||||
"${no_cache_args[@]}" \
|
||||
--pull \
|
||||
--progress "${BUILDKIT_PROGRESS:-plain}" \
|
||||
--file "${DOCKERFILE}" \
|
||||
--build-arg "USE_SCCACHE=${use_sccache}" \
|
||||
"${sccache_args[@]}" \
|
||||
--label "org.opencontainers.image.source=https://github.com/vllm-project/vllm" \
|
||||
--label "org.opencontainers.image.vendor=vLLM" \
|
||||
--label "org.opencontainers.image.title=vLLM ROCm base" \
|
||||
--label "org.opencontainers.image.revision=${BUILDKITE_COMMIT:-}" \
|
||||
--label "vllm.rocm_base.metadata_version=${metadata_version}" \
|
||||
--label "vllm.rocm_base.content_hash=${base_hash}" \
|
||||
--label "vllm.rocm_base.content_files_hash=${content_files_hash}" \
|
||||
--label "vllm.rocm_base.dockerfile=${DOCKERFILE}" \
|
||||
--label "vllm.rocm_base.image.descriptive=${descriptive_tag}" \
|
||||
--label "vllm.rocm_base.image.stable=${stable_tag}" \
|
||||
--label "vllm.rocm_base.git_commit=${BUILDKITE_COMMIT:-}" \
|
||||
--label "vllm.rocm_base.stable_branch=${ROCM_BASE_STABLE_BRANCH:-main}" \
|
||||
--label "vllm.rocm_base.descriptor=${descriptor}" \
|
||||
--label "vllm.rocm_base.dependency_summary=${dependency_summary}" \
|
||||
--label "vllm.rocm_base.base_image=${base_image_arg}" \
|
||||
--label "vllm.rocm_base.base_image_digest=${base_image_digest}" \
|
||||
--label "vllm.rocm_base.dependency.rocm=${rocm_version}" \
|
||||
--label "vllm.rocm_base.dependency.python=${python_version_arg}" \
|
||||
--label "vllm.rocm_base.dependency.pytorch=${pytorch_arg}" \
|
||||
--label "vllm.rocm_base.dependency.torchvision=${pytorch_vision_arg}" \
|
||||
--label "vllm.rocm_base.dependency.torchaudio=${pytorch_audio_arg}" \
|
||||
--label "vllm.rocm_base.dependency.triton=${triton_arg}" \
|
||||
--label "vllm.rocm_base.dependency.flash_attention=${fa_arg}" \
|
||||
--label "vllm.rocm_base.dependency.aiter=${aiter_arg}" \
|
||||
--label "vllm.rocm_base.dependency.mori=${mori_arg}" \
|
||||
--label "vllm.rocm_base.pytorch_rocm_arch=${pytorch_rocm_arch_arg}" \
|
||||
"${tags[@]}" \
|
||||
--push \
|
||||
.
|
||||
|
||||
docker buildx imagetools inspect "${descriptive_tag}" >/dev/null
|
||||
|
||||
metadata_set "rocm-base-refresh" "1"
|
||||
metadata_set "rocm-base-image" "${descriptive_tag}"
|
||||
metadata_set "rocm-base-image-descriptive" "${descriptive_tag}"
|
||||
metadata_set "rocm-base-image-stable" "${stable_tag}"
|
||||
metadata_set "rocm-base-image-ci-descriptive" "${ci_descriptive_tag}"
|
||||
metadata_set "rocm-base-metadata-version" "${metadata_version}"
|
||||
metadata_set "rocm-base-content-hash" "${base_hash}"
|
||||
metadata_set "rocm-base-content-files-hash" "${content_files_hash}"
|
||||
metadata_set "rocm-base-content-files" "${content_files}"
|
||||
metadata_set "rocm-base-content-args" "${content_args}"
|
||||
metadata_set "rocm-base-base-image-digest" "${base_image_digest}"
|
||||
metadata_set "rocm-base-dockerfile" "${DOCKERFILE}"
|
||||
metadata_set "rocm-base-descriptor" "${descriptor}"
|
||||
metadata_set "rocm-base-dependency-summary" "${dependency_summary}"
|
||||
metadata_set "rocm-base-dependency-rocm" "${rocm_version}"
|
||||
metadata_set "rocm-base-dependency-python" "${python_version_arg}"
|
||||
metadata_set "rocm-base-dependency-pytorch" "${pytorch_arg}"
|
||||
metadata_set "rocm-base-dependency-torchvision" "${pytorch_vision_arg}"
|
||||
metadata_set "rocm-base-dependency-torchaudio" "${pytorch_audio_arg}"
|
||||
metadata_set "rocm-base-dependency-triton" "${triton_arg}"
|
||||
metadata_set "rocm-base-dependency-flash-attention" "${fa_arg}"
|
||||
metadata_set "rocm-base-dependency-aiter" "${aiter_arg}"
|
||||
metadata_set "rocm-base-dependency-mori" "${mori_arg}"
|
||||
metadata_set "rocm-base-pytorch-rocm-arch" "${pytorch_rocm_arch_arg}"
|
||||
metadata_set "rocm-ci-image-descriptive" "${ci_descriptive_tag}"
|
||||
|
||||
echo "--- :white_check_mark: ROCm base image published"
|
||||
echo "Use BASE_IMAGE=${descriptive_tag} for downstream ROCm CI builds"
|
||||
}
|
||||
|
||||
main() {
|
||||
metadata_set "rocm-base-refresh" "0"
|
||||
|
||||
if ! rocm_base_changed; then
|
||||
echo "ROCm base Dockerfile did not change; skipping base image refresh"
|
||||
return 0
|
||||
fi
|
||||
|
||||
setup_builder
|
||||
build_base_image
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -1,32 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Fast structural smoke test for the full ROCm CI image.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
image_ref="${VLLM_CI_SMOKE_IMAGE:-rocm/vllm-ci:${BUILDKITE_COMMIT:?BUILDKITE_COMMIT is required}}"
|
||||
|
||||
docker run --rm --network=none --entrypoint /bin/bash "${image_ref}" -ec '
|
||||
if [ ! -d /vllm-workspace ]; then echo Missing directory: /vllm-workspace >&2; exit 1; fi
|
||||
if [ ! -d /vllm-workspace/tests ]; then echo Missing directory: /vllm-workspace/tests >&2; exit 1; fi
|
||||
if [ ! -d /vllm-workspace/src/vllm ]; then echo Missing directory: /vllm-workspace/src/vllm >&2; exit 1; fi
|
||||
if [ ! -x /vllm-workspace/src/vllm/vllm-rs ]; then echo Missing executable: /vllm-workspace/src/vllm/vllm-rs >&2; exit 1; fi
|
||||
|
||||
command -v python3
|
||||
command -v uv
|
||||
command -v pytest
|
||||
|
||||
if ! command -v amd-smi >/dev/null 2>&1 && ! command -v rocminfo >/dev/null 2>&1; then
|
||||
echo No ROCm CLI found in image >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
python3 - <<PY
|
||||
import torch
|
||||
import vllm
|
||||
|
||||
print(torch.__version__)
|
||||
print(vllm.__version__)
|
||||
PY
|
||||
|
||||
echo AMD image smoke OK
|
||||
'
|
||||
@@ -109,9 +109,7 @@ run_nodes() {
|
||||
if [ "$node" -ne 0 ]; then
|
||||
docker exec -d "node$node" /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}"
|
||||
else
|
||||
# Allocate a TTY (-t -i) for the foreground head node so its output
|
||||
# keeps ANSI color in the Buildkite log (see run-amd-test.sh).
|
||||
docker exec -t -i "node$node" /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}"
|
||||
docker exec "node$node" /bin/bash -c "cd $WORKING_DIR ; ${COMMANDS[$node]}"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
@@ -8,12 +8,7 @@ if [[ "$MODE" != "style-clippy" && "$MODE" != "test" ]]; then
|
||||
exit 2
|
||||
fi
|
||||
|
||||
if ROOT_DIR="$(git rev-parse --show-toplevel 2>/dev/null)"; then
|
||||
:
|
||||
else
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
|
||||
ROOT_DIR="$(cd -- "${SCRIPT_DIR}/../.." && pwd -P)"
|
||||
fi
|
||||
ROOT_DIR="$(git rev-parse --show-toplevel)"
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
export CARGO_TERM_COLOR="${CARGO_TERM_COLOR:-always}"
|
||||
@@ -21,20 +16,16 @@ export CARGO_HOME="${CARGO_HOME:-$HOME/.cargo}"
|
||||
export RUSTUP_HOME="${RUSTUP_HOME:-$HOME/.rustup}"
|
||||
export PATH="$CARGO_HOME/bin:$PATH"
|
||||
|
||||
PROTOC_VERSION="${PROTOC_VERSION:-31.1}"
|
||||
CARGO_BINSTALL_VERSION="${CARGO_BINSTALL_VERSION:-1.20.1}"
|
||||
UV_VERSION="${UV_VERSION:-0.11.28}"
|
||||
PYO3_PYTHON_VERSION="${PYO3_PYTHON_VERSION:-3.12}"
|
||||
|
||||
CARGO_SORT_VERSION_REQ="${CARGO_SORT_VERSION_REQ:-2}"
|
||||
CARGO_DENY_VERSION_REQ="${CARGO_DENY_VERSION_REQ:-0.20}"
|
||||
CARGO_NEXTEST_VERSION_REQ="${CARGO_NEXTEST_VERSION_REQ:-0.9}"
|
||||
|
||||
log_section() {
|
||||
echo "--- $*"
|
||||
}
|
||||
|
||||
install_protoc() {
|
||||
if command -v protoc >/dev/null 2>&1; then
|
||||
return
|
||||
fi
|
||||
|
||||
local version="${PROTOC_VERSION:-31.1}"
|
||||
local arch
|
||||
case "$(uname -m)" in
|
||||
x86_64)
|
||||
@@ -49,17 +40,16 @@ install_protoc() {
|
||||
;;
|
||||
esac
|
||||
|
||||
local url="https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-${arch}.zip"
|
||||
local url="https://github.com/protocolbuffers/protobuf/releases/download/v${version}/protoc-${version}-linux-${arch}.zip"
|
||||
local tmp_dir
|
||||
tmp_dir="$(mktemp -d)"
|
||||
|
||||
log_section "Installing protoc ${PROTOC_VERSION}"
|
||||
log_section "Installing protoc ${version}"
|
||||
curl -L --proto '=https' --tlsv1.2 -sSf "$url" -o "$tmp_dir/protoc.zip"
|
||||
mkdir -p "$CARGO_HOME/bin"
|
||||
unzip -q "$tmp_dir/protoc.zip" bin/protoc 'include/*' -d "$CARGO_HOME"
|
||||
chmod +x "$CARGO_HOME/bin/protoc"
|
||||
rm -rf "$tmp_dir"
|
||||
protoc --version
|
||||
}
|
||||
|
||||
rust_toolchain() {
|
||||
@@ -80,48 +70,56 @@ install_rust_toolchain() {
|
||||
}
|
||||
|
||||
install_cargo_binstall() {
|
||||
log_section "Installing cargo-binstall ${CARGO_BINSTALL_VERSION}"
|
||||
if command -v cargo-binstall >/dev/null 2>&1; then
|
||||
return
|
||||
fi
|
||||
|
||||
log_section "Installing cargo-binstall"
|
||||
curl -L --proto '=https' --tlsv1.2 -sSf \
|
||||
"https://raw.githubusercontent.com/cargo-bins/cargo-binstall/v${CARGO_BINSTALL_VERSION}/install-from-binstall-release.sh" \
|
||||
| env BINSTALL_VERSION="$CARGO_BINSTALL_VERSION" bash
|
||||
cargo-binstall -V
|
||||
https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh \
|
||||
| bash
|
||||
}
|
||||
|
||||
install_cargo_sort() {
|
||||
log_section "Installing cargo-sort ${CARGO_SORT_VERSION_REQ}"
|
||||
cargo binstall --no-confirm --force "cargo-sort@${CARGO_SORT_VERSION_REQ}"
|
||||
}
|
||||
if command -v cargo-sort >/dev/null 2>&1; then
|
||||
return
|
||||
fi
|
||||
|
||||
install_cargo_deny() {
|
||||
log_section "Installing cargo-deny ${CARGO_DENY_VERSION_REQ}"
|
||||
cargo binstall --no-confirm --force "cargo-deny@${CARGO_DENY_VERSION_REQ}"
|
||||
log_section "Installing cargo-sort"
|
||||
install_cargo_binstall
|
||||
cargo binstall --no-confirm cargo-sort
|
||||
}
|
||||
|
||||
install_cargo_nextest() {
|
||||
log_section "Installing cargo-nextest ${CARGO_NEXTEST_VERSION_REQ}"
|
||||
cargo binstall \
|
||||
--no-confirm \
|
||||
--force \
|
||||
--secure \
|
||||
"cargo-nextest@${CARGO_NEXTEST_VERSION_REQ}"
|
||||
if command -v cargo-nextest >/dev/null 2>&1; then
|
||||
return
|
||||
fi
|
||||
|
||||
log_section "Installing cargo-nextest"
|
||||
install_cargo_binstall
|
||||
cargo binstall --no-confirm --secure cargo-nextest
|
||||
}
|
||||
|
||||
install_uv() {
|
||||
log_section "Installing uv ${UV_VERSION}"
|
||||
curl -L --proto '=https' --tlsv1.2 -sSf \
|
||||
"https://github.com/astral-sh/uv/releases/download/${UV_VERSION}/uv-installer.sh" \
|
||||
if command -v uv >/dev/null 2>&1; then
|
||||
return
|
||||
fi
|
||||
|
||||
log_section "Installing uv"
|
||||
curl -LsSf --proto '=https' --tlsv1.2 https://astral.sh/uv/install.sh \
|
||||
| env UV_INSTALL_DIR="$CARGO_HOME/bin" sh
|
||||
uv --version
|
||||
}
|
||||
|
||||
setup_pyo3_python() {
|
||||
log_section "Installing Python ${PYO3_PYTHON_VERSION} for PyO3 tests"
|
||||
uv python install "$PYO3_PYTHON_VERSION"
|
||||
local python_version="${PYO3_PYTHON_VERSION:-3.12}"
|
||||
|
||||
log_section "Installing Python ${python_version} for PyO3 tests"
|
||||
uv python install "$python_version"
|
||||
PYO3_PYTHON="$(uv python find \
|
||||
--managed-python \
|
||||
--no-project \
|
||||
--resolve-links \
|
||||
"$PYO3_PYTHON_VERSION")"
|
||||
"$python_version")"
|
||||
export PYO3_PYTHON
|
||||
|
||||
local python_libdir
|
||||
@@ -143,9 +141,7 @@ PY
|
||||
}
|
||||
|
||||
run_style_clippy() {
|
||||
install_cargo_binstall
|
||||
install_cargo_sort
|
||||
install_cargo_deny
|
||||
|
||||
log_section "Checking Rust formatting"
|
||||
cargo fmt --manifest-path rust/Cargo.toml --all -- --check
|
||||
@@ -153,13 +149,6 @@ run_style_clippy() {
|
||||
log_section "Checking Cargo.toml ordering"
|
||||
cargo sort --workspace --check rust
|
||||
|
||||
log_section "Checking Rust dependency bans"
|
||||
cargo deny \
|
||||
--manifest-path rust/Cargo.toml \
|
||||
--config rust/deny.toml \
|
||||
check \
|
||||
bans
|
||||
|
||||
log_section "Running clippy"
|
||||
cargo clippy \
|
||||
--manifest-path rust/Cargo.toml \
|
||||
@@ -174,7 +163,6 @@ run_style_clippy() {
|
||||
run_tests() {
|
||||
install_uv
|
||||
setup_pyo3_python
|
||||
install_cargo_binstall
|
||||
install_cargo_nextest
|
||||
|
||||
log_section "Running cargo nextest"
|
||||
|
||||
@@ -33,14 +33,6 @@ if [[ -n "${ATTENTION_BACKEND:-}" ]]; then
|
||||
EXTRA_ARGS+=(--attention-backend "${ATTENTION_BACKEND}")
|
||||
fi
|
||||
|
||||
# ROCm: run eager to avoid intermittent HIP-graph decode corruption.
|
||||
# See https://github.com/ROCm/clr/issues/279
|
||||
# TODO(aarushjain29): Revert after TheRock 7.14
|
||||
if command -v rocm-smi &> /dev/null || command -v amd-smi &> /dev/null || [[ -d /opt/rocm ]] || [[ -n "${ROCM_PATH:-}" ]]; then
|
||||
echo "ROCm platform detected: adding --enforce-eager to avoid HIP-graph decode corruption"
|
||||
EXTRA_ARGS+=(--enforce-eager)
|
||||
fi
|
||||
|
||||
cleanup() {
|
||||
if [[ -n "${SERVER_PID:-}" ]] && kill -0 "${SERVER_PID}" 2>/dev/null; then
|
||||
kill "${SERVER_PID}" 2>/dev/null || true
|
||||
|
||||
@@ -18,10 +18,6 @@ wait_for_server() {
|
||||
|
||||
MODEL="Qwen/Qwen3-30B-A3B-FP8"
|
||||
BACK="allgather_reducescatter"
|
||||
if command -v rocm-smi &> /dev/null || [[ -d /opt/rocm ]] || [[ -n "${ROCM_PATH:-}" ]]; then
|
||||
# Disable MOE padding for ROCm since it is causing eplb to fail.
|
||||
export VLLM_ROCM_MOE_PADDING=0
|
||||
fi
|
||||
|
||||
cleanup() {
|
||||
if [[ -n "${SERVER_PID:-}" ]] && kill -0 "${SERVER_PID}" 2>/dev/null; then
|
||||
|
||||
@@ -23,7 +23,6 @@ NC='\033[0m' # No Color
|
||||
# Default configuration
|
||||
PIPELINE="ci"
|
||||
DRY_RUN=true
|
||||
TORCH_NIGHTLY=false
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
@@ -35,14 +34,12 @@ Sets RUN_ALL=1 and NIGHTLY=1 environment variables.
|
||||
SAFETY: Dry-run by default. Use --execute to actually trigger a build.
|
||||
|
||||
Options:
|
||||
--execute Actually trigger the build (default: dry-run)
|
||||
--pipeline Buildkite pipeline slug (default: ${PIPELINE})
|
||||
--commit Override commit SHA (default: current HEAD)
|
||||
--branch Override branch name (default: current branch)
|
||||
--message Custom build message (default: auto-generated)
|
||||
--torch-nightly Also build and run the full suite against torch nightly
|
||||
(sets TORCH_NIGHTLY=1)
|
||||
--help Show this help message
|
||||
--execute Actually trigger the build (default: dry-run)
|
||||
--pipeline Buildkite pipeline slug (default: ${PIPELINE})
|
||||
--commit Override commit SHA (default: current HEAD)
|
||||
--branch Override branch name (default: current branch)
|
||||
--message Custom build message (default: auto-generated)
|
||||
--help Show this help message
|
||||
|
||||
Prerequisites:
|
||||
- bk CLI installed: brew tap buildkite/buildkite && brew install buildkite/buildkite/bk
|
||||
@@ -52,7 +49,6 @@ Examples:
|
||||
$(basename "$0") # Dry-run, show what would happen
|
||||
$(basename "$0") --execute # Actually trigger the build
|
||||
$(basename "$0") --pipeline ci-shadow # Dry-run with different pipeline
|
||||
$(basename "$0") --torch-nightly # Dry-run a full torch-nightly run
|
||||
EOF
|
||||
exit 1
|
||||
}
|
||||
@@ -100,10 +96,6 @@ while [[ $# -gt 0 ]]; do
|
||||
MESSAGE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--torch-nightly)
|
||||
TORCH_NIGHTLY=true
|
||||
shift
|
||||
;;
|
||||
--help|-h)
|
||||
usage
|
||||
;;
|
||||
@@ -179,17 +171,11 @@ if [[ $(echo "$REMOTE_BRANCHES" | wc -l) -gt 5 ]]; then
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Environment variables passed to the build.
|
||||
BUILD_ENV=("RUN_ALL=1" "NIGHTLY=1")
|
||||
if [[ "$TORCH_NIGHTLY" == true ]]; then
|
||||
BUILD_ENV+=("TORCH_NIGHTLY=1")
|
||||
fi
|
||||
|
||||
log_info "Pipeline: ${PIPELINE}"
|
||||
log_info "Branch: ${BRANCH}"
|
||||
log_info "Commit: ${COMMIT}"
|
||||
log_info "Message: ${MESSAGE}"
|
||||
log_info "Environment: ${BUILD_ENV[*]}"
|
||||
log_info "Environment: RUN_ALL=1, NIGHTLY=1"
|
||||
echo ""
|
||||
|
||||
# Build the command
|
||||
@@ -201,10 +187,9 @@ CMD=(bk build create
|
||||
--commit "${COMMIT}"
|
||||
--branch "${BRANCH}"
|
||||
--message "${MESSAGE}"
|
||||
--env "RUN_ALL=1"
|
||||
--env "NIGHTLY=1"
|
||||
)
|
||||
for env_var in "${BUILD_ENV[@]}"; do
|
||||
CMD+=(--env "${env_var}")
|
||||
done
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "=========================================="
|
||||
@@ -225,14 +210,8 @@ if [[ "$DRY_RUN" == true ]]; then
|
||||
echo " --commit '$(escape_for_shell "${COMMIT}")' \\"
|
||||
echo " --branch '$(escape_for_shell "${BRANCH}")' \\"
|
||||
echo " --message '$(escape_for_shell "${MESSAGE}")' \\"
|
||||
last_idx=$(( ${#BUILD_ENV[@]} - 1 ))
|
||||
for i in "${!BUILD_ENV[@]}"; do
|
||||
if [[ $i -eq $last_idx ]]; then
|
||||
echo " --env '$(escape_for_shell "${BUILD_ENV[$i]}")'"
|
||||
else
|
||||
echo " --env '$(escape_for_shell "${BUILD_ENV[$i]}")' \\"
|
||||
fi
|
||||
done
|
||||
echo " --env 'RUN_ALL=1' \\"
|
||||
echo " --env 'NIGHTLY=1'"
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo -e "${YELLOW}To actually trigger this build, run:${NC}"
|
||||
|
||||
@@ -6,14 +6,8 @@ set -ex
|
||||
# manylinux platform tag with auditwheel.
|
||||
# Index generation is handled separately by generate-and-upload-nightly-index.sh.
|
||||
|
||||
# auditwheel is Linux-only; macOS wheels already carry a valid tag, so skip the
|
||||
# manylinux retag for them.
|
||||
WHEEL_PLATFORM="${VLLM_WHEEL_PLATFORM:-linux}"
|
||||
|
||||
if [[ "$WHEEL_PLATFORM" == "linux" ]]; then
|
||||
# shellcheck source=lib/manylinux.sh
|
||||
source .buildkite/scripts/lib/manylinux.sh
|
||||
fi
|
||||
# shellcheck source=lib/manylinux.sh
|
||||
source .buildkite/scripts/lib/manylinux.sh
|
||||
|
||||
BUCKET="vllm-wheels"
|
||||
SUBPATH=$BUILDKITE_COMMIT
|
||||
@@ -33,10 +27,8 @@ wheel="${wheel_files[0]}"
|
||||
|
||||
# ========= detect manylinux tag and rename ==========
|
||||
|
||||
if [[ "$WHEEL_PLATFORM" == "linux" ]]; then
|
||||
wheel="$(apply_manylinux_tag "$wheel")"
|
||||
echo "Renamed wheel to: $wheel"
|
||||
fi
|
||||
wheel="$(apply_manylinux_tag "$wheel")"
|
||||
echo "Renamed wheel to: $wheel"
|
||||
|
||||
# Extract the version from the wheel
|
||||
version=$(unzip -p "$wheel" '**/METADATA' | grep '^Version: ' | cut -d' ' -f2)
|
||||
|
||||
@@ -113,8 +113,8 @@ $PYTHON .buildkite/scripts/generate-nightly-index.py \
|
||||
echo "Uploading indices to $S3_COMMIT_PREFIX"
|
||||
aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "$S3_COMMIT_PREFIX"
|
||||
|
||||
# Only scheduled nightly builds should update the moving nightly index.
|
||||
if [[ "${NIGHTLY:-0}" == "1" ]]; then
|
||||
# Update rocm/nightly/ if on main branch and not a PR
|
||||
if [[ "$BUILDKITE_BRANCH" == "main" && "$BUILDKITE_PULL_REQUEST" == "false" ]] || [[ "$NIGHTLY" == "1" ]]; then
|
||||
echo "Updating rocm/nightly/ index..."
|
||||
aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "s3://$BUCKET/rocm/nightly/"
|
||||
fi
|
||||
@@ -147,7 +147,7 @@ echo ""
|
||||
echo "Install command (by commit):"
|
||||
echo " pip install vllm --extra-index-url https://${BUCKET}.s3.amazonaws.com/$ROCM_SUBPATH/"
|
||||
echo ""
|
||||
if [[ "${NIGHTLY:-0}" == "1" ]]; then
|
||||
if [[ "$BUILDKITE_BRANCH" == "main" ]] || [[ "$NIGHTLY" == "1" ]]; then
|
||||
echo "Install command (nightly):"
|
||||
echo " pip install vllm --extra-index-url https://${BUCKET}.s3.amazonaws.com/rocm/nightly/"
|
||||
fi
|
||||
|
||||
@@ -1,13 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
REGISTRY="public.ecr.aws/q9t5s3a7"
|
||||
REPO="vllm-release-repo"
|
||||
ARCH_TAG="${BUILDKITE_COMMIT}-$(uname -m)-xpu"
|
||||
PLATFORM_TAG="${BUILDKITE_COMMIT}-xpu"
|
||||
|
||||
aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin ${REGISTRY}
|
||||
docker manifest rm ${REGISTRY}/${REPO}:${PLATFORM_TAG} || true
|
||||
docker manifest create ${REGISTRY}/${REPO}:${PLATFORM_TAG} ${REGISTRY}/${REPO}:${ARCH_TAG} --amend
|
||||
docker manifest push ${REGISTRY}/${REPO}:${PLATFORM_TAG}
|
||||
@@ -1,21 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -ex
|
||||
|
||||
ORIG_TAG_NAME="$BUILDKITE_COMMIT"
|
||||
REPO="vllm/vllm-openai-xpu"
|
||||
|
||||
echo "Pushing original XPU tag ${ORIG_TAG_NAME}-xpu to nightly tags in ${REPO}"
|
||||
|
||||
aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
|
||||
docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:"$ORIG_TAG_NAME"-x86_64-xpu
|
||||
|
||||
docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:"$ORIG_TAG_NAME"-x86_64-xpu ${REPO}:nightly-x86_64
|
||||
docker push ${REPO}:nightly-x86_64
|
||||
|
||||
docker manifest rm ${REPO}:nightly || true
|
||||
docker manifest rm ${REPO}:nightly-"$BUILDKITE_COMMIT" || true
|
||||
docker manifest create ${REPO}:nightly ${REPO}:nightly-x86_64 --amend
|
||||
docker manifest create ${REPO}:nightly-"$BUILDKITE_COMMIT" ${REPO}:nightly-x86_64 --amend
|
||||
docker manifest push ${REPO}:nightly
|
||||
docker manifest push ${REPO}:nightly-"$BUILDKITE_COMMIT"
|
||||
+609
-1052
File diff suppressed because it is too large
Load Diff
@@ -4,7 +4,7 @@ depends_on:
|
||||
steps:
|
||||
- label: V1 attention (H100-MI300)
|
||||
key: v1-attention-h100-mi300
|
||||
timeout_in_minutes: 85
|
||||
timeout_in_minutes: 30
|
||||
device: h100
|
||||
source_file_dependencies:
|
||||
- vllm/config/attention.py
|
||||
@@ -12,13 +12,11 @@ steps:
|
||||
- vllm/v1/attention
|
||||
- tests/v1/attention
|
||||
commands:
|
||||
- pytest -v -s v1/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
|
||||
parallelism: 2
|
||||
- pytest -v -s v1/attention
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 125
|
||||
device: mi325_1
|
||||
timeout_in_minutes: 70
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
@@ -32,7 +30,7 @@ steps:
|
||||
|
||||
- label: V1 attention (B200)
|
||||
key: v1-attention-b200
|
||||
timeout_in_minutes: 80
|
||||
timeout_in_minutes: 30
|
||||
device: b200-k8s
|
||||
source_file_dependencies:
|
||||
- vllm/config/attention.py
|
||||
@@ -40,5 +38,4 @@ steps:
|
||||
- vllm/v1/attention
|
||||
- tests/v1/attention
|
||||
commands:
|
||||
- pytest -v -s v1/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
|
||||
parallelism: 2
|
||||
- pytest -v -s v1/attention
|
||||
|
||||
@@ -4,7 +4,7 @@ depends_on:
|
||||
steps:
|
||||
- label: Basic Correctness
|
||||
key: basic-correctness
|
||||
timeout_in_minutes: 68
|
||||
timeout_in_minutes: 30
|
||||
device: h200_18gb
|
||||
source_file_dependencies:
|
||||
- vllm/
|
||||
@@ -16,10 +16,3 @@ steps:
|
||||
- pytest -v -s basic_correctness/test_mem.py
|
||||
- pytest -v -s basic_correctness/test_basic_correctness.py
|
||||
- pytest -v -s basic_correctness/test_cpu_offload.py
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 60
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
|
||||
@@ -4,20 +4,13 @@ depends_on:
|
||||
steps:
|
||||
- label: Benchmarks CLI Test
|
||||
key: benchmarks-cli-test
|
||||
timeout_in_minutes: 45
|
||||
timeout_in_minutes: 20
|
||||
device: h200_18gb
|
||||
source_file_dependencies:
|
||||
- vllm/
|
||||
- tests/benchmarks/
|
||||
commands:
|
||||
- pytest -v -s benchmarks/
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 40
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
|
||||
- label: Attention Benchmarks Smoke Test (B200)
|
||||
key: attention-benchmarks-smoke-test-b200
|
||||
@@ -25,7 +18,7 @@ steps:
|
||||
num_gpus: 2
|
||||
optional: true
|
||||
working_dir: "/vllm-workspace/"
|
||||
timeout_in_minutes: 20
|
||||
timeout_in_minutes: 10
|
||||
source_file_dependencies:
|
||||
- benchmarks/attention_benchmarks/
|
||||
- vllm/v1/attention/
|
||||
|
||||
@@ -4,7 +4,7 @@ depends_on:
|
||||
steps:
|
||||
- label: Sequence Parallel Correctness Tests (2 GPUs)
|
||||
key: sequence-parallel-correctness-tests-2-gpus
|
||||
timeout_in_minutes: 80
|
||||
timeout_in_minutes: 50
|
||||
working_dir: "/vllm-workspace/"
|
||||
num_devices: 2
|
||||
source_file_dependencies:
|
||||
@@ -19,7 +19,7 @@ steps:
|
||||
|
||||
- label: Sequence Parallel Correctness Tests (2xH100)
|
||||
key: sequence-parallel-correctness-tests-2xh100
|
||||
timeout_in_minutes: 75
|
||||
timeout_in_minutes: 50
|
||||
working_dir: "/vllm-workspace/"
|
||||
device: h100
|
||||
optional: true
|
||||
@@ -30,7 +30,7 @@ steps:
|
||||
|
||||
- label: AsyncTP Correctness Tests (2xH100)
|
||||
key: asynctp-correctness-tests-2xh100
|
||||
timeout_in_minutes: 30
|
||||
timeout_in_minutes: 50
|
||||
working_dir: "/vllm-workspace/"
|
||||
device: h100
|
||||
optional: true
|
||||
@@ -41,7 +41,7 @@ steps:
|
||||
|
||||
- label: AsyncTP Correctness Tests (B200)
|
||||
key: asynctp-correctness-tests-b200
|
||||
timeout_in_minutes: 30
|
||||
timeout_in_minutes: 50
|
||||
working_dir: "/vllm-workspace/"
|
||||
device: b200-k8s
|
||||
optional: true
|
||||
@@ -52,7 +52,7 @@ steps:
|
||||
|
||||
- label: Distributed Compile Unit Tests (2xH100)
|
||||
key: distributed-compile-unit-tests-2xh100
|
||||
timeout_in_minutes: 45
|
||||
timeout_in_minutes: 20
|
||||
working_dir: "/vllm-workspace/"
|
||||
device: h100
|
||||
num_devices: 2
|
||||
@@ -66,7 +66,7 @@ steps:
|
||||
|
||||
- label: Fusion and Compile Unit Tests (2xB200)
|
||||
key: fusion-and-compile-unit-tests-2xb200
|
||||
timeout_in_minutes: 30
|
||||
timeout_in_minutes: 20
|
||||
working_dir: "/vllm-workspace/"
|
||||
device: b200-k8s
|
||||
source_file_dependencies:
|
||||
@@ -96,7 +96,7 @@ steps:
|
||||
|
||||
- label: Fusion E2E Quick (H100)
|
||||
key: fusion-e2e-quick-h100
|
||||
timeout_in_minutes: 25
|
||||
timeout_in_minutes: 15
|
||||
working_dir: "/vllm-workspace/"
|
||||
device: h100
|
||||
num_devices: 1
|
||||
@@ -115,7 +115,7 @@ steps:
|
||||
|
||||
- label: Fusion E2E Config Sweep (H100)
|
||||
key: fusion-e2e-config-sweep-h100
|
||||
timeout_in_minutes: 25
|
||||
timeout_in_minutes: 30
|
||||
working_dir: "/vllm-workspace/"
|
||||
device: h100
|
||||
num_devices: 1
|
||||
@@ -149,7 +149,7 @@ steps:
|
||||
|
||||
- label: Fusion E2E TP2 Quick (H100)
|
||||
key: fusion-e2e-tp2-quick-h100
|
||||
timeout_in_minutes: 35
|
||||
timeout_in_minutes: 20
|
||||
working_dir: "/vllm-workspace/"
|
||||
device: h100
|
||||
num_devices: 2
|
||||
@@ -167,7 +167,7 @@ steps:
|
||||
|
||||
- label: Fusion E2E TP2 AR-RMS Config Sweep (H100)
|
||||
key: fusion-e2e-tp2-ar-rms-config-sweep-h100
|
||||
timeout_in_minutes: 30
|
||||
timeout_in_minutes: 40
|
||||
working_dir: "/vllm-workspace/"
|
||||
device: h100
|
||||
num_devices: 2
|
||||
@@ -207,7 +207,7 @@ steps:
|
||||
|
||||
- label: Fusion E2E TP2 (B200)
|
||||
key: fusion-e2e-tp2-b200
|
||||
timeout_in_minutes: 45
|
||||
timeout_in_minutes: 20
|
||||
working_dir: "/vllm-workspace/"
|
||||
device: b200-k8s
|
||||
num_devices: 2
|
||||
|
||||
@@ -2,9 +2,9 @@ group: CUDA
|
||||
depends_on:
|
||||
- image-build
|
||||
steps:
|
||||
- label: Platform Tests
|
||||
key: platform-tests
|
||||
timeout_in_minutes: 20
|
||||
- label: Platform Tests (CUDA)
|
||||
key: platform-tests-cuda
|
||||
timeout_in_minutes: 15
|
||||
device: h200_18gb
|
||||
source_file_dependencies:
|
||||
- vllm/envs.py
|
||||
@@ -16,21 +16,16 @@ steps:
|
||||
commands:
|
||||
- pytest -v -s cuda/test_cuda_context.py
|
||||
- pytest -v -s cuda/test_platform_no_cuda_init.py
|
||||
- pytest -v -s cuda/test_cuda_compatibility_path.py
|
||||
|
||||
- label: Cudagraph
|
||||
device: h200_35gb
|
||||
key: cudagraph
|
||||
timeout_in_minutes: 30
|
||||
timeout_in_minutes: 20
|
||||
source_file_dependencies:
|
||||
- tests/v1/cudagraph
|
||||
- vllm/v1/cudagraph_dispatcher.py
|
||||
- vllm/config/compilation.py
|
||||
- vllm/compilation
|
||||
- vllm/v1/worker/encoder_cudagraph.py
|
||||
- vllm/v1/worker/encoder_cudagraph_defs.py
|
||||
commands:
|
||||
- pytest -v -s v1/cudagraph/test_cudagraph_dispatch.py
|
||||
- pytest -v -s v1/cudagraph/test_cudagraph_mode.py
|
||||
- pytest -v -s v1/cudagraph/test_breakable_cudagraph.py
|
||||
- pytest -v -s v1/cudagraph/test_encoder_cudagraph.py
|
||||
- pytest -v -s v1/cudagraph/test_breakable_cudagraph.py
|
||||
@@ -4,7 +4,7 @@ depends_on:
|
||||
steps:
|
||||
- label: Distributed NixlConnector PD accuracy (4 GPUs)
|
||||
key: distributed-nixlconnector-pd-accuracy-4-gpus
|
||||
timeout_in_minutes: 55
|
||||
timeout_in_minutes: 30
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
num_devices: 4
|
||||
source_file_dependencies:
|
||||
@@ -13,24 +13,9 @@ steps:
|
||||
commands:
|
||||
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
|
||||
- bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_4
|
||||
timeout_in_minutes: 60
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
|
||||
- tests/v1/kv_connector/nixl_integration/
|
||||
- vllm/platforms/rocm.py
|
||||
commands:
|
||||
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
|
||||
- ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
|
||||
|
||||
- label: Distributed FlashInfer NixlConnector PD accuracy (4 GPUs)
|
||||
key: distributed-flashinfer-nixlconnector-pd-accuracy-4-gpus
|
||||
timeout_in_minutes: 55
|
||||
timeout_in_minutes: 30
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
num_devices: 4
|
||||
source_file_dependencies:
|
||||
@@ -40,19 +25,6 @@ steps:
|
||||
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
|
||||
- FLASHINFER=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
|
||||
|
||||
- label: Push NixlConnector PP prefill PD accuracy (4 GPUs)
|
||||
key: push-nixlconnector-pp-prefill-pd-accuracy-4-gpus
|
||||
timeout_in_minutes: 30
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
num_devices: 4
|
||||
source_file_dependencies:
|
||||
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
|
||||
- tests/v1/kv_connector/nixl_integration/
|
||||
- tests/v1/kv_connector/nixl_push_integration/
|
||||
commands:
|
||||
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
|
||||
- bash v1/kv_connector/nixl_push_integration/config_sweep_accuracy_test.sh
|
||||
|
||||
- label: DP EP Distributed NixlConnector PD accuracy tests (4 GPUs)
|
||||
key: dp-ep-distributed-nixlconnector-pd-accuracy-tests-4-gpus
|
||||
timeout_in_minutes: 30
|
||||
@@ -64,24 +36,10 @@ steps:
|
||||
commands:
|
||||
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
|
||||
- DP_EP=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_4
|
||||
timeout_in_minutes: 40
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
|
||||
- tests/v1/kv_connector/nixl_integration/
|
||||
- vllm/platforms/rocm.py
|
||||
commands:
|
||||
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
|
||||
- DP_EP=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
|
||||
|
||||
- label: CrossLayer KV layout Distributed NixlConnector PD accuracy tests (4 GPUs)
|
||||
key: crosslayer-kv-layout-distributed-nixlconnector-pd-accuracy-tests-4-gpus
|
||||
timeout_in_minutes: 55
|
||||
timeout_in_minutes: 30
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
num_devices: 4
|
||||
source_file_dependencies:
|
||||
@@ -90,24 +48,10 @@ steps:
|
||||
commands:
|
||||
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
|
||||
- CROSS_LAYERS_BLOCKS=True bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_4
|
||||
timeout_in_minutes: 60
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
|
||||
- tests/v1/kv_connector/nixl_integration/
|
||||
- vllm/platforms/rocm.py
|
||||
commands:
|
||||
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
|
||||
- CROSS_LAYERS_BLOCKS=True ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
|
||||
|
||||
- label: Hybrid SSM NixlConnector PD accuracy tests (4 GPUs)
|
||||
key: hybrid-ssm-nixlconnector-pd-accuracy-tests-4-gpus
|
||||
timeout_in_minutes: 60
|
||||
timeout_in_minutes: 25
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
num_devices: 4
|
||||
source_file_dependencies:
|
||||
@@ -116,36 +60,6 @@ steps:
|
||||
commands:
|
||||
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
|
||||
- HYBRID_SSM=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_4
|
||||
timeout_in_minutes: 55
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
|
||||
- tests/v1/kv_connector/nixl_integration/
|
||||
- vllm/platforms/rocm.py
|
||||
commands:
|
||||
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
|
||||
- HYBRID_SSM=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
|
||||
|
||||
- label: NixlConnector PD edge case test (2 GPUs)
|
||||
key: nixlconnector-pd-edge-cases-2-gpus
|
||||
timeout_in_minutes: 40
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
num_devices: 2
|
||||
source_file_dependencies:
|
||||
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
|
||||
- vllm/v1/core/sched/
|
||||
- tests/v1/kv_connector/nixl_integration/
|
||||
env:
|
||||
PREFILL_GPU_ID: "0"
|
||||
DECODE_GPU_ID: "1"
|
||||
commands:
|
||||
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
|
||||
- bash v1/kv_connector/nixl_integration/run_edge_case_test.sh
|
||||
|
||||
- label: Hybrid SSM NixlConnector PD prefix cache test (2 GPUs)
|
||||
key: hybrid-ssm-nixlconnector-pd-prefix-cache-2-gpus
|
||||
@@ -163,7 +77,7 @@ steps:
|
||||
|
||||
- label: MultiConnector (Nixl+Offloading) PD accuracy (2 GPUs)
|
||||
key: multiconnector-nixl-offloading-pd-accuracy-2-gpus
|
||||
timeout_in_minutes: 40
|
||||
timeout_in_minutes: 30
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
num_devices: 2
|
||||
source_file_dependencies:
|
||||
@@ -178,7 +92,7 @@ steps:
|
||||
|
||||
- label: NixlConnector PD + Spec Decode acceptance (2 GPUs)
|
||||
key: nixlconnector-pd-spec-decode-acceptance-2-gpus
|
||||
timeout_in_minutes: 45
|
||||
timeout_in_minutes: 30
|
||||
device: a100
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
num_devices: 2
|
||||
@@ -189,25 +103,10 @@ steps:
|
||||
commands:
|
||||
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
|
||||
- bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_2
|
||||
timeout_in_minutes: 45
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- vllm/distributed/kv_transfer/kv_connector/v1/nixl/
|
||||
- vllm/v1/worker/kv_connector_model_runner_mixin.py
|
||||
- tests/v1/kv_connector/nixl_integration/
|
||||
- vllm/platforms/rocm.py
|
||||
commands:
|
||||
- uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
|
||||
- KV_CACHE_MEMORY_BYTES=8G ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh
|
||||
|
||||
- label: MultiConnector (Nixl+Offloading) PD edge cases (2 GPUs)
|
||||
key: multiconnector-nixl-offloading-pd-edge-cases-2-gpus
|
||||
timeout_in_minutes: 25
|
||||
timeout_in_minutes: 30
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
num_devices: 2
|
||||
source_file_dependencies:
|
||||
@@ -219,25 +118,3 @@ steps:
|
||||
commands:
|
||||
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
|
||||
- bash v1/kv_connector/nixl_integration/run_multi_connector_edge_case_test.sh
|
||||
|
||||
# P TP 4 - D DPEP 4 test case for DSv4-Flash
|
||||
- label: DSv4-Flash Disaggregated DP EP
|
||||
key: dsv4-flash-disaggregated
|
||||
timeout_in_minutes: 60
|
||||
device: h200
|
||||
optional: true
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
num_devices: 8
|
||||
env:
|
||||
ENABLE_HMA_FLAG: "1"
|
||||
DP_EP: "1"
|
||||
GPU_MEMORY_UTILIZATION: "0.85"
|
||||
PREFILLER_TP_SIZE: "4"
|
||||
DECODER_TP_SIZE: "4"
|
||||
PREFILL_BLOCK_SIZE: "256"
|
||||
DECODE_BLOCK_SIZE: "256"
|
||||
MODEL_NAMES: "deepseek-ai/DeepSeek-V4-Flash"
|
||||
VLLM_SERVE_EXTRA_ARGS: "--trust-remote-code,--kv-cache-dtype,fp8"
|
||||
commands:
|
||||
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
|
||||
- bash v1/kv_connector/nixl_integration/run_accuracy_test.sh
|
||||
|
||||
@@ -4,7 +4,7 @@ depends_on:
|
||||
steps:
|
||||
- label: Distributed Comm Ops
|
||||
key: distributed-comm-ops
|
||||
timeout_in_minutes: 25
|
||||
timeout_in_minutes: 20
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
num_devices: 2
|
||||
source_file_dependencies:
|
||||
@@ -18,7 +18,7 @@ steps:
|
||||
|
||||
- label: Distributed DP Tests (2 GPUs)
|
||||
key: distributed-dp-tests-2-gpus
|
||||
timeout_in_minutes: 35
|
||||
timeout_in_minutes: 20
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
num_devices: 2
|
||||
source_file_dependencies:
|
||||
@@ -37,27 +37,10 @@ steps:
|
||||
- TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
|
||||
- TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py
|
||||
- DP_SIZE=2 pytest -v -s entrypoints/openai/test_multi_api_servers.py
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_2
|
||||
timeout_in_minutes: 45
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- vllm/distributed/
|
||||
- vllm/engine/
|
||||
- vllm/executor/
|
||||
- vllm/worker/worker_base.py
|
||||
- vllm/v1/engine/
|
||||
- vllm/v1/worker/
|
||||
- tests/v1/distributed
|
||||
- tests/entrypoints/openai/test_multi_api_servers.py
|
||||
- vllm/platforms/rocm.py
|
||||
|
||||
- label: Distributed Compile + RPC Tests (2 GPUs)
|
||||
key: distributed-compile-rpc-tests-2-gpus
|
||||
timeout_in_minutes: 65
|
||||
timeout_in_minutes: 20
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
num_devices: 2
|
||||
source_file_dependencies:
|
||||
@@ -80,7 +63,7 @@ steps:
|
||||
|
||||
- label: Distributed Torchrun + Shutdown Tests (2 GPUs)
|
||||
key: distributed-torchrun-shutdown-tests-2-gpus
|
||||
timeout_in_minutes: 30
|
||||
timeout_in_minutes: 20
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
num_devices: 2
|
||||
source_file_dependencies:
|
||||
@@ -135,7 +118,7 @@ steps:
|
||||
|
||||
- label: Distributed DP Tests (4 GPUs)
|
||||
key: distributed-dp-tests-4-gpus
|
||||
timeout_in_minutes: 45
|
||||
timeout_in_minutes: 30
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
num_devices: 4
|
||||
source_file_dependencies:
|
||||
@@ -156,7 +139,7 @@ steps:
|
||||
|
||||
- label: Distributed Compile + Comm (4 GPUs)
|
||||
key: distributed-compile-comm-4-gpus
|
||||
timeout_in_minutes: 70
|
||||
timeout_in_minutes: 30
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
num_devices: 4
|
||||
source_file_dependencies:
|
||||
@@ -176,9 +159,9 @@ steps:
|
||||
# test multi-node TP with multiproc executor (simulated on single node)
|
||||
- pytest -v -s distributed/test_multiproc_executor.py::test_multiproc_executor_multi_node
|
||||
|
||||
- label: Distributed Tests (8xH100)
|
||||
key: distributed-tests-8xh100
|
||||
timeout_in_minutes: 20
|
||||
- label: Distributed Tests (8 GPUs)(H100)
|
||||
key: distributed-tests-8-gpus-h100
|
||||
timeout_in_minutes: 10
|
||||
device: h100
|
||||
num_devices: 8
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
@@ -197,8 +180,8 @@ steps:
|
||||
# test with torchrun tp=2 and dp=4 with ep
|
||||
- torchrun --nproc-per-node=8 ../examples/features/torchrun/torchrun_dp_example_offline.py --tp-size=2 --pp-size=1 --dp-size=4 --enable-ep
|
||||
|
||||
- label: Distributed Tests (4xA100)
|
||||
key: distributed-tests-4xa100
|
||||
- label: Distributed Tests (4 GPUs)(A100)
|
||||
key: distributed-tests-4-gpus-a100
|
||||
device: a100
|
||||
optional: true
|
||||
num_devices: 4
|
||||
@@ -212,9 +195,9 @@ steps:
|
||||
- TARGET_TEST_SUITE=A100 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
|
||||
- pytest -v -s -x lora/test_mixtral.py
|
||||
|
||||
- label: Distributed Tests (2xH100-2xMI300)
|
||||
key: distributed-tests-2xh100-2xmi300
|
||||
timeout_in_minutes: 30
|
||||
- label: Distributed Tests (2 GPUs)(H100)
|
||||
key: distributed-tests-2-gpus-h100
|
||||
timeout_in_minutes: 15
|
||||
device: h100
|
||||
optional: true
|
||||
working_dir: "/vllm-workspace/"
|
||||
@@ -227,15 +210,15 @@ steps:
|
||||
- VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer.py
|
||||
- pytest -v -s tests/distributed/test_packed_tensor.py
|
||||
|
||||
- label: Distributed Tests (2xB200)
|
||||
key: distributed-tests-2xb200
|
||||
- label: Distributed Tests (2 GPUs)(B200)
|
||||
key: distributed-tests-2-gpus-b200
|
||||
device: b200-k8s
|
||||
optional: true
|
||||
working_dir: "/vllm-workspace/"
|
||||
num_devices: 2
|
||||
commands:
|
||||
- pytest -v -s tests/distributed/test_context_parallel.py
|
||||
- pytest -v -s tests/distributed/test_nccl_symm_mem.py
|
||||
- pytest -v -s tests/distributed/test_nccl_symm_mem_allreduce.py
|
||||
- pytest -v -s tests/v1/distributed/test_dbo.py
|
||||
- pytest -v -s tests/distributed/test_mnnvl_alltoall.py
|
||||
|
||||
@@ -261,7 +244,7 @@ steps:
|
||||
|
||||
- label: Pipeline + Context Parallelism (4 GPUs)
|
||||
key: pipeline-context-parallelism-4-gpus
|
||||
timeout_in_minutes: 55
|
||||
timeout_in_minutes: 60
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
num_devices: 4
|
||||
source_file_dependencies:
|
||||
@@ -276,7 +259,7 @@ steps:
|
||||
|
||||
- label: RayExecutorV2 (4 GPUs)
|
||||
key: rayexecutorv2-4-gpus
|
||||
timeout_in_minutes: 45
|
||||
timeout_in_minutes: 60
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
num_devices: 4
|
||||
source_file_dependencies:
|
||||
|
||||
@@ -3,7 +3,7 @@ depends_on:
|
||||
- image-build-cpu
|
||||
steps:
|
||||
- label: Docker Build Metadata
|
||||
timeout_in_minutes: 20
|
||||
timeout_in_minutes: 10
|
||||
device: cpu-small
|
||||
source_file_dependencies:
|
||||
- .buildkite/release-pipeline.yaml
|
||||
|
||||
@@ -2,9 +2,9 @@ group: E2E Integration
|
||||
depends_on:
|
||||
- image-build
|
||||
steps:
|
||||
- label: DeepSeek V2-Lite Sync EPLB Accuracy (4xH100)
|
||||
key: deepseek-v2-lite-sync-eplb-accuracy-4xh100
|
||||
timeout_in_minutes: 25
|
||||
- label: DeepSeek V2-Lite Sync EPLB Accuracy
|
||||
key: deepseek-v2-lite-sync-eplb-accuracy
|
||||
timeout_in_minutes: 60
|
||||
device: h100
|
||||
optional: true
|
||||
num_devices: 4
|
||||
@@ -12,9 +12,9 @@ steps:
|
||||
commands:
|
||||
- bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh 0.25 200 8010
|
||||
|
||||
- label: Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy (4xH100)
|
||||
key: qwen3-30b-a3b-fp8-block-sync-eplb-accuracy-4xh100
|
||||
timeout_in_minutes: 25
|
||||
- label: Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy
|
||||
key: qwen3-30b-a3b-fp8-block-sync-eplb-accuracy
|
||||
timeout_in_minutes: 60
|
||||
device: h100
|
||||
optional: true
|
||||
num_devices: 4
|
||||
@@ -22,9 +22,9 @@ steps:
|
||||
commands:
|
||||
- bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020
|
||||
|
||||
- label: Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy (2xB200)
|
||||
key: qwen3-30b-a3b-fp8-block-sync-eplb-accuracy-2xb200
|
||||
timeout_in_minutes: 20
|
||||
- label: Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy (B200)
|
||||
key: qwen3-30b-a3b-fp8-block-sync-eplb-accuracy-b200
|
||||
timeout_in_minutes: 60
|
||||
device: b200-k8s
|
||||
optional: true
|
||||
num_devices: 2
|
||||
@@ -34,7 +34,7 @@ steps:
|
||||
|
||||
- label: Qwen3-30B-A3B-FP8 DP4 Async EPLB Accuracy
|
||||
key: qwen3-30b-a3b-fp8-dp4-async-eplb-accuracy
|
||||
timeout_in_minutes: 25
|
||||
timeout_in_minutes: 60
|
||||
device: h100
|
||||
optional: true
|
||||
num_devices: 4
|
||||
@@ -44,7 +44,7 @@ steps:
|
||||
|
||||
- label: DeepSeek V2-Lite Prefetch Offload Accuracy (H100)
|
||||
key: deepseek-v2-lite-prefetch-offload-accuracy-h100
|
||||
timeout_in_minutes: 20
|
||||
timeout_in_minutes: 60
|
||||
device: h100
|
||||
optional: true
|
||||
num_devices: 1
|
||||
|
||||
@@ -4,7 +4,7 @@ depends_on:
|
||||
steps:
|
||||
- label: Engine
|
||||
key: engine
|
||||
timeout_in_minutes: 30
|
||||
timeout_in_minutes: 15
|
||||
device: h200_18gb
|
||||
source_file_dependencies:
|
||||
- vllm/compilation/
|
||||
@@ -28,33 +28,30 @@ steps:
|
||||
- pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py test_jit_monitor.py
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 40
|
||||
device: mi325_1
|
||||
timeout_in_minutes: 60
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
|
||||
- label: Engine (1 GPU)
|
||||
key: engine-1-gpu
|
||||
timeout_in_minutes: 45
|
||||
timeout_in_minutes: 30
|
||||
source_file_dependencies:
|
||||
- vllm/v1/engine/
|
||||
- tests/v1/engine/
|
||||
- tests/v1/test_tensor_ipc_queue.py
|
||||
commands:
|
||||
- pytest -v -s v1/engine/test_preprocess_error_handling.py
|
||||
- pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py
|
||||
- pytest -v -s v1/test_tensor_ipc_queue.py
|
||||
mirror:
|
||||
amd:
|
||||
device: mi250_1
|
||||
timeout_in_minutes: 45
|
||||
device: mi325_1
|
||||
timeout_in_minutes: 40
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
|
||||
- label: e2e Scheduling (1 GPU)
|
||||
key: e2e-scheduling-1-gpu
|
||||
timeout_in_minutes: 53
|
||||
timeout_in_minutes: 30
|
||||
device: h200_18gb
|
||||
source_file_dependencies:
|
||||
- vllm/v1/
|
||||
@@ -64,14 +61,14 @@ steps:
|
||||
mirror:
|
||||
amd:
|
||||
device: mi250_1
|
||||
timeout_in_minutes: 55
|
||||
timeout_in_minutes: 60
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
|
||||
- label: e2e Core (1 GPU)
|
||||
device: h200_35gb
|
||||
key: e2e-core-1-gpu
|
||||
timeout_in_minutes: 40
|
||||
timeout_in_minutes: 30
|
||||
source_file_dependencies:
|
||||
- vllm/v1/
|
||||
- tests/v1/e2e/general/
|
||||
@@ -80,7 +77,7 @@ steps:
|
||||
mirror:
|
||||
amd:
|
||||
device: mi250_1
|
||||
timeout_in_minutes: 50
|
||||
timeout_in_minutes: 35
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
@@ -90,7 +87,7 @@ steps:
|
||||
|
||||
- label: V1 e2e (2 GPUs)
|
||||
key: v1-e2e-2-gpus
|
||||
timeout_in_minutes: 25 # TODO: Fix timeout after we have more confidence in the test stability
|
||||
timeout_in_minutes: 60 # TODO: Fix timeout after we have more confidence in the test stability
|
||||
optional: true
|
||||
num_devices: 2
|
||||
source_file_dependencies:
|
||||
@@ -115,17 +112,10 @@ steps:
|
||||
commands:
|
||||
# Only run tests that need exactly 2 GPUs
|
||||
- pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "tensor_parallelism"
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_2
|
||||
timeout_in_minutes: 30
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
|
||||
- label: V1 e2e (4 GPUs)
|
||||
key: v1-e2e-4-gpus
|
||||
timeout_in_minutes: 20 # TODO: Fix timeout after we have more confidence in the test stability
|
||||
timeout_in_minutes: 60 # TODO: Fix timeout after we have more confidence in the test stability
|
||||
optional: true
|
||||
num_devices: 4
|
||||
source_file_dependencies:
|
||||
@@ -153,7 +143,7 @@ steps:
|
||||
|
||||
- label: V1 e2e (4xH100)
|
||||
key: v1-e2e-4xh100
|
||||
timeout_in_minutes: 35
|
||||
timeout_in_minutes: 60
|
||||
device: h100
|
||||
num_devices: 4
|
||||
optional: true
|
||||
|
||||
@@ -3,9 +3,8 @@ depends_on:
|
||||
- image-build
|
||||
steps:
|
||||
- label: Entrypoints Unit Tests
|
||||
device: h200_35gb
|
||||
key: entrypoints-unit-tests
|
||||
timeout_in_minutes: 25
|
||||
timeout_in_minutes: 10
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
source_file_dependencies:
|
||||
- vllm/entrypoints
|
||||
@@ -16,9 +15,8 @@ steps:
|
||||
- pytest -v -s entrypoints/weight_transfer
|
||||
|
||||
- label: Entrypoints Integration (LLM)
|
||||
device: h200_35gb
|
||||
key: entrypoints-integration-llm
|
||||
timeout_in_minutes: 60
|
||||
timeout_in_minutes: 40
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
source_file_dependencies:
|
||||
- vllm/
|
||||
@@ -30,38 +28,31 @@ steps:
|
||||
- pytest -v -s entrypoints/llm/offline_mode # Needs to avoid interference with other tests
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 55
|
||||
device: mi325_1
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
|
||||
- label: Entrypoints Integration (API Server)
|
||||
key: entrypoints-integration-api-server
|
||||
device: h200_35gb
|
||||
timeout_in_minutes: 75
|
||||
timeout_in_minutes: 130
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
source_file_dependencies:
|
||||
- vllm/
|
||||
- tests/entrypoints/serve
|
||||
- tests/entrypoints/scale_out
|
||||
commands:
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc
|
||||
- PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc
|
||||
- pytest -v -s entrypoints/scale_out
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 65
|
||||
device: mi325_1
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
|
||||
- label: Entrypoints Integration (API Server OpenAI - Part 1)
|
||||
device: h200_35gb
|
||||
key: entrypoints-integration-api-server-openai-part-1
|
||||
timeout_in_minutes: 68
|
||||
timeout_in_minutes: 50
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
source_file_dependencies:
|
||||
- vllm/
|
||||
@@ -72,16 +63,14 @@ steps:
|
||||
- pytest -v -s entrypoints/openai --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/correctness
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 65
|
||||
device: mi325_1
|
||||
timeout_in_minutes: 80
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
|
||||
- label: Entrypoints Integration (API Server OpenAI - Part 2)
|
||||
device: h200_35gb
|
||||
key: entrypoints-integration-api-server-openai-part-2
|
||||
timeout_in_minutes: 83
|
||||
timeout_in_minutes: 50
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
source_file_dependencies:
|
||||
- vllm/
|
||||
@@ -93,14 +82,12 @@ steps:
|
||||
- pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 70
|
||||
device: mi325_1
|
||||
timeout_in_minutes: 80
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
|
||||
- label: Entrypoints Integration (API Server Generate)
|
||||
device: h200_35gb
|
||||
key: entrypoints-integration-api-server-generate
|
||||
timeout_in_minutes: 50
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
@@ -117,14 +104,12 @@ steps:
|
||||
- pytest -v -s entrypoints/anthropic
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 65
|
||||
device: mi325_1
|
||||
timeout_in_minutes: 60
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
|
||||
- label: Entrypoints Integration (Responses API)
|
||||
device: h200_35gb
|
||||
key: entrypoints-integration-responses-api
|
||||
timeout_in_minutes: 50
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
@@ -137,7 +122,7 @@ steps:
|
||||
- label: Entrypoints Integration (Speech to Text)
|
||||
device: h200_35gb
|
||||
key: entrypoints-integration-speech_to_text
|
||||
timeout_in_minutes: 45
|
||||
timeout_in_minutes: 50
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
source_file_dependencies:
|
||||
- vllm/
|
||||
@@ -149,7 +134,7 @@ steps:
|
||||
- label: Entrypoints Integration (Multimodal)
|
||||
device: h200_35gb
|
||||
key: entrypoints-integration-multimodal
|
||||
timeout_in_minutes: 45
|
||||
timeout_in_minutes: 50
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
source_file_dependencies:
|
||||
- vllm/
|
||||
@@ -159,9 +144,8 @@ steps:
|
||||
- pytest -v -s entrypoints/multimodal
|
||||
|
||||
- label: Entrypoints Integration (Pooling)
|
||||
device: h200_35gb
|
||||
key: entrypoints-integration-pooling
|
||||
timeout_in_minutes: 75
|
||||
timeout_in_minutes: 50
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
source_file_dependencies:
|
||||
- vllm/
|
||||
@@ -172,7 +156,7 @@ steps:
|
||||
|
||||
- label: OpenAI API Correctness
|
||||
key: openai-api-correctness
|
||||
timeout_in_minutes: 20
|
||||
timeout_in_minutes: 30
|
||||
device: h200_18gb
|
||||
source_file_dependencies:
|
||||
- csrc/
|
||||
@@ -181,9 +165,7 @@ steps:
|
||||
- pytest -s entrypoints/openai/correctness/
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 30
|
||||
device: mi325_1
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
|
||||
@@ -4,7 +4,7 @@ depends_on:
|
||||
steps:
|
||||
- label: EPLB Algorithm
|
||||
key: eplb-algorithm
|
||||
timeout_in_minutes: 20
|
||||
timeout_in_minutes: 15
|
||||
device: h200_18gb
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
source_file_dependencies:
|
||||
@@ -14,22 +14,10 @@ steps:
|
||||
commands:
|
||||
- pytest -v -s distributed/test_eplb_algo.py
|
||||
- pytest -v -s distributed/test_eplb_utils.py
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 30
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- vllm/distributed/eplb
|
||||
- tests/distributed/test_eplb_algo.py
|
||||
- tests/distributed/test_eplb_utils.py
|
||||
- vllm/platforms/rocm.py
|
||||
|
||||
- label: EPLB Execution # 17min
|
||||
key: eplb-execution
|
||||
timeout_in_minutes: 25
|
||||
timeout_in_minutes: 27
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
num_devices: 4
|
||||
source_file_dependencies:
|
||||
@@ -41,7 +29,7 @@ steps:
|
||||
|
||||
- label: Elastic EP Scaling Test
|
||||
key: elastic-ep-scaling-test
|
||||
timeout_in_minutes: 30
|
||||
timeout_in_minutes: 20
|
||||
device: h100
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
num_devices: 4
|
||||
@@ -52,5 +40,4 @@ steps:
|
||||
- vllm/compilation/
|
||||
- tests/distributed/
|
||||
commands:
|
||||
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
|
||||
- pytest -v -s distributed/test_elastic_ep.py
|
||||
|
||||
@@ -1,26 +0,0 @@
|
||||
group: Fault Tolerance
|
||||
depends_on:
|
||||
- image-build
|
||||
steps:
|
||||
- label: Fault Tolerance E2E (2xH100)
|
||||
key: fault-tolerance-e2e-2xh100
|
||||
timeout_in_minutes: 35
|
||||
device: h100
|
||||
num_devices: 2
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
source_file_dependencies:
|
||||
- vllm/v1/fault_tolerance/
|
||||
- vllm/v1/worker/sentinel/
|
||||
- vllm/entrypoints/serve/fault_tolerance/
|
||||
- vllm/distributed/elastic_ep/
|
||||
- vllm/distributed/device_communicators/
|
||||
- vllm/v1/engine/
|
||||
- vllm/v1/worker/
|
||||
- tests/v1/fault_tolerance/
|
||||
- tests/v1/distributed/test_external_lb_dp.py
|
||||
commands:
|
||||
# Base image has no nixl; install it or has_nixl_ep() skips the tests.
|
||||
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
|
||||
# https://github.com/NVIDIA/nccl/issues/1838
|
||||
- export NCCL_CUMEM_HOST_ENABLE=0
|
||||
- pytest -v -s v1/fault_tolerance/test_fault_tolerance_e2e.py
|
||||
@@ -4,7 +4,7 @@ depends_on:
|
||||
steps:
|
||||
- label: vLLM IR Tests
|
||||
key: vllm-ir-tests
|
||||
timeout_in_minutes: 35
|
||||
timeout_in_minutes: 10
|
||||
device: h200_18gb
|
||||
working_dir: "/vllm-workspace/"
|
||||
source_file_dependencies:
|
||||
@@ -15,21 +15,19 @@ steps:
|
||||
- pytest -v -s tests/kernels/ir
|
||||
|
||||
- label: Kernels Core Operation Test
|
||||
device: h200_35gb
|
||||
key: kernels-core-operation-test
|
||||
timeout_in_minutes: 120
|
||||
timeout_in_minutes: 75
|
||||
source_file_dependencies:
|
||||
- csrc/
|
||||
- tests/kernels/core
|
||||
- tests/kernels/test_concat_mla_q.py
|
||||
- tests/kernels/test_fused_qk_norm_rope_gate.py
|
||||
commands:
|
||||
- pytest -v -s kernels/core --ignore=kernels/core/test_minimax_reduce_rms.py kernels/test_concat_mla_q.py kernels/test_fused_qk_norm_rope_gate.py --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
|
||||
parallelism: 3
|
||||
- pytest -v -s kernels/core --ignore=kernels/core/test_minimax_reduce_rms.py kernels/test_concat_mla_q.py kernels/test_fused_qk_norm_rope_gate.py
|
||||
|
||||
- label: Kernels MiniMax Reduce RMS Test (2 GPUs)
|
||||
key: kernels-minimax-reduce-rms-test-2-gpus
|
||||
timeout_in_minutes: 20
|
||||
timeout_in_minutes: 15
|
||||
num_devices: 2
|
||||
device: h100
|
||||
source_file_dependencies:
|
||||
@@ -43,7 +41,7 @@ steps:
|
||||
|
||||
- label: Deepseek V4 Kernel Test (H100)
|
||||
key: deepseek-v4-kernel-test-h100
|
||||
timeout_in_minutes: 30
|
||||
timeout_in_minutes: 15
|
||||
device: h100
|
||||
source_file_dependencies:
|
||||
- csrc/fused_deepseek_v4_qnorm_rope_kv_insert_kernel.cu
|
||||
@@ -56,54 +54,18 @@ steps:
|
||||
|
||||
- label: Deepseek V4 Kernel Test (B200)
|
||||
key: deepseek-v4-kernel-test-b200
|
||||
timeout_in_minutes: 20
|
||||
timeout_in_minutes: 15
|
||||
device: b200-k8s
|
||||
source_file_dependencies:
|
||||
- csrc/fused_deepseek_v4_qnorm_rope_kv_insert_kernel.cu
|
||||
- vllm/models/deepseek_v4/common/ops/
|
||||
- vllm/models/deepseek_v4/nvidia/
|
||||
- tests/kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py
|
||||
- tests/models/test_deepseek_v4_mega_moe.py
|
||||
commands:
|
||||
- pytest -v -s kernels/test_fused_deepseek_v4_*.py
|
||||
- pytest -v -s models/test_deepseek_v4_mega_moe.py
|
||||
|
||||
# Catch-all for test files at the tests/kernels root. This job collects
|
||||
# the whole root so new files are wired by default.
|
||||
# Files with dedicated jobs elsewhere in this file are excluded via --ignore
|
||||
# (test_kda, test_bf16x3_router_gemm_cutedsl and test_ll_bf16_gemm run in
|
||||
# their own jobs / Kernels (B200)).
|
||||
- label: Kernels Root Misc Test (B200)
|
||||
key: kernels-root-misc-test-b200
|
||||
timeout_in_minutes: 45
|
||||
device: b200-k8s
|
||||
source_file_dependencies:
|
||||
- csrc/
|
||||
- vllm/
|
||||
- tests/kernels/
|
||||
commands:
|
||||
- pytest -v -s kernels/
|
||||
--ignore=kernels/attention
|
||||
--ignore=kernels/core
|
||||
--ignore=kernels/helion
|
||||
--ignore=kernels/ir
|
||||
--ignore=kernels/mamba
|
||||
--ignore=kernels/moe
|
||||
--ignore=kernels/quantization
|
||||
--ignore=kernels/test_concat_mla_q.py
|
||||
--ignore=kernels/test_fused_qk_norm_rope_gate.py
|
||||
--ignore=kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py
|
||||
--ignore=kernels/test_top_k_per_row.py
|
||||
--ignore=kernels/test_kda.py
|
||||
--ignore=kernels/test_bf16x3_router_gemm_cutedsl.py
|
||||
--ignore=kernels/test_ll_bf16_gemm.py
|
||||
--ignore=kernels/test_shuffle_rows.py
|
||||
# BROKEN on main, pending kernel fixes (B200):
|
||||
# test_shuffle_rows.py (1: test_shuffle_rows_edge_cases)
|
||||
|
||||
- label: Kernels Attention Test %N
|
||||
key: kernels-attention-test
|
||||
timeout_in_minutes: 65
|
||||
timeout_in_minutes: 35
|
||||
source_file_dependencies:
|
||||
- csrc/attention/
|
||||
- vllm/v1/attention
|
||||
@@ -116,9 +78,8 @@ steps:
|
||||
parallelism: 2
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 90
|
||||
device: mi325_1
|
||||
timeout_in_minutes: 55
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
@@ -145,7 +106,7 @@ steps:
|
||||
|
||||
- label: Kernels Quantization Test %N
|
||||
key: kernels-quantization-test
|
||||
timeout_in_minutes: 60
|
||||
timeout_in_minutes: 90
|
||||
source_file_dependencies:
|
||||
- csrc/quantization/
|
||||
- vllm/model_executor/layers/quantization
|
||||
@@ -155,9 +116,7 @@ steps:
|
||||
parallelism: 2
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 120
|
||||
device: mi325_1
|
||||
source_file_dependencies:
|
||||
- csrc/quantization/
|
||||
- vllm/model_executor/layers/quantization
|
||||
@@ -172,7 +131,7 @@ steps:
|
||||
|
||||
- label: Kernels MoE Test %N
|
||||
key: kernels-moe-test
|
||||
timeout_in_minutes: 50
|
||||
timeout_in_minutes: 25
|
||||
source_file_dependencies:
|
||||
- csrc/quantization/cutlass_w8a8/moe/
|
||||
- csrc/moe/
|
||||
@@ -187,9 +146,8 @@ steps:
|
||||
parallelism: 5
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 55
|
||||
device: mi325_1
|
||||
timeout_in_minutes: 50
|
||||
source_file_dependencies:
|
||||
- csrc/quantization/cutlass_w8a8/moe/
|
||||
- csrc/moe/
|
||||
@@ -204,9 +162,8 @@ steps:
|
||||
- image-build-amd
|
||||
|
||||
- label: Kernels Mamba Test
|
||||
device: h200_35gb
|
||||
key: kernels-mamba-test
|
||||
timeout_in_minutes: 40
|
||||
timeout_in_minutes: 45
|
||||
source_file_dependencies:
|
||||
- csrc/mamba/
|
||||
- tests/kernels/mamba
|
||||
@@ -214,9 +171,20 @@ steps:
|
||||
commands:
|
||||
- pytest -v -s kernels/mamba
|
||||
|
||||
- label: Kernels KDA Test
|
||||
timeout_in_minutes: 20
|
||||
device: h200_18gb
|
||||
source_file_dependencies:
|
||||
- vllm/model_executor/layers/fla/ops/kda.py
|
||||
- vllm/model_executor/layers/fla/ops/chunk_delta_h.py
|
||||
- vllm/model_executor/layers/fla/ops/l2norm.py
|
||||
- tests/kernels/test_kda.py
|
||||
commands:
|
||||
- pytest -v -s kernels/test_kda.py
|
||||
|
||||
- label: Kernels DeepGEMM Test (H100)
|
||||
key: kernels-deepgemm-test-h100
|
||||
timeout_in_minutes: 35
|
||||
timeout_in_minutes: 45
|
||||
device: h100
|
||||
num_devices: 1
|
||||
source_file_dependencies:
|
||||
@@ -243,7 +211,7 @@ steps:
|
||||
|
||||
- label: Kernels (B200)
|
||||
key: kernels-b200
|
||||
timeout_in_minutes: 80
|
||||
timeout_in_minutes: 30
|
||||
working_dir: "/vllm-workspace/"
|
||||
device: b200-k8s
|
||||
# optional: true
|
||||
@@ -263,15 +231,6 @@ steps:
|
||||
- vllm/v1/attention/backends/mla/flashinfer_mla.py
|
||||
- vllm/v1/attention/selector.py
|
||||
- vllm/platforms/cuda.py
|
||||
- vllm/model_executor/kernels/linear/cute_dsl/ll_bf16.py
|
||||
- vllm/model_executor/kernels/linear/cute_dsl/_ll_bf16_dotprod.py
|
||||
- vllm/model_executor/kernels/linear/cute_dsl/_ll_bf16_splitk.py
|
||||
- vllm/cute_utils/
|
||||
- vllm/model_executor/layers/mamba/ops/gdn_chunk_cutedsl/
|
||||
- vllm/model_executor/layers/fused_moe/router/bf16x3_router_gemm_cutedsl.py
|
||||
- tests/kernels/mamba/test_gdn_prefill_cutedsl.py
|
||||
- tests/kernels/test_bf16x3_router_gemm_cutedsl.py
|
||||
- tests/kernels/test_ll_bf16_gemm.py
|
||||
- tests/kernels/test_top_k_per_row.py
|
||||
commands:
|
||||
- nvidia-smi
|
||||
@@ -300,28 +259,24 @@ steps:
|
||||
- pytest -v -s tests/kernels/moe/test_flashinfer_moe.py
|
||||
- pytest -v -s tests/kernels/moe/test_trtllm_nvfp4_moe.py
|
||||
- pytest -v -s tests/kernels/moe/test_cutedsl_moe.py
|
||||
- pytest -v -s tests/kernels/mamba/test_gdn_prefill_cutedsl.py
|
||||
- pytest -v -s tests/kernels/test_bf16x3_router_gemm_cutedsl.py
|
||||
- pytest -v -s tests/kernels/test_ll_bf16_gemm.py
|
||||
# e2e
|
||||
- pytest -v -s tests/models/quantization/test_nvfp4.py
|
||||
|
||||
- label: Kernels Helion Test
|
||||
key: kernels-helion-test
|
||||
timeout_in_minutes: 115
|
||||
timeout_in_minutes: 30
|
||||
device: h100
|
||||
source_file_dependencies:
|
||||
- vllm/utils/import_utils.py
|
||||
- tests/kernels/helion/
|
||||
commands:
|
||||
- pip install helion==1.1.0
|
||||
- pytest -v -s kernels/helion/ --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT
|
||||
parallelism: 2
|
||||
- pytest -v -s kernels/helion/
|
||||
|
||||
|
||||
- label: Kernels FP8 MoE Test (1xH100)
|
||||
key: kernels-fp8-moe-test-1xh100
|
||||
timeout_in_minutes: 40
|
||||
- label: Kernels FP8 MoE Test (1 H100)
|
||||
key: kernels-fp8-moe-test-1-h100
|
||||
timeout_in_minutes: 90
|
||||
device: h100
|
||||
num_devices: 1
|
||||
optional: true
|
||||
@@ -336,9 +291,9 @@ steps:
|
||||
- pytest -v -s kernels/moe/test_triton_moe_no_act_mul.py
|
||||
- pytest -v -s kernels/moe/test_triton_moe_ptpc_fp8.py
|
||||
|
||||
- label: Kernels FP8 MoE Test (2xH100)
|
||||
key: kernels-fp8-moe-test-2xh100
|
||||
timeout_in_minutes: 45
|
||||
- label: Kernels FP8 MoE Test (2 H100s)
|
||||
key: kernels-fp8-moe-test-2-h100s
|
||||
timeout_in_minutes: 90
|
||||
device: h100
|
||||
num_devices: 2
|
||||
optional: true
|
||||
@@ -348,7 +303,7 @@ steps:
|
||||
|
||||
- label: Kernels Fp4 MoE Test (B200)
|
||||
key: kernels-fp4-moe-test-b200
|
||||
timeout_in_minutes: 25
|
||||
timeout_in_minutes: 60
|
||||
device: b200-k8s
|
||||
num_devices: 1
|
||||
optional: true
|
||||
@@ -361,7 +316,7 @@ steps:
|
||||
|
||||
- label: Kernels FusedMoE Layer Test (2 H100s)
|
||||
key: kernels-fusedmoe-layer-test-2-h100s
|
||||
timeout_in_minutes: 30
|
||||
timeout_in_minutes: 90
|
||||
device: h100
|
||||
num_devices: 2
|
||||
source_file_dependencies:
|
||||
|
||||
@@ -5,7 +5,7 @@ steps:
|
||||
- label: LM Eval Small Models
|
||||
device: h200_35gb
|
||||
key: lm-eval-small-models
|
||||
timeout_in_minutes: 45
|
||||
timeout_in_minutes: 75
|
||||
source_file_dependencies:
|
||||
- csrc/
|
||||
- vllm/model_executor/layers/quantization
|
||||
@@ -14,9 +14,8 @@ steps:
|
||||
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small.txt
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 45
|
||||
device: mi325_1
|
||||
timeout_in_minutes: 55
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
@@ -29,8 +28,7 @@ steps:
|
||||
- vllm/_aiter_ops.py
|
||||
- vllm/platforms/rocm.py
|
||||
|
||||
# - label: LM Eval Large Models (4xA100)
|
||||
# key: lm-eval-large-models-4xa100
|
||||
# - label: LM Eval Large Models (4 GPUs)(A100)
|
||||
# device: a100
|
||||
# optional: true
|
||||
# num_devices: 4
|
||||
@@ -42,8 +40,8 @@ steps:
|
||||
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
# - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large.txt --tp-size=4
|
||||
|
||||
- label: LM Eval Large Models (4xH100)
|
||||
key: lm-eval-large-models-4xh100
|
||||
- label: LM Eval Large Models (4 GPUs)(H100)
|
||||
key: lm-eval-large-models-4-gpus-h100
|
||||
device: h100
|
||||
optional: true
|
||||
num_devices: 4
|
||||
@@ -55,9 +53,9 @@ steps:
|
||||
- export VLLM_USE_DEEP_GEMM=0 # We found Triton is faster than DeepGEMM for H100
|
||||
- pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-hopper.txt --tp-size=4
|
||||
|
||||
- label: LM Eval Small Models (1xB200)
|
||||
key: lm-eval-small-models-1xb200
|
||||
timeout_in_minutes: 50
|
||||
- label: LM Eval Small Models (B200)
|
||||
key: lm-eval-small-models-b200
|
||||
timeout_in_minutes: 120
|
||||
device: b200-k8s
|
||||
optional: true
|
||||
source_file_dependencies:
|
||||
@@ -66,45 +64,10 @@ steps:
|
||||
commands:
|
||||
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell.txt
|
||||
|
||||
- label: LM Eval Small Models Distributed (2xB200)
|
||||
key: lm-eval-small-models-distributed-2xb200
|
||||
- label: LM Eval Large Models (B200, EP)
|
||||
key: lm-eval-large-models-b200-ep
|
||||
timeout_in_minutes: 120
|
||||
device: b200-k8s
|
||||
num_devices: 2
|
||||
optional: true
|
||||
source_file_dependencies:
|
||||
- csrc/
|
||||
- vllm/model_executor/layers/quantization
|
||||
autorun_on_main: true
|
||||
commands:
|
||||
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small-tp.txt
|
||||
|
||||
- label: LM Eval PCP (4xB200)
|
||||
key: lm-eval-pcp-4xb200
|
||||
timeout_in_minutes: 360
|
||||
device: b200-k8s
|
||||
num_devices: 4
|
||||
optional: true
|
||||
source_file_dependencies:
|
||||
- csrc/
|
||||
- tests/evals/gsm8k/configs/GLM-5.2-NVFP4-TP2-PCP2-EP.yaml
|
||||
- tests/evals/gsm8k/configs/GLM-5.2-NVFP4-TP1-PCP4-EP.yaml
|
||||
- tests/evals/gsm8k/configs/models-pcp.txt
|
||||
- vllm/model_executor/layers/quantization
|
||||
- vllm/config/parallel.py
|
||||
- vllm/distributed/parallel_state.py
|
||||
- vllm/model_executor/layers/attention/mla_attention.py
|
||||
- vllm/model_executor/layers/attention/pcp.py
|
||||
- vllm/v1/worker/gpu/model_runner.py
|
||||
- vllm/v1/worker/gpu/pcp_manager.py
|
||||
autorun_on_main: true
|
||||
commands:
|
||||
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-pcp.txt
|
||||
|
||||
- label: LM Eval Large Models EP (2xB200)
|
||||
key: lm-eval-large-models-ep-2xb200
|
||||
timeout_in_minutes: 60
|
||||
device: b200-k8s
|
||||
optional: true
|
||||
num_devices: 2
|
||||
source_file_dependencies:
|
||||
@@ -113,9 +76,9 @@ steps:
|
||||
commands:
|
||||
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell-ep.txt
|
||||
|
||||
- label: LM Eval Qwen3.5 Models (2xB200)
|
||||
key: lm-eval-qwen3-5-models-2xb200
|
||||
timeout_in_minutes: 45
|
||||
- label: LM Eval Qwen3.5 Models (B200)
|
||||
key: lm-eval-qwen3-5-models-b200
|
||||
timeout_in_minutes: 120
|
||||
device: b200-k8s
|
||||
optional: true
|
||||
num_devices: 2
|
||||
@@ -126,13 +89,13 @@ steps:
|
||||
- vllm/transformers_utils/configs/qwen3_5_moe.py
|
||||
- vllm/model_executor/models/qwen3_next.py
|
||||
- vllm/model_executor/models/qwen3_next_mtp.py
|
||||
- vllm/third_party/flash_linear_attention/ops/
|
||||
- vllm/model_executor/layers/fla/ops/
|
||||
commands:
|
||||
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-qwen35-blackwell.txt
|
||||
|
||||
- label: LM Eval Large Models (8xH200)
|
||||
key: lm-eval-large-models-8xh200
|
||||
timeout_in_minutes: 50
|
||||
- label: LM Eval Large Models (H200)
|
||||
key: lm-eval-large-models-h200
|
||||
timeout_in_minutes: 60
|
||||
device: h200
|
||||
optional: true
|
||||
num_devices: 8
|
||||
@@ -140,9 +103,8 @@ steps:
|
||||
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200.txt
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_8
|
||||
timeout_in_minutes: 40
|
||||
timeout_in_minutes: 180
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
commands:
|
||||
@@ -174,9 +136,9 @@ steps:
|
||||
commands:
|
||||
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor-dp-ep/config-b200.txt
|
||||
|
||||
- label: LM Eval Humming f16 (A100 - TEMPORARY)
|
||||
key: lm-eval-humming-f16-a100
|
||||
timeout_in_minutes: 75
|
||||
- label: LM Eval Humming (A100 - TEMPORARY)
|
||||
key: lm-eval-humming-a100
|
||||
timeout_in_minutes: 30
|
||||
device: a100
|
||||
optional: true
|
||||
num_devices: 1
|
||||
@@ -184,29 +146,13 @@ steps:
|
||||
- vllm/model_executor/layers/quantization/humming.py
|
||||
- vllm/model_executor/layers/quantization/utils/humming_utils.py
|
||||
- vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
|
||||
- vllm/model_executor/layers/fused_moe/oracle/
|
||||
- vllm/model_executor/kernels/linear/
|
||||
- vllm/model_executor/layers/fused_moe/oracle/mxfp4.py
|
||||
commands:
|
||||
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config.txt
|
||||
|
||||
- label: LM Eval Humming Act int8 (A100 - TEMPORARY)
|
||||
key: lm-eval-humming-act-a100
|
||||
timeout_in_minutes: 45
|
||||
device: a100
|
||||
optional: true
|
||||
num_devices: 1
|
||||
source_file_dependencies:
|
||||
- vllm/model_executor/layers/quantization/humming.py
|
||||
- vllm/model_executor/layers/quantization/utils/humming_utils.py
|
||||
- vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
|
||||
- vllm/model_executor/layers/fused_moe/oracle/
|
||||
- vllm/model_executor/kernels/linear/
|
||||
commands:
|
||||
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt
|
||||
|
||||
- label: LM Eval Humming f16 (H100 - TEMPORARY)
|
||||
key: lm-eval-humming-f16-h100
|
||||
timeout_in_minutes: 70
|
||||
- label: LM Eval Humming (H100 - TEMPORARY)
|
||||
key: lm-eval-humming-h100
|
||||
timeout_in_minutes: 30
|
||||
device: h100
|
||||
optional: true
|
||||
num_devices: 1
|
||||
@@ -214,30 +160,14 @@ steps:
|
||||
- vllm/model_executor/layers/quantization/humming.py
|
||||
- vllm/model_executor/layers/quantization/utils/humming_utils.py
|
||||
- vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
|
||||
- vllm/model_executor/layers/fused_moe/oracle/
|
||||
- vllm/model_executor/kernels/linear/
|
||||
- vllm/model_executor/layers/fused_moe/oracle/mxfp4.py
|
||||
commands:
|
||||
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config.txt
|
||||
|
||||
- label: LM Eval Humming Act fp8/int8 (H100 - TEMPORARY)
|
||||
key: lm-eval-humming-act-h100
|
||||
timeout_in_minutes: 70
|
||||
device: h100
|
||||
optional: true
|
||||
num_devices: 1
|
||||
source_file_dependencies:
|
||||
- vllm/model_executor/layers/quantization/humming.py
|
||||
- vllm/model_executor/layers/quantization/utils/humming_utils.py
|
||||
- vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
|
||||
- vllm/model_executor/layers/fused_moe/oracle/
|
||||
- vllm/model_executor/kernels/linear/
|
||||
commands:
|
||||
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-fp8.txt
|
||||
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt
|
||||
|
||||
- label: LM Eval Humming f16 (B200 - TEMPORARY)
|
||||
key: lm-eval-humming-f16-b200
|
||||
timeout_in_minutes: 50
|
||||
- label: LM Eval Humming (B200 - TEMPORARY)
|
||||
key: lm-eval-humming-b200
|
||||
timeout_in_minutes: 30
|
||||
device: b200-k8s
|
||||
optional: true
|
||||
num_devices: 1
|
||||
@@ -245,30 +175,14 @@ steps:
|
||||
- vllm/model_executor/layers/quantization/humming.py
|
||||
- vllm/model_executor/layers/quantization/utils/humming_utils.py
|
||||
- vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
|
||||
- vllm/model_executor/layers/fused_moe/oracle/
|
||||
- vllm/model_executor/kernels/linear/
|
||||
- vllm/model_executor/layers/fused_moe/oracle/mxfp4.py
|
||||
commands:
|
||||
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config.txt
|
||||
|
||||
- label: LM Eval Humming Act fp8/int8 (B200 - TEMPORARY)
|
||||
key: lm-eval-humming-act-b200
|
||||
timeout_in_minutes: 50
|
||||
device: b200-k8s
|
||||
optional: true
|
||||
num_devices: 1
|
||||
source_file_dependencies:
|
||||
- vllm/model_executor/layers/quantization/humming.py
|
||||
- vllm/model_executor/layers/quantization/utils/humming_utils.py
|
||||
- vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
|
||||
- vllm/model_executor/layers/fused_moe/oracle/
|
||||
- vllm/model_executor/kernels/linear/
|
||||
commands:
|
||||
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-fp8.txt
|
||||
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/humming/config-act-int8.txt
|
||||
|
||||
- label: LM Eval TurboQuant KV Cache
|
||||
key: lm-eval-turboquant-kv-cache
|
||||
timeout_in_minutes: 55
|
||||
timeout_in_minutes: 75
|
||||
device: h200_18gb
|
||||
source_file_dependencies:
|
||||
- vllm/model_executor/layers/quantization/turboquant/
|
||||
@@ -278,9 +192,9 @@ steps:
|
||||
commands:
|
||||
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant.txt
|
||||
|
||||
- label: GPQA Eval (GPT-OSS) (2xH100)
|
||||
key: gpqa-eval-gpt-oss-2xh100
|
||||
timeout_in_minutes: 35
|
||||
- label: GPQA Eval (GPT-OSS) (H100)
|
||||
key: gpqa-eval-gpt-oss-h100
|
||||
timeout_in_minutes: 120
|
||||
device: h100
|
||||
optional: true
|
||||
num_devices: 2
|
||||
@@ -292,9 +206,9 @@ steps:
|
||||
- uv pip install --system 'gpt-oss[eval]==0.0.5'
|
||||
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-h100.txt
|
||||
|
||||
- label: GPQA Eval (GPT-OSS) (2xB200)
|
||||
key: gpqa-eval-gpt-oss-2xb200
|
||||
timeout_in_minutes: 30
|
||||
- label: GPQA Eval (GPT-OSS) (B200)
|
||||
key: gpqa-eval-gpt-oss-b200
|
||||
timeout_in_minutes: 120
|
||||
device: b200-k8s
|
||||
optional: true
|
||||
num_devices: 2
|
||||
@@ -308,12 +222,10 @@ steps:
|
||||
|
||||
- label: GPQA Eval (GPT-OSS) (DGX Spark)
|
||||
key: gpqa-eval-gpt-oss-spark
|
||||
timeout_in_minutes: 35
|
||||
timeout_in_minutes: 120
|
||||
device: dgx-spark
|
||||
optional: true
|
||||
num_devices: 1
|
||||
depends_on:
|
||||
- arm64-image-build
|
||||
source_file_dependencies:
|
||||
- csrc/
|
||||
- vllm/model_executor/layers/quantization
|
||||
@@ -322,50 +234,9 @@ steps:
|
||||
- uv pip install --system 'gpt-oss[eval]==0.0.5'
|
||||
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-spark.txt
|
||||
|
||||
- label: LM Eval KV-Offload (1xH200)
|
||||
key: kv-offload-small
|
||||
timeout_in_minutes: 30
|
||||
device: h200_35gb
|
||||
source_file_dependencies:
|
||||
- vllm/distributed/kv_transfer/kv_connector/v1/offloading/
|
||||
- vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py
|
||||
- vllm/v1/kv_offload/
|
||||
- vllm/v1/simple_kv_offload/
|
||||
- tests/evals/gsm8k/test_gsm8k_offloading.py
|
||||
commands:
|
||||
- pytest -s -v evals/gsm8k/test_gsm8k_offloading.py -k "nemotron-h-8b or gemma-4-e4b-it"
|
||||
|
||||
- label: LM Eval KV-Offload (2xH100)
|
||||
key: kv-offload-medium
|
||||
timeout_in_minutes: 45
|
||||
device: h100
|
||||
num_devices: 2
|
||||
source_file_dependencies:
|
||||
- vllm/distributed/kv_transfer/kv_connector/v1/offloading/
|
||||
- vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py
|
||||
- vllm/v1/kv_offload/
|
||||
- vllm/v1/simple_kv_offload/
|
||||
- tests/evals/gsm8k/test_gsm8k_offloading.py
|
||||
commands:
|
||||
- pytest -s -v evals/gsm8k/test_gsm8k_offloading.py -k "qwen3.5-35b or deepseek-v2-lite"
|
||||
|
||||
- label: LM Eval KV-Offload (4xH100)
|
||||
key: kv-offload-large
|
||||
timeout_in_minutes: 40
|
||||
device: h100
|
||||
num_devices: 4
|
||||
source_file_dependencies:
|
||||
- vllm/distributed/kv_transfer/kv_connector/v1/offloading/
|
||||
- vllm/distributed/kv_transfer/kv_connector/v1/simple_cpu_offload_connector.py
|
||||
- vllm/v1/kv_offload/
|
||||
- vllm/v1/simple_kv_offload/
|
||||
- tests/evals/gsm8k/test_gsm8k_offloading.py
|
||||
commands:
|
||||
- pytest -s -v evals/gsm8k/test_gsm8k_offloading.py -k "deepseek-v4-flash"
|
||||
|
||||
- label: MRCR Eval Small Models
|
||||
device: h200_35gb
|
||||
timeout_in_minutes: 25
|
||||
timeout_in_minutes: 30
|
||||
source_file_dependencies:
|
||||
- tests/evals/mrcr/
|
||||
commands:
|
||||
|
||||
@@ -5,7 +5,7 @@ steps:
|
||||
- label: LoRA %N
|
||||
device: h200_35gb
|
||||
key: lora
|
||||
timeout_in_minutes: 40
|
||||
timeout_in_minutes: 30
|
||||
source_file_dependencies:
|
||||
- vllm/lora
|
||||
- tests/lora
|
||||
@@ -14,10 +14,9 @@ steps:
|
||||
parallelism: 4
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_1
|
||||
device: mi325_1
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
timeout_in_minutes: 85
|
||||
timeout_in_minutes: 60
|
||||
source_file_dependencies:
|
||||
- vllm/lora
|
||||
- tests/lora
|
||||
@@ -28,7 +27,7 @@ steps:
|
||||
|
||||
- label: LoRA TP (Distributed)
|
||||
key: lora-tp-distributed
|
||||
timeout_in_minutes: 60
|
||||
timeout_in_minutes: 30
|
||||
num_devices: 4
|
||||
source_file_dependencies:
|
||||
- vllm/lora
|
||||
@@ -47,4 +46,4 @@ steps:
|
||||
- pytest -v -s -x lora/test_qwen3_with_multi_loras.py
|
||||
- pytest -v -s -x lora/test_olmoe_tp.py
|
||||
- pytest -v -s -x lora/test_gptoss_tp.py
|
||||
- pytest -v -s -x lora/test_qwen35_densemodel_lora.py
|
||||
- pytest -v -s -x lora/test_qwen35_densemodel_lora.py
|
||||
@@ -5,7 +5,7 @@ steps:
|
||||
- label: V1 Spec Decode
|
||||
device: h200_35gb
|
||||
key: v1-spec-decode
|
||||
timeout_in_minutes: 40
|
||||
timeout_in_minutes: 30
|
||||
source_file_dependencies:
|
||||
- vllm/config/
|
||||
- vllm/distributed/
|
||||
@@ -23,15 +23,14 @@ steps:
|
||||
- pytest -v -s -m 'not slow_test' v1/spec_decode
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 50
|
||||
timeout_in_minutes: 65
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
|
||||
- label: V1 Sample + Logits
|
||||
key: v1-sample-logits
|
||||
timeout_in_minutes: 83
|
||||
timeout_in_minutes: 30
|
||||
device: h200_18gb
|
||||
source_file_dependencies:
|
||||
- vllm/config/
|
||||
@@ -59,16 +58,13 @@ steps:
|
||||
- pytest -v -s v1/test_outputs.py
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 70
|
||||
device: mi325_1
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
|
||||
- label: V1 Core + KV + Metrics
|
||||
device: h200_35gb
|
||||
key: v1-core-kv-metrics
|
||||
timeout_in_minutes: 80
|
||||
timeout_in_minutes: 30
|
||||
source_file_dependencies:
|
||||
- vllm/config/
|
||||
- vllm/distributed/
|
||||
@@ -92,9 +88,7 @@ steps:
|
||||
- tests/v1/kv_offload
|
||||
- tests/v1/simple_kv_offload
|
||||
- tests/v1/worker
|
||||
- tests/v1/streaming_input
|
||||
- tests/v1/kv_connector/unit
|
||||
- tests/v1/ec_connector/unit
|
||||
- tests/v1/metrics
|
||||
- tests/entrypoints/openai/correctness/test_lmeval.py
|
||||
commands:
|
||||
@@ -106,18 +100,15 @@ steps:
|
||||
- pytest -v -s v1/kv_offload
|
||||
- pytest -v -s v1/simple_kv_offload
|
||||
- pytest -v -s v1/worker
|
||||
- pytest -v -s v1/streaming_input
|
||||
- pytest -v -s -m 'not cpu_test' v1/kv_connector/unit
|
||||
- pytest -v -s -m 'not cpu_test' v1/ec_connector/unit
|
||||
- pytest -v -s -m 'not cpu_test' v1/metrics
|
||||
# Integration test for streaming correctness (requires special branch).
|
||||
- pip install -U git+https://github.com/vllm-project/lm-evaluation-harness.git@streaming-api
|
||||
- pip install -U git+https://github.com/robertgshaw2-redhat/lm-evaluation-harness.git@streaming-api
|
||||
- pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 65
|
||||
device: mi325_1
|
||||
timeout_in_minutes: 60
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
|
||||
@@ -148,8 +139,6 @@ steps:
|
||||
- pytest -v -s -m 'cpu_test' v1/core
|
||||
- pytest -v -s v1/structured_output
|
||||
- pytest -v -s v1/test_serial_utils.py
|
||||
- pytest -v -s v1/test_kv_cache_spec_registry.py
|
||||
- pytest -v -s v1/cudagraph/test_cudagraph_manager.py
|
||||
- pytest -v -s -m 'cpu_test' v1/kv_connector/unit
|
||||
- pytest -v -s -m 'cpu_test' v1/metrics
|
||||
|
||||
@@ -183,7 +172,7 @@ steps:
|
||||
|
||||
- label: Regression
|
||||
key: regression
|
||||
timeout_in_minutes: 30
|
||||
timeout_in_minutes: 20
|
||||
device: h200_18gb
|
||||
source_file_dependencies:
|
||||
- vllm/config/
|
||||
@@ -199,21 +188,21 @@ steps:
|
||||
- vllm/v1/
|
||||
- tests/test_regression
|
||||
commands:
|
||||
- pip install 'modelscope<1.38'
|
||||
- pip install modelscope
|
||||
- pytest -v -s test_regression.py
|
||||
working_dir: "/vllm-workspace/tests" # optional
|
||||
|
||||
- label: Examples
|
||||
device: h200_35gb
|
||||
key: examples
|
||||
timeout_in_minutes: 40
|
||||
timeout_in_minutes: 45
|
||||
working_dir: "/vllm-workspace/examples"
|
||||
source_file_dependencies:
|
||||
- vllm/entrypoints
|
||||
- vllm/multimodal
|
||||
- examples/
|
||||
commands:
|
||||
- pip install --no-deps tensorizer # for tensorizer test
|
||||
- pip install tensorizer # for tensorizer test
|
||||
# for basic
|
||||
- python3 basic/offline_inference/chat.py
|
||||
- python3 basic/offline_inference/generate.py --model facebook/opt-125m
|
||||
@@ -235,22 +224,10 @@ steps:
|
||||
- python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048
|
||||
# https://github.com/vllm-project/vllm/pull/26682 uses slightly more memory in PyTorch 2.9+ causing this test to OOM in 1xL4 GPU
|
||||
- python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 75
|
||||
source_file_dependencies:
|
||||
- vllm/entrypoints
|
||||
- vllm/multimodal
|
||||
- examples/
|
||||
- vllm/platforms/rocm.py
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
|
||||
- label: Metrics, Tracing (2 GPUs)
|
||||
key: metrics-tracing-2-gpus
|
||||
timeout_in_minutes: 25
|
||||
timeout_in_minutes: 20
|
||||
num_devices: 2
|
||||
source_file_dependencies:
|
||||
- vllm/config/
|
||||
@@ -266,7 +243,6 @@ steps:
|
||||
- vllm/utils/
|
||||
- vllm/v1/
|
||||
- tests/v1/tracing
|
||||
- tests/tracing/
|
||||
commands:
|
||||
- "pip install \
|
||||
'opentelemetry-sdk>=1.26.0' \
|
||||
@@ -274,14 +250,6 @@ steps:
|
||||
'opentelemetry-exporter-otlp>=1.26.0' \
|
||||
'opentelemetry-semantic-conventions-ai>=0.4.1'"
|
||||
- pytest -v -s v1/tracing
|
||||
- pytest -v -s tracing
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_2
|
||||
timeout_in_minutes: 30
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
|
||||
- label: Python-only Installation
|
||||
key: python-only-installation
|
||||
@@ -294,22 +262,11 @@ steps:
|
||||
- setup.py
|
||||
commands:
|
||||
- bash standalone_tests/python_only_compile.sh
|
||||
mirror:
|
||||
amd:
|
||||
device: mi250_1
|
||||
timeout_in_minutes: 55
|
||||
soft_fail: true
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- tests/standalone_tests/python_only_compile.sh
|
||||
- setup.py
|
||||
- vllm/platforms/rocm.py
|
||||
|
||||
- label: Async Engine, Inputs, Utils, Worker
|
||||
device: h200_35gb
|
||||
key: async-engine-inputs-utils-worker
|
||||
timeout_in_minutes: 25
|
||||
timeout_in_minutes: 50
|
||||
source_file_dependencies:
|
||||
- vllm/assets/
|
||||
- vllm/config/
|
||||
@@ -336,7 +293,7 @@ steps:
|
||||
key: async-engine-inputs-utils-worker-config-cpu
|
||||
depends_on:
|
||||
- image-build-cpu
|
||||
timeout_in_minutes: 65
|
||||
timeout_in_minutes: 30
|
||||
source_file_dependencies:
|
||||
- vllm/assets/
|
||||
- vllm/config/
|
||||
@@ -368,7 +325,6 @@ steps:
|
||||
- tests/test_outputs.py
|
||||
- tests/test_pooling_params.py
|
||||
- tests/test_ray_env.py
|
||||
- tests/test_sampling_params.py
|
||||
- tests/multimodal
|
||||
- tests/renderers
|
||||
- tests/standalone_tests/lazy_imports.py
|
||||
@@ -386,10 +342,9 @@ steps:
|
||||
- pytest -v -s test_outputs.py
|
||||
- pytest -v -s test_pooling_params.py
|
||||
- pytest -v -s test_ray_env.py
|
||||
- pytest -v -s test_sampling_params.py
|
||||
- pytest -v -s -m 'cpu_test' multimodal
|
||||
- pytest -v -s renderers
|
||||
- pytest -v -s reasoning
|
||||
- pytest -v -s reasoning --ignore=reasoning/test_seedoss_reasoning_parser.py --ignore=reasoning/test_glm4_moe_reasoning_parser.py
|
||||
- pytest -v -s tool_parsers
|
||||
- pytest -v -s tokenizers_
|
||||
- pytest -v -s parser
|
||||
@@ -398,7 +353,7 @@ steps:
|
||||
|
||||
- label: Batch Invariance (A100)
|
||||
key: batch-invariance-a100
|
||||
timeout_in_minutes: 60
|
||||
timeout_in_minutes: 30
|
||||
device: a100
|
||||
source_file_dependencies:
|
||||
- vllm/v1/attention
|
||||
@@ -408,11 +363,11 @@ steps:
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- pip install pytest-timeout pytest-forked
|
||||
- pytest -v -s v1/determinism/test_batch_invariance.py
|
||||
- VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k TRITON_MLA
|
||||
- VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle[TRITON_MLA]
|
||||
|
||||
- label: Batch Invariance (H100)
|
||||
key: batch-invariance-h100
|
||||
timeout_in_minutes: 60
|
||||
timeout_in_minutes: 30
|
||||
device: h100
|
||||
source_file_dependencies:
|
||||
- vllm/v1/attention
|
||||
@@ -423,12 +378,12 @@ steps:
|
||||
- pip install pytest-timeout pytest-forked
|
||||
- pytest -v -s v1/determinism/test_batch_invariance.py
|
||||
- pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py
|
||||
- VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k TRITON_MLA
|
||||
- VLLM_TEST_MODEL=Qwen/Qwen3-30B-A3B-Thinking-2507-FP8 pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k FLASH_ATTN
|
||||
- VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle[TRITON_MLA]
|
||||
- VLLM_TEST_MODEL=Qwen/Qwen3-30B-A3B-Thinking-2507-FP8 pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle[FLASH_ATTN]
|
||||
|
||||
- label: Batch Invariance (B200)
|
||||
key: batch-invariance-b200
|
||||
timeout_in_minutes: 45
|
||||
timeout_in_minutes: 30
|
||||
device: b200-k8s
|
||||
source_file_dependencies:
|
||||
- vllm/v1/attention
|
||||
@@ -439,18 +394,15 @@ steps:
|
||||
- pip install pytest-timeout pytest-forked
|
||||
- pytest -v -s v1/determinism/test_batch_invariance.py
|
||||
- pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py
|
||||
- VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k TRITON_MLA
|
||||
- VLLM_TEST_MODEL=Qwen/Qwen3-30B-A3B-Thinking-2507-FP8 pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k FLASH_ATTN
|
||||
- VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle[TRITON_MLA]
|
||||
- VLLM_TEST_MODEL=Qwen/Qwen3-30B-A3B-Thinking-2507-FP8 pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle[FLASH_ATTN]
|
||||
- pytest -v -s v1/determinism/test_nvfp4_batch_invariant.py
|
||||
- pytest -v -s v1/determinism/test_nvfp4_batch_invariant_scaled_mm.py
|
||||
- pytest -v -s v1/determinism/test_matmul_batch_invariant.py
|
||||
- pytest -v -s v1/determinism/test_cutlass_batch_invariance.py
|
||||
- pytest -v -s v1/determinism/test_online_batch_invariance.py
|
||||
|
||||
|
||||
- label: Acceptance Length Test (Large Models) # optional
|
||||
device: h200_35gb
|
||||
key: acceptance-length-test-large-models
|
||||
timeout_in_minutes: 20
|
||||
timeout_in_minutes: 25
|
||||
gpu: h100
|
||||
optional: true
|
||||
num_gpus: 1
|
||||
|
||||
@@ -3,16 +3,13 @@ depends_on:
|
||||
- image-build
|
||||
steps:
|
||||
- label: Model Executor
|
||||
device: h200_35gb
|
||||
key: model-executor
|
||||
timeout_in_minutes: 60
|
||||
timeout_in_minutes: 35
|
||||
source_file_dependencies:
|
||||
- vllm/engine/arg_utils.py
|
||||
- vllm/config/model.py
|
||||
- vllm/model_executor
|
||||
- vllm/model_executor/warmup
|
||||
- tests/model_executor
|
||||
- tests/model_executor/test_jit_warmup.py
|
||||
- tests/entrypoints/openai/completion/test_tensorizer_entrypoint.py
|
||||
commands:
|
||||
- apt-get update && apt-get install -y curl libsodium23
|
||||
@@ -26,20 +23,3 @@ steps:
|
||||
# calls that the signal method cannot interrupt.
|
||||
- pytest -v -s model_executor -m '(not slow_test)' --timeout=900 --timeout-method=thread
|
||||
- pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py --timeout=900 --timeout-method=thread
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 60
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- vllm/engine/arg_utils.py
|
||||
- vllm/config/model.py
|
||||
- vllm/model_executor
|
||||
- vllm/model_executor/warmup
|
||||
- tests/model_executor
|
||||
- tests/model_executor/test_jit_warmup.py
|
||||
- tests/entrypoints/openai/completion/test_tensorizer_entrypoint.py
|
||||
- vllm/_aiter_ops.py
|
||||
- vllm/platforms/rocm.py
|
||||
|
||||
@@ -5,7 +5,7 @@ steps:
|
||||
- label: Model Runner V2 Core Tests
|
||||
device: h200_35gb
|
||||
key: model-runner-v2-core-tests
|
||||
timeout_in_minutes: 35
|
||||
timeout_in_minutes: 45
|
||||
source_file_dependencies:
|
||||
- vllm/v1/worker/gpu/
|
||||
- vllm/v1/worker/gpu_worker.py
|
||||
@@ -18,7 +18,9 @@ steps:
|
||||
- set -x
|
||||
- export VLLM_USE_V2_MODEL_RUNNER=1
|
||||
- pytest -v -s v1/engine/test_llm_engine.py -k "not test_engine_metrics"
|
||||
- pytest -v -s v1/e2e/general/test_async_scheduling.py -k "not ngram"
|
||||
# This requires eager until we sort out CG correctness issues.
|
||||
# TODO: remove ENFORCE_EAGER here after https://github.com/vllm-project/vllm/pull/32936 is merged.
|
||||
- ENFORCE_EAGER=1 pytest -v -s v1/e2e/general/test_async_scheduling.py -k "not ngram"
|
||||
- pytest -v -s v1/e2e/general/test_context_length.py
|
||||
- pytest -v -s v1/e2e/general/test_min_tokens.py
|
||||
# Temporary hack filter to exclude ngram spec decoding based tests.
|
||||
@@ -27,7 +29,7 @@ steps:
|
||||
- label: Model Runner V2 Examples
|
||||
device: h200_35gb
|
||||
key: model-runner-v2-examples
|
||||
timeout_in_minutes: 35
|
||||
timeout_in_minutes: 45
|
||||
working_dir: "/vllm-workspace/examples"
|
||||
source_file_dependencies:
|
||||
- vllm/v1/worker/gpu/
|
||||
@@ -41,7 +43,7 @@ steps:
|
||||
commands:
|
||||
- set -x
|
||||
- export VLLM_USE_V2_MODEL_RUNNER=1
|
||||
- pip install --no-deps tensorizer # for tensorizer test
|
||||
- pip install tensorizer # for tensorizer test
|
||||
- python3 basic/offline_inference/chat.py # for basic
|
||||
- python3 basic/offline_inference/generate.py --model facebook/opt-125m
|
||||
#- python3 basic/offline_inference/generate.py --model meta-llama/Llama-2-13b-chat-hf --cpu-offload-gb 10 # TODO
|
||||
@@ -63,7 +65,7 @@ steps:
|
||||
|
||||
- label: Model Runner V2 Distributed (2 GPUs)
|
||||
key: model-runner-v2-distributed-2-gpus
|
||||
timeout_in_minutes: 30
|
||||
timeout_in_minutes: 45
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
num_devices: 2
|
||||
source_file_dependencies:
|
||||
@@ -84,7 +86,7 @@ steps:
|
||||
|
||||
- label: Model Runner V2 Pipeline Parallelism (4 GPUs)
|
||||
key: model-runner-v2-pipeline-parallelism-4-gpus
|
||||
timeout_in_minutes: 50
|
||||
timeout_in_minutes: 60
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
num_devices: 4
|
||||
source_file_dependencies:
|
||||
|
||||
@@ -4,8 +4,9 @@ depends_on:
|
||||
steps:
|
||||
- label: Basic Models Tests (Initialization)
|
||||
key: basic-models-tests-initialization
|
||||
timeout_in_minutes: 25
|
||||
timeout_in_minutes: 45
|
||||
device: h200_18gb
|
||||
torch_nightly: true
|
||||
source_file_dependencies:
|
||||
- vllm/
|
||||
- tests/models/test_initialization.py
|
||||
@@ -13,11 +14,13 @@ steps:
|
||||
commands:
|
||||
# Run a subset of model initialization tests
|
||||
- pytest -v -s models/test_initialization.py::test_can_initialize_small_subset
|
||||
mirror:
|
||||
torch_nightly: {}
|
||||
|
||||
- label: Basic Models Tests (Extra Initialization) %N
|
||||
device: h200_35gb
|
||||
key: basic-models-tests-extra-initialization
|
||||
timeout_in_minutes: 100
|
||||
timeout_in_minutes: 45
|
||||
source_file_dependencies:
|
||||
- vllm/model_executor/models/
|
||||
- tests/models/test_initialization.py
|
||||
@@ -27,65 +30,31 @@ steps:
|
||||
# subset of supported models (the complement of the small subset in the above
|
||||
# test.) Also run if model initialization test file is modified
|
||||
- pytest -v -s models/test_initialization.py -k 'not test_can_initialize_small_subset' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
|
||||
parallelism: 4
|
||||
parallelism: 2
|
||||
mirror:
|
||||
torch_nightly: {}
|
||||
|
||||
- label: Basic Models Tests (Other)
|
||||
device: h200_35gb
|
||||
key: basic-models-tests-other
|
||||
timeout_in_minutes: 35
|
||||
timeout_in_minutes: 45
|
||||
source_file_dependencies:
|
||||
- vllm/
|
||||
- tests/models/test_terratorch.py
|
||||
- tests/models/transformers/test_backend.py
|
||||
- tests/models/test_transformers.py
|
||||
- tests/models/test_registry.py
|
||||
commands:
|
||||
- pytest -v -s models/test_terratorch.py models/transformers/test_backend.py models/test_registry.py
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 50
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
|
||||
- label: Inkling Unit Tests (B200)
|
||||
key: inkling-unit-tests-b200
|
||||
timeout_in_minutes: 40
|
||||
device: b200-k8s
|
||||
source_file_dependencies:
|
||||
- vllm/models/inkling/
|
||||
- vllm/cute_utils/
|
||||
- cmake/external_projects/tml_fa4.cmake
|
||||
- tests/models/inkling/
|
||||
commands:
|
||||
# FA4 kernel tests require SM100; the suite skips them elsewhere.
|
||||
- pytest -v -s models/inkling
|
||||
|
||||
- label: Kimi K3 Unit Tests (B200)
|
||||
key: kimi-k3-unit-tests-b200
|
||||
timeout_in_minutes: 40
|
||||
device: b200-k8s
|
||||
source_file_dependencies:
|
||||
- vllm/models/kimi_k3/
|
||||
- csrc/libtorch_stable/kimi_k3/
|
||||
- tests/models/kimi_k3/
|
||||
- tests/kernels/attention/test_kimi_k3_mla_fused_epilogue.py
|
||||
- tests/kernels/test_bf16_skinny_gemm.py
|
||||
commands:
|
||||
# The native NVIDIA Kimi K3 kernels require the SM100 family.
|
||||
- pytest -v -s models/kimi_k3 kernels/attention/test_kimi_k3_mla_fused_epilogue.py kernels/test_bf16_skinny_gemm.py
|
||||
- pytest -v -s models/test_terratorch.py models/test_transformers.py models/test_registry.py
|
||||
|
||||
- label: Basic Models Test (Other CPU) # 5min
|
||||
key: basic-models-test-other-cpu
|
||||
depends_on:
|
||||
- image-build-cpu
|
||||
timeout_in_minutes: 20
|
||||
timeout_in_minutes: 10
|
||||
source_file_dependencies:
|
||||
- vllm/
|
||||
- tests/models/test_utils.py
|
||||
- tests/models/test_vision.py
|
||||
- tests/models/test_adapters.py
|
||||
- tests/models/transformers/fusers/
|
||||
device: cpu-small
|
||||
commands:
|
||||
- pytest -v -s models/test_utils.py models/test_vision.py models/test_adapters.py models/transformers/fusers/
|
||||
- pytest -v -s models/test_utils.py models/test_vision.py
|
||||
|
||||
@@ -4,7 +4,7 @@ depends_on:
|
||||
steps:
|
||||
- label: Distributed Model Tests (2 GPUs)
|
||||
key: distributed-model-tests-2-gpus
|
||||
timeout_in_minutes: 60
|
||||
timeout_in_minutes: 50
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
num_devices: 2
|
||||
source_file_dependencies:
|
||||
@@ -17,7 +17,7 @@ steps:
|
||||
- TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
|
||||
- CUDA_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)'
|
||||
# Avoid importing model tests that cause CUDA reinitialization error
|
||||
- pytest models/transformers/test_backend.py -v -s -m 'distributed(num_gpus=2)'
|
||||
- pytest models/test_transformers.py -v -s -m 'distributed(num_gpus=2)'
|
||||
- pytest models/language -v -s -m 'distributed(num_gpus=2)'
|
||||
- pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)'
|
||||
- pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py
|
||||
|
||||
@@ -4,7 +4,7 @@ depends_on:
|
||||
steps:
|
||||
- label: Language Models Tests (Standard)
|
||||
key: language-models-tests-standard
|
||||
timeout_in_minutes: 30
|
||||
timeout_in_minutes: 25
|
||||
device: h200_18gb
|
||||
source_file_dependencies:
|
||||
- vllm/
|
||||
@@ -14,17 +14,11 @@ steps:
|
||||
- pip freeze | grep -E 'torch'
|
||||
- pytest -v -s models/language -m 'core_model and (not slow_test)'
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 45
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
torch_nightly: {}
|
||||
|
||||
- label: Language Models Tests (Extra Standard) %N
|
||||
device: h200_35gb
|
||||
key: language-models-tests-extra-standard
|
||||
timeout_in_minutes: 40
|
||||
timeout_in_minutes: 45
|
||||
source_file_dependencies:
|
||||
- vllm/model_executor/models/
|
||||
- tests/models/language/pooling/test_embedding.py
|
||||
@@ -37,42 +31,27 @@ steps:
|
||||
- pytest -v -s models/language -m 'core_model and slow_test' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
|
||||
parallelism: 2
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 40
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- vllm/model_executor/models/
|
||||
- vllm/model_executor/model_loader/
|
||||
- vllm/model_executor/layers/
|
||||
- vllm/v1/attention/backends/
|
||||
- vllm/v1/attention/selector.py
|
||||
- tests/models/language/pooling/test_embedding.py
|
||||
- tests/models/language/generation/test_common.py
|
||||
- tests/models/language/pooling/test_classification.py
|
||||
- vllm/_aiter_ops.py
|
||||
- vllm/platforms/rocm.py
|
||||
torch_nightly: {}
|
||||
|
||||
- label: Language Models Tests (Hybrid) %N
|
||||
device: h200_35gb
|
||||
key: language-models-tests-hybrid
|
||||
timeout_in_minutes: 65
|
||||
timeout_in_minutes: 75
|
||||
source_file_dependencies:
|
||||
- vllm/
|
||||
- tests/models/language/generation
|
||||
commands:
|
||||
# Install fast path packages for testing against transformers
|
||||
# Note: also needed to run plamo2 model in vLLM
|
||||
- uv pip install --system --no-build-isolation 'git+https://github.com/state-spaces/mamba@v2.3.0'
|
||||
- uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0'
|
||||
# Shard the hybrid language model tests that are numerically stable on Hopper.
|
||||
- pytest -v -s models/language/generation -m hybrid_model -k 'not granite-4.0-tiny-preview' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
|
||||
# Shard hybrid language model tests
|
||||
- pytest -v -s models/language/generation -m hybrid_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
|
||||
parallelism: 2
|
||||
mirror:
|
||||
torch_nightly: {}
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 60
|
||||
device: mi325_1
|
||||
timeout_in_minutes: 90
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
commands:
|
||||
@@ -80,37 +59,24 @@ steps:
|
||||
- uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0'
|
||||
- pytest -v -s models/language/generation -m hybrid_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
|
||||
|
||||
# Granite 4 hybrid generation is sensitive to hardware-specific Triton SSD
|
||||
# autotuning (https://github.com/vllm-project/vllm/issues/25194). Keep this one
|
||||
# correctness test on L4 until its H200 output matches the Transformers reference.
|
||||
- label: Language Models Tests (Granite L4 Compatibility)
|
||||
key: language-models-tests-granite-l4-compatibility
|
||||
timeout_in_minutes: 65
|
||||
source_file_dependencies:
|
||||
- vllm/
|
||||
- tests/models/language/generation
|
||||
commands:
|
||||
- uv pip install --system --no-build-isolation 'git+https://github.com/state-spaces/mamba@v2.3.0'
|
||||
- uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0'
|
||||
- pytest -v -s models/language/generation -m hybrid_model -k 'granite-4.0-tiny-preview'
|
||||
|
||||
- label: Language Models Test (Extended Generation) # 80min
|
||||
device: h200_35gb
|
||||
key: language-models-test-extended-generation
|
||||
timeout_in_minutes: 65
|
||||
timeout_in_minutes: 110
|
||||
optional: true
|
||||
source_file_dependencies:
|
||||
- vllm/
|
||||
- tests/models/language/generation
|
||||
commands:
|
||||
# Install fast path packages for testing against transformers
|
||||
# Note: also needed to run plamo2 model in vLLM
|
||||
- uv pip install --system --no-build-isolation 'git+https://github.com/state-spaces/mamba@v2.3.0'
|
||||
- uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0'
|
||||
- pytest -v -s models/language/generation -m '(not core_model) and (not hybrid_model)'
|
||||
|
||||
- label: Language Models Test (PPL)
|
||||
key: language-models-test-ppl
|
||||
timeout_in_minutes: 30
|
||||
timeout_in_minutes: 110
|
||||
device: h200_18gb
|
||||
optional: true
|
||||
source_file_dependencies:
|
||||
@@ -119,10 +85,10 @@ steps:
|
||||
commands:
|
||||
- pytest -v -s models/language/generation_ppl_test
|
||||
|
||||
- label: Language Models Test (Extended Pooling)
|
||||
- label: Language Models Test (Extended Pooling) # 36min
|
||||
device: h200_35gb
|
||||
key: language-models-test-extended-pooling
|
||||
timeout_in_minutes: 120
|
||||
timeout_in_minutes: 50
|
||||
optional: true
|
||||
source_file_dependencies:
|
||||
- vllm/
|
||||
@@ -131,15 +97,14 @@ steps:
|
||||
- pytest -v -s models/language/pooling -m 'not core_model'
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 95
|
||||
device: mi325_1
|
||||
timeout_in_minutes: 100
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
|
||||
- label: Language Models Test (MTEB)
|
||||
key: language-models-test-mteb
|
||||
timeout_in_minutes: 68
|
||||
timeout_in_minutes: 110
|
||||
device: h200_18gb
|
||||
optional: true
|
||||
source_file_dependencies:
|
||||
|
||||
@@ -4,107 +4,104 @@ depends_on:
|
||||
steps:
|
||||
- label: "Multi-Modal Models (Standard) 1: qwen2"
|
||||
key: multi-modal-models-standard-1-qwen2
|
||||
timeout_in_minutes: 68
|
||||
timeout_in_minutes: 45
|
||||
device: h200_18gb
|
||||
source_file_dependencies:
|
||||
- vllm/
|
||||
- tests/models/multimodal
|
||||
commands:
|
||||
- pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
|
||||
- pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2"
|
||||
- pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 65
|
||||
device: mi325_1
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
|
||||
- label: "Multi-Modal Models (Standard) 2: qwen3 + gemma"
|
||||
key: multi-modal-models-standard-2-qwen3-gemma
|
||||
timeout_in_minutes: 75
|
||||
timeout_in_minutes: 45
|
||||
device: h200_18gb
|
||||
source_file_dependencies:
|
||||
- vllm/
|
||||
- tests/models/multimodal
|
||||
commands:
|
||||
- pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
|
||||
- pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen3 or gemma"
|
||||
- pytest -v -s models/multimodal/generation/test_mm_prefix_lm.py -m core_model
|
||||
- pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 55
|
||||
device: mi325_1
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
|
||||
- label: "Multi-Modal Models (Standard) 3: llava + qwen2_vl"
|
||||
device: h200_35gb
|
||||
key: multi-modal-models-standard-3-llava-qwen2-vl
|
||||
timeout_in_minutes: 40
|
||||
timeout_in_minutes: 45
|
||||
source_file_dependencies:
|
||||
- vllm/
|
||||
- tests/models/multimodal
|
||||
commands:
|
||||
- pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
|
||||
- pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma"
|
||||
- pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model
|
||||
mirror:
|
||||
amd:
|
||||
device: mi250_1
|
||||
timeout_in_minutes: 55
|
||||
device: mi325_1
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
|
||||
- label: "Multi-Modal Models (Standard) 4: other + whisper"
|
||||
device: h200_35gb
|
||||
key: multi-modal-models-standard-4-other-whisper
|
||||
timeout_in_minutes: 75
|
||||
timeout_in_minutes: 45
|
||||
source_file_dependencies:
|
||||
- vllm/
|
||||
- tests/models/multimodal
|
||||
commands:
|
||||
- pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_mm_prefix_lm.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/generation/test_vit_cudagraph.py --ignore models/multimodal/processing
|
||||
- pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
|
||||
- pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/generation/test_vit_cudagraph.py --ignore models/multimodal/processing
|
||||
- pytest -v -s models/multimodal/generation/test_vit_cudagraph.py -m core_model
|
||||
- pytest models/multimodal/generation/test_memory_leak.py -m core_model
|
||||
- cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model # Otherwise, mp_method="spawn" doesn't work
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 50
|
||||
device: mi325_1
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
|
||||
- label: Multi-Modal Processor (CPU) %N
|
||||
- label: Multi-Modal Processor (CPU)
|
||||
key: multi-modal-processor-cpu
|
||||
depends_on:
|
||||
- image-build-cpu
|
||||
timeout_in_minutes: 125
|
||||
timeout_in_minutes: 60
|
||||
source_file_dependencies:
|
||||
- vllm/
|
||||
- tests/models/multimodal
|
||||
- tests/models/registry.py
|
||||
device: cpu-medium
|
||||
commands:
|
||||
- pytest -v -s models/multimodal/processing --ignore models/multimodal/processing/test_tensor_schema.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
|
||||
parallelism: 4
|
||||
- pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
|
||||
- pytest -v -s models/multimodal/processing --ignore models/multimodal/processing/test_tensor_schema.py
|
||||
|
||||
- label: Multi-Modal Processor # 44min
|
||||
key: multi-modal-processor
|
||||
timeout_in_minutes: 98
|
||||
timeout_in_minutes: 60
|
||||
device: h200_18gb
|
||||
source_file_dependencies:
|
||||
- vllm/
|
||||
- tests/models/multimodal
|
||||
- tests/models/registry.py
|
||||
commands:
|
||||
- pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
|
||||
- pytest -v -s models/multimodal/processing/test_tensor_schema.py
|
||||
|
||||
- label: Multi-Modal Accuracy Eval (Small Models) # 50min
|
||||
device: h200_35gb
|
||||
key: multi-modal-accuracy-eval-small-models
|
||||
timeout_in_minutes: 30
|
||||
timeout_in_minutes: 70
|
||||
working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
|
||||
source_file_dependencies:
|
||||
- vllm/multimodal/
|
||||
@@ -112,22 +109,8 @@ steps:
|
||||
- vllm/v1/core/
|
||||
commands:
|
||||
- pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-mm-small.txt --tp-size=1
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 35
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- vllm/multimodal/
|
||||
- vllm/inputs/
|
||||
- vllm/v1/core/
|
||||
- vllm/platforms/rocm.py
|
||||
- vllm/model_executor/model_loader/
|
||||
|
||||
- label: Multi-Modal Models (Extended Generation 1)
|
||||
device: h200_35gb
|
||||
key: multi-modal-models-extended-generation-1
|
||||
optional: true
|
||||
source_file_dependencies:
|
||||
@@ -135,13 +118,12 @@ steps:
|
||||
- tests/models/multimodal/generation
|
||||
- tests/models/multimodal/test_mapping.py
|
||||
commands:
|
||||
- pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
|
||||
- pytest -v -s models/multimodal/generation -m 'not core_model' --ignore models/multimodal/generation/test_common.py
|
||||
- pytest -v -s models/multimodal/test_mapping.py
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 90
|
||||
device: mi325_1
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
|
||||
@@ -153,6 +135,7 @@ steps:
|
||||
- vllm/
|
||||
- tests/models/multimodal/generation
|
||||
commands:
|
||||
- pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
|
||||
- pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=0) and not core_model'
|
||||
|
||||
- label: Multi-Modal Models (Extended Generation 3)
|
||||
@@ -163,6 +146,7 @@ steps:
|
||||
- vllm/
|
||||
- tests/models/multimodal/generation
|
||||
commands:
|
||||
- pip install git+https://github.com/TIGER-AI-Lab/Mantis.git
|
||||
- pytest -v -s models/multimodal/generation/test_common.py -m 'split(group=1) and not core_model'
|
||||
|
||||
- label: Multi-Modal Models (Extended Pooling)
|
||||
@@ -176,8 +160,7 @@ steps:
|
||||
- pytest -v -s models/multimodal/pooling -m 'not core_model'
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_1
|
||||
device: mi325_1
|
||||
timeout_in_minutes: 60
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
|
||||
@@ -4,7 +4,7 @@ depends_on:
|
||||
steps:
|
||||
- label: Plugin Tests (2 GPUs)
|
||||
key: plugin-tests-2-gpus
|
||||
timeout_in_minutes: 35
|
||||
timeout_in_minutes: 60
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
num_devices: 2
|
||||
source_file_dependencies:
|
||||
@@ -37,11 +37,6 @@ steps:
|
||||
- pytest -v -s plugins_tests/test_stats_logger_plugins.py
|
||||
- pip uninstall dummy_stat_logger -y
|
||||
# end stat_logger plugins test
|
||||
# begin endpoint plugins test
|
||||
- pip install -e ./plugins/vllm_add_dummy_endpoint_plugin
|
||||
- pytest -v -s plugins_tests/test_endpoint_plugins.py
|
||||
- pip uninstall vllm_add_dummy_endpoint_plugin -y
|
||||
# end endpoint plugins test
|
||||
# other tests continue here:
|
||||
- pytest -v -s plugins_tests/test_scheduler_plugins.py
|
||||
- pip install -e ./plugins/vllm_add_dummy_model
|
||||
|
||||
@@ -5,7 +5,7 @@ steps:
|
||||
- label: PyTorch Compilation Unit Tests
|
||||
device: h200_35gb
|
||||
key: pytorch-compilation-unit-tests
|
||||
timeout_in_minutes: 150
|
||||
timeout_in_minutes: 10
|
||||
source_file_dependencies:
|
||||
- vllm/__init__.py
|
||||
- vllm/_aiter_ops.py
|
||||
@@ -78,7 +78,7 @@ steps:
|
||||
|
||||
- label: PyTorch Compilation Passes Unit Tests
|
||||
key: pytorch-compilation-passes-unit-tests
|
||||
timeout_in_minutes: 45
|
||||
timeout_in_minutes: 20
|
||||
source_file_dependencies:
|
||||
- vllm/__init__.py
|
||||
- vllm/_aiter_ops.py
|
||||
@@ -107,11 +107,16 @@ steps:
|
||||
- tests/compile/passes
|
||||
commands:
|
||||
- pytest -s -v compile/passes --ignore compile/passes/distributed
|
||||
mirror:
|
||||
amd:
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 180
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
|
||||
- label: PyTorch Fullgraph Smoke Test
|
||||
device: h200_35gb
|
||||
key: pytorch-fullgraph-smoke-test
|
||||
timeout_in_minutes: 90
|
||||
timeout_in_minutes: 35
|
||||
source_file_dependencies:
|
||||
- vllm/__init__.py
|
||||
- vllm/_aiter_ops.py
|
||||
@@ -143,46 +148,11 @@ steps:
|
||||
# as it is a heavy test that is covered in other steps.
|
||||
# Use `find` to launch multiple instances of pytest so that
|
||||
# they do not suffer from https://github.com/vllm-project/vllm/issues/28965
|
||||
- "find compile/fullgraph/ -name 'test_*.py' -not -name 'test_full_cudagraph.py' -not -name 'test_full_graph.py' -print0 | xargs -0 -n1 -I{} pytest -s -v '{}'"
|
||||
|
||||
# Hopper-only DeepSeek-V2-Lite cases in this file require two 29.3-GiB model
|
||||
# instances and cannot fit a 35GB MIG slice. L4 retains the original coverage:
|
||||
# those SM90 cases skip while the architecture-compatible cases still run.
|
||||
- label: PyTorch Fullgraph CUDAGraph (L4 Compatibility)
|
||||
key: pytorch-fullgraph-cudagraph-l4-compatibility
|
||||
timeout_in_minutes: 60
|
||||
source_file_dependencies:
|
||||
- vllm/__init__.py
|
||||
- vllm/_aiter_ops.py
|
||||
- vllm/_custom_ops.py
|
||||
- vllm/compilation/
|
||||
- vllm/config/
|
||||
- vllm/distributed/
|
||||
- vllm/engine/
|
||||
- vllm/env_override.py
|
||||
- vllm/envs.py
|
||||
- vllm/forward_context.py
|
||||
- vllm/inputs/
|
||||
- vllm/ir/
|
||||
- vllm/kernels/
|
||||
- vllm/logger.py
|
||||
- vllm/model_executor/
|
||||
- vllm/multimodal/
|
||||
- vllm/platforms/
|
||||
- vllm/plugins/
|
||||
- vllm/sampling_params.py
|
||||
- vllm/sequence.py
|
||||
- vllm/transformers_utils/
|
||||
- vllm/triton_utils/
|
||||
- vllm/utils/
|
||||
- vllm/v1/
|
||||
- tests/compile
|
||||
commands:
|
||||
- pytest -s -v compile/fullgraph/test_full_cudagraph.py
|
||||
- "find compile/fullgraph/ -name 'test_*.py' -not -name 'test_full_graph.py' -print0 | xargs -0 -n1 -I{} pytest -s -v '{}'"
|
||||
|
||||
- label: PyTorch Fullgraph
|
||||
key: pytorch-fullgraph
|
||||
timeout_in_minutes: 40
|
||||
timeout_in_minutes: 30
|
||||
device: h200_18gb
|
||||
source_file_dependencies:
|
||||
- vllm/__init__.py
|
||||
@@ -225,13 +195,3 @@ steps:
|
||||
- requirements/test/nightly-torch.txt
|
||||
commands:
|
||||
- bash standalone_tests/pytorch_nightly_dependency.sh
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 30
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- requirements/test/nightly-torch.txt
|
||||
- vllm/platforms/rocm.py
|
||||
|
||||
@@ -3,11 +3,8 @@ depends_on:
|
||||
- image-build
|
||||
steps:
|
||||
- label: Quantization
|
||||
device: h200_35gb
|
||||
key: quantization
|
||||
timeout_in_minutes: 75
|
||||
env:
|
||||
VLLM_USE_V2_MODEL_RUNNER: "0"
|
||||
timeout_in_minutes: 90
|
||||
source_file_dependencies:
|
||||
- csrc/
|
||||
- vllm/model_executor/layers/quantization
|
||||
@@ -22,14 +19,11 @@ steps:
|
||||
# TODO(jerryzh168): resolve the above comment
|
||||
- uv pip install --system torchao==0.17.0 --index-url https://download.pytorch.org/whl/cu130
|
||||
- uv pip install --system conch-triton-kernels
|
||||
# The SM90-only checkpoint currently contains a removed weight_chan_scale
|
||||
# parameter. It was not exercised by the previous L4 job.
|
||||
- VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py -k 'not test_compressed_tensors_w4a8_fp8'
|
||||
- VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py
|
||||
|
||||
- label: Quantized Fusions
|
||||
device: h200_35gb
|
||||
key: quantized-fusions
|
||||
timeout_in_minutes: 20
|
||||
timeout_in_minutes: 30
|
||||
source_file_dependencies:
|
||||
- tests/fusion
|
||||
- vllm/model_executor/layers/fusion
|
||||
@@ -41,7 +35,7 @@ steps:
|
||||
|
||||
- label: Quantized MoE Test (B200)
|
||||
key: quantized-moe-test-b200
|
||||
timeout_in_minutes: 120
|
||||
timeout_in_minutes: 60
|
||||
working_dir: "/vllm-workspace/"
|
||||
device: b200-k8s
|
||||
source_file_dependencies:
|
||||
@@ -58,11 +52,8 @@ steps:
|
||||
- pytest -s -v tests/quantization/test_blackwell_moe.py
|
||||
|
||||
- label: Quantized Models Test
|
||||
device: h200_35gb
|
||||
key: quantized-models-test
|
||||
timeout_in_minutes: 65
|
||||
env:
|
||||
VLLM_USE_V2_MODEL_RUNNER: "0"
|
||||
timeout_in_minutes: 60
|
||||
source_file_dependencies:
|
||||
- vllm/model_executor/layers/quantization
|
||||
- tests/models/quantization
|
||||
|
||||
@@ -3,7 +3,7 @@ depends_on:
|
||||
- image-build
|
||||
steps:
|
||||
- label: Rust Frontend OpenAI Coverage
|
||||
timeout_in_minutes: 30
|
||||
timeout_in_minutes: 90
|
||||
device: h200_18gb
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
source_file_dependencies:
|
||||
@@ -15,30 +15,28 @@ steps:
|
||||
- tests/utils.py
|
||||
- tests/benchmarks/test_serve_cli.py
|
||||
- tests/entrypoints/openai/chat_completion/test_chat_completion.py
|
||||
- tests/entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py
|
||||
# - tests/entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py
|
||||
|
||||
# - tests/entrypoints/openai/completion/test_prompt_validation.py
|
||||
- tests/entrypoints/openai/completion/test_shutdown.py
|
||||
- tests/entrypoints/openai/test_return_token_ids.py
|
||||
- tests/entrypoints/openai/test_uds.py
|
||||
# - tests/entrypoints/openai/test_return_token_ids.py
|
||||
# - tests/entrypoints/openai/test_uds.py
|
||||
- tests/v1/sample/test_logprobs_e2e.py
|
||||
commands:
|
||||
- export VLLM_USE_RUST_FRONTEND=1
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- pytest -v -s benchmarks/test_serve_cli.py -k "not insecure and not (test_bench_serve and not test_bench_serve_chat)"
|
||||
- pytest -v -s entrypoints/openai/chat_completion/test_chat_completion.py -k "not test_invalid_json_schema and not test_invalid_regex and not test_kv_transfer_prompt_token_ids_round_trip and not test_kv_transfer_prompt_token_ids_streaming"
|
||||
- pytest -v -s entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py -k "not multiple"
|
||||
- pytest -v -s entrypoints/openai/chat_completion/test_chat_completion.py
|
||||
# - pytest -v -s entrypoints/openai/chat_completion/test_chat_logit_bias_validation.py -k "not invalid"
|
||||
|
||||
# - pytest -v -s entrypoints/openai/completion/test_prompt_validation.py -k "not prompt_embeds"
|
||||
- pytest -v -s entrypoints/openai/completion/test_shutdown.py -k "not engine_failure and not test_abort_timeout_exits_quickly"
|
||||
# test_comparison streams differently: Rust emits a separate first (prompt_token_ids) chunk and
|
||||
# finish chunk without logprobs, while the test reads `logprobs.tokens` on every chunk.
|
||||
- pytest -v -s entrypoints/openai/test_return_token_ids.py -k "not test_comparison"
|
||||
- pytest -v -s entrypoints/openai/test_uds.py
|
||||
# - pytest -v -s entrypoints/openai/test_return_token_ids.py
|
||||
# - pytest -v -s entrypoints/openai/test_uds.py
|
||||
- pytest -v -s v1/sample/test_logprobs_e2e.py -k "test_prompt_logprobs_e2e_server"
|
||||
|
||||
- label: Rust Frontend Serve/Admin Coverage
|
||||
timeout_in_minutes: 25
|
||||
timeout_in_minutes: 60
|
||||
device: h200_18gb
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
source_file_dependencies:
|
||||
@@ -47,27 +45,22 @@ steps:
|
||||
- vllm/entrypoints/serve/
|
||||
- vllm/v1/engine/
|
||||
- tests/utils.py
|
||||
- tests/entrypoints/serve/dev/rpc/test_collective_rpc.py
|
||||
- tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py
|
||||
# - tests/entrypoints/serve/dev/rpc/test_collective_rpc.py
|
||||
- tests/entrypoints/serve/disagg/test_serving_tokens.py
|
||||
- tests/entrypoints/serve/instrumentator/test_basic.py
|
||||
- tests/entrypoints/serve/instrumentator/test_metrics.py
|
||||
# - tests/entrypoints/serve/dev/test_sleep.py
|
||||
- tests/entrypoints/serve/tokenize/test_tokenization.py
|
||||
commands:
|
||||
- export VLLM_USE_RUST_FRONTEND=1
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc/test_collective_rpc.py
|
||||
# server_load can be flaky under the Rust frontend; keep it excluded for now.
|
||||
# - pytest -v -s entrypoints/serve/dev/rpc/test_collective_rpc.py
|
||||
- pytest -v -s entrypoints/serve/instrumentator/test_basic.py -k "not show_version and not server_load"
|
||||
# test_generate_logprobs expects Python-style top_logprobs truncation (dedup sampled + cap at max(k, 1)).
|
||||
- pytest -v -s entrypoints/scale_out/token_in_token_out/test_serving_tokens.py -k "not stream and not lora and not test_generate_logprobs and not stop_string_workflow"
|
||||
- pytest -v -s entrypoints/serve/disagg/test_serving_tokens.py -k "not stream and not lora and not test_generate_logprobs and not stop_string_workflow"
|
||||
- pytest -v -s entrypoints/serve/instrumentator/test_metrics.py -k "text and not show and not run_batch and not test_metrics_counts and not test_metrics_exist"
|
||||
# - pytest -v -s entrypoints/serve/dev/test_sleep.py
|
||||
# /tokenizer_info is not implemented in the Rust frontend (the CLI flag is accepted as a no-op).
|
||||
- pytest -v -s entrypoints/serve/tokenize/test_tokenization.py -k "not tokenizer_info"
|
||||
|
||||
- label: Rust Frontend Core Correctness
|
||||
timeout_in_minutes: 20
|
||||
timeout_in_minutes: 30
|
||||
device: h200_18gb
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
source_file_dependencies:
|
||||
@@ -81,8 +74,7 @@ steps:
|
||||
- pytest -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine
|
||||
|
||||
- label: Rust Frontend Tool Use
|
||||
device: h200_35gb
|
||||
timeout_in_minutes: 25
|
||||
timeout_in_minutes: 60
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
source_file_dependencies:
|
||||
- rust/
|
||||
@@ -96,7 +88,7 @@ steps:
|
||||
- pytest -v -s tool_use --ignore=tool_use/mistral --models llama3.2 -k "not test_response_format_with_tool_choice_required and not test_parallel_tool_calls_false and not test_tool_call_and_choice"
|
||||
|
||||
- label: Rust Frontend Distributed
|
||||
timeout_in_minutes: 25
|
||||
timeout_in_minutes: 30
|
||||
num_devices: 4
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
source_file_dependencies:
|
||||
|
||||
@@ -4,7 +4,7 @@ steps:
|
||||
- label: Rust Frontend Cargo Style + Clippy
|
||||
key: rust-frontend-cargo-style-clippy
|
||||
depends_on: []
|
||||
timeout_in_minutes: 20
|
||||
timeout_in_minutes: 30
|
||||
device: cpu-medium
|
||||
no_plugin: true
|
||||
source_file_dependencies:
|
||||
@@ -18,7 +18,7 @@ steps:
|
||||
- label: Rust Frontend Cargo Tests
|
||||
key: rust-frontend-cargo-tests
|
||||
depends_on: []
|
||||
timeout_in_minutes: 20
|
||||
timeout_in_minutes: 30
|
||||
device: cpu-medium
|
||||
no_plugin: true
|
||||
source_file_dependencies:
|
||||
|
||||
@@ -5,7 +5,7 @@ steps:
|
||||
- label: Samplers Test
|
||||
device: h200_35gb
|
||||
key: samplers-test
|
||||
timeout_in_minutes: 40
|
||||
timeout_in_minutes: 75
|
||||
source_file_dependencies:
|
||||
- vllm/model_executor/layers
|
||||
- vllm/sampling_metadata.py
|
||||
@@ -20,17 +20,7 @@ steps:
|
||||
mirror:
|
||||
amd:
|
||||
device: mi250_1
|
||||
timeout_in_minutes: 40
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- vllm/model_executor/layers
|
||||
- vllm/sampling_metadata.py
|
||||
- vllm/v1/sample/
|
||||
- vllm/entrypoints/generate/beam_search/
|
||||
- tests/samplers
|
||||
- tests/conftest.py
|
||||
- vllm/_aiter_ops.py
|
||||
- vllm/platforms/rocm.py
|
||||
commands:
|
||||
- pytest -v -s samplers
|
||||
|
||||
@@ -4,7 +4,7 @@ depends_on:
|
||||
steps:
|
||||
- label: Spec Decode Eagle
|
||||
key: spec-decode-eagle
|
||||
timeout_in_minutes: 25
|
||||
timeout_in_minutes: 30
|
||||
device: h200_18gb
|
||||
source_file_dependencies:
|
||||
- vllm/v1/spec_decode/
|
||||
@@ -14,9 +14,8 @@ steps:
|
||||
- pytest -v -s v1/e2e/spec_decode -k "eagle_correctness"
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 55
|
||||
device: mi325_1
|
||||
timeout_in_minutes: 45
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
@@ -30,7 +29,7 @@ steps:
|
||||
|
||||
- label: Spec Decode Eagle Nightly B200
|
||||
key: spec-decode-eagle-nightly-b200
|
||||
timeout_in_minutes: 25
|
||||
timeout_in_minutes: 30
|
||||
device: b200-k8s
|
||||
optional: true
|
||||
source_file_dependencies:
|
||||
@@ -42,7 +41,7 @@ steps:
|
||||
|
||||
- label: Spec Decode Speculators + MTP
|
||||
key: spec-decode-speculators-mtp
|
||||
timeout_in_minutes: 20
|
||||
timeout_in_minutes: 30
|
||||
device: h200_18gb
|
||||
source_file_dependencies:
|
||||
- vllm/v1/spec_decode/
|
||||
@@ -54,9 +53,8 @@ steps:
|
||||
- pytest -v -s v1/e2e/spec_decode -k "speculators or mtp_correctness"
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 75
|
||||
device: mi325_1
|
||||
timeout_in_minutes: 65
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
@@ -84,21 +82,18 @@ steps:
|
||||
|
||||
- label: Spec Decode Ngram + Suffix
|
||||
key: spec-decode-ngram-suffix
|
||||
timeout_in_minutes: 20
|
||||
timeout_in_minutes: 30
|
||||
device: h200_18gb
|
||||
source_file_dependencies:
|
||||
- vllm/v1/spec_decode/
|
||||
- vllm/v1/worker/gpu/spec_decode/
|
||||
- tests/v1/e2e/spec_decode/
|
||||
- tests/spec_decode/
|
||||
commands:
|
||||
- pytest -v -s v1/e2e/spec_decode -k "ngram or suffix"
|
||||
- python3 spec_decode/test_custom_proposer.py
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 35
|
||||
device: mi325_1
|
||||
timeout_in_minutes: 65
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
@@ -112,7 +107,7 @@ steps:
|
||||
|
||||
- label: Spec Decode Draft Model
|
||||
key: spec-decode-draft-model
|
||||
timeout_in_minutes: 45
|
||||
timeout_in_minutes: 30
|
||||
device: h200_18gb
|
||||
source_file_dependencies:
|
||||
- vllm/v1/spec_decode/
|
||||
@@ -122,9 +117,8 @@ steps:
|
||||
- pytest -v -s v1/e2e/spec_decode -k "draft_model or no_sync or batch_inference"
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_1
|
||||
timeout_in_minutes: 55
|
||||
device: mi325_1
|
||||
timeout_in_minutes: 50
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
@@ -138,7 +132,7 @@ steps:
|
||||
|
||||
- label: Spec Decode Draft Model Nightly B200
|
||||
key: spec-decode-draft-model-nightly-b200
|
||||
timeout_in_minutes: 40
|
||||
timeout_in_minutes: 30
|
||||
device: b200-k8s
|
||||
optional: true
|
||||
source_file_dependencies:
|
||||
@@ -150,7 +144,7 @@ steps:
|
||||
|
||||
- label: Speculators Correctness
|
||||
key: speculators-correctness
|
||||
timeout_in_minutes: 30
|
||||
timeout_in_minutes: 60
|
||||
device: h100
|
||||
optional: true
|
||||
num_devices: 1
|
||||
@@ -163,7 +157,7 @@ steps:
|
||||
- pytest -v -s v1/spec_decode/test_speculators_correctness.py -m slow_test
|
||||
|
||||
- label: Spec Decode MTP hybrid (B200)
|
||||
timeout_in_minutes: 20
|
||||
timeout_in_minutes: 30
|
||||
device: b200-k8s
|
||||
optional: true
|
||||
source_file_dependencies:
|
||||
@@ -174,31 +168,3 @@ steps:
|
||||
- tests/v1/e2e/spec_decode/
|
||||
commands:
|
||||
- pytest -v -s v1/e2e/spec_decode -k "qwen3_5-hybrid"
|
||||
|
||||
- label: Spec Decode DeepSeek MTP Parallel Load (B200)
|
||||
key: spec-decode-deepseek-mtp-parallel-load-b200
|
||||
timeout_in_minutes: 30
|
||||
device: b200-k8s
|
||||
optional: true
|
||||
num_devices: 2
|
||||
source_file_dependencies:
|
||||
- vllm/v1/spec_decode/llm_base_proposer.py
|
||||
- vllm/v1/spec_decode/eagle.py
|
||||
- vllm/v1/worker/gpu/spec_decode/eagle/
|
||||
- vllm/model_executor/models/deepseek_mtp.py
|
||||
- vllm/model_executor/models/deepseek_v2.py
|
||||
- tests/v1/e2e/spec_decode/test_mtp_parallel_load.py
|
||||
commands:
|
||||
- pytest -v -s v1/e2e/spec_decode/test_mtp_parallel_load.py
|
||||
|
||||
- label: Spec Decode Acceptance Rates Nightly
|
||||
key: spec-decode-acceptance-rates-nightly
|
||||
timeout_in_minutes: 60
|
||||
device: h200_35gb
|
||||
optional: true
|
||||
source_file_dependencies:
|
||||
- vllm/v1/spec_decode/
|
||||
- vllm/v1/worker/gpu/spec_decode/
|
||||
- tests/v1/e2e/spec_decode/
|
||||
commands:
|
||||
- pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "acceptance_rates"
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
group: Torch ABI
|
||||
depends_on:
|
||||
- image-build
|
||||
steps:
|
||||
- label: Torch Stable ABI Audit
|
||||
key: torch-stable-abi-audit
|
||||
timeout_in_minutes: 5
|
||||
source_file_dependencies:
|
||||
- .buildkite/check-torch-abi.py
|
||||
- csrc/
|
||||
- cmake/
|
||||
- setup.py
|
||||
commands:
|
||||
- python3 /vllm-workspace/.buildkite/check-torch-abi.py
|
||||
@@ -4,7 +4,7 @@ depends_on:
|
||||
steps:
|
||||
- label: Weight Loading Multiple GPU # 33min
|
||||
key: weight-loading-multiple-gpu
|
||||
timeout_in_minutes: 50
|
||||
timeout_in_minutes: 45
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
num_devices: 2
|
||||
optional: true
|
||||
@@ -13,15 +13,6 @@ steps:
|
||||
- tests/weight_loading
|
||||
commands:
|
||||
- bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models.txt
|
||||
mirror:
|
||||
amd:
|
||||
dind: false
|
||||
device: mi300_2
|
||||
timeout_in_minutes: 35
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
commands:
|
||||
- bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-amd.txt
|
||||
|
||||
# - label: Weight Loading Multiple GPU - Large Models # optional
|
||||
# working_dir: "/vllm-workspace/tests"
|
||||
|
||||
+10
-13
@@ -3,7 +3,7 @@
|
||||
|
||||
# This lists cover the "core" components of vLLM that require careful review
|
||||
/vllm/compilation @zou3519 @youkaichao @ProExpertProg @BoyuanFeng
|
||||
/vllm/distributed/kv_transfer @NickLucche @ApostaC @orozery @xuechendi @ivanium
|
||||
/vllm/distributed/kv_transfer @NickLucche @ApostaC @orozery @xuechendi
|
||||
/vllm/lora @jeejeelee
|
||||
/vllm/model_executor/layers/attention @LucasWilkinson @MatthewBonanni
|
||||
/vllm/model_executor/layers/fused_moe @mgoin @pavanimajety @zyongye
|
||||
@@ -11,7 +11,7 @@
|
||||
/vllm/model_executor/layers/mamba @tdoublep @tomeras91
|
||||
/vllm/model_executor/layers/mamba/gdn/qwen_gdn_linear_attn.py @tdoublep @ZJY0516 @vadiklyutiy
|
||||
/vllm/model_executor/model_loader @22quinn
|
||||
/vllm/model_executor/layers/batch_invariant.py @yewentao256
|
||||
/vllm/model_executor/layers/batch_invariant.py @yewentao256
|
||||
/vllm/ir @ProExpertProg
|
||||
/vllm/kernels/ @ProExpertProg @tjtanaa
|
||||
/vllm/kernels/helion @ProExpertProg @zou3519
|
||||
@@ -23,7 +23,7 @@
|
||||
# Any change to the VllmConfig changes can have a large user-facing impact,
|
||||
# so spam a lot of people
|
||||
/vllm/config @WoosukKwon @youkaichao @robertgshaw2-redhat @mgoin @tlrmchlsmth @houseroad @yewentao256 @ProExpertProg
|
||||
/vllm/config/cache.py @heheda12345 @ivanium
|
||||
/vllm/config/cache.py @heheda12345
|
||||
|
||||
# Config utils
|
||||
/vllm/config/utils.py @hmellor
|
||||
@@ -47,7 +47,6 @@
|
||||
|
||||
# Rust Frontend
|
||||
/rust/ @BugenZhao @njhill
|
||||
/rust/src/bench @esmeetu
|
||||
/build_rust.sh @BugenZhao @njhill
|
||||
/rust-toolchain.toml @BugenZhao @njhill
|
||||
/.buildkite/test_areas/rust* @BugenZhao @njhill
|
||||
@@ -68,17 +67,16 @@
|
||||
/vllm/v1/attention/backends/flashinfer.py @mgoin @pavanimajety @vadiklyutiy
|
||||
/vllm/v1/attention/backends/triton_attn.py @tdoublep
|
||||
/vllm/v1/attention/backends/gdn_attn.py @ZJY0516 @vadiklyutiy
|
||||
/vllm/v1/core @WoosukKwon @robertgshaw2-redhat @njhill @ywang96 @alexm-redhat @heheda12345 @ApostaC @orozery @ivanium
|
||||
/vllm/v1/core @WoosukKwon @robertgshaw2-redhat @njhill @ywang96 @alexm-redhat @heheda12345 @ApostaC @orozery
|
||||
/vllm/v1/sample @22quinn @houseroad @njhill
|
||||
/vllm/v1/spec_decode @benchislett @luccafong @MatthewBonanni
|
||||
/vllm/v1/structured_output @mgoin @russellb @aarnphm @benchislett
|
||||
/vllm/v1/kv_cache_interface.py @heheda12345 @ivanium
|
||||
/vllm/v1/kv_cache_interface.py @heheda12345
|
||||
/vllm/v1/kv_offload @ApostaC @orozery
|
||||
/vllm/v1/simple_kv_offload @ivanium
|
||||
/vllm/v1/engine @njhill
|
||||
/vllm/v1/executor @njhill
|
||||
/vllm/v1/worker @njhill
|
||||
/vllm/v1/worker/kv_connector_model_runner_mixin.py @orozery @NickLucche @ivanium
|
||||
/vllm/v1/worker/kv_connector_model_runner_mixin.py @orozery @NickLucche
|
||||
|
||||
# Model runner V2
|
||||
/vllm/v1/worker/gpu @WoosukKwon @njhill @yewentao256
|
||||
@@ -105,14 +103,13 @@
|
||||
/tests/test_inputs.py @DarkLight1337 @ywang96
|
||||
/tests/entrypoints/llm/test_struct_output_generate.py @mgoin @russellb @aarnphm
|
||||
/tests/v1/structured_output @mgoin @russellb @aarnphm
|
||||
/tests/v1/core @WoosukKwon @robertgshaw2-redhat @njhill @ywang96 @alexm-redhat @heheda12345 @ApostaC @orozery @ivanium
|
||||
/tests/v1/core @WoosukKwon @robertgshaw2-redhat @njhill @ywang96 @alexm-redhat @heheda12345 @ApostaC @orozery
|
||||
/tests/weight_loading @mgoin @youkaichao @yewentao256
|
||||
/tests/lora @jeejeelee
|
||||
/tests/models/language/generation/test_hybrid.py @tdoublep @tomeras91
|
||||
/tests/v1/kv_connector/nixl_integration @NickLucche
|
||||
/tests/v1/kv_connector @ApostaC @orozery @ivanium
|
||||
/tests/v1/kv_connector @ApostaC @orozery
|
||||
/tests/v1/kv_offload @ApostaC @orozery
|
||||
/tests/v1/simple_kv_offload @ivanium
|
||||
/tests/v1/determinism @yewentao256
|
||||
/tests/reasoning @aarnphm @chaunceyjiang @sfeng33 @bbrowning
|
||||
/tests/tool_parsers @aarnphm @chaunceyjiang @sfeng33 @bbrowning
|
||||
@@ -120,7 +117,7 @@
|
||||
|
||||
# Transformers modeling backend
|
||||
/vllm/model_executor/models/transformers @hmellor
|
||||
/tests/models/transformers @hmellor
|
||||
/tests/models/test_transformers.py @hmellor
|
||||
|
||||
# Docs
|
||||
/docs/mkdocs @hmellor
|
||||
@@ -173,7 +170,7 @@ mkdocs.yaml @hmellor
|
||||
# Kernels
|
||||
/vllm/v1/attention/ops/chunked_prefill_paged_decode.py @tdoublep
|
||||
/vllm/v1/attention/ops/triton_unified_attention.py @tdoublep
|
||||
/vllm/third_party/flash_linear_attention @ZJY0516 @vadiklyutiy
|
||||
/vllm/model_executor/layers/fla @ZJY0516 @vadiklyutiy
|
||||
|
||||
# ROCm related: specify owner with write access to notify AMD folks for careful code review
|
||||
/vllm/**/*rocm* @tjtanaa @dllehr-amd
|
||||
|
||||
@@ -3,5 +3,3 @@
|
||||
self-hosted-runner:
|
||||
labels:
|
||||
- vllm-runners
|
||||
# Not yet in actionlint's known-label set.
|
||||
- macos-26
|
||||
|
||||
@@ -19,7 +19,6 @@ pull_request_rules:
|
||||
description: Comment on PR when pre-commit check fails
|
||||
conditions:
|
||||
- check-failure=pre-commit
|
||||
- -check-cancelled=pre-commit
|
||||
- -closed
|
||||
- -draft
|
||||
- or:
|
||||
@@ -182,18 +181,6 @@ pull_request_rules:
|
||||
add:
|
||||
- performance
|
||||
|
||||
- name: label-quantization
|
||||
description: Automatically apply quantization label
|
||||
conditions:
|
||||
- label != stale
|
||||
- or:
|
||||
- files~=^vllm/model_executor/layers/quantization/
|
||||
- title~=(?i)quant
|
||||
actions:
|
||||
label:
|
||||
add:
|
||||
- quantization
|
||||
|
||||
- name: label-qwen
|
||||
description: Automatically apply qwen label
|
||||
conditions:
|
||||
@@ -233,31 +220,6 @@ pull_request_rules:
|
||||
add:
|
||||
- gpt-oss
|
||||
|
||||
- name: label-kimi
|
||||
description: Automatically apply kimi label
|
||||
conditions:
|
||||
- label != stale
|
||||
- or:
|
||||
- files~=(?i)kimi
|
||||
- files~=(?i)moonshot
|
||||
- title~=(?i)(?:kimi|moonshot)
|
||||
actions:
|
||||
label:
|
||||
add:
|
||||
- kimi
|
||||
|
||||
- name: label-k3
|
||||
description: Automatically apply k3 label (launch triage; retire after ramp-down)
|
||||
conditions:
|
||||
- label != stale
|
||||
- or:
|
||||
- files~=(?i)kimi[-_]?k3
|
||||
- title~=(?i)(?:kimi[-\s]?k3|\bk3\b)
|
||||
actions:
|
||||
label:
|
||||
add:
|
||||
- k3
|
||||
|
||||
- name: label-nvidia
|
||||
description: Automatically apply nvidia label
|
||||
conditions:
|
||||
|
||||
@@ -130,66 +130,6 @@ jobs:
|
||||
},
|
||||
],
|
||||
},
|
||||
kimi: {
|
||||
keywords: [
|
||||
{ term: "Kimi", searchIn: "both" },
|
||||
{ term: "Moonshot", searchIn: "both" },
|
||||
],
|
||||
substrings: [
|
||||
{ term: "moonshotai/", searchIn: "both" },
|
||||
{ term: "kimi", searchIn: "title" },
|
||||
],
|
||||
},
|
||||
k3: {
|
||||
keywords: [
|
||||
{ term: "Kimi K3", searchIn: "both" },
|
||||
{ term: "K3", searchIn: "title" },
|
||||
],
|
||||
substrings: [
|
||||
{ term: "moonshotai/kimi-k3", searchIn: "both" },
|
||||
],
|
||||
},
|
||||
quantization: {
|
||||
keywords: [
|
||||
{
|
||||
term: "quantization",
|
||||
searchIn: "both"
|
||||
},
|
||||
{
|
||||
term: "quantized",
|
||||
searchIn: "both"
|
||||
},
|
||||
],
|
||||
},
|
||||
"intel-gpu": {
|
||||
// Keyword search - matches whole words only (with word boundaries)
|
||||
keywords: [
|
||||
{
|
||||
term: "B50",
|
||||
searchIn: "both"
|
||||
},
|
||||
{
|
||||
term: "B60",
|
||||
searchIn: "both"
|
||||
},
|
||||
{
|
||||
term: "B70",
|
||||
searchIn: "both"
|
||||
},
|
||||
{
|
||||
term: "intel gpu",
|
||||
searchIn: "both"
|
||||
},
|
||||
{
|
||||
term: "Arc GPU",
|
||||
searchIn: "both"
|
||||
},
|
||||
{
|
||||
term: "BMG",
|
||||
searchIn: "both"
|
||||
},
|
||||
],
|
||||
},
|
||||
// Add more label configurations here as needed
|
||||
// example: {
|
||||
// keywords: [...],
|
||||
@@ -383,11 +323,11 @@ jobs:
|
||||
// {users} will be replaced with @mentions
|
||||
const ccConfig = {
|
||||
rocm: {
|
||||
users: ['hongxiayang', 'tjtanaa', 'vllmellm', 'giuseppegrossi'],
|
||||
users: ['hongxiayang', 'tjtanaa', 'vllmellm'],
|
||||
message: 'CC {users} for ROCm-related issue',
|
||||
},
|
||||
mistral: {
|
||||
users: ['patrickvonplaten', 'juliendenize', 'andylolu2', 'NickLucche'],
|
||||
users: ['patrickvonplaten', 'juliendenize', 'andylolu2'],
|
||||
message: 'CC {users} for Mistral-related issue',
|
||||
},
|
||||
// Add more label -> user mappings here
|
||||
@@ -551,4 +491,4 @@ jobs:
|
||||
issue_number: context.issue.number,
|
||||
body: message,
|
||||
});
|
||||
core.notice(`Requested missing ROCm info from @${author}: ${missing.map(m => m.name).join(', ')}`);
|
||||
core.notice(`Requested missing ROCm info from @${author}: ${missing.map(m => m.name).join(', ')}`);
|
||||
@@ -11,25 +11,13 @@ permissions:
|
||||
|
||||
jobs:
|
||||
macos-m1-smoke-test:
|
||||
# macos-26 (the supported target) is still a preview runner, so gate on GA
|
||||
# macos-15 and keep macos-26 non-blocking.
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- os: macos-15
|
||||
required: true
|
||||
- os: macos-26
|
||||
required: false
|
||||
name: macos-m1-smoke-test (${{ matrix.os }})
|
||||
runs-on: ${{ matrix.os }}
|
||||
continue-on-error: ${{ !matrix.required }}
|
||||
runs-on: macos-latest
|
||||
timeout-minutes: 30
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
- uses: actions/checkout@v6.0.1
|
||||
|
||||
- uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
|
||||
- uses: astral-sh/setup-uv@v7
|
||||
with:
|
||||
enable-cache: true
|
||||
cache-dependency-glob: |
|
||||
@@ -84,11 +72,14 @@ jobs:
|
||||
# Test health endpoint
|
||||
curl -f http://localhost:8000/health
|
||||
|
||||
# Long prompt: hits the split-KV path that short prompts skip (#46769).
|
||||
PAYLOAD=$(python -c "import json; print(json.dumps({'model': 'Qwen/Qwen3-0.6B', 'prompt': 'The quick brown fox jumps over the lazy dog. ' * 24, 'max_tokens': 16}))")
|
||||
curl -f --max-time 120 http://localhost:8000/v1/completions \
|
||||
# Test completion
|
||||
curl -f http://localhost:8000/v1/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$PAYLOAD"
|
||||
-d '{
|
||||
"model": "Qwen/Qwen3-0.6B",
|
||||
"prompt": "Hello",
|
||||
"max_tokens": 5
|
||||
}'
|
||||
|
||||
# Cleanup
|
||||
kill "$SERVER_PID"
|
||||
|
||||
@@ -80,9 +80,9 @@ jobs:
|
||||
'',
|
||||
'\u{1f4ac} Join our developer Slack at https://slack.vllm.ai to discuss your PR in `#pr-reviews`, coordinate on features in `#feat-` channels, or join special interest groups in `#sig-` channels.',
|
||||
'',
|
||||
'PRs do not trigger a full CI run by default. Reviewers with write access and configured trusted contributors can comment `/ci run` whenever CI signals are needed.',
|
||||
'PRs do not trigger a full CI run by default. Once the PR is approved and ready to go, your PR reviewer(s) can run CI to test the changes comprehensively before merging.',
|
||||
'',
|
||||
'Once the PR is approved or has the `ready` label, the PR author can also use `/ci run` or `/ci retry`. New commits do not start CI automatically.',
|
||||
'To run CI, PR reviewers can either: Add `ready` label to the PR or enable auto-merge.',
|
||||
'',
|
||||
'If you have any questions, please reach out to us on Slack at https://slack.vllm.ai.',
|
||||
'',
|
||||
|
||||
@@ -28,8 +28,7 @@ jobs:
|
||||
pull_number: context.payload.pull_request.number,
|
||||
});
|
||||
|
||||
const readyLabels = ['ready', 'ready-run-all-tests'];
|
||||
const hasReadyLabel = pr.labels.some(l => readyLabels.includes(l.name));
|
||||
const hasReadyLabel = pr.labels.some(l => l.name === 'ready');
|
||||
const hasVerifiedLabel = pr.labels.some(l => l.name === 'verified');
|
||||
|
||||
const { data: mergedPRs } = await github.rest.search.issuesAndPullRequests({
|
||||
@@ -41,7 +40,7 @@ jobs:
|
||||
if (hasReadyLabel || hasVerifiedLabel || mergedCount >= 4) {
|
||||
core.info(`Check passed: verified label=${hasVerifiedLabel}, ready label=${hasReadyLabel}, 4+ merged PRs=${mergedCount >= 4}`);
|
||||
} else {
|
||||
core.setFailed(`PR must have the 'verified', 'ready', or 'ready-run-all-tests' label to run pre-commit, or the author must have at least 4 merged PRs (found ${mergedCount}).`);
|
||||
core.setFailed(`PR must have the 'verified' or 'ready' (which also triggers tests) label or the author must have at least 4 merged PRs (found ${mergedCount}).`);
|
||||
}
|
||||
|
||||
pre-commit:
|
||||
@@ -49,7 +48,7 @@ jobs:
|
||||
if: always() && (needs.pre-run-check.result == 'success' || needs.pre-run-check.result == 'skipped')
|
||||
runs-on: [self-hosted, linux, x64, vllm-runners]
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
- uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
@@ -1,40 +0,0 @@
|
||||
name: Run CI from PR comment
|
||||
|
||||
on:
|
||||
issue_comment:
|
||||
types: [created]
|
||||
|
||||
concurrency:
|
||||
group: run-ci-comment-${{ github.event.issue.number }}
|
||||
cancel-in-progress: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
run-ci-command:
|
||||
if: >-
|
||||
github.event.issue.pull_request &&
|
||||
(github.event.comment.body == '/ci run' ||
|
||||
github.event.comment.body == '/ci retry')
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
with:
|
||||
ref: ${{ github.event.repository.default_branch }}
|
||||
persist-credentials: false
|
||||
- uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- name: Authorize and run CI command
|
||||
run: >-
|
||||
uv run --no-project --python 3.12
|
||||
.github/workflows/scripts/run_ci_command.py
|
||||
env:
|
||||
BUILDKITE_API_TOKEN: ${{ secrets.BUILDKITE_API_TOKEN }}
|
||||
BUILDKITE_ORGANIZATION: vllm
|
||||
BUILDKITE_PIPELINE: ci
|
||||
CI_TRUSTED_USERS: ${{ vars.CI_TRUSTED_USERS }}
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
@@ -1,581 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any
|
||||
|
||||
COMMAND_RUN_CI = "/ci run"
|
||||
COMMAND_RETRY_FAILED = "/ci retry"
|
||||
READY_LABELS = {"ready", "ready-run-all-tests"}
|
||||
TRUSTED_PERMISSIONS = {"admin", "maintain", "write"}
|
||||
ACTIVE_BUILD_STATES = {
|
||||
"blocked",
|
||||
"creating",
|
||||
"scheduled",
|
||||
"running",
|
||||
"failing",
|
||||
"canceling",
|
||||
"waiting",
|
||||
"waiting_failed",
|
||||
}
|
||||
RETRY_STATES = "failed,timed_out,expired"
|
||||
|
||||
|
||||
class ApiError(RuntimeError):
|
||||
def __init__(self, status: int | None, message: str) -> None:
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
|
||||
|
||||
class HttpTransport:
|
||||
def request(
|
||||
self,
|
||||
url: str,
|
||||
*,
|
||||
body: Mapping[str, Any] | None = None,
|
||||
headers: Mapping[str, str] | None = None,
|
||||
method: str = "GET",
|
||||
) -> Any:
|
||||
data = None if body is None else json.dumps(body).encode()
|
||||
request = urllib.request.Request(
|
||||
url,
|
||||
data=data,
|
||||
headers=dict(headers or {}),
|
||||
method=method,
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=30) as response:
|
||||
response_body = response.read().decode()
|
||||
except urllib.error.HTTPError as error:
|
||||
response_body = error.read().decode()
|
||||
message = self._error_message(response_body, error.reason)
|
||||
raise ApiError(
|
||||
error.code,
|
||||
f"API returned {error.code}: {message}",
|
||||
) from error
|
||||
except urllib.error.URLError as error:
|
||||
raise ApiError(None, f"API request failed: {error.reason}") from error
|
||||
|
||||
if not response_body:
|
||||
return None
|
||||
try:
|
||||
return json.loads(response_body)
|
||||
except json.JSONDecodeError as error:
|
||||
raise ApiError(None, "API returned a non-JSON response.") from error
|
||||
|
||||
@staticmethod
|
||||
def _error_message(response_body: str, fallback: str) -> str:
|
||||
try:
|
||||
parsed = json.loads(response_body)
|
||||
except json.JSONDecodeError:
|
||||
return fallback
|
||||
return str(parsed.get("message", fallback))
|
||||
|
||||
|
||||
class GitHubClient:
|
||||
def __init__(
|
||||
self,
|
||||
token: str,
|
||||
repository: str,
|
||||
transport: HttpTransport | None = None,
|
||||
) -> None:
|
||||
if not token:
|
||||
raise RuntimeError("GH_TOKEN is not set.")
|
||||
self.owner, self.repo = repository.split("/", maxsplit=1)
|
||||
self.transport = transport or HttpTransport()
|
||||
self.headers = {
|
||||
"Accept": "application/vnd.github+json",
|
||||
"Authorization": f"Bearer {token}",
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": "vllm-ci-command",
|
||||
"X-GitHub-Api-Version": "2022-11-28",
|
||||
}
|
||||
|
||||
def _request(
|
||||
self,
|
||||
path: str,
|
||||
*,
|
||||
body: Mapping[str, Any] | None = None,
|
||||
method: str = "GET",
|
||||
) -> Any:
|
||||
return self.transport.request(
|
||||
f"https://api.github.com{path}",
|
||||
body=body,
|
||||
headers=self.headers,
|
||||
method=method,
|
||||
)
|
||||
|
||||
def _repo_path(self, suffix: str) -> str:
|
||||
owner = urllib.parse.quote(self.owner, safe="")
|
||||
repo = urllib.parse.quote(self.repo, safe="")
|
||||
return f"/repos/{owner}/{repo}{suffix}"
|
||||
|
||||
def _paginate(self, path: str) -> list[dict[str, Any]]:
|
||||
results: list[dict[str, Any]] = []
|
||||
separator = "&" if "?" in path else "?"
|
||||
for page in range(1, 101):
|
||||
response = self._request(f"{path}{separator}per_page=100&page={page}")
|
||||
if not isinstance(response, list):
|
||||
raise ApiError(None, "GitHub API returned an invalid list response.")
|
||||
results.extend(response)
|
||||
if len(response) < 100:
|
||||
return results
|
||||
raise ApiError(None, "GitHub API pagination exceeded 10,000 results.")
|
||||
|
||||
def get_pr(self, number: int) -> dict[str, Any]:
|
||||
return self._request(self._repo_path(f"/pulls/{number}"))
|
||||
|
||||
def get_permission(self, actor: str) -> str:
|
||||
username = urllib.parse.quote(actor, safe="")
|
||||
try:
|
||||
response = self._request(
|
||||
self._repo_path(f"/collaborators/{username}/permission")
|
||||
)
|
||||
except ApiError as error:
|
||||
if error.status == 404:
|
||||
return "none"
|
||||
raise
|
||||
return str(response["permission"])
|
||||
|
||||
def get_review_decision(self, number: int) -> str | None:
|
||||
query = """
|
||||
query($owner: String!, $repo: String!, $number: Int!) {
|
||||
repository(owner: $owner, name: $repo) {
|
||||
pullRequest(number: $number) {
|
||||
reviewDecision
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
response = self._request(
|
||||
"/graphql",
|
||||
body={
|
||||
"query": query,
|
||||
"variables": {
|
||||
"number": number,
|
||||
"owner": self.owner,
|
||||
"repo": self.repo,
|
||||
},
|
||||
},
|
||||
method="POST",
|
||||
)
|
||||
return response["data"]["repository"]["pullRequest"]["reviewDecision"]
|
||||
|
||||
def list_reviews(self, number: int) -> list[dict[str, Any]]:
|
||||
return self._paginate(self._repo_path(f"/pulls/{number}/reviews"))
|
||||
|
||||
def list_reactions(self, comment_id: int) -> list[dict[str, Any]]:
|
||||
return self._paginate(
|
||||
self._repo_path(f"/issues/comments/{comment_id}/reactions")
|
||||
)
|
||||
|
||||
def add_reaction(self, comment_id: int, content: str) -> None:
|
||||
self._request(
|
||||
self._repo_path(f"/issues/comments/{comment_id}/reactions"),
|
||||
body={"content": content},
|
||||
method="POST",
|
||||
)
|
||||
|
||||
def add_comment(self, issue_number: int, body: str) -> None:
|
||||
self._request(
|
||||
self._repo_path(f"/issues/{issue_number}/comments"),
|
||||
body={"body": body},
|
||||
method="POST",
|
||||
)
|
||||
|
||||
|
||||
class BuildkiteClient:
|
||||
def __init__(
|
||||
self,
|
||||
token: str,
|
||||
organization: str,
|
||||
pipeline: str,
|
||||
transport: HttpTransport | None = None,
|
||||
) -> None:
|
||||
self.token = token
|
||||
self.transport = transport or HttpTransport()
|
||||
organization = urllib.parse.quote(organization, safe="")
|
||||
pipeline = urllib.parse.quote(pipeline, safe="")
|
||||
self.base_url = (
|
||||
"https://api.buildkite.com/v2/organizations/"
|
||||
f"{organization}/pipelines/{pipeline}/builds"
|
||||
)
|
||||
|
||||
def _request(
|
||||
self,
|
||||
*,
|
||||
body: Mapping[str, Any] | None = None,
|
||||
method: str = "GET",
|
||||
path: str = "",
|
||||
query: Sequence[tuple[str, str]] = (),
|
||||
) -> Any:
|
||||
if not self.token:
|
||||
raise RuntimeError("The BUILDKITE_API_TOKEN repository secret is not set.")
|
||||
url = f"{self.base_url}{path}"
|
||||
if query:
|
||||
url = f"{url}?{urllib.parse.urlencode(query)}"
|
||||
return self.transport.request(
|
||||
url,
|
||||
body=body,
|
||||
headers={
|
||||
"Authorization": f"Bearer {self.token}",
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": "vllm-ci-command",
|
||||
},
|
||||
method=method,
|
||||
)
|
||||
|
||||
def list_builds(
|
||||
self,
|
||||
commit: str,
|
||||
*,
|
||||
metadata: tuple[str, str] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
query = [
|
||||
("commit", commit),
|
||||
("exclude_jobs", "true"),
|
||||
("exclude_pipeline", "true"),
|
||||
("per_page", "100"),
|
||||
]
|
||||
if metadata:
|
||||
key, value = metadata
|
||||
query.append((f"meta_data[{key}]", value))
|
||||
response = self._request(query=query)
|
||||
if not isinstance(response, list):
|
||||
raise ApiError(None, "Buildkite API returned an invalid build list.")
|
||||
return response
|
||||
|
||||
def create_build(self, body: Mapping[str, Any]) -> dict[str, Any]:
|
||||
return self._request(body=body, method="POST")
|
||||
|
||||
def retry_failed_jobs(
|
||||
self,
|
||||
build_number: int,
|
||||
states: str,
|
||||
) -> dict[str, Any]:
|
||||
number = urllib.parse.quote(str(build_number), safe="")
|
||||
return self._request(
|
||||
body={"states": states},
|
||||
method="PUT",
|
||||
path=f"/{number}/retry_failed_jobs",
|
||||
)
|
||||
|
||||
|
||||
def parse_command(body: str) -> str | None:
|
||||
if body in {COMMAND_RUN_CI, COMMAND_RETRY_FAILED}:
|
||||
return body
|
||||
return None
|
||||
|
||||
|
||||
def parse_trusted_users(value: str = "") -> set[str]:
|
||||
return {
|
||||
user.casefold() for item in value.split(",") for user in item.split() if user
|
||||
}
|
||||
|
||||
|
||||
def has_ready_label(pr: Mapping[str, Any]) -> bool:
|
||||
return any(label["name"] in READY_LABELS for label in pr["labels"])
|
||||
|
||||
|
||||
def is_trusted_permission(permission: str) -> bool:
|
||||
return permission in TRUSTED_PERMISSIONS
|
||||
|
||||
|
||||
def authorize(
|
||||
*,
|
||||
actor: str,
|
||||
permission: str,
|
||||
pr: Mapping[str, Any],
|
||||
trusted_approval: bool = False,
|
||||
trusted_users: set[str] | None = None,
|
||||
) -> tuple[bool, str]:
|
||||
trusted_users = trusted_users or set()
|
||||
if is_trusted_permission(permission):
|
||||
return True, f"repository {permission} permission"
|
||||
if actor.casefold() in trusted_users:
|
||||
return True, "configured trusted contributor"
|
||||
if actor.casefold() != pr["user"]["login"].casefold():
|
||||
return (
|
||||
False,
|
||||
"Only reviewers with write access can run CI before it is "
|
||||
"delegated to the PR author.",
|
||||
)
|
||||
if pr["draft"]:
|
||||
return False, "PR authors cannot run CI while the PR is a draft."
|
||||
if has_ready_label(pr):
|
||||
return True, "ready label"
|
||||
if trusted_approval:
|
||||
return True, "approval from a trusted reviewer"
|
||||
return (
|
||||
False,
|
||||
"A reviewer with write access must run `/ci run`, approve the PR, "
|
||||
"or add the `ready` label first.",
|
||||
)
|
||||
|
||||
|
||||
def has_trusted_approval(
|
||||
github: GitHubClient,
|
||||
number: int,
|
||||
trusted_users: set[str],
|
||||
) -> bool:
|
||||
if github.get_review_decision(number) != "APPROVED":
|
||||
return False
|
||||
|
||||
latest_review_states: dict[str, tuple[str, str]] = {}
|
||||
for review in github.list_reviews(number):
|
||||
user = review.get("user") or {}
|
||||
login = user.get("login")
|
||||
state = review.get("state")
|
||||
if login and state in {"APPROVED", "CHANGES_REQUESTED", "DISMISSED"}:
|
||||
latest_review_states[login.casefold()] = (login, state)
|
||||
|
||||
for login, state in latest_review_states.values():
|
||||
if state != "APPROVED":
|
||||
continue
|
||||
if login.casefold() in trusted_users:
|
||||
return True
|
||||
if is_trusted_permission(github.get_permission(login)):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_build_for_pr(build: Mapping[str, Any], pr_number: int) -> bool:
|
||||
pull_request = build.get("pull_request")
|
||||
if isinstance(pull_request, Mapping):
|
||||
build_pr_number = pull_request.get("id", pull_request.get("number"))
|
||||
if build_pr_number is not None:
|
||||
return str(build_pr_number) == str(pr_number)
|
||||
metadata = build.get("meta_data") or {}
|
||||
return str(metadata.get("github-pr-number")) == str(pr_number)
|
||||
|
||||
|
||||
def is_active_build(build: Mapping[str, Any]) -> bool:
|
||||
return bool(build.get("blocked")) or build.get("state") in ACTIVE_BUILD_STATES
|
||||
|
||||
|
||||
def select_latest_build(
|
||||
builds: Sequence[dict[str, Any]],
|
||||
pr_number: int,
|
||||
) -> dict[str, Any] | None:
|
||||
matching = [build for build in builds if is_build_for_pr(build, pr_number)]
|
||||
return max(matching, key=lambda build: build.get("created_at", ""), default=None)
|
||||
|
||||
|
||||
def create_build_payload(
|
||||
*,
|
||||
actor: str,
|
||||
comment_id: int,
|
||||
pr: Mapping[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"commit": pr["head"]["sha"],
|
||||
"branch": pr["head"]["ref"],
|
||||
"message": f"PR #{pr['number']} {COMMAND_RUN_CI} by @{actor}",
|
||||
"pull_request_id": pr["number"],
|
||||
"pull_request_base_branch": pr["base"]["ref"],
|
||||
"pull_request_repository": pr["head"]["repo"]["clone_url"],
|
||||
"pull_request_labels": [label["name"] for label in pr["labels"]],
|
||||
"ignore_pipeline_branch_filters": True,
|
||||
"env": {
|
||||
"VLLM_CI_GITHUB_COMMENT_ID": str(comment_id),
|
||||
"VLLM_CI_TRIGGERED_BY": actor,
|
||||
},
|
||||
"meta_data": {
|
||||
"github-comment-id": str(comment_id),
|
||||
"github-pr-number": str(pr["number"]),
|
||||
"github-triggered-by": actor,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def add_reaction_safely(
|
||||
github: GitHubClient,
|
||||
comment_id: int,
|
||||
content: str,
|
||||
) -> None:
|
||||
try:
|
||||
github.add_reaction(comment_id, content)
|
||||
except Exception as error:
|
||||
print(f"Could not add {content} reaction: {error}", file=sys.stderr)
|
||||
|
||||
|
||||
def is_already_handled(github: GitHubClient, comment_id: int) -> bool:
|
||||
return any(
|
||||
reaction.get("content") in {"rocket", "-1"}
|
||||
and (reaction.get("user") or {}).get("login") == "github-actions[bot]"
|
||||
for reaction in github.list_reactions(comment_id)
|
||||
)
|
||||
|
||||
|
||||
def handle_run_ci(
|
||||
*,
|
||||
actor: str,
|
||||
buildkite: BuildkiteClient,
|
||||
comment_id: int,
|
||||
github: GitHubClient,
|
||||
pr: Mapping[str, Any],
|
||||
) -> str:
|
||||
duplicate_builds = buildkite.list_builds(
|
||||
pr["head"]["sha"],
|
||||
metadata=("github-comment-id", str(comment_id)),
|
||||
)
|
||||
duplicate = select_latest_build(duplicate_builds, pr["number"])
|
||||
if duplicate:
|
||||
return f"CI was already requested by this comment: {duplicate['web_url']}"
|
||||
|
||||
current_builds = buildkite.list_builds(pr["head"]["sha"])
|
||||
active_build = next(
|
||||
(
|
||||
build
|
||||
for build in current_builds
|
||||
if is_build_for_pr(build, pr["number"]) and is_active_build(build)
|
||||
),
|
||||
None,
|
||||
)
|
||||
if active_build:
|
||||
return f"CI is already running for this commit: {active_build['web_url']}"
|
||||
|
||||
current_pr = github.get_pr(pr["number"])
|
||||
if current_pr["state"] != "open" or current_pr["head"]["sha"] != pr["head"]["sha"]:
|
||||
return (
|
||||
"The PR head changed while processing the command. Comment `/ci run` again."
|
||||
)
|
||||
|
||||
build = buildkite.create_build(
|
||||
create_build_payload(
|
||||
actor=actor,
|
||||
comment_id=comment_id,
|
||||
pr=current_pr,
|
||||
)
|
||||
)
|
||||
return (
|
||||
f"Triggered [Buildkite CI #{build['number']}]({build['web_url']}) "
|
||||
f"for commit `{current_pr['head']['sha'][:12]}`."
|
||||
)
|
||||
|
||||
|
||||
def handle_retry_failed(
|
||||
*,
|
||||
buildkite: BuildkiteClient,
|
||||
pr: Mapping[str, Any],
|
||||
) -> str:
|
||||
builds = buildkite.list_builds(pr["head"]["sha"])
|
||||
build = select_latest_build(builds, pr["number"])
|
||||
if not build:
|
||||
return "No CI build exists for the current PR commit. Use `/ci run` first."
|
||||
if not build.get("finished_at") or is_active_build(build):
|
||||
return f"CI is still running for this commit: {build['web_url']}"
|
||||
|
||||
retried = buildkite.retry_failed_jobs(build["number"], RETRY_STATES)
|
||||
if retried["retried_jobs_count"] == 0:
|
||||
return (
|
||||
f"No failed, timed-out, or expired jobs need retrying: {build['web_url']}"
|
||||
)
|
||||
return (
|
||||
f"Queued {retried['retried_jobs_count']} failed job(s) for retry in "
|
||||
f"[Buildkite CI #{build['number']}]({build['web_url']})."
|
||||
)
|
||||
|
||||
|
||||
def run(
|
||||
event: Mapping[str, Any],
|
||||
github: GitHubClient,
|
||||
buildkite: BuildkiteClient,
|
||||
trusted_users_value: str = "",
|
||||
) -> None:
|
||||
command = parse_command(event["comment"]["body"])
|
||||
if not command or "pull_request" not in event["issue"]:
|
||||
return
|
||||
|
||||
issue_number = event["issue"]["number"]
|
||||
comment_id = event["comment"]["id"]
|
||||
actor = event["comment"]["user"]["login"]
|
||||
|
||||
if is_already_handled(github, comment_id):
|
||||
print(f"Comment {comment_id} was already handled.")
|
||||
return
|
||||
add_reaction_safely(github, comment_id, "eyes")
|
||||
|
||||
try:
|
||||
pr = github.get_pr(issue_number)
|
||||
permission = github.get_permission(actor)
|
||||
if pr["state"] != "open":
|
||||
github.add_comment(issue_number, "CI commands require an open PR.")
|
||||
return
|
||||
|
||||
trusted_users = parse_trusted_users(trusted_users_value)
|
||||
should_check_approval = (
|
||||
not is_trusted_permission(permission)
|
||||
and actor.casefold() not in trusted_users
|
||||
and actor.casefold() == pr["user"]["login"].casefold()
|
||||
and not pr["draft"]
|
||||
and not has_ready_label(pr)
|
||||
)
|
||||
trusted_approval = should_check_approval and has_trusted_approval(
|
||||
github,
|
||||
issue_number,
|
||||
trusted_users,
|
||||
)
|
||||
allowed, reason = authorize(
|
||||
actor=actor,
|
||||
permission=permission,
|
||||
pr=pr,
|
||||
trusted_approval=trusted_approval,
|
||||
trusted_users=trusted_users,
|
||||
)
|
||||
if not allowed:
|
||||
add_reaction_safely(github, comment_id, "-1")
|
||||
github.add_comment(issue_number, f"@{actor}, {reason}")
|
||||
return
|
||||
|
||||
print(f"Authorized @{actor}: {reason}")
|
||||
if command == COMMAND_RUN_CI:
|
||||
message = handle_run_ci(
|
||||
actor=actor,
|
||||
buildkite=buildkite,
|
||||
comment_id=comment_id,
|
||||
github=github,
|
||||
pr=pr,
|
||||
)
|
||||
else:
|
||||
message = handle_retry_failed(buildkite=buildkite, pr=pr)
|
||||
add_reaction_safely(github, comment_id, "rocket")
|
||||
github.add_comment(issue_number, message)
|
||||
except Exception:
|
||||
add_reaction_safely(github, comment_id, "confused")
|
||||
raise
|
||||
|
||||
|
||||
def main() -> None:
|
||||
event_path = os.environ["GITHUB_EVENT_PATH"]
|
||||
with open(event_path, encoding="utf-8") as event_file:
|
||||
event = json.load(event_file)
|
||||
|
||||
if not parse_command(event["comment"]["body"]):
|
||||
return
|
||||
|
||||
github = GitHubClient(
|
||||
os.environ.get("GH_TOKEN", ""),
|
||||
os.environ["GITHUB_REPOSITORY"],
|
||||
)
|
||||
buildkite = BuildkiteClient(
|
||||
os.environ.get("BUILDKITE_API_TOKEN", ""),
|
||||
os.environ.get("BUILDKITE_ORGANIZATION", "vllm"),
|
||||
os.environ.get("BUILDKITE_PIPELINE", "ci"),
|
||||
)
|
||||
run(
|
||||
event,
|
||||
github,
|
||||
buildkite,
|
||||
os.environ.get("CI_TRUSTED_USERS", ""),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,363 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import unittest
|
||||
from typing import Any
|
||||
|
||||
from run_ci_command import (
|
||||
COMMAND_RETRY_FAILED,
|
||||
COMMAND_RUN_CI,
|
||||
RETRY_STATES,
|
||||
BuildkiteClient,
|
||||
authorize,
|
||||
create_build_payload,
|
||||
has_trusted_approval,
|
||||
is_active_build,
|
||||
is_build_for_pr,
|
||||
parse_command,
|
||||
parse_trusted_users,
|
||||
run,
|
||||
select_latest_build,
|
||||
)
|
||||
|
||||
|
||||
def make_pr(**overrides: Any) -> dict[str, Any]:
|
||||
pr = {
|
||||
"base": {"ref": "main"},
|
||||
"draft": False,
|
||||
"head": {
|
||||
"ref": "feature",
|
||||
"repo": {"clone_url": "https://github.com/contributor/vllm.git"},
|
||||
"sha": "0123456789abcdef",
|
||||
},
|
||||
"labels": [],
|
||||
"number": 42,
|
||||
"state": "open",
|
||||
"user": {"login": "author"},
|
||||
}
|
||||
pr.update(overrides)
|
||||
return pr
|
||||
|
||||
|
||||
def make_event(command: str, actor: str = "reviewer") -> dict[str, Any]:
|
||||
return {
|
||||
"comment": {
|
||||
"body": command,
|
||||
"id": 99,
|
||||
"user": {"login": actor},
|
||||
},
|
||||
"issue": {
|
||||
"number": 42,
|
||||
"pull_request": {},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class FakeGitHub:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
permission: str = "write",
|
||||
permissions: dict[str, str] | None = None,
|
||||
pr: dict[str, Any] | None = None,
|
||||
review_decision: str = "REVIEW_REQUIRED",
|
||||
reviews: list[dict[str, Any]] | None = None,
|
||||
) -> None:
|
||||
self.comments: list[str] = []
|
||||
self.permission = permission
|
||||
self.permissions = permissions or {}
|
||||
self.pr = pr or make_pr()
|
||||
self.reactions: list[str] = []
|
||||
self.review_decision = review_decision
|
||||
self.reviews = reviews or []
|
||||
|
||||
def get_pr(self, number: int) -> dict[str, Any]:
|
||||
return self.pr
|
||||
|
||||
def get_permission(self, actor: str) -> str:
|
||||
return self.permissions.get(actor, self.permission)
|
||||
|
||||
def get_review_decision(self, number: int) -> str:
|
||||
return self.review_decision
|
||||
|
||||
def list_reviews(self, number: int) -> list[dict[str, Any]]:
|
||||
return self.reviews
|
||||
|
||||
def list_reactions(self, comment_id: int) -> list[dict[str, Any]]:
|
||||
return []
|
||||
|
||||
def add_reaction(self, comment_id: int, content: str) -> None:
|
||||
self.reactions.append(content)
|
||||
|
||||
def add_comment(self, issue_number: int, body: str) -> None:
|
||||
self.comments.append(body)
|
||||
|
||||
|
||||
class FakeBuildkite:
|
||||
def __init__(
|
||||
self,
|
||||
build_lists: list[list[dict[str, Any]]] | None = None,
|
||||
) -> None:
|
||||
self.build_lists = build_lists or []
|
||||
self.created_builds: list[dict[str, Any]] = []
|
||||
self.list_calls: list[tuple[str, tuple[str, str] | None]] = []
|
||||
self.retry_calls: list[tuple[int, str]] = []
|
||||
|
||||
def list_builds(
|
||||
self,
|
||||
commit: str,
|
||||
*,
|
||||
metadata: tuple[str, str] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
self.list_calls.append((commit, metadata))
|
||||
return self.build_lists.pop(0)
|
||||
|
||||
def create_build(self, body: dict[str, Any]) -> dict[str, Any]:
|
||||
self.created_builds.append(body)
|
||||
return {
|
||||
"number": 123,
|
||||
"web_url": "https://buildkite.example/builds/123",
|
||||
}
|
||||
|
||||
def retry_failed_jobs(
|
||||
self,
|
||||
build_number: int,
|
||||
states: str,
|
||||
) -> dict[str, Any]:
|
||||
self.retry_calls.append((build_number, states))
|
||||
return {"retried_jobs_count": 3}
|
||||
|
||||
|
||||
class FakeTransport:
|
||||
def __init__(self, response: Any) -> None:
|
||||
self.calls: list[dict[str, Any]] = []
|
||||
self.response = response
|
||||
|
||||
def request(self, url: str, **kwargs: Any) -> Any:
|
||||
self.calls.append({"url": url, **kwargs})
|
||||
return self.response
|
||||
|
||||
|
||||
class RunCiCommandTest(unittest.TestCase):
|
||||
def test_only_exact_ci_commands_are_accepted(self) -> None:
|
||||
self.assertEqual(parse_command(COMMAND_RUN_CI), COMMAND_RUN_CI)
|
||||
self.assertEqual(
|
||||
parse_command(COMMAND_RETRY_FAILED),
|
||||
COMMAND_RETRY_FAILED,
|
||||
)
|
||||
self.assertIsNone(parse_command("/ci run please"))
|
||||
self.assertIsNone(parse_command(" /ci run"))
|
||||
|
||||
def test_write_access_authorizes_reviewers_and_authors(self) -> None:
|
||||
allowed, _ = authorize(
|
||||
actor="reviewer",
|
||||
permission="write",
|
||||
pr=make_pr(),
|
||||
)
|
||||
self.assertTrue(allowed)
|
||||
|
||||
def test_configured_trusted_contributors_can_run_ci(self) -> None:
|
||||
trusted_users = parse_trusted_users("trusted-one, TRUSTED-TWO")
|
||||
allowed, _ = authorize(
|
||||
actor="trusted-two",
|
||||
permission="read",
|
||||
pr=make_pr(),
|
||||
trusted_users=trusted_users,
|
||||
)
|
||||
self.assertTrue(allowed)
|
||||
|
||||
def test_authors_need_an_approval_or_ready_label(self) -> None:
|
||||
pending, _ = authorize(
|
||||
actor="author",
|
||||
permission="read",
|
||||
pr=make_pr(),
|
||||
)
|
||||
approved, _ = authorize(
|
||||
actor="author",
|
||||
permission="read",
|
||||
pr=make_pr(),
|
||||
trusted_approval=True,
|
||||
)
|
||||
ready, _ = authorize(
|
||||
actor="author",
|
||||
permission="read",
|
||||
pr=make_pr(labels=[{"name": "ready"}]),
|
||||
)
|
||||
self.assertFalse(pending)
|
||||
self.assertTrue(approved)
|
||||
self.assertTrue(ready)
|
||||
|
||||
def test_non_author_contributors_without_write_are_denied(self) -> None:
|
||||
allowed, _ = authorize(
|
||||
actor="contributor",
|
||||
permission="read",
|
||||
pr=make_pr(),
|
||||
trusted_approval=True,
|
||||
)
|
||||
self.assertFalse(allowed)
|
||||
|
||||
def test_authors_cannot_use_ready_state_on_draft_prs(self) -> None:
|
||||
allowed, _ = authorize(
|
||||
actor="author",
|
||||
permission="read",
|
||||
pr=make_pr(draft=True, labels=[{"name": "ready"}]),
|
||||
trusted_approval=True,
|
||||
)
|
||||
self.assertFalse(allowed)
|
||||
|
||||
def test_only_trusted_reviewers_can_delegate_through_approval(self) -> None:
|
||||
approved_review = {
|
||||
"state": "APPROVED",
|
||||
"user": {"login": "reviewer"},
|
||||
}
|
||||
trusted = FakeGitHub(
|
||||
permission="read",
|
||||
permissions={"reviewer": "write"},
|
||||
review_decision="APPROVED",
|
||||
reviews=[approved_review],
|
||||
)
|
||||
untrusted = FakeGitHub(
|
||||
permission="read",
|
||||
review_decision="APPROVED",
|
||||
reviews=[approved_review],
|
||||
)
|
||||
self.assertTrue(has_trusted_approval(trusted, 42, set()))
|
||||
self.assertFalse(has_trusted_approval(untrusted, 42, set()))
|
||||
|
||||
def test_build_matching_is_scoped_to_the_pr(self) -> None:
|
||||
self.assertTrue(is_build_for_pr({"pull_request": {"id": 42}}, 42))
|
||||
self.assertFalse(is_build_for_pr({"pull_request": {"id": 43}}, 42))
|
||||
self.assertTrue(
|
||||
is_build_for_pr(
|
||||
{"meta_data": {"github-pr-number": "42"}},
|
||||
42,
|
||||
)
|
||||
)
|
||||
|
||||
def test_latest_build_selection_ignores_other_prs(self) -> None:
|
||||
latest = select_latest_build(
|
||||
[
|
||||
{
|
||||
"created_at": "2026-07-28T02:00:00Z",
|
||||
"number": 3,
|
||||
"pull_request": {"id": 43},
|
||||
},
|
||||
{
|
||||
"created_at": "2026-07-28T01:00:00Z",
|
||||
"number": 2,
|
||||
"pull_request": {"id": 42},
|
||||
},
|
||||
{
|
||||
"created_at": "2026-07-28T00:00:00Z",
|
||||
"number": 1,
|
||||
"pull_request": {"id": 42},
|
||||
},
|
||||
],
|
||||
42,
|
||||
)
|
||||
self.assertEqual(latest["number"], 2)
|
||||
|
||||
def test_active_build_states_prevent_duplicate_runs(self) -> None:
|
||||
self.assertTrue(is_active_build({"state": "scheduled"}))
|
||||
self.assertTrue(is_active_build({"state": "running"}))
|
||||
self.assertTrue(is_active_build({"state": "waiting"}))
|
||||
self.assertTrue(is_active_build({"blocked": True, "state": "passed"}))
|
||||
self.assertFalse(is_active_build({"state": "failed"}))
|
||||
|
||||
def test_build_payload_preserves_pr_context(self) -> None:
|
||||
payload = create_build_payload(
|
||||
actor="reviewer",
|
||||
comment_id=99,
|
||||
pr=make_pr(labels=[{"name": "ready"}, {"name": "v1"}]),
|
||||
)
|
||||
self.assertEqual(
|
||||
payload,
|
||||
{
|
||||
"commit": "0123456789abcdef",
|
||||
"branch": "feature",
|
||||
"message": "PR #42 /ci run by @reviewer",
|
||||
"pull_request_id": 42,
|
||||
"pull_request_base_branch": "main",
|
||||
"pull_request_repository": ("https://github.com/contributor/vllm.git"),
|
||||
"pull_request_labels": ["ready", "v1"],
|
||||
"ignore_pipeline_branch_filters": True,
|
||||
"env": {
|
||||
"VLLM_CI_GITHUB_COMMENT_ID": "99",
|
||||
"VLLM_CI_TRIGGERED_BY": "reviewer",
|
||||
},
|
||||
"meta_data": {
|
||||
"github-comment-id": "99",
|
||||
"github-pr-number": "42",
|
||||
"github-triggered-by": "reviewer",
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
def test_ci_run_dispatches_build_with_current_pr_metadata(self) -> None:
|
||||
github = FakeGitHub()
|
||||
buildkite = FakeBuildkite([[], []])
|
||||
run(make_event(COMMAND_RUN_CI), github, buildkite)
|
||||
|
||||
self.assertEqual(len(buildkite.created_builds), 1)
|
||||
self.assertEqual(
|
||||
buildkite.created_builds[0]["message"],
|
||||
"PR #42 /ci run by @reviewer",
|
||||
)
|
||||
self.assertEqual(github.reactions, ["eyes", "rocket"])
|
||||
self.assertIn("Buildkite CI #123", github.comments[0])
|
||||
|
||||
def test_unapproved_authors_are_denied_without_buildkite(self) -> None:
|
||||
github = FakeGitHub(
|
||||
permission="read",
|
||||
pr=make_pr(),
|
||||
review_decision="REVIEW_REQUIRED",
|
||||
)
|
||||
buildkite = FakeBuildkite()
|
||||
run(make_event(COMMAND_RUN_CI, "author"), github, buildkite)
|
||||
|
||||
self.assertEqual(buildkite.list_calls, [])
|
||||
self.assertEqual(github.reactions, ["eyes", "-1"])
|
||||
self.assertIn("approve the PR", github.comments[0])
|
||||
|
||||
def test_ci_retry_uses_latest_current_sha_build(self) -> None:
|
||||
github = FakeGitHub(
|
||||
permission="read",
|
||||
pr=make_pr(labels=[{"name": "ready"}]),
|
||||
)
|
||||
buildkite = FakeBuildkite(
|
||||
[
|
||||
[
|
||||
{
|
||||
"created_at": "2026-07-28T01:00:00Z",
|
||||
"finished_at": "2026-07-28T02:00:00Z",
|
||||
"number": 123,
|
||||
"pull_request": {"id": 42},
|
||||
"state": "failed",
|
||||
"web_url": "https://buildkite.example/builds/123",
|
||||
}
|
||||
]
|
||||
]
|
||||
)
|
||||
run(make_event(COMMAND_RETRY_FAILED, "author"), github, buildkite)
|
||||
|
||||
self.assertEqual(buildkite.retry_calls, [(123, RETRY_STATES)])
|
||||
self.assertIn("Queued 3 failed job", github.comments[0])
|
||||
|
||||
def test_buildkite_retry_uses_retry_failed_jobs_endpoint(self) -> None:
|
||||
transport = FakeTransport({"retried_jobs_count": 2})
|
||||
client = BuildkiteClient(
|
||||
"secret",
|
||||
"vllm",
|
||||
"ci",
|
||||
transport=transport,
|
||||
)
|
||||
client.retry_failed_jobs(123, RETRY_STATES)
|
||||
|
||||
call = transport.calls[0]
|
||||
self.assertEqual(call["method"], "PUT")
|
||||
self.assertTrue(call["url"].endswith("/123/retry_failed_jobs"))
|
||||
self.assertEqual(call["body"], {"states": RETRY_STATES})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user