diff --git a/.buildkite/lm-eval-harness/configs/Meta-Llama-4-Maverick-17B-128E-Instruct-FP8.yaml b/.buildkite/lm-eval-harness/configs/Meta-Llama-4-Maverick-17B-128E-Instruct-FP8.yaml index 9a5af854011..a9c9857fc30 100644 --- a/.buildkite/lm-eval-harness/configs/Meta-Llama-4-Maverick-17B-128E-Instruct-FP8.yaml +++ b/.buildkite/lm-eval-harness/configs/Meta-Llama-4-Maverick-17B-128E-Instruct-FP8.yaml @@ -1,6 +1,7 @@ # For hf script, without -t option (tensor parallel size). # bash .buildkite/lm-eval-harness/run-lm-eval-mmlupro-vllm-baseline.sh -m meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 -l 250 -t 8 -f 5 model_name: "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8" +rocm_safetensors_load_strategy: lazy required_gpu_arch: - gfx942 - gfx950 diff --git a/.buildkite/lm-eval-harness/test_lm_eval_correctness.py b/.buildkite/lm-eval-harness/test_lm_eval_correctness.py index dd2fd5f05b4..9335ce02900 100644 --- a/.buildkite/lm-eval-harness/test_lm_eval_correctness.py +++ b/.buildkite/lm-eval-harness/test_lm_eval_correctness.py @@ -72,6 +72,11 @@ def launch_lm_eval(eval_config, tp_size): if moe_backend is not None: model_args += f"moe_backend={moe_backend}," + if current_platform.is_rocm(): + rocm_load_strategy = eval_config.get("rocm_safetensors_load_strategy") + if rocm_load_strategy is not None: + model_args += f"safetensors_load_strategy={rocm_load_strategy}," + env_vars = eval_config.get("env_vars", None) with scoped_env_vars(env_vars): results = lm_eval.simple_evaluate( diff --git a/.buildkite/scripts/ci-bake-rocm.sh b/.buildkite/scripts/ci-bake-rocm.sh index 22391291b44..2891291105f 100644 --- a/.buildkite/scripts/ci-bake-rocm.sh +++ b/.buildkite/scripts/ci-bake-rocm.sh @@ -1898,8 +1898,8 @@ confirm_remote_image_push() { fi if [[ -z "${remote_revision}" \ - && ${IMAGE_EXISTED_BEFORE_BUILD} -eq 0 \ - && image_tag_is_commit_scoped ]]; then + && ${IMAGE_EXISTED_BEFORE_BUILD} -eq 0 ]] \ + && image_tag_is_commit_scoped; then echo "Remote image exists under a commit-scoped tag; accepting push despite missing revision label." return 0 fi @@ -2029,36 +2029,57 @@ upload_wheel_artifacts_if_present() { local wheel_dir="./wheel-export" local artifact_dir="artifacts/vllm-rocm-install" local archive_name="vllm-rocm-install.tar.gz" + local metadata_dir="${wheel_dir}/.vllm-ci-artifact" + local native_base_image="" local whl="" local whl_name="" + local -a wheels=() if ! should_upload_wheel_artifacts; then return 0 fi - if [[ ! -d "${wheel_dir}" ]] || ! ls "${wheel_dir}"/*.whl >/dev/null 2>&1; then - echo "No ROCm wheel artifacts found in ${wheel_dir}" - return 0 + if [[ -d "${wheel_dir}" ]]; then + mapfile -t wheels < <(find "${wheel_dir}" -maxdepth 1 -type f -name '*.whl' -print) + fi + if [[ ${#wheels[@]} -ne 1 ]]; then + echo "Expected exactly one ROCm wheel in ${wheel_dir}; found ${#wheels[@]}" >&2 + return 1 + fi + whl="${wheels[0]}" + whl_name=$(basename "${whl}") + native_base_image="${CI_BASE_IMAGE_TAG_COMMIT_REF:-${CI_BASE_IMAGE:-}}" + if [[ -z "${native_base_image}" ]]; then + echo "Native ROCm artifact requires a ci_base image reference" >&2 + return 1 fi echo "--- :package: Uploading ROCm vLLM install artifact" - mkdir -p "${artifact_dir}" + rm -rf "${artifact_dir}" "${metadata_dir}" + mkdir -p "${artifact_dir}" "${metadata_dir}" + + printf '%s\n' "${BUILDKITE_COMMIT:-local}" > "${metadata_dir}/commit.txt" + printf '%s\n' "${native_base_image}" > "${metadata_dir}/native-base-image.txt" + printf '%s\n' "${CI_BASE_IMAGE:-}" > "${metadata_dir}/ci-base-image.txt" + printf '%s\n' "${IMAGE_TAG:-}" > "${metadata_dir}/fallback-image.txt" + printf '%s\n' "${whl_name}" > "${metadata_dir}/wheel-filename.txt" tar -C "${wheel_dir}" -czf "${artifact_dir}/${archive_name}" . + ( + cd "${artifact_dir}" + sha256sum "${archive_name}" > "${archive_name}.sha256" + ) echo "Created ${archive_name}: $(du -sh "${artifact_dir}/${archive_name}" | cut -f1)" - printf '%s\n' "${CI_BASE_IMAGE:-}" > "${artifact_dir}/ci-base-image.txt" - printf '%s\n' "${IMAGE_TAG:-}" > "${artifact_dir}/fallback-image.txt" - - for whl in "${wheel_dir}"/*.whl; do - [[ -f "${whl}" ]] || continue - whl_name=$(basename "${whl}") - cp "${whl}" "${artifact_dir}/${whl_name}" - echo "Copied ${whl_name}: $(du -sh "${artifact_dir}/${whl_name}" | cut -f1)" - done + cp "${metadata_dir}"/*.txt "${artifact_dir}/" + cp "${whl}" "${artifact_dir}/${whl_name}" + echo "Copied ${whl_name}: $(du -sh "${artifact_dir}/${whl_name}" | cut -f1)" if command -v buildkite-agent >/dev/null 2>&1; then - buildkite-agent artifact upload "${artifact_dir}/*" + buildkite-agent artifact upload "${artifact_dir}/*" || return 1 echo "ROCm vLLM install artifacts uploaded to ${artifact_dir}/" + elif [[ "${BUILDKITE:-false}" == "true" ]]; then + echo "buildkite-agent not found; cannot upload required ROCm artifacts" >&2 + return 1 else echo "Not in Buildkite, skipping artifact upload" fi @@ -2090,6 +2111,11 @@ main() { echo "BAKE_PRINT_ONLY=1 set; skipping build" return 0 fi + if should_upload_wheel_artifacts; then + # wheel-export is an output directory, not a BuildKit cache. Starting + # clean prevents a failed/retried export from packaging a stale wheel. + rm -rf ./wheel-export + fi seed_dependency_caches_if_needed run_bake upload_wheel_artifacts_if_present diff --git a/.buildkite/scripts/hardware_ci/run-amd-test.sh b/.buildkite/scripts/hardware_ci/run-amd-test.sh index fee9ab04f4b..4e8735c841a 100755 --- a/.buildkite/scripts/hardware_ci/run-amd-test.sh +++ b/.buildkite/scripts/hardware_ci/run-amd-test.sh @@ -1,7 +1,7 @@ #!/bin/bash -# This script runs tests inside the corresponding ROCm docker container. -# It handles both single-node and multi-node test configurations. +# This script runs ROCm tests either directly in a native CI pod or inside the +# corresponding Docker container. Multi-node tests continue to use Docker. # # Multi-node detection: Instead of matching on fragile group names, we detect # multi-node jobs structurally by looking for the bracket command syntax @@ -34,10 +34,27 @@ set -o pipefail : "${CLICOLOR_FORCE:=1}" : "${PY_COLORS:=1}" : "${ROCM_DOCKER_TTY:=1}" +: "${PYTHONFAULTHANDLER:=1}" +: "${PYTEST_TIMEOUT:=2100}" if [[ " ${PYTEST_ADDOPTS:-} " != *" --color"* ]]; then PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--color=yes" fi -export BUILDKIT_PROGRESS TERM FORCE_COLOR CLICOLOR_FORCE PY_COLORS PYTEST_ADDOPTS ROCM_DOCKER_TTY +if [[ " ${PYTEST_ADDOPTS:-} " != *" --durations="* ]]; then + PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--durations=25" +fi +if [[ " ${PYTEST_ADDOPTS:-} " != *" --durations-min="* ]]; then + PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--durations-min=1.0" +fi +# Dump stacks after 15 minutes, then stop an individual test after 35 minutes. +if [[ " ${PYTEST_ADDOPTS:-} " != *" faulthandler_timeout="* ]]; then + PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }-o faulthandler_timeout=900" +fi +if [[ " ${PYTEST_ADDOPTS:-} " != *" --timeout-method="* && + " ${PYTEST_ADDOPTS:-} " != *" --timeout-method "* ]]; then + PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--timeout-method=thread" +fi +export BUILDKIT_PROGRESS TERM FORCE_COLOR CLICOLOR_FORCE PY_COLORS PYTEST_ADDOPTS PYTEST_TIMEOUT ROCM_DOCKER_TTY +export PYTHONFAULTHANDLER # Export Python path for commands that run directly on the host. Containerized # tests set this to /vllm-workspace below so spawned Python processes do not @@ -53,6 +70,28 @@ report_docker_usage() { docker system df || true } +clear_ci_orchestration_env() { + unset -v \ + VLLM_TEST_GROUP_NAME \ + VLLM_CI_REQUIRE_PERSISTENT_HF_CACHE \ + VLLM_CI_ARTIFACT_STEP \ + VLLM_TEST_CACHE \ + VLLM_CI_EXECUTION_MODE \ + VLLM_CI_WORKSPACE \ + VLLM_CI_REQUIRE_WORKSPACE_MOUNT \ + VLLM_TEST_COMMANDS \ + VLLM_CI_BRANCH \ + VLLM_CI_BASE_IMAGE \ + VLLM_CI_FALLBACK_IMAGE \ + VLLM_CI_DOCKER_DISABLED \ + VLLM_CI_ARTIFACT_GLOB \ + VLLM_CI_ARTIFACT_CHECKSUM_GLOB \ + VLLM_CI_EXPECTED_GPU_COUNT \ + VLLM_CI_USE_ARTIFACTS \ + VLLM_CI_RESULTS_ROOT \ + VLLM_ALLOW_DEPRECATED_BEAM_SEARCH +} + cleanup_network() { local max_nodes=${NUM_NODES:-2} for node in $(seq 0 $((max_nodes - 1))); do @@ -145,7 +184,11 @@ prepare_artifact_image() { fi cp "${wheel_dir}"/*.whl "${context_dir}/wheels/" || return 1 - tar -C "${wheel_dir}" --exclude='*.whl' -cf - . \ + tar -C "${wheel_dir}" \ + --exclude='*.whl' \ + --exclude='.vllm-ci-artifact' \ + --exclude='./.vllm-ci-artifact' \ + -cf - . \ | tar -C "${workspace_dir}" -xf - || return 1 cat > "${context_dir}/Dockerfile" <<'EOF' ARG BASE_IMAGE @@ -168,6 +211,257 @@ EOF return 0 } +is_native_runtime() { + [[ "${AMD_CI_RUNTIME:-}" == "native" || "${NATIVE_CI:-}" == "true" ]] +} + +validate_native_workspace() { + local workspace_dir="${VLLM_CI_WORKSPACE:-/vllm-workspace}" + local workspace_real="" + local checkout_real="" + local workspace_mount="" + + mkdir -p "${workspace_dir}" || return 1 + workspace_real=$(readlink -m "${workspace_dir}") || return 1 + if [[ -n "${BUILDKITE_BUILD_CHECKOUT_PATH:-}" ]]; then + checkout_real=$(readlink -m "${BUILDKITE_BUILD_CHECKOUT_PATH}") || return 1 + if [[ "${checkout_real}" == "${workspace_real}" \ + || "${checkout_real}" == "${workspace_real}/"* \ + || "${workspace_real}" == "${checkout_real}/"* ]]; then + echo "Refusing to replace ${workspace_real}; it overlaps the Buildkite checkout ${checkout_real}" >&2 + return 1 + fi + fi + if [[ "${VLLM_CI_REQUIRE_WORKSPACE_MOUNT:-1}" == "1" ]]; then + if ! command -v findmnt >/dev/null 2>&1; then + echo "findmnt is required to verify the native workspace mount" >&2 + return 1 + fi + workspace_mount=$(findmnt -n -T "${workspace_real}" -o TARGET 2>/dev/null || true) + if [[ "$(readlink -m "${workspace_mount:-/}")" != "${workspace_real}" ]]; then + echo "Native CI requires a dedicated volume mounted at ${workspace_real}" >&2 + return 1 + fi + fi +} + +prepare_native_workspace() { + if [[ "${VLLM_CI_USE_ARTIFACTS:-0}" != "1" ]]; then + echo "Native CI requires VLLM_CI_USE_ARTIFACTS=1" + return 1 + fi + if ! command -v buildkite-agent >/dev/null 2>&1; then + echo "buildkite-agent not found; cannot download ROCm wheel artifact" + return 1 + fi + validate_native_workspace || return 1 + + local artifact_glob="${VLLM_CI_ARTIFACT_GLOB:-artifacts/vllm-rocm-install/vllm-rocm-install.tar.gz}" + local artifact_checksum_glob="${VLLM_CI_ARTIFACT_CHECKSUM_GLOB:-${artifact_glob}.sha256}" + local artifact_step="${VLLM_CI_ARTIFACT_STEP:-image-build-amd}" + local archive="" + local checksum="" + local download_dir="" + local metadata_dir="" + local recorded_base="" + local recorded_commit="" + local recorded_wheel="" + local workspace_dir="${VLLM_CI_WORKSPACE:-/vllm-workspace}" + local wheel_dir="" + local attempt=0 + local attempt_dir="" + local -a archives=() + local -a checksums=() + local -a wheels=() + + artifact_work_dir=$(mktemp -d -t vllm-rocm-artifact.XXXXXX) || return 1 + wheel_dir="${artifact_work_dir}/wheels" + mkdir -p "${wheel_dir}" || return 1 + + echo "--- Downloading ROCm wheel artifact from ${artifact_step} (native in-pod)" + for attempt in 1 2 3; do + attempt_dir="${artifact_work_dir}/download-${attempt}" + rm -rf "${attempt_dir}" || return 1 + mkdir -p "${attempt_dir}" || return 1 + if buildkite-agent artifact download \ + "${artifact_glob}" "${attempt_dir}" --step "${artifact_step}" \ + && buildkite-agent artifact download \ + "${artifact_checksum_glob}" "${attempt_dir}" --step "${artifact_step}"; then + download_dir="${attempt_dir}" + break + fi + echo "Artifact download attempt ${attempt}/3 failed" + if [[ "${attempt}" -lt 3 ]]; then + sleep $((attempt * 2)) + fi + done + if [[ -z "${download_dir}" ]]; then + echo "Failed to download ${artifact_glob} and ${artifact_checksum_glob} from ${artifact_step}" + return 1 + fi + + mapfile -t archives < <( + find "${download_dir}" -name "vllm-rocm-install.tar.gz" -type f -print + ) + mapfile -t checksums < <( + find "${download_dir}" -name "vllm-rocm-install.tar.gz.sha256" -type f -print + ) + if [[ ${#archives[@]} -ne 1 || ${#checksums[@]} -ne 1 ]]; then + echo "Expected exactly one ROCm archive and checksum; found ${#archives[@]} archive(s) and ${#checksums[@]} checksum(s)" >&2 + return 1 + fi + archive="${archives[0]}" + checksum="${checksums[0]}" + if [[ "$(dirname "${archive}")" != "$(dirname "${checksum}")" ]]; then + echo "ROCm archive and checksum were downloaded to different directories" >&2 + return 1 + fi + ( + cd "$(dirname "${archive}")" + sha256sum -c "$(basename "${checksum}")" + ) || return 1 + + tar --no-same-owner -xzf "${archive}" -C "${wheel_dir}" || return 1 + mapfile -t wheels < <( + find "${wheel_dir}" -maxdepth 1 -type f -name '*.whl' -print + ) + if [[ ${#wheels[@]} -ne 1 ]]; then + echo "ROCm artifact must contain exactly one top-level wheel; found ${#wheels[@]}" >&2 + return 1 + fi + metadata_dir="${wheel_dir}/.vllm-ci-artifact" + for metadata_file in commit.txt native-base-image.txt wheel-filename.txt; do + if [[ ! -s "${metadata_dir}/${metadata_file}" ]]; then + echo "ROCm artifact metadata is missing ${metadata_file}" >&2 + return 1 + fi + done + for metadata_file in ci-base-image.txt fallback-image.txt; do + if [[ ! -f "${metadata_dir}/${metadata_file}" ]]; then + echo "ROCm artifact metadata is missing ${metadata_file}" >&2 + return 1 + fi + done + + recorded_commit=$(tr -d '\r\n' < "${metadata_dir}/commit.txt") + recorded_base=$(tr -d '\r\n' < "${metadata_dir}/native-base-image.txt") + recorded_wheel=$(tr -d '\r\n' < "${metadata_dir}/wheel-filename.txt") + if [[ -z "${BUILDKITE_COMMIT:-}" || "${recorded_commit}" != "${BUILDKITE_COMMIT}" ]]; then + echo "ROCm artifact commit ${recorded_commit} does not match ${BUILDKITE_COMMIT:-unset}" >&2 + return 1 + fi + if [[ -z "${VLLM_CI_BASE_IMAGE:-}" || "${recorded_base}" != "${VLLM_CI_BASE_IMAGE}" ]]; then + echo "ROCm artifact base ${recorded_base} does not match ${VLLM_CI_BASE_IMAGE:-unset}" >&2 + return 1 + fi + if [[ "${recorded_wheel}" != "$(basename "${wheels[0]}")" ]]; then + echo "ROCm artifact wheel manifest ${recorded_wheel} does not match $(basename "${wheels[0]}")" >&2 + return 1 + fi + for required_dir in tests .buildkite requirements; do + if [[ ! -d "${wheel_dir}/${required_dir}" ]]; then + echo "ROCm wheel artifact did not contain ${required_dir}/" >&2 + return 1 + fi + done + + echo "--- Installing ROCm wheel into pod environment" + python3 -m pip install --no-deps --force-reinstall "${wheels[0]}" || return 1 + + echo "--- Preparing ${workspace_dir} from artifact" + find "${workspace_dir}" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + || return 1 + tar -C "${wheel_dir}" \ + --exclude='*.whl' \ + --exclude='.vllm-ci-artifact' \ + --exclude='./.vllm-ci-artifact' \ + -cf - . | tar --no-same-owner -C "${workspace_dir}" -xf - || return 1 + if [[ ! -d "${workspace_dir}/tests" ]]; then + echo "Failed to stage the native test workspace" >&2 + return 1 + fi + + return 0 +} + +initialize_native_environment() { + local job_id="${BUILDKITE_JOB_ID:-${BUILDKITE_PARALLEL_JOB:-local}}" + local job_id_suffix="" + local native_root="" + local hf_mount="" + + if [[ "$(id -u)" -ne 0 ]]; then + echo "Native ROCm CI currently requires the ci_base container to run as root" >&2 + return 1 + fi + + job_id="${job_id//[^A-Za-z0-9_.-]/_}" + job_id_suffix="${job_id##*-}" + job_id_suffix="${job_id_suffix:0:12}" + native_root="/tmp/vllm-native-${job_id}" + TMPDIR="/tmp/vllm-${job_id_suffix}/tmp" + VLLM_RPC_BASE_PATH="/tmp" + : "${TORCHINDUCTOR_CACHE_DIR:=${native_root}/cache/torchinductor}" + : "${TRITON_CACHE_DIR:=${native_root}/cache/triton}" + : "${VLLM_CACHE_ROOT:=${native_root}/cache/vllm}" + : "${XDG_CACHE_HOME:=${native_root}/cache/xdg}" + : "${HF_HOME:=/home/buildkite-agent/huggingface}" + : "${HF_HUB_DOWNLOAD_TIMEOUT:=300}" + : "${HF_HUB_ETAG_TIMEOUT:=60}" + export TMPDIR VLLM_RPC_BASE_PATH + export TORCHINDUCTOR_CACHE_DIR TRITON_CACHE_DIR VLLM_CACHE_ROOT XDG_CACHE_HOME + export HF_HOME HF_HUB_DOWNLOAD_TIMEOUT HF_HUB_ETAG_TIMEOUT + export PYTORCH_ROCM_ARCH="" + + mkdir -p "${TMPDIR}" \ + "${TORCHINDUCTOR_CACHE_DIR}" \ + "${TRITON_CACHE_DIR}" \ + "${VLLM_CACHE_ROOT}" \ + "${XDG_CACHE_HOME}" \ + "${HF_HOME}" || return 1 + + if [[ "${VLLM_CI_REQUIRE_PERSISTENT_HF_CACHE:-0}" == "1" ]]; then + if ! command -v findmnt >/dev/null 2>&1; then + echo "findmnt is required to verify the native Hugging Face cache mount" >&2 + return 1 + fi + hf_mount=$(findmnt -n -T "${HF_HOME}" -o TARGET 2>/dev/null || true) + if [[ -z "${hf_mount}" || "${hf_mount}" == "/" ]]; then + echo "Native CI requires a persistent volume mounted at or above ${HF_HOME}" >&2 + return 1 + fi + fi +} + +run_native_preflight() { + local expected_gpus="${VLLM_CI_EXPECTED_GPU_COUNT:-1}" + + if [[ ! "${expected_gpus}" =~ ^[0-9]+$ ]]; then + echo "Invalid VLLM_CI_EXPECTED_GPU_COUNT=${expected_gpus}" >&2 + return 1 + fi + + python3 -c "import encodings, importlib.metadata as im, importlib.util as iu; [im.version(d) for d in ('transformers', 'torch', 'ray', 'sympy', 'markupsafe', 'vllm')]; missing=[m for m in ('torch.utils.model_zoo', 'transformers.models.nomic_bert', 'ray.dag', 'sympy.physics', 'markupsafe._speedups') if iu.find_spec(m) is None]; assert not missing, missing" || return 1 + + if [[ "${expected_gpus}" == "0" ]]; then + echo "Native CPU-only AMD job: skipping ROCm device validation" + return 0 + fi + + echo "--- ROCm info" + rocminfo || return 1 + VLLM_CI_EXPECTED_GPU_COUNT="${expected_gpus}" python3 - <<'PY' +import os + +import torch + +expected = int(os.environ["VLLM_CI_EXPECTED_GPU_COUNT"]) +assert torch.version.hip, "PyTorch is not a ROCm build" +assert torch.cuda.is_available(), "ROCm GPU is not available to PyTorch" +actual = torch.cuda.device_count() +assert actual == expected, f"Expected {expected} ROCm GPU(s), found {actual}" +PY +} + is_multi_node() { local cmds="$1" # Primary signal: NUM_NODES environment variable set by the pipeline @@ -350,7 +644,58 @@ re_quote_pytest_markers() { # Main ############################################################################### -# --- GPU initialization --- +if is_native_runtime; then + echo "--- Native in-pod ROCm CI (AMD_CI_RUNTIME=${AMD_CI_RUNTIME:-unset}, NATIVE_CI=${NATIVE_CI:-unset})" + artifact_work_dir="" + + cleanup_native_workspace() { + if [[ -n "${artifact_work_dir}" ]]; then + rm -rf "${artifact_work_dir}" + fi + } + trap cleanup_native_workspace EXIT + + if [[ -n "${VLLM_TEST_COMMANDS:-}" ]]; then + commands="${VLLM_TEST_COMMANDS}" + commands_source="env" + else + commands="$*" + commands_source="argv" + if [[ -z "$commands" ]]; then + echo "Error: No test commands provided for native CI." >&2 + exit 1 + fi + fi + + if [[ "$commands_source" == "argv" ]]; then + commands=$(re_quote_pytest_markers "$commands") + fi + + if is_multi_node "$commands"; then + echo "Native CI does not support multi-node jobs yet." + exit 1 + fi + + if ! initialize_native_environment; then + echo "Failed to initialize the native test environment" + exit 1 + fi + if ! prepare_native_workspace; then + echo "Failed to prepare native test workspace" + exit 1 + fi + + export PYTHONPATH="${VLLM_CI_WORKSPACE:-/vllm-workspace}" + + echo "Native test commands: $commands" + run_native_preflight || exit 1 + # Keep AMD CI orchestration variables out of vLLM's runtime environment. + clear_ci_orchestration_env + /bin/bash -o pipefail -c "${commands}" + handle_pytest_exit "$?" +fi + +# --- GPU initialization for legacy Docker execution --- echo "--- ROCm info" rocminfo @@ -452,25 +797,27 @@ fi echo "Final commands: $commands" -# The ROCm test image often ships /vllm-workspace without .git (artifact tarball unpack). -# tests/standalone_tests/python_only_compile.sh uses merge-base(HEAD, origin/main) for -# wheels.vllm.ai; compute on the agent (full git checkout) and pass into the container. -vllm_standalone_merge_base="" -checkout="${BUILDKITE_BUILD_CHECKOUT_PATH:-}" -if [[ -z "${checkout}" || ! -d "${checkout}" ]]; then - checkout="." +standalone_merge_base_env=() +if [[ "$commands" == *python_only_compile.sh* ]]; then + # The ROCm test image often ships /vllm-workspace without .git. Resolve the + # wheels.vllm.ai commit from the agent checkout for this test only. + vllm_standalone_merge_base="" + checkout="${BUILDKITE_BUILD_CHECKOUT_PATH:-}" + if [[ -z "${checkout}" || ! -d "${checkout}" ]]; then + checkout="." + fi + # Pass safe.directory per-command because Buildkite uses mixed user IDs. + if git -c "safe.directory=${checkout}" -C "${checkout}" rev-parse --is-inside-work-tree >/dev/null 2>&1; then + vllm_standalone_merge_base="$( + git -c "safe.directory=${checkout}" -C "${checkout}" merge-base HEAD origin/main 2>/dev/null || true + )" + fi + if [[ -z "${vllm_standalone_merge_base}" ]]; then + vllm_standalone_merge_base="${BUILDKITE_COMMIT:-}" + fi + echo "INFO: passing CI_STANDALONE_MERGE_BASE into container: ${vllm_standalone_merge_base}" + standalone_merge_base_env=(-e "CI_STANDALONE_MERGE_BASE=${vllm_standalone_merge_base}") fi -# Pass safe.directory per-command (-c) because buildkite runs will always fail -# the next check on git 2.35.2+ due to mixed uses of root and buildkite-agent/uids. -if git -c "safe.directory=${checkout}" -C "${checkout}" rev-parse --is-inside-work-tree >/dev/null 2>&1; then - vllm_standalone_merge_base="$( - git -c "safe.directory=${checkout}" -C "${checkout}" merge-base HEAD origin/main 2>/dev/null || true - )" -fi -if [[ -z "${vllm_standalone_merge_base}" ]]; then - vllm_standalone_merge_base="${BUILDKITE_COMMIT:-}" -fi -echo "INFO: passing VLLM_STANDALONE_MERGE_BASE into container: ${vllm_standalone_merge_base}" MYPYTHONPATH="/vllm-workspace" @@ -501,6 +848,7 @@ else fi # --- Route: multi-node vs single-node --- +clear_ci_orchestration_env if is_multi_node "$commands"; then echo "--- Multi-node job detected" export DCKR_VER=$(docker --version | sed 's/Docker version \(.*\), build .*/\1/') @@ -589,7 +937,9 @@ else -e FORCE_COLOR \ -e CLICOLOR_FORCE \ -e PY_COLORS \ + -e PYTHONFAULTHANDLER \ -e PYTEST_ADDOPTS \ + -e PYTEST_TIMEOUT \ -v "${HF_CACHE}:${HF_MOUNT}" \ -e "HF_HOME=${HF_MOUNT}" \ -e "PYTHONPATH=${MYPYTHONPATH}" \ @@ -599,7 +949,7 @@ else -e "VLLM_CACHE_ROOT=${CONTAINER_CACHE_ROOT}/vllm" \ -e "XDG_CACHE_HOME=${CONTAINER_CACHE_ROOT}/xdg" \ -e "PYTORCH_ROCM_ARCH=" \ - -e "VLLM_STANDALONE_MERGE_BASE=${vllm_standalone_merge_base}" \ + "${standalone_merge_base_env[@]}" \ --name "${container_name}" \ "${image_name}" \ /bin/bash -c "${CONTAINER_PREFLIGHT} && ${commands}" diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index ba7823e420a..ce6ee3857bd 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -14,6 +14,8 @@ # command(str): the single command to run for tests. incompatible with commands. # commands(list): the list of commands to run for the test. incompatible with command. # mirror_hardwares(list): the list of hardware to run the test on as well. currently only supports [amdexperimental] +# dind(bool): when false, run the job directly in the AMD Kubernetes pod. +# When true or omitted, use the legacy Docker-in-Docker path. # num_gpus(int): override the number of GPUs for the test. defaults to 1 GPU. currently supports 2,4. # num_nodes(int): whether to simulate multi-node setup by launching multiple containers on one host, # in this case, commands must be specified. the first command runs on the first host, the second @@ -469,6 +471,7 @@ steps: - label: Basic Correctness # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 fast_check: true working_dir: "/vllm-workspace/tests" @@ -486,6 +489,7 @@ steps: - label: Distributed Model Tests (2 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_2 num_gpus: 2 optional: true @@ -515,6 +519,7 @@ steps: - label: Benchmarks CLI Test # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -529,6 +534,7 @@ steps: - label: PyTorch Compilation Unit Tests # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -548,6 +554,7 @@ steps: - label: Fusion E2E Config Sweep (H100-MI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 num_gpus: 1 working_dir: "/vllm-workspace/" @@ -568,6 +575,7 @@ steps: - label: Fusion E2E Quick (H100-MI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 num_gpus: 1 working_dir: "/vllm-workspace/" @@ -589,6 +597,7 @@ steps: - label: PyTorch Compilation Passes Unit Tests # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -600,6 +609,7 @@ steps: - label: PyTorch Fullgraph # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -617,6 +627,7 @@ steps: - label: Pytorch Nightly Dependency Override Check # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true soft_fail: true @@ -630,6 +641,7 @@ steps: - label: Distributed Compile Unit Tests (2xH100-2xMI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_2 num_gpus: 2 working_dir: "/vllm-workspace/" @@ -650,6 +662,7 @@ steps: - label: Platform Tests # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -664,6 +677,7 @@ steps: - label: Async Engine, Inputs, Utils, Worker # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -681,6 +695,7 @@ steps: - label: Distributed Comm Ops # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_2 num_gpus: 2 working_dir: "/vllm-workspace/tests" @@ -697,12 +712,14 @@ steps: - label: EPLB Algorithm # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: - vllm/distributed/eplb - tests/distributed/test_eplb_algo.py + - tests/distributed/test_eplb_utils.py - vllm/platforms/rocm.py commands: - pytest -v -s distributed/test_eplb_algo.py @@ -711,6 +728,7 @@ steps: - label: EPLB Execution # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_4 num_gpus: 4 working_dir: "/vllm-workspace/tests" @@ -726,6 +744,7 @@ steps: - label: Distributed Tests (2xH100-2xMI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_2 num_gpus: 2 working_dir: "/vllm-workspace/" @@ -752,6 +771,7 @@ steps: - label: Distributed Tests (4xA100-4xMI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_4 num_gpus: 4 optional: true @@ -767,6 +787,7 @@ steps: - label: Distributed Torchrun + Examples (4 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_4 num_gpus: 4 working_dir: "/vllm-workspace/tests" @@ -792,6 +813,7 @@ steps: - label: Elastic EP Scaling Test # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_4 num_gpus: 4 optional: true @@ -809,6 +831,7 @@ steps: - label: RayExecutorV2 (4 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_4 num_gpus: 4 working_dir: "/vllm-workspace/tests" @@ -831,6 +854,7 @@ steps: - label: Distributed Tests (8xH100-8xMI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_8 num_gpus: 8 optional: true @@ -851,6 +875,7 @@ steps: - label: Entrypoints Unit Tests # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 fast_check: true working_dir: "/vllm-workspace/tests" @@ -866,6 +891,7 @@ steps: - label: Entrypoints Integration (LLM) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true fast_check: true @@ -882,6 +908,7 @@ steps: - label: Entrypoints Integration (API Server) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true fast_check: true @@ -899,6 +926,7 @@ steps: - label: Entrypoints Integration (API Server OpenAI - Part 1) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 fast_check: true optional: true @@ -914,6 +942,7 @@ steps: - label: Entrypoints Integration (API Server OpenAI - Part 2) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 fast_check: true optional: true @@ -930,6 +959,7 @@ steps: - label: Entrypoints Integration (API Server Generate) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true fast_check: true @@ -950,6 +980,7 @@ steps: - label: Entrypoints Integration (Responses API) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 fast_check: true working_dir: "/vllm-workspace/tests" @@ -963,6 +994,7 @@ steps: - label: Entrypoints Integration (Speech to Text) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 fast_check: true working_dir: "/vllm-workspace/tests" @@ -976,6 +1008,7 @@ steps: - label: Entrypoints Integration (Multimodal) timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 fast_check: true working_dir: "/vllm-workspace/tests" @@ -989,6 +1022,7 @@ steps: - label: Entrypoints Integration (Pooling) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 fast_check: true working_dir: "/vllm-workspace/tests" @@ -1002,6 +1036,7 @@ steps: - label: OpenAI API correctness # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1023,6 +1058,7 @@ steps: - label: DeepSeek V2-Lite Prefetch Offload Accuracy (H100-MI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 num_gpus: 1 working_dir: "/vllm-workspace" @@ -1043,6 +1079,7 @@ steps: - label: LM Eval Small Models # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -1060,6 +1097,7 @@ steps: - label: MRCR Eval Small Models # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1077,6 +1115,7 @@ steps: - label: LM Eval Small Models (MI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/.buildkite/lm-eval-harness" @@ -1095,6 +1134,7 @@ steps: - label: Multi-Modal Accuracy Eval (Small Models) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/.buildkite/lm-eval-harness" @@ -1110,6 +1150,7 @@ steps: - label: GPQA Eval (GPT-OSS) (2xH100-2xMI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_2 num_gpus: 2 optional: true @@ -1132,7 +1173,9 @@ steps: - label: LM Eval Small Models (2xB200-2xMI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_2 + num_gpus: 2 working_dir: "/vllm-workspace/tests" source_file_dependencies: - csrc/ @@ -1149,6 +1192,7 @@ steps: - label: DeepSeek V2-Lite Sync EPLB Accuracy (4xH100-4xMI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_4 num_gpus: 4 optional: true @@ -1171,6 +1215,7 @@ steps: - label: LM Eval Large Models (4xA100-4xMI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_4 num_gpus: 4 optional: true @@ -1191,7 +1236,9 @@ steps: - label: Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy (4xH100-4xMI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_4 + num_gpus: 4 optional: true working_dir: "/vllm-workspace" source_file_dependencies: @@ -1211,6 +1258,7 @@ steps: - label: Qwen3-30B-A3B-FP8 DP4 Async EPLB Accuracy (4xH100-4xMI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_4 num_gpus: 4 optional: true @@ -1232,6 +1280,7 @@ steps: - label: Qwen3-Next-80B-A3B-Instruct MTP Async EPLB Accuracy # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_4 num_gpus: 4 optional: true @@ -1254,6 +1303,7 @@ steps: - label: LM Eval Large Models (8xH200-8xMI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_8 optional: true num_gpus: 8 @@ -1276,6 +1326,7 @@ steps: - label: ROCm LM Eval Large Models (8 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_8 optional: true num_gpus: 8 @@ -1299,6 +1350,7 @@ steps: - label: Examples # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/examples" @@ -1335,6 +1387,7 @@ steps: - label: vLLM IR Tests # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/" @@ -1350,6 +1403,7 @@ steps: - label: Kernels Attention Test %N # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true parallelism: 2 @@ -1368,6 +1422,7 @@ steps: - label: Kernels Core Operation Test # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -1384,6 +1439,7 @@ steps: - label: Kernels KDA Test # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1399,6 +1455,7 @@ steps: - label: Kernels MoE Test %N # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true parallelism: 5 @@ -1420,6 +1477,7 @@ steps: - label: Kernels Quantization Test %N # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true parallelism: 2 @@ -1438,7 +1496,9 @@ steps: - label: Kernels FP8 MoE Test (2xH100-2xMI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_2 + num_gpus: 2 optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -1457,6 +1517,7 @@ steps: - label: LoRA %N # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 parallelism: 4 optional: true @@ -1471,6 +1532,7 @@ steps: - label: LoRA TP (Distributed) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_4 num_gpus: 4 working_dir: "/vllm-workspace/tests" @@ -1491,6 +1553,7 @@ steps: - label: Model Executor # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1513,6 +1576,7 @@ steps: - label: Model Runner V2 Core Tests # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1537,6 +1601,7 @@ steps: - label: Model Runner V2 Examples # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/examples" @@ -1571,6 +1636,7 @@ steps: - label: Model Runner V2 Distributed (2 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_2 num_gpus: 2 optional: true @@ -1592,6 +1658,7 @@ steps: - label: Model Runner V2 Pipeline Parallelism (4 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_4 num_gpus: 4 optional: true @@ -1611,6 +1678,7 @@ steps: - label: Model Runner V2 Spec Decode # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1635,6 +1703,7 @@ steps: - label: Basic Models Tests (Extra Initialization) %N # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 parallelism: 6 working_dir: "/vllm-workspace/tests" @@ -1651,6 +1720,7 @@ steps: - label: Basic Models Tests (Initialization) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -1663,6 +1733,7 @@ steps: - label: Basic Models Tests (Other) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1679,6 +1750,7 @@ steps: - label: Language Models Test (Extended Pooling) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1691,6 +1763,7 @@ steps: - label: Language Models Tests (Standard) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1704,6 +1777,7 @@ steps: - label: Language Models Tests (Extra Standard) %N # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true parallelism: 2 @@ -1728,6 +1802,7 @@ steps: - label: Multi-Modal Models (Extended Generation 1) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1742,6 +1817,7 @@ steps: - label: Multi-Modal Models (Extended Generation 2) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1755,6 +1831,7 @@ steps: - label: Multi-Modal Models (Extended Generation 3) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1767,6 +1844,7 @@ steps: - label: "Multi-Modal Models (Standard) 1: qwen2" # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1780,6 +1858,7 @@ steps: - label: "Multi-Modal Models (Standard) 3: llava + qwen2_vl" # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1794,6 +1873,7 @@ steps: - label: "Multi-Modal Models (Standard) 4: other + whisper" # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -1808,6 +1888,7 @@ steps: - label: Multi-Modal Processor # 1h 42m timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -1821,9 +1902,9 @@ steps: - label: Multi-Modal Processor (CPU) %N # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 parallelism: 4 - no_gpu: true optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -1838,6 +1919,7 @@ steps: - label: Quantized Models Test # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -1854,6 +1936,7 @@ steps: - label: Transformers Nightly Models (Shardable) %N # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 parallelism: 4 optional: true @@ -1876,6 +1959,7 @@ steps: - label: Transformers Nightly Models (Single) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/" @@ -1903,6 +1987,7 @@ steps: - label: Plugin Tests (2 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_2 num_gpus: 2 optional: true @@ -1954,6 +2039,7 @@ steps: - label: GGUF Plugin # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true soft_fail: true @@ -1972,6 +2058,7 @@ steps: - label: Rust Frontend OpenAI Coverage # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2004,6 +2091,7 @@ steps: - label: Rust Frontend Serve Admin Coverage # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2031,6 +2119,7 @@ steps: - label: Rust Frontend Core Correctness # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2048,6 +2137,7 @@ steps: - label: Rust Frontend Tool Use # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2066,6 +2156,7 @@ steps: - label: Rust Frontend Distributed # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_4 num_gpus: 4 optional: true @@ -2094,6 +2185,7 @@ steps: - label: Quantization # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -2120,6 +2212,7 @@ steps: - label: ROCm AITER Ops Test # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -2137,6 +2230,7 @@ steps: - label: Samplers Test # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2154,22 +2248,10 @@ steps: #------------------------------------------------------------ mi300 · misc ------------------------------------------------------------# -- label: Python-only Installation # TBD - timeout_in_minutes: 180 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - agent_pool: mi300_1 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - tests/standalone_tests/python_only_compile.sh - - setup.py - - vllm/platforms/rocm.py - commands: - - bash standalone_tests/python_only_compile.sh - - label: Regression # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -2184,6 +2266,7 @@ steps: - label: Ray Dependency Compatibility Check # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 working_dir: "/" source_file_dependencies: @@ -2198,6 +2281,7 @@ steps: - label: Acceptance Length Test (Large Models) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2213,6 +2297,7 @@ steps: - label: e2e Scheduling (1 GPU) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2226,6 +2311,7 @@ steps: - label: Engine (1 GPU) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2240,6 +2326,7 @@ steps: - label: Spec Decode Draft Model # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2257,6 +2344,7 @@ steps: - label: Spec Decode Eagle # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2274,6 +2362,7 @@ steps: - label: Spec Decode Ngram + Suffix # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2291,6 +2380,7 @@ steps: - label: Spec Decode Speculators + MTP # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2309,6 +2399,7 @@ steps: - label: Speculators Correctness # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2337,9 +2428,22 @@ steps: - export VLLM_ALLOW_INSECURE_SERIALIZATION=1 - pytest -v -s v1/spec_decode/test_speculators_correctness.py -m slow_test +- label: V1 Spec Decode # TBD + timeout_in_minutes: 180 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_1 + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/ + - tests/v1/spec_decode + commands: + - pytest -v -s -m 'not slow_test' v1/spec_decode + - label: Extract Hidden States Integration # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_2 num_gpus: 2 working_dir: "/vllm-workspace/tests" @@ -2371,6 +2475,7 @@ steps: - label: V1 attention (H100-MI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2388,6 +2493,7 @@ steps: - label: V1 Core + KV + Metrics # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -2414,8 +2520,8 @@ steps: - label: V1 others (CPU) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 - no_gpu: true optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -2431,6 +2537,7 @@ steps: - label: V1 Sample + Logits # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_1 optional: true working_dir: "/vllm-workspace/tests" @@ -2451,6 +2558,7 @@ steps: - label: Distributed DP Tests (2 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_2 optional: true num_gpus: 2 @@ -2466,6 +2574,7 @@ steps: - tests/entrypoints/openai/test_multi_api_servers.py - vllm/platforms/rocm.py commands: + - export NCCL_CUMEM_HOST_ENABLE=0 - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py @@ -2474,6 +2583,7 @@ steps: - label: Metrics, Tracing (2 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_2 optional: true num_gpus: 2 @@ -2492,7 +2602,9 @@ steps: - label: V1 e2e (2 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_2 + num_gpus: 2 optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -2504,6 +2616,7 @@ steps: - label: NixlConnector PD + Spec Decode acceptance (2 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_2 num_gpus: 2 optional: true @@ -2515,17 +2628,18 @@ steps: - vllm/platforms/rocm.py commands: - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt - - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh + - KV_CACHE_MEMORY_BYTES=8G ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh - label: CrossLayer KV layout Distributed NixlConnector PD accuracy tests (4 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_4 num_gpus: 4 optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: - - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py + - vllm/distributed/kv_transfer/kv_connector/v1/nixl/ - tests/v1/kv_connector/nixl_integration/ - vllm/platforms/rocm.py commands: @@ -2535,6 +2649,7 @@ steps: - label: Distributed DP Tests (4 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_4 num_gpus: 4 working_dir: "/vllm-workspace/tests" @@ -2545,6 +2660,7 @@ steps: - tests/distributed/test_utils - vllm/platforms/rocm.py commands: + - export NCCL_CUMEM_HOST_ENABLE=0 - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py @@ -2556,6 +2672,7 @@ steps: - label: Distributed NixlConnector PD accuracy (4 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_4 num_gpus: 4 optional: true @@ -2571,6 +2688,7 @@ steps: - label: DP EP Distributed NixlConnector PD accuracy tests (4 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_4 num_gpus: 4 optional: true @@ -2586,12 +2704,13 @@ steps: - label: Hybrid SSM NixlConnector PD accuracy tests (4 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_4 num_gpus: 4 optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: - - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py + - vllm/distributed/kv_transfer/kv_connector/v1/nixl/ - tests/v1/kv_connector/nixl_integration/ - vllm/platforms/rocm.py commands: @@ -2601,6 +2720,7 @@ steps: - label: Hybrid SSM NixlConnector PD prefix cache test (2 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_2 num_gpus: 2 optional: true @@ -2618,6 +2738,7 @@ steps: - label: MultiConnector (Nixl+Offloading) PD accuracy (2 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_2 num_gpus: 2 optional: true @@ -2636,6 +2757,7 @@ steps: - label: MultiConnector (Nixl+Offloading) PD edge cases (2 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_2 num_gpus: 2 optional: true @@ -2654,7 +2776,9 @@ steps: - label: V1 e2e (4 GPUs) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_4 + num_gpus: 4 optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -2666,7 +2790,9 @@ steps: - label: V1 e2e (4xH100-4xMI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_4 + num_gpus: 4 optional: true source_file_dependencies: - vllm/v1/attention/backends/utils.py @@ -2680,6 +2806,7 @@ steps: - label: Weight Loading Multiple GPU # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_2 num_gpus: 2 working_dir: "/vllm-workspace/tests" @@ -2692,6 +2819,7 @@ steps: - label: Weight Loading Multiple GPU - Large Models # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false agent_pool: mi300_2 num_gpus: 2 optional: true @@ -2876,18 +3004,20 @@ steps: - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen3 or gemma" - pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model -#------------------------------------------------------------ mi325 · v1 -------------------------------------------------------------# +#----------------------------------------------------------- mi325 · misc ------------------------------------------------------------# -- label: V1 Spec Decode # TBD +- label: Python-only Installation # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325] agent_pool: mi325_1 + optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: - - vllm/ - - tests/v1/spec_decode + - tests/standalone_tests/python_only_compile.sh + - setup.py + - vllm/platforms/rocm.py commands: - - pytest -v -s -m 'not slow_test' v1/spec_decode + - bash standalone_tests/python_only_compile.sh ######################################################################################################################################### # # @@ -3114,6 +3244,7 @@ steps: timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] agent_pool: mi355_2 + num_gpus: 2 optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -3288,6 +3419,7 @@ steps: timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] agent_pool: mi355_2 + num_gpus: 2 working_dir: "/vllm-workspace/tests" source_file_dependencies: - csrc/moe/ diff --git a/.buildkite/test_areas/benchmarks.yaml b/.buildkite/test_areas/benchmarks.yaml index 7c26a8d6e98..4b709f3962f 100644 --- a/.buildkite/test_areas/benchmarks.yaml +++ b/.buildkite/test_areas/benchmarks.yaml @@ -13,6 +13,7 @@ steps: - pytest -v -s benchmarks/ mirror: amd: + dind: false device: mi300_1 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/disaggregated.yaml b/.buildkite/test_areas/disaggregated.yaml index 38020d91ad0..b25884a96f5 100644 --- a/.buildkite/test_areas/disaggregated.yaml +++ b/.buildkite/test_areas/disaggregated.yaml @@ -15,6 +15,7 @@ steps: - bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh mirror: amd: + dind: false device: mi300_4 timeout_in_minutes: 85 depends_on: @@ -65,6 +66,7 @@ steps: - DP_EP=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh mirror: amd: + dind: false device: mi300_4 timeout_in_minutes: 60 depends_on: @@ -90,6 +92,7 @@ steps: - CROSS_LAYERS_BLOCKS=True bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh mirror: amd: + dind: false device: mi300_4 timeout_in_minutes: 85 depends_on: @@ -115,6 +118,7 @@ steps: - HYBRID_SSM=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh mirror: amd: + dind: false device: mi300_4 timeout_in_minutes: 80 depends_on: @@ -171,6 +175,7 @@ steps: - bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh mirror: amd: + dind: false device: mi300_2 timeout_in_minutes: 70 depends_on: @@ -182,7 +187,7 @@ steps: - vllm/platforms/rocm.py commands: - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt - - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh + - KV_CACHE_MEMORY_BYTES=8G ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh - label: MultiConnector (Nixl+Offloading) PD edge cases (2 GPUs) key: multiconnector-nixl-offloading-pd-edge-cases-2-gpus diff --git a/.buildkite/test_areas/distributed.yaml b/.buildkite/test_areas/distributed.yaml index b519132c6f1..d9fcff19f05 100644 --- a/.buildkite/test_areas/distributed.yaml +++ b/.buildkite/test_areas/distributed.yaml @@ -39,6 +39,7 @@ steps: - DP_SIZE=2 pytest -v -s entrypoints/openai/test_multi_api_servers.py mirror: amd: + dind: false device: mi300_2 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/engine.yaml b/.buildkite/test_areas/engine.yaml index 1c50ab7a113..4557aa0fcbe 100644 --- a/.buildkite/test_areas/engine.yaml +++ b/.buildkite/test_areas/engine.yaml @@ -114,6 +114,7 @@ steps: - pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "tensor_parallelism" mirror: amd: + dind: false device: mi300_2 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/expert_parallelism.yaml b/.buildkite/test_areas/expert_parallelism.yaml index d02ffeb7748..d2278408fac 100644 --- a/.buildkite/test_areas/expert_parallelism.yaml +++ b/.buildkite/test_areas/expert_parallelism.yaml @@ -16,6 +16,7 @@ steps: - pytest -v -s distributed/test_eplb_utils.py mirror: amd: + dind: false device: mi300_1 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index 1f7d2787536..d2cee7b6365 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -117,6 +117,7 @@ steps: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200.txt mirror: amd: + dind: false device: mi300_8 timeout_in_minutes: 60 depends_on: diff --git a/.buildkite/test_areas/misc.yaml b/.buildkite/test_areas/misc.yaml index 3365d09c7b8..fb08373c9a6 100644 --- a/.buildkite/test_areas/misc.yaml +++ b/.buildkite/test_areas/misc.yaml @@ -23,6 +23,7 @@ steps: - pytest -v -s -m 'not slow_test' v1/spec_decode mirror: amd: + dind: false device: mi300_1 timeout_in_minutes: 75 depends_on: diff --git a/.buildkite/test_areas/model_executor.yaml b/.buildkite/test_areas/model_executor.yaml index 4280e600df4..1f511590ffb 100644 --- a/.buildkite/test_areas/model_executor.yaml +++ b/.buildkite/test_areas/model_executor.yaml @@ -25,6 +25,7 @@ steps: - pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py --timeout=900 --timeout-method=thread mirror: amd: + dind: false device: mi300_1 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/models_language.yaml b/.buildkite/test_areas/models_language.yaml index d89d3fabaec..83d2fa26c22 100644 --- a/.buildkite/test_areas/models_language.yaml +++ b/.buildkite/test_areas/models_language.yaml @@ -15,6 +15,7 @@ steps: - pytest -v -s models/language -m 'core_model and (not slow_test)' mirror: amd: + dind: false device: mi300_1 depends_on: - image-build-amd @@ -35,6 +36,7 @@ steps: parallelism: 2 mirror: amd: + dind: false device: mi300_1 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/models_multimodal.yaml b/.buildkite/test_areas/models_multimodal.yaml index 473192f03f9..32a27a56fde 100644 --- a/.buildkite/test_areas/models_multimodal.yaml +++ b/.buildkite/test_areas/models_multimodal.yaml @@ -107,6 +107,7 @@ steps: - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-mm-small.txt --tp-size=1 mirror: amd: + dind: false device: mi300_1 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/pytorch.yaml b/.buildkite/test_areas/pytorch.yaml index 72cb0069664..cdc5a1fd044 100644 --- a/.buildkite/test_areas/pytorch.yaml +++ b/.buildkite/test_areas/pytorch.yaml @@ -109,6 +109,7 @@ steps: - pytest -s -v compile/passes --ignore compile/passes/distributed mirror: amd: + dind: false device: mi300_1 timeout_in_minutes: 65 depends_on: @@ -197,6 +198,7 @@ steps: - bash standalone_tests/pytorch_nightly_dependency.sh mirror: amd: + dind: false device: mi300_1 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/weight_loading.yaml b/.buildkite/test_areas/weight_loading.yaml index eeb24a49d8d..46a874178b5 100644 --- a/.buildkite/test_areas/weight_loading.yaml +++ b/.buildkite/test_areas/weight_loading.yaml @@ -15,6 +15,7 @@ steps: - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models.txt mirror: amd: + dind: false device: mi300_2 depends_on: - image-build-amd diff --git a/docker/Dockerfile.rocm b/docker/Dockerfile.rocm index 93daf9d150d..a7cc8c0870b 100644 --- a/docker/Dockerfile.rocm +++ b/docker/Dockerfile.rocm @@ -674,6 +674,7 @@ RUN apt-get update -q -y && apt-get install -q -y --no-install-recommends \ libibverbs1 \ ibverbs-providers \ ibverbs-utils \ + util-linux \ unzip \ pkg-config ffmpeg libavcodec-dev libavformat-dev libavutil-dev \ libswscale-dev libavdevice-dev libavfilter-dev libswresample-dev \ diff --git a/tests/basic_correctness/test_mem.py b/tests/basic_correctness/test_mem.py index c0f8a592223..e4b3fb65842 100644 --- a/tests/basic_correctness/test_mem.py +++ b/tests/basic_correctness/test_mem.py @@ -31,7 +31,7 @@ def test_python_error(): x = torch.empty(alloc_bytes, dtype=torch.uint8, device=DEVICE_TYPE) tensors.append(x) # release the memory - allocator.sleep() + allocator.sleep(offload_tags=()) # allocate more memory than the total memory y = torch.empty(alloc_bytes, dtype=torch.uint8, device=DEVICE_TYPE) diff --git a/tests/standalone_tests/python_only_compile.sh b/tests/standalone_tests/python_only_compile.sh index ea9d2441ca0..14f563ae6fb 100644 --- a/tests/standalone_tests/python_only_compile.sh +++ b/tests/standalone_tests/python_only_compile.sh @@ -5,16 +5,16 @@ set -e # ROCm CI runs this script inside `run-amd-test.sh` where /vllm-workspace often has no .git -# (wheel artifact layout). The wrapper passes VLLM_STANDALONE_MERGE_BASE from the agent checkout. +# (wheel artifact layout). The wrapper passes CI_STANDALONE_MERGE_BASE from the agent checkout. merge_base_commit="" -if [[ -n "${VLLM_STANDALONE_MERGE_BASE:-}" ]]; then - merge_base_commit="${VLLM_STANDALONE_MERGE_BASE}" +if [[ -n "${CI_STANDALONE_MERGE_BASE:-}" ]]; then + merge_base_commit="${CI_STANDALONE_MERGE_BASE}" elif merge_base_commit="$(git -C /vllm-workspace merge-base HEAD origin/main 2>/dev/null)"; then : elif merge_base_commit="$(git merge-base HEAD origin/main 2>/dev/null)"; then : else - echo "ERROR: need a git checkout or VLLM_STANDALONE_MERGE_BASE to resolve wheels.vllm.ai commit." >&2 + echo "ERROR: need a git checkout or CI_STANDALONE_MERGE_BASE to resolve wheels.vllm.ai commit." >&2 exit 1 fi diff --git a/tests/v1/kv_connector/nixl_integration/spec_decode_acceptance_test.sh b/tests/v1/kv_connector/nixl_integration/spec_decode_acceptance_test.sh index 10e119d48a9..501ec332d69 100755 --- a/tests/v1/kv_connector/nixl_integration/spec_decode_acceptance_test.sh +++ b/tests/v1/kv_connector/nixl_integration/spec_decode_acceptance_test.sh @@ -20,7 +20,8 @@ # SD_MODEL - drafter model path # MODEL_NAME - target model (default: meta-llama/Llama-3.1-8B-Instruct) # NUM_SPEC_TOKENS - number of speculative tokens (default: 3) -# GPU_MEMORY_UTILIZATION - (default: 0.7) +# GPU_MEMORY_UTILIZATION - used when KV_CACHE_MEMORY_BYTES is unset (default: 0.7) +# KV_CACHE_MEMORY_BYTES - optional KV cache size per server # ATTENTION_BACKEND - attention backend to use # Default: TRITON_ATTN on ROCm, FLASH_ATTN on NVIDIA # ROCm options: TRITON_ATTN, ROCM_ATTN, ROCM_AITER_FA, @@ -52,10 +53,17 @@ NUM_DECODE_INSTANCES=${NUM_DECODE_INSTANCES:-1} PREFILLER_TP_SIZE=${PREFILLER_TP_SIZE:-1} DECODER_TP_SIZE=${DECODER_TP_SIZE:-1} GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.7} +KV_CACHE_MEMORY_BYTES=${KV_CACHE_MEMORY_BYTES:-} BLOCK_SIZE=${BLOCK_SIZE:-16} SERVER_HOST="${SERVER_HOST:-127.0.0.1}" NIXL_SIDE_CHANNEL_HOST="${NIXL_SIDE_CHANNEL_HOST:-$SERVER_HOST}" +if [[ -n "$KV_CACHE_MEMORY_BYTES" ]]; then + KV_CACHE_ARGS=(--kv-cache-memory-bytes "$KV_CACHE_MEMORY_BYTES") +else + KV_CACHE_ARGS=(--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION") +fi + # Resolve the repository root from the script location instead of `.git`. SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" GIT_ROOT="${GIT_ROOT:-$(cd -- "${SCRIPT_DIR}/../../../.." && pwd -P)}" @@ -209,6 +217,7 @@ run_test_for_device() { echo "SD model: ${SD_MODEL}" echo "Spec tokens: ${NUM_SPEC_TOKENS}" echo "KV buffer device: ${kv_device}" + echo "KV cache memory: ${KV_CACHE_MEMORY_BYTES:-auto-sized}" echo "Attention backend: ${ATTENTION_BACKEND}" echo "GPU platform: ${GPU_PLATFORM}" echo "Server host: ${SERVER_HOST}" @@ -248,7 +257,7 @@ run_test_for_device() { --enforce-eager \ --max-model-len $MAX_MODEL_LEN \ --block-size ${BLOCK_SIZE} \ - --gpu-memory-utilization $GPU_MEMORY_UTILIZATION \ + "${KV_CACHE_ARGS[@]}" \ --tensor-parallel-size $PREFILLER_TP_SIZE \ --kv-transfer-config "$kv_config_p" \ --speculative-config "$PREFILL_SPEC_CONFIG" \ @@ -287,7 +296,7 @@ run_test_for_device() { --enforce-eager \ --max-model-len $MAX_MODEL_LEN \ --block-size ${BLOCK_SIZE} \ - --gpu-memory-utilization $GPU_MEMORY_UTILIZATION \ + "${KV_CACHE_ARGS[@]}" \ --tensor-parallel-size $DECODER_TP_SIZE \ --kv-transfer-config "$kv_config_d" \ --speculative-config "$DECODE_SPEC_CONFIG" \ diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_hma_e2e.py b/tests/v1/kv_connector/unit/test_mooncake_store_hma_e2e.py index d0a50f46579..f09e0a24729 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_hma_e2e.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_hma_e2e.py @@ -97,14 +97,16 @@ def _build_worker_with_dict_store(vllm_config, kv_cache_config, store): with ( patch.dict(sys.modules, {"mooncake.store": fake_mooncake_store}), patch.object(mooncake_store_worker, "MooncakeStoreConfig") as MCfg, + patch.object(mooncake_store_worker, "LookupKeyServer"), ): - sc = MCfg.load_from_env.return_value + sc = MCfg.load_from_config.return_value sc.metadata_server = "" sc.global_segment_size = 1 << 20 sc.local_buffer_size = 1 << 20 sc.protocol = "tcp" sc.device_name = "" sc.master_server_address = "" + sc.mode = "embedded" sc.enable_offload = False with ( patch( diff --git a/vllm/model_executor/model_loader/weight_utils.py b/vllm/model_executor/model_loader/weight_utils.py index ef25a156d9e..8e673ae7fcb 100644 --- a/vllm/model_executor/model_loader/weight_utils.py +++ b/vllm/model_executor/model_loader/weight_utils.py @@ -694,10 +694,22 @@ def _get_checkpoints_size_bytes(files: list[str]) -> int: def _get_available_ram_bytes() -> int: - """Return the available RAM in bytes.""" + """Return available RAM, honoring cgroup limits on ROCm.""" import psutil - return psutil.virtual_memory().available + host_available = psutil.virtual_memory().available + if not current_platform.is_rocm(): + return host_available + + from vllm.utils.cpu_resource_utils import get_cgroup_memory_limit + + cgroup_limit, cgroup_usage = get_cgroup_memory_limit() + if cgroup_limit is None: + return host_available + cgroup_available = ( + cgroup_limit if cgroup_usage is None else max(0, cgroup_limit - cgroup_usage) + ) + return min(host_available, cgroup_available) def _get_fs_type(files: list[str]) -> str: