[CI][AMD] Configure MI300 tests for native execution without DinD (#48387)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
2026-07-15 02:14:16 +00:00
parent 4f7fffb92f
commit 3ad85e0de4
23 changed files with 631 additions and 75 deletions
@@ -1,6 +1,7 @@
 # For hf script, without -t option (tensor parallel size).
 # bash .buildkite/lm-eval-harness/run-lm-eval-mmlupro-vllm-baseline.sh -m meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 -l 250 -t 8 -f 5
 model_name: "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8"
+rocm_safetensors_load_strategy: lazy
 required_gpu_arch:
  - gfx942
  - gfx950
@@ -72,6 +72,11 @@ def launch_lm_eval(eval_config, tp_size):
    if moe_backend is not None:
        model_args += f"moe_backend={moe_backend},"

+    if current_platform.is_rocm():
+        rocm_load_strategy = eval_config.get("rocm_safetensors_load_strategy")
+        if rocm_load_strategy is not None:
+            model_args += f"safetensors_load_strategy={rocm_load_strategy},"
+
    env_vars = eval_config.get("env_vars", None)
    with scoped_env_vars(env_vars):
        results = lm_eval.simple_evaluate(
@@ -1898,8 +1898,8 @@ confirm_remote_image_push() {
        fi

        if [[ -z "${remote_revision}" \
-              && ${IMAGE_EXISTED_BEFORE_BUILD} -eq 0 \
-              && image_tag_is_commit_scoped ]]; then
+              && ${IMAGE_EXISTED_BEFORE_BUILD} -eq 0 ]] \
+            && image_tag_is_commit_scoped; then
            echo "Remote image exists under a commit-scoped tag; accepting push despite missing revision label."
            return 0
        fi
@@ -2029,36 +2029,57 @@ upload_wheel_artifacts_if_present() {
    local wheel_dir="./wheel-export"
    local artifact_dir="artifacts/vllm-rocm-install"
    local archive_name="vllm-rocm-install.tar.gz"
+    local metadata_dir="${wheel_dir}/.vllm-ci-artifact"
+    local native_base_image=""
    local whl=""
    local whl_name=""
+    local -a wheels=()

    if ! should_upload_wheel_artifacts; then
        return 0
    fi

-    if [[ ! -d "${wheel_dir}" ]] || ! ls "${wheel_dir}"/*.whl >/dev/null 2>&1; then
-        echo "No ROCm wheel artifacts found in ${wheel_dir}"
-        return 0
+    if [[ -d "${wheel_dir}" ]]; then
+        mapfile -t wheels < <(find "${wheel_dir}" -maxdepth 1 -type f -name '*.whl' -print)
+    fi
+    if [[ ${#wheels[@]} -ne 1 ]]; then
+        echo "Expected exactly one ROCm wheel in ${wheel_dir}; found ${#wheels[@]}" >&2
+        return 1
+    fi
+    whl="${wheels[0]}"
+    whl_name=$(basename "${whl}")
+    native_base_image="${CI_BASE_IMAGE_TAG_COMMIT_REF:-${CI_BASE_IMAGE:-}}"
+    if [[ -z "${native_base_image}" ]]; then
+        echo "Native ROCm artifact requires a ci_base image reference" >&2
+        return 1
    fi

    echo "--- :package: Uploading ROCm vLLM install artifact"
-    mkdir -p "${artifact_dir}"
+    rm -rf "${artifact_dir}" "${metadata_dir}"
+    mkdir -p "${artifact_dir}" "${metadata_dir}"
+
+    printf '%s\n' "${BUILDKITE_COMMIT:-local}" > "${metadata_dir}/commit.txt"
+    printf '%s\n' "${native_base_image}" > "${metadata_dir}/native-base-image.txt"
+    printf '%s\n' "${CI_BASE_IMAGE:-}" > "${metadata_dir}/ci-base-image.txt"
+    printf '%s\n' "${IMAGE_TAG:-}" > "${metadata_dir}/fallback-image.txt"
+    printf '%s\n' "${whl_name}" > "${metadata_dir}/wheel-filename.txt"

    tar -C "${wheel_dir}" -czf "${artifact_dir}/${archive_name}" .
+    (
+        cd "${artifact_dir}"
+        sha256sum "${archive_name}" > "${archive_name}.sha256"
+    )
    echo "Created ${archive_name}: $(du -sh "${artifact_dir}/${archive_name}" | cut -f1)"
-    printf '%s\n' "${CI_BASE_IMAGE:-}" > "${artifact_dir}/ci-base-image.txt"
-    printf '%s\n' "${IMAGE_TAG:-}" > "${artifact_dir}/fallback-image.txt"
-
-    for whl in "${wheel_dir}"/*.whl; do
-        [[ -f "${whl}" ]] || continue
-        whl_name=$(basename "${whl}")
-        cp "${whl}" "${artifact_dir}/${whl_name}"
-        echo "Copied ${whl_name}: $(du -sh "${artifact_dir}/${whl_name}" | cut -f1)"
-    done
+    cp "${metadata_dir}"/*.txt "${artifact_dir}/"
+    cp "${whl}" "${artifact_dir}/${whl_name}"
+    echo "Copied ${whl_name}: $(du -sh "${artifact_dir}/${whl_name}" | cut -f1)"

    if command -v buildkite-agent >/dev/null 2>&1; then
-        buildkite-agent artifact upload "${artifact_dir}/*"
+        buildkite-agent artifact upload "${artifact_dir}/*" || return 1
        echo "ROCm vLLM install artifacts uploaded to ${artifact_dir}/"
+    elif [[ "${BUILDKITE:-false}" == "true" ]]; then
+        echo "buildkite-agent not found; cannot upload required ROCm artifacts" >&2
+        return 1
    else
        echo "Not in Buildkite, skipping artifact upload"
    fi
@@ -2090,6 +2111,11 @@ main() {
        echo "BAKE_PRINT_ONLY=1 set; skipping build"
        return 0
    fi
+    if should_upload_wheel_artifacts; then
+        # wheel-export is an output directory, not a BuildKit cache. Starting
+        # clean prevents a failed/retried export from packaging a stale wheel.
+        rm -rf ./wheel-export
+    fi
    seed_dependency_caches_if_needed
    run_bake
    upload_wheel_artifacts_if_present
@@ -1,7 +1,7 @@
 #!/bin/bash

-# This script runs tests inside the corresponding ROCm docker container.
-# It handles both single-node and multi-node test configurations.
+# This script runs ROCm tests either directly in a native CI pod or inside the
+# corresponding Docker container. Multi-node tests continue to use Docker.
 #
 # Multi-node detection: Instead of matching on fragile group names, we detect
 # multi-node jobs structurally by looking for the bracket command syntax
@@ -34,10 +34,27 @@ set -o pipefail
 : "${CLICOLOR_FORCE:=1}"
 : "${PY_COLORS:=1}"
 : "${ROCM_DOCKER_TTY:=1}"
+: "${PYTHONFAULTHANDLER:=1}"
+: "${PYTEST_TIMEOUT:=2100}"
 if [[ " ${PYTEST_ADDOPTS:-} " != *" --color"* ]]; then
  PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--color=yes"
 fi
-export BUILDKIT_PROGRESS TERM FORCE_COLOR CLICOLOR_FORCE PY_COLORS PYTEST_ADDOPTS ROCM_DOCKER_TTY
+if [[ " ${PYTEST_ADDOPTS:-} " != *" --durations="* ]]; then
+  PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--durations=25"
+fi
+if [[ " ${PYTEST_ADDOPTS:-} " != *" --durations-min="* ]]; then
+  PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--durations-min=1.0"
+fi
+# Dump stacks after 15 minutes, then stop an individual test after 35 minutes.
+if [[ " ${PYTEST_ADDOPTS:-} " != *" faulthandler_timeout="* ]]; then
+  PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }-o faulthandler_timeout=900"
+fi
+if [[ " ${PYTEST_ADDOPTS:-} " != *" --timeout-method="* &&
+  " ${PYTEST_ADDOPTS:-} " != *" --timeout-method "* ]]; then
+  PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--timeout-method=thread"
+fi
+export BUILDKIT_PROGRESS TERM FORCE_COLOR CLICOLOR_FORCE PY_COLORS PYTEST_ADDOPTS PYTEST_TIMEOUT ROCM_DOCKER_TTY
+export PYTHONFAULTHANDLER

 # Export Python path for commands that run directly on the host. Containerized
 # tests set this to /vllm-workspace below so spawned Python processes do not
@@ -53,6 +70,28 @@ report_docker_usage() {
  docker system df || true
 }

+clear_ci_orchestration_env() {
+  unset -v \
+    VLLM_TEST_GROUP_NAME \
+    VLLM_CI_REQUIRE_PERSISTENT_HF_CACHE \
+    VLLM_CI_ARTIFACT_STEP \
+    VLLM_TEST_CACHE \
+    VLLM_CI_EXECUTION_MODE \
+    VLLM_CI_WORKSPACE \
+    VLLM_CI_REQUIRE_WORKSPACE_MOUNT \
+    VLLM_TEST_COMMANDS \
+    VLLM_CI_BRANCH \
+    VLLM_CI_BASE_IMAGE \
+    VLLM_CI_FALLBACK_IMAGE \
+    VLLM_CI_DOCKER_DISABLED \
+    VLLM_CI_ARTIFACT_GLOB \
+    VLLM_CI_ARTIFACT_CHECKSUM_GLOB \
+    VLLM_CI_EXPECTED_GPU_COUNT \
+    VLLM_CI_USE_ARTIFACTS \
+    VLLM_CI_RESULTS_ROOT \
+    VLLM_ALLOW_DEPRECATED_BEAM_SEARCH
+}
+
 cleanup_network() {
  local max_nodes=${NUM_NODES:-2}
  for node in $(seq 0 $((max_nodes - 1))); do
@@ -145,7 +184,11 @@ prepare_artifact_image() {
  fi

  cp "${wheel_dir}"/*.whl "${context_dir}/wheels/" || return 1
-  tar -C "${wheel_dir}" --exclude='*.whl' -cf - . \
+  tar -C "${wheel_dir}" \
+    --exclude='*.whl' \
+    --exclude='.vllm-ci-artifact' \
+    --exclude='./.vllm-ci-artifact' \
+    -cf - . \
    | tar -C "${workspace_dir}" -xf - || return 1
  cat > "${context_dir}/Dockerfile" <<'EOF'
 ARG BASE_IMAGE
@@ -168,6 +211,257 @@ EOF
  return 0
 }

+is_native_runtime() {
+  [[ "${AMD_CI_RUNTIME:-}" == "native" || "${NATIVE_CI:-}" == "true" ]]
+}
+
+validate_native_workspace() {
+  local workspace_dir="${VLLM_CI_WORKSPACE:-/vllm-workspace}"
+  local workspace_real=""
+  local checkout_real=""
+  local workspace_mount=""
+
+  mkdir -p "${workspace_dir}" || return 1
+  workspace_real=$(readlink -m "${workspace_dir}") || return 1
+  if [[ -n "${BUILDKITE_BUILD_CHECKOUT_PATH:-}" ]]; then
+    checkout_real=$(readlink -m "${BUILDKITE_BUILD_CHECKOUT_PATH}") || return 1
+    if [[ "${checkout_real}" == "${workspace_real}" \
+      || "${checkout_real}" == "${workspace_real}/"* \
+      || "${workspace_real}" == "${checkout_real}/"* ]]; then
+      echo "Refusing to replace ${workspace_real}; it overlaps the Buildkite checkout ${checkout_real}" >&2
+      return 1
+    fi
+  fi
+  if [[ "${VLLM_CI_REQUIRE_WORKSPACE_MOUNT:-1}" == "1" ]]; then
+    if ! command -v findmnt >/dev/null 2>&1; then
+      echo "findmnt is required to verify the native workspace mount" >&2
+      return 1
+    fi
+    workspace_mount=$(findmnt -n -T "${workspace_real}" -o TARGET 2>/dev/null || true)
+    if [[ "$(readlink -m "${workspace_mount:-/}")" != "${workspace_real}" ]]; then
+      echo "Native CI requires a dedicated volume mounted at ${workspace_real}" >&2
+      return 1
+    fi
+  fi
+}
+
+prepare_native_workspace() {
+  if [[ "${VLLM_CI_USE_ARTIFACTS:-0}" != "1" ]]; then
+    echo "Native CI requires VLLM_CI_USE_ARTIFACTS=1"
+    return 1
+  fi
+  if ! command -v buildkite-agent >/dev/null 2>&1; then
+    echo "buildkite-agent not found; cannot download ROCm wheel artifact"
+    return 1
+  fi
+  validate_native_workspace || return 1
+
+  local artifact_glob="${VLLM_CI_ARTIFACT_GLOB:-artifacts/vllm-rocm-install/vllm-rocm-install.tar.gz}"
+  local artifact_checksum_glob="${VLLM_CI_ARTIFACT_CHECKSUM_GLOB:-${artifact_glob}.sha256}"
+  local artifact_step="${VLLM_CI_ARTIFACT_STEP:-image-build-amd}"
+  local archive=""
+  local checksum=""
+  local download_dir=""
+  local metadata_dir=""
+  local recorded_base=""
+  local recorded_commit=""
+  local recorded_wheel=""
+  local workspace_dir="${VLLM_CI_WORKSPACE:-/vllm-workspace}"
+  local wheel_dir=""
+  local attempt=0
+  local attempt_dir=""
+  local -a archives=()
+  local -a checksums=()
+  local -a wheels=()
+
+  artifact_work_dir=$(mktemp -d -t vllm-rocm-artifact.XXXXXX) || return 1
+  wheel_dir="${artifact_work_dir}/wheels"
+  mkdir -p "${wheel_dir}" || return 1
+
+  echo "--- Downloading ROCm wheel artifact from ${artifact_step} (native in-pod)"
+  for attempt in 1 2 3; do
+    attempt_dir="${artifact_work_dir}/download-${attempt}"
+    rm -rf "${attempt_dir}" || return 1
+    mkdir -p "${attempt_dir}" || return 1
+    if buildkite-agent artifact download \
+      "${artifact_glob}" "${attempt_dir}" --step "${artifact_step}" \
+      && buildkite-agent artifact download \
+        "${artifact_checksum_glob}" "${attempt_dir}" --step "${artifact_step}"; then
+      download_dir="${attempt_dir}"
+      break
+    fi
+    echo "Artifact download attempt ${attempt}/3 failed"
+    if [[ "${attempt}" -lt 3 ]]; then
+      sleep $((attempt * 2))
+    fi
+  done
+  if [[ -z "${download_dir}" ]]; then
+    echo "Failed to download ${artifact_glob} and ${artifact_checksum_glob} from ${artifact_step}"
+    return 1
+  fi
+
+  mapfile -t archives < <(
+    find "${download_dir}" -name "vllm-rocm-install.tar.gz" -type f -print
+  )
+  mapfile -t checksums < <(
+    find "${download_dir}" -name "vllm-rocm-install.tar.gz.sha256" -type f -print
+  )
+  if [[ ${#archives[@]} -ne 1 || ${#checksums[@]} -ne 1 ]]; then
+    echo "Expected exactly one ROCm archive and checksum; found ${#archives[@]} archive(s) and ${#checksums[@]} checksum(s)" >&2
+    return 1
+  fi
+  archive="${archives[0]}"
+  checksum="${checksums[0]}"
+  if [[ "$(dirname "${archive}")" != "$(dirname "${checksum}")" ]]; then
+    echo "ROCm archive and checksum were downloaded to different directories" >&2
+    return 1
+  fi
+  (
+    cd "$(dirname "${archive}")"
+    sha256sum -c "$(basename "${checksum}")"
+  ) || return 1
+
+  tar --no-same-owner -xzf "${archive}" -C "${wheel_dir}" || return 1
+  mapfile -t wheels < <(
+    find "${wheel_dir}" -maxdepth 1 -type f -name '*.whl' -print
+  )
+  if [[ ${#wheels[@]} -ne 1 ]]; then
+    echo "ROCm artifact must contain exactly one top-level wheel; found ${#wheels[@]}" >&2
+    return 1
+  fi
+  metadata_dir="${wheel_dir}/.vllm-ci-artifact"
+  for metadata_file in commit.txt native-base-image.txt wheel-filename.txt; do
+    if [[ ! -s "${metadata_dir}/${metadata_file}" ]]; then
+      echo "ROCm artifact metadata is missing ${metadata_file}" >&2
+      return 1
+    fi
+  done
+  for metadata_file in ci-base-image.txt fallback-image.txt; do
+    if [[ ! -f "${metadata_dir}/${metadata_file}" ]]; then
+      echo "ROCm artifact metadata is missing ${metadata_file}" >&2
+      return 1
+    fi
+  done
+
+  recorded_commit=$(tr -d '\r\n' < "${metadata_dir}/commit.txt")
+  recorded_base=$(tr -d '\r\n' < "${metadata_dir}/native-base-image.txt")
+  recorded_wheel=$(tr -d '\r\n' < "${metadata_dir}/wheel-filename.txt")
+  if [[ -z "${BUILDKITE_COMMIT:-}" || "${recorded_commit}" != "${BUILDKITE_COMMIT}" ]]; then
+    echo "ROCm artifact commit ${recorded_commit} does not match ${BUILDKITE_COMMIT:-unset}" >&2
+    return 1
+  fi
+  if [[ -z "${VLLM_CI_BASE_IMAGE:-}" || "${recorded_base}" != "${VLLM_CI_BASE_IMAGE}" ]]; then
+    echo "ROCm artifact base ${recorded_base} does not match ${VLLM_CI_BASE_IMAGE:-unset}" >&2
+    return 1
+  fi
+  if [[ "${recorded_wheel}" != "$(basename "${wheels[0]}")" ]]; then
+    echo "ROCm artifact wheel manifest ${recorded_wheel} does not match $(basename "${wheels[0]}")" >&2
+    return 1
+  fi
+  for required_dir in tests .buildkite requirements; do
+    if [[ ! -d "${wheel_dir}/${required_dir}" ]]; then
+      echo "ROCm wheel artifact did not contain ${required_dir}/" >&2
+      return 1
+    fi
+  done
+
+  echo "--- Installing ROCm wheel into pod environment"
+  python3 -m pip install --no-deps --force-reinstall "${wheels[0]}" || return 1
+
+  echo "--- Preparing ${workspace_dir} from artifact"
+  find "${workspace_dir}" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + || return 1
+  tar -C "${wheel_dir}" \
+    --exclude='*.whl' \
+    --exclude='.vllm-ci-artifact' \
+    --exclude='./.vllm-ci-artifact' \
+    -cf - . | tar --no-same-owner -C "${workspace_dir}" -xf - || return 1
+  if [[ ! -d "${workspace_dir}/tests" ]]; then
+    echo "Failed to stage the native test workspace" >&2
+    return 1
+  fi
+
+  return 0
+}
+
+initialize_native_environment() {
+  local job_id="${BUILDKITE_JOB_ID:-${BUILDKITE_PARALLEL_JOB:-local}}"
+  local job_id_suffix=""
+  local native_root=""
+  local hf_mount=""
+
+  if [[ "$(id -u)" -ne 0 ]]; then
+    echo "Native ROCm CI currently requires the ci_base container to run as root" >&2
+    return 1
+  fi
+
+  job_id="${job_id//[^A-Za-z0-9_.-]/_}"
+  job_id_suffix="${job_id##*-}"
+  job_id_suffix="${job_id_suffix:0:12}"
+  native_root="/tmp/vllm-native-${job_id}"
+  TMPDIR="/tmp/vllm-${job_id_suffix}/tmp"
+  VLLM_RPC_BASE_PATH="/tmp"
+  : "${TORCHINDUCTOR_CACHE_DIR:=${native_root}/cache/torchinductor}"
+  : "${TRITON_CACHE_DIR:=${native_root}/cache/triton}"
+  : "${VLLM_CACHE_ROOT:=${native_root}/cache/vllm}"
+  : "${XDG_CACHE_HOME:=${native_root}/cache/xdg}"
+  : "${HF_HOME:=/home/buildkite-agent/huggingface}"
+  : "${HF_HUB_DOWNLOAD_TIMEOUT:=300}"
+  : "${HF_HUB_ETAG_TIMEOUT:=60}"
+  export TMPDIR VLLM_RPC_BASE_PATH
+  export TORCHINDUCTOR_CACHE_DIR TRITON_CACHE_DIR VLLM_CACHE_ROOT XDG_CACHE_HOME
+  export HF_HOME HF_HUB_DOWNLOAD_TIMEOUT HF_HUB_ETAG_TIMEOUT
+  export PYTORCH_ROCM_ARCH=""
+
+  mkdir -p "${TMPDIR}" \
+    "${TORCHINDUCTOR_CACHE_DIR}" \
+    "${TRITON_CACHE_DIR}" \
+    "${VLLM_CACHE_ROOT}" \
+    "${XDG_CACHE_HOME}" \
+    "${HF_HOME}" || return 1
+
+  if [[ "${VLLM_CI_REQUIRE_PERSISTENT_HF_CACHE:-0}" == "1" ]]; then
+    if ! command -v findmnt >/dev/null 2>&1; then
+      echo "findmnt is required to verify the native Hugging Face cache mount" >&2
+      return 1
+    fi
+    hf_mount=$(findmnt -n -T "${HF_HOME}" -o TARGET 2>/dev/null || true)
+    if [[ -z "${hf_mount}" || "${hf_mount}" == "/" ]]; then
+      echo "Native CI requires a persistent volume mounted at or above ${HF_HOME}" >&2
+      return 1
+    fi
+  fi
+}
+
+run_native_preflight() {
+  local expected_gpus="${VLLM_CI_EXPECTED_GPU_COUNT:-1}"
+
+  if [[ ! "${expected_gpus}" =~ ^[0-9]+$ ]]; then
+    echo "Invalid VLLM_CI_EXPECTED_GPU_COUNT=${expected_gpus}" >&2
+    return 1
+  fi
+
+  python3 -c "import encodings, importlib.metadata as im, importlib.util as iu; [im.version(d) for d in ('transformers', 'torch', 'ray', 'sympy', 'markupsafe', 'vllm')]; missing=[m for m in ('torch.utils.model_zoo', 'transformers.models.nomic_bert', 'ray.dag', 'sympy.physics', 'markupsafe._speedups') if iu.find_spec(m) is None]; assert not missing, missing" || return 1
+
+  if [[ "${expected_gpus}" == "0" ]]; then
+    echo "Native CPU-only AMD job: skipping ROCm device validation"
+    return 0
+  fi
+
+  echo "--- ROCm info"
+  rocminfo || return 1
+  VLLM_CI_EXPECTED_GPU_COUNT="${expected_gpus}" python3 - <<'PY'
+import os
+
+import torch
+
+expected = int(os.environ["VLLM_CI_EXPECTED_GPU_COUNT"])
+assert torch.version.hip, "PyTorch is not a ROCm build"
+assert torch.cuda.is_available(), "ROCm GPU is not available to PyTorch"
+actual = torch.cuda.device_count()
+assert actual == expected, f"Expected {expected} ROCm GPU(s), found {actual}"
+PY
+}
+
 is_multi_node() {
  local cmds="$1"
  # Primary signal: NUM_NODES environment variable set by the pipeline
@@ -350,7 +644,58 @@ re_quote_pytest_markers() {
 # Main
 ###############################################################################

-# --- GPU initialization ---
+if is_native_runtime; then
+  echo "--- Native in-pod ROCm CI (AMD_CI_RUNTIME=${AMD_CI_RUNTIME:-unset}, NATIVE_CI=${NATIVE_CI:-unset})"
+  artifact_work_dir=""
+
+  cleanup_native_workspace() {
+    if [[ -n "${artifact_work_dir}" ]]; then
+      rm -rf "${artifact_work_dir}"
+    fi
+  }
+  trap cleanup_native_workspace EXIT
+
+  if [[ -n "${VLLM_TEST_COMMANDS:-}" ]]; then
+    commands="${VLLM_TEST_COMMANDS}"
+    commands_source="env"
+  else
+    commands="$*"
+    commands_source="argv"
+    if [[ -z "$commands" ]]; then
+      echo "Error: No test commands provided for native CI." >&2
+      exit 1
+    fi
+  fi
+
+  if [[ "$commands_source" == "argv" ]]; then
+    commands=$(re_quote_pytest_markers "$commands")
+  fi
+
+  if is_multi_node "$commands"; then
+    echo "Native CI does not support multi-node jobs yet."
+    exit 1
+  fi
+
+  if ! initialize_native_environment; then
+    echo "Failed to initialize the native test environment"
+    exit 1
+  fi
+  if ! prepare_native_workspace; then
+    echo "Failed to prepare native test workspace"
+    exit 1
+  fi
+
+  export PYTHONPATH="${VLLM_CI_WORKSPACE:-/vllm-workspace}"
+
+  echo "Native test commands: $commands"
+  run_native_preflight || exit 1
+  # Keep AMD CI orchestration variables out of vLLM's runtime environment.
+  clear_ci_orchestration_env
+  /bin/bash -o pipefail -c "${commands}"
+  handle_pytest_exit "$?"
+fi
+
+# --- GPU initialization for legacy Docker execution ---
 echo "--- ROCm info"
 rocminfo

@@ -452,25 +797,27 @@ fi

 echo "Final commands: $commands"

-# The ROCm test image often ships /vllm-workspace without .git (artifact tarball unpack).
-# tests/standalone_tests/python_only_compile.sh uses merge-base(HEAD, origin/main) for
-# wheels.vllm.ai; compute on the agent (full git checkout) and pass into the container.
-vllm_standalone_merge_base=""
-checkout="${BUILDKITE_BUILD_CHECKOUT_PATH:-}"
-if [[ -z "${checkout}" || ! -d "${checkout}" ]]; then
-  checkout="."
+standalone_merge_base_env=()
+if [[ "$commands" == *python_only_compile.sh* ]]; then
+  # The ROCm test image often ships /vllm-workspace without .git. Resolve the
+  # wheels.vllm.ai commit from the agent checkout for this test only.
+  vllm_standalone_merge_base=""
+  checkout="${BUILDKITE_BUILD_CHECKOUT_PATH:-}"
+  if [[ -z "${checkout}" || ! -d "${checkout}" ]]; then
+    checkout="."
+  fi
+  # Pass safe.directory per-command because Buildkite uses mixed user IDs.
+  if git -c "safe.directory=${checkout}" -C "${checkout}" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
+    vllm_standalone_merge_base="$(
+      git -c "safe.directory=${checkout}" -C "${checkout}" merge-base HEAD origin/main 2>/dev/null || true
+    )"
+  fi
+  if [[ -z "${vllm_standalone_merge_base}" ]]; then
+    vllm_standalone_merge_base="${BUILDKITE_COMMIT:-}"
+  fi
+  echo "INFO: passing CI_STANDALONE_MERGE_BASE into container: ${vllm_standalone_merge_base}"
+  standalone_merge_base_env=(-e "CI_STANDALONE_MERGE_BASE=${vllm_standalone_merge_base}")
 fi
-# Pass safe.directory per-command (-c) because buildkite runs will always fail
-# the next check on git 2.35.2+ due to mixed uses of root and buildkite-agent/uids.
-if git -c "safe.directory=${checkout}" -C "${checkout}" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
-  vllm_standalone_merge_base="$(
-    git -c "safe.directory=${checkout}" -C "${checkout}" merge-base HEAD origin/main 2>/dev/null || true
-  )"
-fi
-if [[ -z "${vllm_standalone_merge_base}" ]]; then
-  vllm_standalone_merge_base="${BUILDKITE_COMMIT:-}"
-fi
-echo "INFO: passing VLLM_STANDALONE_MERGE_BASE into container: ${vllm_standalone_merge_base}"

 MYPYTHONPATH="/vllm-workspace"

@@ -501,6 +848,7 @@ else
 fi

 # --- Route: multi-node vs single-node ---
+clear_ci_orchestration_env
 if is_multi_node "$commands"; then
  echo "--- Multi-node job detected"
  export DCKR_VER=$(docker --version | sed 's/Docker version \(.*\), build .*/\1/')
@@ -589,7 +937,9 @@ else
    -e FORCE_COLOR \
    -e CLICOLOR_FORCE \
    -e PY_COLORS \
+    -e PYTHONFAULTHANDLER \
    -e PYTEST_ADDOPTS \
+    -e PYTEST_TIMEOUT \
    -v "${HF_CACHE}:${HF_MOUNT}" \
    -e "HF_HOME=${HF_MOUNT}" \
    -e "PYTHONPATH=${MYPYTHONPATH}" \
@@ -599,7 +949,7 @@ else
    -e "VLLM_CACHE_ROOT=${CONTAINER_CACHE_ROOT}/vllm" \
    -e "XDG_CACHE_HOME=${CONTAINER_CACHE_ROOT}/xdg" \
    -e "PYTORCH_ROCM_ARCH=" \
-    -e "VLLM_STANDALONE_MERGE_BASE=${vllm_standalone_merge_base}" \
+    "${standalone_merge_base_env[@]}" \
    --name "${container_name}" \
    "${image_name}" \
    /bin/bash -c "${CONTAINER_PREFLIGHT} && ${commands}"
@@ -14,6 +14,8 @@
 # command(str): the single command to run for tests. incompatible with commands.
 # commands(list): the list of commands to run for the test. incompatible with command.
 # mirror_hardwares(list): the list of hardware to run the test on as well. currently only supports [amdexperimental]
+# dind(bool): when false, run the job directly in the AMD Kubernetes pod.
+#     When true or omitted, use the legacy Docker-in-Docker path.
 # num_gpus(int): override the number of GPUs for the test. defaults to 1 GPU. currently supports 2,4.
 # num_nodes(int): whether to simulate multi-node setup by launching multiple containers on one host,
 #     in this case, commands must be specified. the first command runs on the first host, the second
@@ -469,6 +471,7 @@ steps:
 - label: Basic Correctness # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  fast_check: true
  working_dir: "/vllm-workspace/tests"
@@ -486,6 +489,7 @@ steps:
 - label: Distributed Model Tests (2 GPUs) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_2
  num_gpus: 2
  optional: true
@@ -515,6 +519,7 @@ steps:
 - label: Benchmarks CLI Test # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  working_dir: "/vllm-workspace/tests"
@@ -529,6 +534,7 @@ steps:
 - label: PyTorch Compilation Unit Tests # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  working_dir: "/vllm-workspace/tests"
@@ -548,6 +554,7 @@ steps:
 - label: Fusion E2E Config Sweep (H100-MI300) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  num_gpus: 1
  working_dir: "/vllm-workspace/"
@@ -568,6 +575,7 @@ steps:
 - label: Fusion E2E Quick (H100-MI300) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  num_gpus: 1
  working_dir: "/vllm-workspace/"
@@ -589,6 +597,7 @@ steps:
 - label: PyTorch Compilation Passes Unit Tests # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  working_dir: "/vllm-workspace/tests"
  source_file_dependencies:
@@ -600,6 +609,7 @@ steps:
 - label: PyTorch Fullgraph # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  working_dir: "/vllm-workspace/tests"
@@ -617,6 +627,7 @@ steps:
 - label: Pytorch Nightly Dependency Override Check # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  soft_fail: true
@@ -630,6 +641,7 @@ steps:
 - label: Distributed Compile Unit Tests (2xH100-2xMI300) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_2
  num_gpus: 2
  working_dir: "/vllm-workspace/"
@@ -650,6 +662,7 @@ steps:
 - label: Platform Tests # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  working_dir: "/vllm-workspace/tests"
  source_file_dependencies:
@@ -664,6 +677,7 @@ steps:
 - label: Async Engine, Inputs, Utils, Worker # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  working_dir: "/vllm-workspace/tests"
  source_file_dependencies:
@@ -681,6 +695,7 @@ steps:
 - label: Distributed Comm Ops # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_2
  num_gpus: 2
  working_dir: "/vllm-workspace/tests"
@@ -697,12 +712,14 @@ steps:
 - label: EPLB Algorithm # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  working_dir: "/vllm-workspace/tests"
  source_file_dependencies:
  - vllm/distributed/eplb
  - tests/distributed/test_eplb_algo.py
+  - tests/distributed/test_eplb_utils.py
  - vllm/platforms/rocm.py
  commands:
  - pytest -v -s distributed/test_eplb_algo.py
@@ -711,6 +728,7 @@ steps:
 - label: EPLB Execution # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_4
  num_gpus: 4
  working_dir: "/vllm-workspace/tests"
@@ -726,6 +744,7 @@ steps:
 - label: Distributed Tests (2xH100-2xMI300) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_2
  num_gpus: 2
  working_dir: "/vllm-workspace/"
@@ -752,6 +771,7 @@ steps:
 - label: Distributed Tests (4xA100-4xMI300) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_4
  num_gpus: 4
  optional: true
@@ -767,6 +787,7 @@ steps:
 - label: Distributed Torchrun + Examples (4 GPUs) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_4
  num_gpus: 4
  working_dir: "/vllm-workspace/tests"
@@ -792,6 +813,7 @@ steps:
 - label: Elastic EP Scaling Test # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_4
  num_gpus: 4
  optional: true
@@ -809,6 +831,7 @@ steps:
 - label: RayExecutorV2 (4 GPUs) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_4
  num_gpus: 4
  working_dir: "/vllm-workspace/tests"
@@ -831,6 +854,7 @@ steps:
 - label: Distributed Tests (8xH100-8xMI300) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_8
  num_gpus: 8
  optional: true
@@ -851,6 +875,7 @@ steps:
 - label: Entrypoints Unit Tests # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  fast_check: true
  working_dir: "/vllm-workspace/tests"
@@ -866,6 +891,7 @@ steps:
 - label: Entrypoints Integration (LLM) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  fast_check: true
@@ -882,6 +908,7 @@ steps:
 - label: Entrypoints Integration (API Server) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  fast_check: true
@@ -899,6 +926,7 @@ steps:
 - label: Entrypoints Integration (API Server OpenAI - Part 1) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  fast_check: true
  optional: true
@@ -914,6 +942,7 @@ steps:
 - label: Entrypoints Integration (API Server OpenAI - Part 2) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  fast_check: true
  optional: true
@@ -930,6 +959,7 @@ steps:
 - label: Entrypoints Integration (API Server Generate) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  fast_check: true
@@ -950,6 +980,7 @@ steps:
 - label: Entrypoints Integration (Responses API) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  fast_check: true
  working_dir: "/vllm-workspace/tests"
@@ -963,6 +994,7 @@ steps:
 - label: Entrypoints Integration (Speech to Text) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  fast_check: true
  working_dir: "/vllm-workspace/tests"
@@ -976,6 +1008,7 @@ steps:
 - label: Entrypoints Integration (Multimodal)
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  fast_check: true
  working_dir: "/vllm-workspace/tests"
@@ -989,6 +1022,7 @@ steps:
 - label: Entrypoints Integration (Pooling) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  fast_check: true
  working_dir: "/vllm-workspace/tests"
@@ -1002,6 +1036,7 @@ steps:
 - label: OpenAI API correctness # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  working_dir: "/vllm-workspace/tests"
@@ -1023,6 +1058,7 @@ steps:
 - label: DeepSeek V2-Lite Prefetch Offload Accuracy (H100-MI300) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  num_gpus: 1
  working_dir: "/vllm-workspace"
@@ -1043,6 +1079,7 @@ steps:
 - label: LM Eval Small Models # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  working_dir: "/vllm-workspace/tests"
  source_file_dependencies:
@@ -1060,6 +1097,7 @@ steps:
 - label: MRCR Eval Small Models # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  working_dir: "/vllm-workspace/tests"
@@ -1077,6 +1115,7 @@ steps:
 - label: LM Eval Small Models (MI300) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
@@ -1095,6 +1134,7 @@ steps:
 - label: Multi-Modal Accuracy Eval (Small Models) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
@@ -1110,6 +1150,7 @@ steps:
 - label: GPQA Eval (GPT-OSS) (2xH100-2xMI300) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_2
  num_gpus: 2
  optional: true
@@ -1132,7 +1173,9 @@ steps:
 - label: LM Eval Small Models (2xB200-2xMI300) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_2
+  num_gpus: 2
  working_dir: "/vllm-workspace/tests"
  source_file_dependencies:
  - csrc/
@@ -1149,6 +1192,7 @@ steps:
 - label: DeepSeek V2-Lite Sync EPLB Accuracy (4xH100-4xMI300) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_4
  num_gpus: 4
  optional: true
@@ -1171,6 +1215,7 @@ steps:
 - label: LM Eval Large Models (4xA100-4xMI300) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_4
  num_gpus: 4
  optional: true
@@ -1191,7 +1236,9 @@ steps:
 - label: Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy (4xH100-4xMI300) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_4
+  num_gpus: 4
  optional: true
  working_dir: "/vllm-workspace"
  source_file_dependencies:
@@ -1211,6 +1258,7 @@ steps:
 - label: Qwen3-30B-A3B-FP8 DP4 Async EPLB Accuracy (4xH100-4xMI300) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_4
  num_gpus: 4
  optional: true
@@ -1232,6 +1280,7 @@ steps:
 - label: Qwen3-Next-80B-A3B-Instruct MTP Async EPLB Accuracy # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_4
  num_gpus: 4
  optional: true
@@ -1254,6 +1303,7 @@ steps:
 - label: LM Eval Large Models (8xH200-8xMI300) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_8
  optional: true
  num_gpus: 8
@@ -1276,6 +1326,7 @@ steps:
 - label: ROCm LM Eval Large Models (8 GPUs) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_8
  optional: true
  num_gpus: 8
@@ -1299,6 +1350,7 @@ steps:
 - label: Examples # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  working_dir: "/vllm-workspace/examples"
@@ -1335,6 +1387,7 @@ steps:
 - label: vLLM IR Tests # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  working_dir: "/vllm-workspace/"
@@ -1350,6 +1403,7 @@ steps:
 - label: Kernels Attention Test %N # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  parallelism: 2
@@ -1368,6 +1422,7 @@ steps:
 - label: Kernels Core Operation Test # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  working_dir: "/vllm-workspace/tests"
  source_file_dependencies:
@@ -1384,6 +1439,7 @@ steps:
 - label: Kernels KDA Test # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  working_dir: "/vllm-workspace/tests"
@@ -1399,6 +1455,7 @@ steps:
 - label: Kernels MoE Test %N # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  parallelism: 5
@@ -1420,6 +1477,7 @@ steps:
 - label: Kernels Quantization Test %N # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  parallelism: 2
@@ -1438,7 +1496,9 @@ steps:
 - label: Kernels FP8 MoE Test (2xH100-2xMI300) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_2
+  num_gpus: 2
  optional: true
  working_dir: "/vllm-workspace/tests"
  source_file_dependencies:
@@ -1457,6 +1517,7 @@ steps:
 - label: LoRA %N # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  parallelism: 4
  optional: true
@@ -1471,6 +1532,7 @@ steps:
 - label: LoRA TP (Distributed) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_4
  num_gpus: 4
  working_dir: "/vllm-workspace/tests"
@@ -1491,6 +1553,7 @@ steps:
 - label: Model Executor # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  working_dir: "/vllm-workspace/tests"
@@ -1513,6 +1576,7 @@ steps:
 - label: Model Runner V2 Core Tests # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  working_dir: "/vllm-workspace/tests"
@@ -1537,6 +1601,7 @@ steps:
 - label: Model Runner V2 Examples # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  working_dir: "/vllm-workspace/examples"
@@ -1571,6 +1636,7 @@ steps:
 - label: Model Runner V2 Distributed (2 GPUs) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_2
  num_gpus: 2
  optional: true
@@ -1592,6 +1658,7 @@ steps:
 - label: Model Runner V2 Pipeline Parallelism (4 GPUs) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_4
  num_gpus: 4
  optional: true
@@ -1611,6 +1678,7 @@ steps:
 - label: Model Runner V2 Spec Decode # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  working_dir: "/vllm-workspace/tests"
@@ -1635,6 +1703,7 @@ steps:
 - label: Basic Models Tests (Extra Initialization) %N # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  parallelism: 6
  working_dir: "/vllm-workspace/tests"
@@ -1651,6 +1720,7 @@ steps:
 - label: Basic Models Tests (Initialization) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  working_dir: "/vllm-workspace/tests"
  source_file_dependencies:
@@ -1663,6 +1733,7 @@ steps:
 - label: Basic Models Tests (Other) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  working_dir: "/vllm-workspace/tests"
@@ -1679,6 +1750,7 @@ steps:
 - label: Language Models Test (Extended Pooling)  # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  working_dir: "/vllm-workspace/tests"
@@ -1691,6 +1763,7 @@ steps:
 - label: Language Models Tests (Standard) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  working_dir: "/vllm-workspace/tests"
@@ -1704,6 +1777,7 @@ steps:
 - label: Language Models Tests (Extra Standard) %N # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  parallelism: 2
@@ -1728,6 +1802,7 @@ steps:
 - label: Multi-Modal Models (Extended Generation 1) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  working_dir: "/vllm-workspace/tests"
@@ -1742,6 +1817,7 @@ steps:
 - label: Multi-Modal Models (Extended Generation 2) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  working_dir: "/vllm-workspace/tests"
@@ -1755,6 +1831,7 @@ steps:
 - label: Multi-Modal Models (Extended Generation 3) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  working_dir: "/vllm-workspace/tests"
@@ -1767,6 +1844,7 @@ steps:
 - label: "Multi-Modal Models (Standard) 1: qwen2" # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  working_dir: "/vllm-workspace/tests"
@@ -1780,6 +1858,7 @@ steps:
 - label: "Multi-Modal Models (Standard) 3: llava + qwen2_vl" # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  working_dir: "/vllm-workspace/tests"
@@ -1794,6 +1873,7 @@ steps:
 - label: "Multi-Modal Models (Standard) 4: other + whisper" # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  working_dir: "/vllm-workspace/tests"
  source_file_dependencies:
@@ -1808,6 +1888,7 @@ steps:
 - label: Multi-Modal Processor # 1h 42m
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  working_dir: "/vllm-workspace/tests"
@@ -1821,9 +1902,9 @@ steps:
 - label: Multi-Modal Processor (CPU) %N # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  parallelism: 4
-  no_gpu: true
  optional: true
  working_dir: "/vllm-workspace/tests"
  source_file_dependencies:
@@ -1838,6 +1919,7 @@ steps:
 - label: Quantized Models Test # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  working_dir: "/vllm-workspace/tests"
  source_file_dependencies:
@@ -1854,6 +1936,7 @@ steps:
 - label: Transformers Nightly Models (Shardable) %N # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  parallelism: 4
  optional: true
@@ -1876,6 +1959,7 @@ steps:
 - label: Transformers Nightly Models (Single) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  working_dir: "/vllm-workspace/"
@@ -1903,6 +1987,7 @@ steps:
 - label: Plugin Tests (2 GPUs) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_2
  num_gpus: 2
  optional: true
@@ -1954,6 +2039,7 @@ steps:
 - label: GGUF Plugin # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  soft_fail: true
@@ -1972,6 +2058,7 @@ steps:
 - label: Rust Frontend OpenAI Coverage # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  working_dir: "/vllm-workspace/tests"
@@ -2004,6 +2091,7 @@ steps:
 - label: Rust Frontend Serve Admin Coverage # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  working_dir: "/vllm-workspace/tests"
@@ -2031,6 +2119,7 @@ steps:
 - label: Rust Frontend Core Correctness # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  working_dir: "/vllm-workspace/tests"
@@ -2048,6 +2137,7 @@ steps:
 - label: Rust Frontend Tool Use # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  working_dir: "/vllm-workspace/tests"
@@ -2066,6 +2156,7 @@ steps:
 - label: Rust Frontend Distributed # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_4
  num_gpus: 4
  optional: true
@@ -2094,6 +2185,7 @@ steps:
 - label: Quantization # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  working_dir: "/vllm-workspace/tests"
  source_file_dependencies:
@@ -2120,6 +2212,7 @@ steps:
 - label: ROCm AITER Ops Test # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  working_dir: "/vllm-workspace/tests"
  source_file_dependencies:
@@ -2137,6 +2230,7 @@ steps:
 - label: Samplers Test # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  working_dir: "/vllm-workspace/tests"
@@ -2154,22 +2248,10 @@ steps:

 #------------------------------------------------------------  mi300 · misc  ------------------------------------------------------------#

- label: Python-only Installation # TBD
-  timeout_in_minutes: 180
-  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
-  agent_pool: mi300_1
-  optional: true
-  working_dir: "/vllm-workspace/tests"
-  source_file_dependencies:
-  - tests/standalone_tests/python_only_compile.sh
-  - setup.py
-  - vllm/platforms/rocm.py
-  commands:
-  - bash standalone_tests/python_only_compile.sh
-
 - label: Regression # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  working_dir: "/vllm-workspace/tests"
  source_file_dependencies:
@@ -2184,6 +2266,7 @@ steps:
 - label: Ray Dependency Compatibility Check # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  working_dir: "/"
  source_file_dependencies:
@@ -2198,6 +2281,7 @@ steps:
 - label: Acceptance Length Test (Large Models) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  working_dir: "/vllm-workspace/tests"
@@ -2213,6 +2297,7 @@ steps:
 - label: e2e Scheduling (1 GPU) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  working_dir: "/vllm-workspace/tests"
@@ -2226,6 +2311,7 @@ steps:
 - label: Engine (1 GPU) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  working_dir: "/vllm-workspace/tests"
@@ -2240,6 +2326,7 @@ steps:
 - label: Spec Decode Draft Model # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  working_dir: "/vllm-workspace/tests"
@@ -2257,6 +2344,7 @@ steps:
 - label: Spec Decode Eagle # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  working_dir: "/vllm-workspace/tests"
@@ -2274,6 +2362,7 @@ steps:
 - label: Spec Decode Ngram + Suffix # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  working_dir: "/vllm-workspace/tests"
@@ -2291,6 +2380,7 @@ steps:
 - label: Spec Decode Speculators + MTP # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  working_dir: "/vllm-workspace/tests"
@@ -2309,6 +2399,7 @@ steps:
 - label: Speculators Correctness # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  working_dir: "/vllm-workspace/tests"
@@ -2337,9 +2428,22 @@ steps:
  - export VLLM_ALLOW_INSECURE_SERIALIZATION=1
  - pytest -v -s v1/spec_decode/test_speculators_correctness.py -m slow_test

+- label: V1 Spec Decode # TBD
+  timeout_in_minutes: 180
+  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
+  agent_pool: mi300_1
+  working_dir: "/vllm-workspace/tests"
+  source_file_dependencies:
+  - vllm/
+  - tests/v1/spec_decode
+  commands:
+  - pytest -v -s -m 'not slow_test' v1/spec_decode
+
 - label: Extract Hidden States Integration # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_2
  num_gpus: 2
  working_dir: "/vllm-workspace/tests"
@@ -2371,6 +2475,7 @@ steps:
 - label: V1 attention (H100-MI300) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  working_dir: "/vllm-workspace/tests"
@@ -2388,6 +2493,7 @@ steps:
 - label: V1 Core + KV + Metrics # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  working_dir: "/vllm-workspace/tests"
  source_file_dependencies:
@@ -2414,8 +2520,8 @@ steps:
 - label: V1 others (CPU) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
-  no_gpu: true
  optional: true
  working_dir: "/vllm-workspace/tests"
  source_file_dependencies:
@@ -2431,6 +2537,7 @@ steps:
 - label: V1 Sample + Logits # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_1
  optional: true
  working_dir: "/vllm-workspace/tests"
@@ -2451,6 +2558,7 @@ steps:
 - label: Distributed DP Tests (2 GPUs) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_2
  optional: true
  num_gpus: 2
@@ -2466,6 +2574,7 @@ steps:
  - tests/entrypoints/openai/test_multi_api_servers.py
  - vllm/platforms/rocm.py
  commands:
+  - export NCCL_CUMEM_HOST_ENABLE=0
  - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py
  - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
  - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py
@@ -2474,6 +2583,7 @@ steps:
 - label: Metrics, Tracing (2 GPUs) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_2
  optional: true
  num_gpus: 2
@@ -2492,7 +2602,9 @@ steps:
 - label: V1 e2e (2 GPUs) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_2
+  num_gpus: 2
  optional: true
  working_dir: "/vllm-workspace/tests"
  source_file_dependencies:
@@ -2504,6 +2616,7 @@ steps:
 - label: NixlConnector PD + Spec Decode acceptance (2 GPUs) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_2
  num_gpus: 2
  optional: true
@@ -2515,17 +2628,18 @@ steps:
  - vllm/platforms/rocm.py
  commands:
  - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
-  - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh
+  - KV_CACHE_MEMORY_BYTES=8G ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh

 - label: CrossLayer KV layout Distributed NixlConnector PD accuracy tests (4 GPUs) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_4
  num_gpus: 4
  optional: true
  working_dir: "/vllm-workspace/tests"
  source_file_dependencies:
-  - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
+  - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
  - tests/v1/kv_connector/nixl_integration/
  - vllm/platforms/rocm.py
  commands:
@@ -2535,6 +2649,7 @@ steps:
 - label: Distributed DP Tests (4 GPUs) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_4
  num_gpus: 4
  working_dir: "/vllm-workspace/tests"
@@ -2545,6 +2660,7 @@ steps:
  - tests/distributed/test_utils
  - vllm/platforms/rocm.py
  commands:
+  - export NCCL_CUMEM_HOST_ENABLE=0
  - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_async_llm_dp.py
  - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_eagle_dp.py
  - TP_SIZE=2 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py
@@ -2556,6 +2672,7 @@ steps:
 - label: Distributed NixlConnector PD accuracy (4 GPUs)  # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_4
  num_gpus: 4
  optional: true
@@ -2571,6 +2688,7 @@ steps:
 - label: DP EP Distributed NixlConnector PD accuracy tests (4 GPUs) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_4
  num_gpus: 4
  optional: true
@@ -2586,12 +2704,13 @@ steps:
 - label: Hybrid SSM NixlConnector PD accuracy tests (4 GPUs) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_4
  num_gpus: 4
  optional: true
  working_dir: "/vllm-workspace/tests"
  source_file_dependencies:
-  - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
+  - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
  - tests/v1/kv_connector/nixl_integration/
  - vllm/platforms/rocm.py
  commands:
@@ -2601,6 +2720,7 @@ steps:
 - label: Hybrid SSM NixlConnector PD prefix cache test (2 GPUs) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_2
  num_gpus: 2
  optional: true
@@ -2618,6 +2738,7 @@ steps:
 - label: MultiConnector (Nixl+Offloading) PD accuracy (2 GPUs) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_2
  num_gpus: 2
  optional: true
@@ -2636,6 +2757,7 @@ steps:
 - label: MultiConnector (Nixl+Offloading) PD edge cases (2 GPUs) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_2
  num_gpus: 2
  optional: true
@@ -2654,7 +2776,9 @@ steps:
 - label: V1 e2e (4 GPUs) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_4
+  num_gpus: 4
  optional: true
  working_dir: "/vllm-workspace/tests"
  source_file_dependencies:
@@ -2666,7 +2790,9 @@ steps:
 - label: V1 e2e (4xH100-4xMI300) # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_4
+  num_gpus: 4
  optional: true
  source_file_dependencies:
    - vllm/v1/attention/backends/utils.py
@@ -2680,6 +2806,7 @@ steps:
 - label: Weight Loading Multiple GPU # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_2
  num_gpus: 2
  working_dir: "/vllm-workspace/tests"
@@ -2692,6 +2819,7 @@ steps:
 - label: Weight Loading Multiple GPU - Large Models # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  dind: false
  agent_pool: mi300_2
  num_gpus: 2
  optional: true
@@ -2876,18 +3004,20 @@ steps:
  - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen3 or gemma"
  - pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model

-#------------------------------------------------------------  mi325 · v1  -------------------------------------------------------------#
+#-----------------------------------------------------------  mi325 · misc  ------------------------------------------------------------#

- label: V1 Spec Decode # TBD
+- label: Python-only Installation # TBD
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
  agent_pool: mi325_1
+  optional: true
  working_dir: "/vllm-workspace/tests"
  source_file_dependencies:
-  - vllm/
-  - tests/v1/spec_decode
+  - tests/standalone_tests/python_only_compile.sh
+  - setup.py
+  - vllm/platforms/rocm.py
  commands:
-  - pytest -v -s -m 'not slow_test' v1/spec_decode
+  - bash standalone_tests/python_only_compile.sh

 #########################################################################################################################################
 #                                                                                                                                       #
@@ -3114,6 +3244,7 @@ steps:
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
  agent_pool: mi355_2
+  num_gpus: 2
  optional: true
  working_dir: "/vllm-workspace/tests"
  source_file_dependencies:
@@ -3288,6 +3419,7 @@ steps:
  timeout_in_minutes: 180
  mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
  agent_pool: mi355_2
+  num_gpus: 2
  working_dir: "/vllm-workspace/tests"
  source_file_dependencies:
  - csrc/moe/
@@ -13,6 +13,7 @@ steps:
  - pytest -v -s benchmarks/
  mirror:
    amd:
+      dind: false
      device: mi300_1
      depends_on:
      - image-build-amd
@@ -15,6 +15,7 @@ steps:
    - bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
  mirror:
    amd:
+      dind: false
      device: mi300_4
      timeout_in_minutes: 85
      depends_on:
@@ -65,6 +66,7 @@ steps:
    - DP_EP=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
  mirror:
    amd:
+      dind: false
      device: mi300_4
      timeout_in_minutes: 60
      depends_on:
@@ -90,6 +92,7 @@ steps:
    - CROSS_LAYERS_BLOCKS=True bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
  mirror:
    amd:
+      dind: false
      device: mi300_4
      timeout_in_minutes: 85
      depends_on:
@@ -115,6 +118,7 @@ steps:
    - HYBRID_SSM=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
  mirror:
    amd:
+      dind: false
      device: mi300_4
      timeout_in_minutes: 80
      depends_on:
@@ -171,6 +175,7 @@ steps:
    - bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh
  mirror:
    amd:
+      dind: false
      device: mi300_2
      timeout_in_minutes: 70
      depends_on:
@@ -182,7 +187,7 @@ steps:
        - vllm/platforms/rocm.py
      commands:
        - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
-        - ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh
+        - KV_CACHE_MEMORY_BYTES=8G ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_spec_decode_test.sh

 - label: MultiConnector (Nixl+Offloading) PD edge cases (2 GPUs)
  key: multiconnector-nixl-offloading-pd-edge-cases-2-gpus
@@ -39,6 +39,7 @@ steps:
  - DP_SIZE=2 pytest -v -s entrypoints/openai/test_multi_api_servers.py
  mirror:
    amd:
+      dind: false
      device: mi300_2
      depends_on:
      - image-build-amd
@@ -114,6 +114,7 @@ steps:
    - pytest -v -s v1/e2e/spec_decode/test_spec_decode.py -k "tensor_parallelism"
  mirror:
    amd:
+      dind: false
      device: mi300_2
      depends_on:
      - image-build-amd
@@ -16,6 +16,7 @@ steps:
  - pytest -v -s distributed/test_eplb_utils.py
  mirror:
    amd:
+      dind: false
      device: mi300_1
      depends_on:
      - image-build-amd
@@ -117,6 +117,7 @@ steps:
    - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200.txt
  mirror:
    amd:
+      dind: false
      device: mi300_8
      timeout_in_minutes: 60
      depends_on:
@@ -23,6 +23,7 @@ steps:
    - pytest -v -s -m 'not slow_test' v1/spec_decode
  mirror:
    amd:
+      dind: false
      device: mi300_1
      timeout_in_minutes: 75
      depends_on:
@@ -25,6 +25,7 @@ steps:
    - pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py --timeout=900 --timeout-method=thread
  mirror:
    amd:
+      dind: false
      device: mi300_1
      depends_on:
      - image-build-amd
@@ -15,6 +15,7 @@ steps:
    - pytest -v -s models/language -m 'core_model and (not slow_test)'
  mirror:
    amd:
+      dind: false
      device: mi300_1
      depends_on:
      - image-build-amd
@@ -35,6 +36,7 @@ steps:
  parallelism: 2
  mirror:
    amd:
+      dind: false
      device: mi300_1
      depends_on:
      - image-build-amd
@@ -107,6 +107,7 @@ steps:
  - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-mm-small.txt --tp-size=1
  mirror:
    amd:
+      dind: false
      device: mi300_1
      depends_on:
      - image-build-amd
@@ -109,6 +109,7 @@ steps:
  - pytest -s -v compile/passes --ignore compile/passes/distributed
  mirror:
    amd:
+      dind: false
      device: mi300_1
      timeout_in_minutes: 65
      depends_on:
@@ -197,6 +198,7 @@ steps:
  - bash standalone_tests/pytorch_nightly_dependency.sh
  mirror:
    amd:
+      dind: false
      device: mi300_1
      depends_on:
      - image-build-amd
@@ -15,6 +15,7 @@ steps:
    - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models.txt
  mirror:
    amd:
+      dind: false
      device: mi300_2
      depends_on:
      - image-build-amd
@@ -674,6 +674,7 @@ RUN apt-get update -q -y && apt-get install -q -y --no-install-recommends \
    libibverbs1 \
    ibverbs-providers \
    ibverbs-utils \
+    util-linux \
    unzip \
    pkg-config ffmpeg libavcodec-dev libavformat-dev libavutil-dev \
    libswscale-dev libavdevice-dev libavfilter-dev libswresample-dev \
@@ -31,7 +31,7 @@ def test_python_error():
        x = torch.empty(alloc_bytes, dtype=torch.uint8, device=DEVICE_TYPE)
        tensors.append(x)
    # release the memory
-    allocator.sleep()
+    allocator.sleep(offload_tags=())

    # allocate more memory than the total memory
    y = torch.empty(alloc_bytes, dtype=torch.uint8, device=DEVICE_TYPE)
@@ -5,16 +5,16 @@
 set -e

 # ROCm CI runs this script inside `run-amd-test.sh` where /vllm-workspace often has no .git
-# (wheel artifact layout). The wrapper passes VLLM_STANDALONE_MERGE_BASE from the agent checkout.
+# (wheel artifact layout). The wrapper passes CI_STANDALONE_MERGE_BASE from the agent checkout.
 merge_base_commit=""
-if [[ -n "${VLLM_STANDALONE_MERGE_BASE:-}" ]]; then
-    merge_base_commit="${VLLM_STANDALONE_MERGE_BASE}"
+if [[ -n "${CI_STANDALONE_MERGE_BASE:-}" ]]; then
+    merge_base_commit="${CI_STANDALONE_MERGE_BASE}"
 elif merge_base_commit="$(git -C /vllm-workspace merge-base HEAD origin/main 2>/dev/null)"; then
    :
 elif merge_base_commit="$(git merge-base HEAD origin/main 2>/dev/null)"; then
    :
 else
-    echo "ERROR: need a git checkout or VLLM_STANDALONE_MERGE_BASE to resolve wheels.vllm.ai commit." >&2
+    echo "ERROR: need a git checkout or CI_STANDALONE_MERGE_BASE to resolve wheels.vllm.ai commit." >&2
    exit 1
 fi

@@ -20,7 +20,8 @@
 #   SD_MODEL            - drafter model path
 #   MODEL_NAME          - target model (default: meta-llama/Llama-3.1-8B-Instruct)
 #   NUM_SPEC_TOKENS     - number of speculative tokens (default: 3)
-#   GPU_MEMORY_UTILIZATION - (default: 0.7)
+#   GPU_MEMORY_UTILIZATION - used when KV_CACHE_MEMORY_BYTES is unset (default: 0.7)
+#   KV_CACHE_MEMORY_BYTES - optional KV cache size per server
 #   ATTENTION_BACKEND   - attention backend to use
 #                         Default: TRITON_ATTN on ROCm, FLASH_ATTN on NVIDIA
 #                         ROCm options: TRITON_ATTN, ROCM_ATTN, ROCM_AITER_FA,
@@ -52,10 +53,17 @@ NUM_DECODE_INSTANCES=${NUM_DECODE_INSTANCES:-1}
 PREFILLER_TP_SIZE=${PREFILLER_TP_SIZE:-1}
 DECODER_TP_SIZE=${DECODER_TP_SIZE:-1}
 GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.7}
+KV_CACHE_MEMORY_BYTES=${KV_CACHE_MEMORY_BYTES:-}
 BLOCK_SIZE=${BLOCK_SIZE:-16}
 SERVER_HOST="${SERVER_HOST:-127.0.0.1}"
 NIXL_SIDE_CHANNEL_HOST="${NIXL_SIDE_CHANNEL_HOST:-$SERVER_HOST}"

+if [[ -n "$KV_CACHE_MEMORY_BYTES" ]]; then
+  KV_CACHE_ARGS=(--kv-cache-memory-bytes "$KV_CACHE_MEMORY_BYTES")
+else
+  KV_CACHE_ARGS=(--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION")
+fi
+
 # Resolve the repository root from the script location instead of `.git`.
 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
 GIT_ROOT="${GIT_ROOT:-$(cd -- "${SCRIPT_DIR}/../../../.." && pwd -P)}"
@@ -209,6 +217,7 @@ run_test_for_device() {
  echo "SD model:           ${SD_MODEL}"
  echo "Spec tokens:        ${NUM_SPEC_TOKENS}"
  echo "KV buffer device:   ${kv_device}"
+  echo "KV cache memory:    ${KV_CACHE_MEMORY_BYTES:-auto-sized}"
  echo "Attention backend:  ${ATTENTION_BACKEND}"
  echo "GPU platform:       ${GPU_PLATFORM}"
  echo "Server host:        ${SERVER_HOST}"
@@ -248,7 +257,7 @@ run_test_for_device() {
      --enforce-eager \
      --max-model-len $MAX_MODEL_LEN \
      --block-size ${BLOCK_SIZE} \
-      --gpu-memory-utilization $GPU_MEMORY_UTILIZATION \
+      "${KV_CACHE_ARGS[@]}" \
      --tensor-parallel-size $PREFILLER_TP_SIZE \
      --kv-transfer-config "$kv_config_p" \
      --speculative-config "$PREFILL_SPEC_CONFIG" \
@@ -287,7 +296,7 @@ run_test_for_device() {
      --enforce-eager \
      --max-model-len $MAX_MODEL_LEN \
      --block-size ${BLOCK_SIZE} \
-      --gpu-memory-utilization $GPU_MEMORY_UTILIZATION \
+      "${KV_CACHE_ARGS[@]}" \
      --tensor-parallel-size $DECODER_TP_SIZE \
      --kv-transfer-config "$kv_config_d" \
      --speculative-config "$DECODE_SPEC_CONFIG" \
@@ -97,14 +97,16 @@ def _build_worker_with_dict_store(vllm_config, kv_cache_config, store):
    with (
        patch.dict(sys.modules, {"mooncake.store": fake_mooncake_store}),
        patch.object(mooncake_store_worker, "MooncakeStoreConfig") as MCfg,
+        patch.object(mooncake_store_worker, "LookupKeyServer"),
    ):
-        sc = MCfg.load_from_env.return_value
+        sc = MCfg.load_from_config.return_value
        sc.metadata_server = ""
        sc.global_segment_size = 1 << 20
        sc.local_buffer_size = 1 << 20
        sc.protocol = "tcp"
        sc.device_name = ""
        sc.master_server_address = ""
+        sc.mode = "embedded"
        sc.enable_offload = False
        with (
            patch(
@@ -694,10 +694,22 @@ def _get_checkpoints_size_bytes(files: list[str]) -> int:


 def _get_available_ram_bytes() -> int:
-    """Return the available RAM in bytes."""
+    """Return available RAM, honoring cgroup limits on ROCm."""
    import psutil

-    return psutil.virtual_memory().available
+    host_available = psutil.virtual_memory().available
+    if not current_platform.is_rocm():
+        return host_available
+
+    from vllm.utils.cpu_resource_utils import get_cgroup_memory_limit
+
+    cgroup_limit, cgroup_usage = get_cgroup_memory_limit()
+    if cgroup_limit is None:
+        return host_available
+    cgroup_available = (
+        cgroup_limit if cgroup_usage is None else max(0, cgroup_limit - cgroup_usage)
+    )
+    return min(host_available, cgroup_available)


 def _get_fs_type(files: list[str]) -> str: