[CI][AMD] Retry Hugging Face tests online

Signed-off-by: Andreas Karatzas <akaratza@amd.com>

.buildkite/scripts/hardware_ci/run-amd-test.sh

@@ -56,6 +56,15 @@ fi
 export BUILDKIT_PROGRESS TERM FORCE_COLOR CLICOLOR_FORCE PY_COLORS PYTEST_ADDOPTS PYTEST_TIMEOUT ROCM_DOCKER_TTY
 export PYTHONFAULTHANDLER
 
+# The AMD pipeline template sets this for eligible pilot jobs. Capture it
+# before clear_ci_orchestration_env removes CI-only controls from test processes.
+hf_offline_retry_enabled="${VLLM_CI_HF_OFFLINE_RETRY:-0}"
+if [[ "${hf_offline_retry_enabled}" != "0" && "${hf_offline_retry_enabled}" != "1" ]]; then
+  echo "VLLM_CI_HF_OFFLINE_RETRY must be 0 or 1" >&2
+  exit 2
+fi
+hf_retry_command_file=""
+
 # Export Python path for commands that run directly on the host. Containerized
 # tests set this to /vllm-workspace below so spawned Python processes do not
 # depend on their current working directory.
@@ -73,6 +82,7 @@ report_docker_usage() {
 clear_ci_orchestration_env() {
   unset -v \
     VLLM_TEST_GROUP_NAME \
+    VLLM_CI_HF_OFFLINE_RETRY \
     VLLM_CI_REQUIRE_PERSISTENT_HF_CACHE \
     VLLM_CI_ARTIFACT_STEP \
     VLLM_TEST_CACHE \
@@ -92,6 +102,29 @@ clear_ci_orchestration_env() {
     VLLM_ALLOW_DEPRECATED_BEAM_SEARCH
 }
 
+prepare_hf_retry_command_file() {
+  local command_text=$1
+
+  if ! hf_retry_command_file=$(mktemp "${TMPDIR:-/tmp}/vllm-hf-offline-command.XXXXXX"); then
+    echo "Failed to create the Hugging Face retry command file" >&2
+    return 1
+  fi
+  if ! printf '%s\n' "${command_text}" >"${hf_retry_command_file}"; then
+    echo "Failed to write the Hugging Face retry command file" >&2
+    rm -f -- "${hf_retry_command_file}"
+    hf_retry_command_file=""
+    return 1
+  fi
+}
+
+# shellcheck disable=SC2329  # Called by cleanup functions registered as traps.
+cleanup_hf_retry_command_file() {
+  if [[ -n "${hf_retry_command_file}" ]]; then
+    rm -f -- "${hf_retry_command_file}"
+    hf_retry_command_file=""
+  fi
+}
+
 cleanup_network() {
   local max_nodes=${NUM_NODES:-2}
   for node in $(seq 0 $((max_nodes - 1))); do
@@ -649,6 +682,7 @@ if is_native_runtime; then
   artifact_work_dir=""
 
   cleanup_native_workspace() {
+    cleanup_hf_retry_command_file
     if [[ -n "${artifact_work_dir}" ]]; then
       rm -rf "${artifact_work_dir}"
     fi
@@ -691,7 +725,13 @@ if is_native_runtime; then
   run_native_preflight || exit 1
   # Keep AMD CI orchestration variables out of vLLM's runtime environment.
   clear_ci_orchestration_env
-  /bin/bash -o pipefail -c "${commands}"
+  if [[ "${hf_offline_retry_enabled}" == "1" ]]; then
+    prepare_hf_retry_command_file "${commands}" || exit 1
+    bash "${VLLM_CI_WORKSPACE:-/vllm-workspace}/.buildkite/scripts/hf-offline-retry.sh" \
+      "${hf_retry_command_file}"
+  else
+    /bin/bash -o pipefail -c "${commands}"
+  fi
   handle_pytest_exit "$?"
 fi
 
@@ -709,6 +749,7 @@ artifact_work_dir=""
 container_name="rocm_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10; echo)"
 
 remove_docker_container() {
+  cleanup_hf_retry_command_file
   if docker container inspect "${container_name}" >/dev/null 2>&1; then
     docker rm -f "${container_name}" || true
   fi
@@ -850,6 +891,10 @@ fi
 # --- Route: multi-node vs single-node ---
 clear_ci_orchestration_env
 if is_multi_node "$commands"; then
+  if [[ "${hf_offline_retry_enabled}" == "1" ]]; then
+    echo "Hugging Face offline retry is not enabled for AMD multi-node jobs yet" >&2
+    exit 2
+  fi
   echo "--- Multi-node job detected"
   export DCKR_VER=$(docker --version | sed 's/Docker version \(.*\), build .*/\1/')
 
@@ -894,6 +939,16 @@ if is_multi_node "$commands"; then
   fi
 else
   echo "--- Single-node job"
+  hf_retry_container_args=()
+  container_test_command="${commands}"
+  if [[ "${hf_offline_retry_enabled}" == "1" ]]; then
+    prepare_hf_retry_command_file "${commands}" || exit 1
+    hf_retry_container_path="/tmp/vllm-hf-offline-command.sh"
+    hf_retry_container_args=(
+      -v "${hf_retry_command_file}:${hf_retry_container_path}:ro"
+    )
+    container_test_command="bash /vllm-workspace/.buildkite/scripts/hf-offline-retry.sh ${hf_retry_container_path}"
+  fi
   echo "Render devices: $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES"
   docker_run_terminal_args=(-i)
   if [[ "${ROCM_DOCKER_TTY}" == "1" ]]; then
@@ -950,9 +1005,10 @@ else
     -e "XDG_CACHE_HOME=${CONTAINER_CACHE_ROOT}/xdg" \
     -e "PYTORCH_ROCM_ARCH=" \
     "${standalone_merge_base_env[@]}" \
+    "${hf_retry_container_args[@]}" \
     --name "${container_name}" \
     "${image_name}" \
-    /bin/bash -c "${CONTAINER_PREFLIGHT} && ${commands}"
+    /bin/bash -c "${CONTAINER_PREFLIGHT} && ${container_test_command}"
 
   exit_code=$?
   handle_pytest_exit "$exit_code"

.buildkite/scripts/hf-offline-retry-self-test.sh

@@ -0,0 +1,122 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
+helper="${script_dir}/hf-offline-retry.sh"
+test_root=$(mktemp -d "${TMPDIR:-/tmp}/vllm-hf-offline-retry-test.XXXXXX")
+
+cleanup() {
+  rm -rf -- "${test_root}"
+}
+trap cleanup EXIT
+
+assert_equal() {
+  local expected=$1
+  local actual=$2
+  local description=$3
+
+  if [[ "${actual}" != "${expected}" ]]; then
+    printf 'FAIL: %s\n  expected: %q\n  actual:   %q\n' \
+      "${description}" "${expected}" "${actual}" >&2
+    exit 1
+  fi
+}
+
+status_command_file="${test_root}/status-command.sh"
+cat >"${status_command_file}" <<'VLLM_TEST_COMMAND'
+if [[ "${HF_HUB_OFFLINE-}" == "1" &&
+      "${TRANSFORMERS_OFFLINE-}" == "1" &&
+      "${HF_DATASETS_OFFLINE-}" == "1" ]]; then
+  printf '%s\n' offline >>"${ATTEMPTS_FILE}"
+  exit "${OFFLINE_STATUS}"
+fi
+
+if [[ -z "${HF_HUB_OFFLINE+x}" &&
+      -z "${TRANSFORMERS_OFFLINE+x}" &&
+      -z "${HF_DATASETS_OFFLINE+x}" ]]; then
+  printf '%s\n' online >>"${ATTEMPTS_FILE}"
+  exit "${ONLINE_STATUS}"
+fi
+
+exit 91
+VLLM_TEST_COMMAND
+
+run_status_case() {
+  local name=$1
+  local offline_status=$2
+  local online_status=$3
+  local expected_status=$4
+  local expected_attempts=$5
+  local attempts_file="${test_root}/${name}.attempts"
+  local actual_status
+  local actual_attempts
+
+  set +e
+  ATTEMPTS_FILE="${attempts_file}" \
+    OFFLINE_STATUS="${offline_status}" \
+    ONLINE_STATUS="${online_status}" \
+    bash "${helper}" "${status_command_file}"
+  actual_status=$?
+  set -e
+
+  actual_attempts=$(<"${attempts_file}")
+  assert_equal "${expected_status}" "${actual_status}" "${name} exit status"
+  assert_equal "${expected_attempts}" "${actual_attempts}" "${name} attempts"
+}
+
+run_status_case offline_success 0 99 0 "offline"
+run_status_case retry_status_1 1 0 0 $'offline\nonline'
+run_status_case retry_status_2 2 0 0 $'offline\nonline'
+run_status_case retry_status_123 123 0 0 $'offline\nonline'
+run_status_case no_retry_status_42 42 0 42 "offline"
+run_status_case online_failure 1 7 7 $'offline\nonline'
+
+transport_file="${test_root}/transport.out"
+transport_command_file="${test_root}/transport-command.sh"
+cat >"${transport_command_file}" <<'VLLM_TRANSPORT_COMMAND'
+cat >"${OUTPUT_FILE}" <<'INNER_PAYLOAD'
+literal:$HOME
+quotes:'single' "double"
+two command-file lines
+INNER_PAYLOAD
+printf 'expanded:%s\n' "${TRANSPORT_VALUE}" >>"${OUTPUT_FILE}"
+VLLM_TRANSPORT_COMMAND
+
+OUTPUT_FILE="${transport_file}" \
+  TRANSPORT_VALUE='value with spaces, $dollars, and "quotes"' \
+  bash "${helper}" "${transport_command_file}"
+
+transport_output=$(<"${transport_file}")
+assert_equal \
+  $'literal:$HOME\nquotes:\'single\' "double"\ntwo command-file lines\nexpanded:value with spaces, $dollars, and "quotes"' \
+  "${transport_output}" \
+  "quotes, dollars, and newlines"
+
+stdin_file="${test_root}/stdin.out"
+stdin_payload="stdin payload with \$dollars and \"quotes\""
+stdin_command_file="${test_root}/stdin-command.sh"
+cat >"${stdin_command_file}" <<'VLLM_STDIN_COMMAND'
+IFS= read -r input
+printf '%s\n' "${input}" >"${OUTPUT_FILE}"
+VLLM_STDIN_COMMAND
+
+printf '%s\n' "${stdin_payload}" |
+  OUTPUT_FILE="${stdin_file}" bash "${helper}" "${stdin_command_file}"
+
+stdin_output=$(<"${stdin_file}")
+assert_equal \
+  "${stdin_payload}" \
+  "${stdin_output}" \
+  "stdin preservation"
+
+set +e
+bash "${helper}" >/dev/null 2>&1
+usage_status=$?
+bash "${helper}" "${test_root}/missing-command" >/dev/null 2>&1
+missing_status=$?
+set -e
+assert_equal "64" "${usage_status}" "missing argument exit status"
+assert_equal "66" "${missing_status}" "unreadable command file exit status"
+
+echo "PASS: hf-offline-retry"

.buildkite/scripts/hf-offline-retry.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+set -uo pipefail
+
+if [[ $# -ne 1 ]]; then
+  echo "Usage: $0 COMMAND_FILE" >&2
+  exit 64
+fi
+
+command_file=$1
+if [[ ! -f "${command_file}" || ! -r "${command_file}" ]]; then
+  echo "Command file is not readable: ${command_file}" >&2
+  exit 66
+fi
+
+run_attempt() (
+  if [[ "$1" == "offline" ]]; then
+    export HF_HUB_OFFLINE=1
+    export TRANSFORMERS_OFFLINE=1
+    export HF_DATASETS_OFFLINE=1
+  else
+    unset HF_HUB_OFFLINE
+    unset TRANSFORMERS_OFFLINE
+    unset HF_DATASETS_OFFLINE
+  fi
+
+  bash -e -o pipefail -- "${command_file}"
+)
+
+echo "--- :package: Hugging Face offline attempt"
+run_attempt offline
+offline_status=$?
+
+if [[ ${offline_status} -eq 0 ]]; then
+  exit 0
+fi
+
+case "${offline_status}" in
+  1 | 2 | 123)
+    echo "--- :globe_with_meridians: Hugging Face offline attempt failed; retrying online"
+    run_attempt online
+    online_status=$?
+    exit "${online_status}"
+    ;;
+  *)
+    exit "${offline_status}"
+    ;;
+esac

.buildkite/test-amd.yaml

@@ -25,6 +25,9 @@
 #     and $$BUILDKITE_PARALLEL_JOB_COUNT environment variables.
 # working_dir(str): specify the place where the command should execute, default to /vllm-workspace/tests
 # source_file_dependencies(list): the list of prefixes to opt-in the test for, if empty, the test will always run.
+# hf_offline_retry(bool): for single-node run-amd-test jobs, run offline first
+#     and retry eligible failures online. May be set as the file default or a
+#     per-step override.
 
 # When adding a test
 # - If the test belongs to an existing group, add it there
@@ -105,6 +108,8 @@
 #####################################################################################################################################
 
 
+# AMD-only pilot: test jobs inherit this; non-test jobs opt out below.
+hf_offline_retry: true
 steps:
 
 #########################################################################################################################################
@@ -411,6 +416,7 @@ steps:
 #------------------------------------------------------------  mi250 · rust  -----------------------------------------------------------#
 
 - label: Rust Frontend Cargo Style + Clippy # TBD
+  hf_offline_retry: false
   timeout_in_minutes: 180
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
   agent_pool: mi250_1
@@ -425,6 +431,7 @@ steps:
   - bash .buildkite/scripts/run-rust-frontend-cargo-ci.sh style-clippy
 
 - label: Rust Frontend Cargo Tests # TBD
+  hf_offline_retry: false
   timeout_in_minutes: 180
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
   agent_pool: mi250_1
@@ -441,6 +448,7 @@ steps:
 #-----------------------------------------------------------  mi250 · docker  ----------------------------------------------------------#
 
 - label: Docker Build Metadata (ROCm) # TBD
+  hf_offline_retry: false
   timeout_in_minutes: 180
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
   agent_pool: mi250_1
@@ -2264,6 +2272,7 @@ steps:
 #---------------------------------------------------------  mi300 · ray_compat  ---------------------------------------------------------#
 
 - label: Ray Dependency Compatibility Check # TBD
+  hf_offline_retry: false
   timeout_in_minutes: 180
   mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
   dind: false