From d7cbffd5f8c7fba3f147532bf2fbc2e4ab3f3c2d Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Thu, 23 Jul 2026 03:30:26 +0000 Subject: [PATCH] [CI][AMD] Retry Hugging Face tests online Signed-off-by: Andreas Karatzas --- .../scripts/hardware_ci/run-amd-test.sh | 60 ++++++++- .../scripts/hf-offline-retry-self-test.sh | 122 ++++++++++++++++++ .buildkite/scripts/hf-offline-retry.sh | 48 +++++++ .buildkite/test-amd.yaml | 9 ++ 4 files changed, 237 insertions(+), 2 deletions(-) create mode 100644 .buildkite/scripts/hf-offline-retry-self-test.sh create mode 100644 .buildkite/scripts/hf-offline-retry.sh diff --git a/.buildkite/scripts/hardware_ci/run-amd-test.sh b/.buildkite/scripts/hardware_ci/run-amd-test.sh index 4e8735c841a..0de9249983c 100755 --- a/.buildkite/scripts/hardware_ci/run-amd-test.sh +++ b/.buildkite/scripts/hardware_ci/run-amd-test.sh @@ -56,6 +56,15 @@ fi export BUILDKIT_PROGRESS TERM FORCE_COLOR CLICOLOR_FORCE PY_COLORS PYTEST_ADDOPTS PYTEST_TIMEOUT ROCM_DOCKER_TTY export PYTHONFAULTHANDLER +# The AMD pipeline template sets this for eligible pilot jobs. Capture it +# before clear_ci_orchestration_env removes CI-only controls from test processes. +hf_offline_retry_enabled="${VLLM_CI_HF_OFFLINE_RETRY:-0}" +if [[ "${hf_offline_retry_enabled}" != "0" && "${hf_offline_retry_enabled}" != "1" ]]; then + echo "VLLM_CI_HF_OFFLINE_RETRY must be 0 or 1" >&2 + exit 2 +fi +hf_retry_command_file="" + # Export Python path for commands that run directly on the host. Containerized # tests set this to /vllm-workspace below so spawned Python processes do not # depend on their current working directory. @@ -73,6 +82,7 @@ report_docker_usage() { clear_ci_orchestration_env() { unset -v \ VLLM_TEST_GROUP_NAME \ + VLLM_CI_HF_OFFLINE_RETRY \ VLLM_CI_REQUIRE_PERSISTENT_HF_CACHE \ VLLM_CI_ARTIFACT_STEP \ VLLM_TEST_CACHE \ @@ -92,6 +102,29 @@ clear_ci_orchestration_env() { VLLM_ALLOW_DEPRECATED_BEAM_SEARCH } +prepare_hf_retry_command_file() { + local command_text=$1 + + if ! hf_retry_command_file=$(mktemp "${TMPDIR:-/tmp}/vllm-hf-offline-command.XXXXXX"); then + echo "Failed to create the Hugging Face retry command file" >&2 + return 1 + fi + if ! printf '%s\n' "${command_text}" >"${hf_retry_command_file}"; then + echo "Failed to write the Hugging Face retry command file" >&2 + rm -f -- "${hf_retry_command_file}" + hf_retry_command_file="" + return 1 + fi +} + +# shellcheck disable=SC2329 # Called by cleanup functions registered as traps. +cleanup_hf_retry_command_file() { + if [[ -n "${hf_retry_command_file}" ]]; then + rm -f -- "${hf_retry_command_file}" + hf_retry_command_file="" + fi +} + cleanup_network() { local max_nodes=${NUM_NODES:-2} for node in $(seq 0 $((max_nodes - 1))); do @@ -649,6 +682,7 @@ if is_native_runtime; then artifact_work_dir="" cleanup_native_workspace() { + cleanup_hf_retry_command_file if [[ -n "${artifact_work_dir}" ]]; then rm -rf "${artifact_work_dir}" fi @@ -691,7 +725,13 @@ if is_native_runtime; then run_native_preflight || exit 1 # Keep AMD CI orchestration variables out of vLLM's runtime environment. clear_ci_orchestration_env - /bin/bash -o pipefail -c "${commands}" + if [[ "${hf_offline_retry_enabled}" == "1" ]]; then + prepare_hf_retry_command_file "${commands}" || exit 1 + bash "${VLLM_CI_WORKSPACE:-/vllm-workspace}/.buildkite/scripts/hf-offline-retry.sh" \ + "${hf_retry_command_file}" + else + /bin/bash -o pipefail -c "${commands}" + fi handle_pytest_exit "$?" fi @@ -709,6 +749,7 @@ artifact_work_dir="" container_name="rocm_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10; echo)" remove_docker_container() { + cleanup_hf_retry_command_file if docker container inspect "${container_name}" >/dev/null 2>&1; then docker rm -f "${container_name}" || true fi @@ -850,6 +891,10 @@ fi # --- Route: multi-node vs single-node --- clear_ci_orchestration_env if is_multi_node "$commands"; then + if [[ "${hf_offline_retry_enabled}" == "1" ]]; then + echo "Hugging Face offline retry is not enabled for AMD multi-node jobs yet" >&2 + exit 2 + fi echo "--- Multi-node job detected" export DCKR_VER=$(docker --version | sed 's/Docker version \(.*\), build .*/\1/') @@ -894,6 +939,16 @@ if is_multi_node "$commands"; then fi else echo "--- Single-node job" + hf_retry_container_args=() + container_test_command="${commands}" + if [[ "${hf_offline_retry_enabled}" == "1" ]]; then + prepare_hf_retry_command_file "${commands}" || exit 1 + hf_retry_container_path="/tmp/vllm-hf-offline-command.sh" + hf_retry_container_args=( + -v "${hf_retry_command_file}:${hf_retry_container_path}:ro" + ) + container_test_command="bash /vllm-workspace/.buildkite/scripts/hf-offline-retry.sh ${hf_retry_container_path}" + fi echo "Render devices: $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES" docker_run_terminal_args=(-i) if [[ "${ROCM_DOCKER_TTY}" == "1" ]]; then @@ -950,9 +1005,10 @@ else -e "XDG_CACHE_HOME=${CONTAINER_CACHE_ROOT}/xdg" \ -e "PYTORCH_ROCM_ARCH=" \ "${standalone_merge_base_env[@]}" \ + "${hf_retry_container_args[@]}" \ --name "${container_name}" \ "${image_name}" \ - /bin/bash -c "${CONTAINER_PREFLIGHT} && ${commands}" + /bin/bash -c "${CONTAINER_PREFLIGHT} && ${container_test_command}" exit_code=$? handle_pytest_exit "$exit_code" diff --git a/.buildkite/scripts/hf-offline-retry-self-test.sh b/.buildkite/scripts/hf-offline-retry-self-test.sh new file mode 100644 index 00000000000..7a9e3e6c24b --- /dev/null +++ b/.buildkite/scripts/hf-offline-retry-self-test.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash + +set -euo pipefail + +script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +helper="${script_dir}/hf-offline-retry.sh" +test_root=$(mktemp -d "${TMPDIR:-/tmp}/vllm-hf-offline-retry-test.XXXXXX") + +cleanup() { + rm -rf -- "${test_root}" +} +trap cleanup EXIT + +assert_equal() { + local expected=$1 + local actual=$2 + local description=$3 + + if [[ "${actual}" != "${expected}" ]]; then + printf 'FAIL: %s\n expected: %q\n actual: %q\n' \ + "${description}" "${expected}" "${actual}" >&2 + exit 1 + fi +} + +status_command_file="${test_root}/status-command.sh" +cat >"${status_command_file}" <<'VLLM_TEST_COMMAND' +if [[ "${HF_HUB_OFFLINE-}" == "1" && + "${TRANSFORMERS_OFFLINE-}" == "1" && + "${HF_DATASETS_OFFLINE-}" == "1" ]]; then + printf '%s\n' offline >>"${ATTEMPTS_FILE}" + exit "${OFFLINE_STATUS}" +fi + +if [[ -z "${HF_HUB_OFFLINE+x}" && + -z "${TRANSFORMERS_OFFLINE+x}" && + -z "${HF_DATASETS_OFFLINE+x}" ]]; then + printf '%s\n' online >>"${ATTEMPTS_FILE}" + exit "${ONLINE_STATUS}" +fi + +exit 91 +VLLM_TEST_COMMAND + +run_status_case() { + local name=$1 + local offline_status=$2 + local online_status=$3 + local expected_status=$4 + local expected_attempts=$5 + local attempts_file="${test_root}/${name}.attempts" + local actual_status + local actual_attempts + + set +e + ATTEMPTS_FILE="${attempts_file}" \ + OFFLINE_STATUS="${offline_status}" \ + ONLINE_STATUS="${online_status}" \ + bash "${helper}" "${status_command_file}" + actual_status=$? + set -e + + actual_attempts=$(<"${attempts_file}") + assert_equal "${expected_status}" "${actual_status}" "${name} exit status" + assert_equal "${expected_attempts}" "${actual_attempts}" "${name} attempts" +} + +run_status_case offline_success 0 99 0 "offline" +run_status_case retry_status_1 1 0 0 $'offline\nonline' +run_status_case retry_status_2 2 0 0 $'offline\nonline' +run_status_case retry_status_123 123 0 0 $'offline\nonline' +run_status_case no_retry_status_42 42 0 42 "offline" +run_status_case online_failure 1 7 7 $'offline\nonline' + +transport_file="${test_root}/transport.out" +transport_command_file="${test_root}/transport-command.sh" +cat >"${transport_command_file}" <<'VLLM_TRANSPORT_COMMAND' +cat >"${OUTPUT_FILE}" <<'INNER_PAYLOAD' +literal:$HOME +quotes:'single' "double" +two command-file lines +INNER_PAYLOAD +printf 'expanded:%s\n' "${TRANSPORT_VALUE}" >>"${OUTPUT_FILE}" +VLLM_TRANSPORT_COMMAND + +OUTPUT_FILE="${transport_file}" \ + TRANSPORT_VALUE='value with spaces, $dollars, and "quotes"' \ + bash "${helper}" "${transport_command_file}" + +transport_output=$(<"${transport_file}") +assert_equal \ + $'literal:$HOME\nquotes:\'single\' "double"\ntwo command-file lines\nexpanded:value with spaces, $dollars, and "quotes"' \ + "${transport_output}" \ + "quotes, dollars, and newlines" + +stdin_file="${test_root}/stdin.out" +stdin_payload="stdin payload with \$dollars and \"quotes\"" +stdin_command_file="${test_root}/stdin-command.sh" +cat >"${stdin_command_file}" <<'VLLM_STDIN_COMMAND' +IFS= read -r input +printf '%s\n' "${input}" >"${OUTPUT_FILE}" +VLLM_STDIN_COMMAND + +printf '%s\n' "${stdin_payload}" | + OUTPUT_FILE="${stdin_file}" bash "${helper}" "${stdin_command_file}" + +stdin_output=$(<"${stdin_file}") +assert_equal \ + "${stdin_payload}" \ + "${stdin_output}" \ + "stdin preservation" + +set +e +bash "${helper}" >/dev/null 2>&1 +usage_status=$? +bash "${helper}" "${test_root}/missing-command" >/dev/null 2>&1 +missing_status=$? +set -e +assert_equal "64" "${usage_status}" "missing argument exit status" +assert_equal "66" "${missing_status}" "unreadable command file exit status" + +echo "PASS: hf-offline-retry" diff --git a/.buildkite/scripts/hf-offline-retry.sh b/.buildkite/scripts/hf-offline-retry.sh new file mode 100644 index 00000000000..1861ba761b6 --- /dev/null +++ b/.buildkite/scripts/hf-offline-retry.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +set -uo pipefail + +if [[ $# -ne 1 ]]; then + echo "Usage: $0 COMMAND_FILE" >&2 + exit 64 +fi + +command_file=$1 +if [[ ! -f "${command_file}" || ! -r "${command_file}" ]]; then + echo "Command file is not readable: ${command_file}" >&2 + exit 66 +fi + +run_attempt() ( + if [[ "$1" == "offline" ]]; then + export HF_HUB_OFFLINE=1 + export TRANSFORMERS_OFFLINE=1 + export HF_DATASETS_OFFLINE=1 + else + unset HF_HUB_OFFLINE + unset TRANSFORMERS_OFFLINE + unset HF_DATASETS_OFFLINE + fi + + bash -e -o pipefail -- "${command_file}" +) + +echo "--- :package: Hugging Face offline attempt" +run_attempt offline +offline_status=$? + +if [[ ${offline_status} -eq 0 ]]; then + exit 0 +fi + +case "${offline_status}" in + 1 | 2 | 123) + echo "--- :globe_with_meridians: Hugging Face offline attempt failed; retrying online" + run_attempt online + online_status=$? + exit "${online_status}" + ;; + *) + exit "${offline_status}" + ;; +esac diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index bbde36e970b..ee0f5faddb3 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -25,6 +25,9 @@ # and $$BUILDKITE_PARALLEL_JOB_COUNT environment variables. # working_dir(str): specify the place where the command should execute, default to /vllm-workspace/tests # source_file_dependencies(list): the list of prefixes to opt-in the test for, if empty, the test will always run. +# hf_offline_retry(bool): for single-node run-amd-test jobs, run offline first +# and retry eligible failures online. May be set as the file default or a +# per-step override. # When adding a test # - If the test belongs to an existing group, add it there @@ -105,6 +108,8 @@ ##################################################################################################################################### +# AMD-only pilot: test jobs inherit this; non-test jobs opt out below. +hf_offline_retry: true steps: ######################################################################################################################################### @@ -411,6 +416,7 @@ steps: #------------------------------------------------------------ mi250 · rust -----------------------------------------------------------# - label: Rust Frontend Cargo Style + Clippy # TBD + hf_offline_retry: false timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] agent_pool: mi250_1 @@ -425,6 +431,7 @@ steps: - bash .buildkite/scripts/run-rust-frontend-cargo-ci.sh style-clippy - label: Rust Frontend Cargo Tests # TBD + hf_offline_retry: false timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] agent_pool: mi250_1 @@ -441,6 +448,7 @@ steps: #----------------------------------------------------------- mi250 · docker ----------------------------------------------------------# - label: Docker Build Metadata (ROCm) # TBD + hf_offline_retry: false timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] agent_pool: mi250_1 @@ -2264,6 +2272,7 @@ steps: #--------------------------------------------------------- mi300 · ray_compat ---------------------------------------------------------# - label: Ray Dependency Compatibility Check # TBD + hf_offline_retry: false timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false