From da99ffcc13362ab446a22a9bc6ed36c4c14942e3 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Sun, 26 Jul 2026 22:31:24 -0500 Subject: [PATCH] [ROCm][CI] Keep native datasets cache off shared NFS (#49516) Signed-off-by: Andreas Karatzas --- .buildkite/scripts/hardware_ci/run-amd-test.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.buildkite/scripts/hardware_ci/run-amd-test.sh b/.buildkite/scripts/hardware_ci/run-amd-test.sh index 0f6bb2564c9..ed4a9ba3a2e 100755 --- a/.buildkite/scripts/hardware_ci/run-amd-test.sh +++ b/.buildkite/scripts/hardware_ci/run-amd-test.sh @@ -405,11 +405,14 @@ initialize_native_environment() { VLLM_CACHE_ROOT="${native_root}/cache/vllm" XDG_CACHE_HOME="${native_root}/cache/xdg" : "${HF_HOME:=/home/buildkite-agent/huggingface}" + # datasets uses POSIX locks that are unsupported by the shared HF NFS cache. + # Keep processed datasets job-local while retaining the persistent Hub cache. + HF_DATASETS_CACHE="${native_root}/cache/huggingface/datasets" : "${HF_HUB_DOWNLOAD_TIMEOUT:=300}" : "${HF_HUB_ETAG_TIMEOUT:=60}" export TMPDIR VLLM_RPC_BASE_PATH export TORCHINDUCTOR_CACHE_DIR TRITON_CACHE_DIR VLLM_CACHE_ROOT XDG_CACHE_HOME - export HF_HOME HF_HUB_DOWNLOAD_TIMEOUT HF_HUB_ETAG_TIMEOUT + export HF_HOME HF_DATASETS_CACHE HF_HUB_DOWNLOAD_TIMEOUT HF_HUB_ETAG_TIMEOUT export PYTORCH_ROCM_ARCH="" mkdir -p "${TMPDIR}" \ @@ -417,7 +420,8 @@ initialize_native_environment() { "${TRITON_CACHE_DIR}" \ "${VLLM_CACHE_ROOT}" \ "${XDG_CACHE_HOME}" \ - "${HF_HOME}" || return 1 + "${HF_HOME}" \ + "${HF_DATASETS_CACHE}" || return 1 echo "Native compile caches: VLLM_CACHE_ROOT=${VLLM_CACHE_ROOT} TORCHINDUCTOR_CACHE_DIR=${TORCHINDUCTOR_CACHE_DIR}"