[CI] Reuse loaded config for cached tokenizer (#49509)

Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>

tests/tokenizers_/test_registry.py

@@ -109,6 +109,8 @@ def test_cached_tokenizer_from_config_registers_local_config(tmp_path: Path):
     try:
 
         def fake_from_pretrained(path_or_repo_id: str, *args, **kwargs):
+            passed_config = kwargs.pop("config")
+            assert isinstance(passed_config, Qwen3_5MoeConfig)
             loaded_config = AutoConfig.from_pretrained(
                 path_or_repo_id,
                 trust_remote_code=False,

vllm/tokenizers/registry.py

@@ -18,6 +18,7 @@ from vllm.transformers_utils.repo_utils import (
 )
 from vllm.utils.import_utils import resolve_obj_by_qualname
 
+from .hf import CachedHfTokenizer
 from .protocol import TokenizerLike
 
 if TYPE_CHECKING:
@@ -233,6 +234,12 @@ def get_tokenizer(
     else:
         tokenizer_cls_ = tokenizer_cls
 
+    if config is not None and tokenizer_cls_ is CachedHfTokenizer:
+        # AutoTokenizer otherwise reloads config.json internally. Reuse the
+        # config that get_config just loaded successfully so a concurrent Hub
+        # cache refresh cannot invalidate the file between the two reads.
+        kwargs.setdefault("config", config)
+
     tokenizer = tokenizer_cls_.from_pretrained(tokenizer_name, *args, **kwargs)
     if model_type in _MODEL_TYPES_WITH_INCORRECT_TOKENIZER_CLASS:
         from vllm.tokenizers.hf import get_cached_tokenizer