[Misc] Remove redundant engine arg tokens_only

Signed-off-by: Zhuohan Li <zhuohan123@gmail.com>
2025-12-01 17:28:00 -08:00
1 changed files with 0 additions and 5 deletions
@@ -571,7 +571,6 @@ class EngineArgs:
    kv_offloading_backend: KVOffloadingBackend | None = (
        CacheConfig.kv_offloading_backend
    )
-    tokens_only: bool = False

    def __post_init__(self):
        # support `EngineArgs(compilation_config={...})`
@@ -1571,10 +1570,6 @@ class EngineArgs:
            else ParallelConfig.data_parallel_rpc_port
        )

-        if self.tokens_only and not model_config.skip_tokenizer_init:
-            model_config.skip_tokenizer_init = True
-            logger.info("Skipping tokenizer initialization for tokens-only mode.")
-
        if self.async_scheduling and not self.disable_nccl_for_dp_synchronization:
            logger.info(
                "Disabling NCCL for DP synchronization when using async scheduling."