diff --git a/vllm/v1/worker/cpu_worker.py b/vllm/v1/worker/cpu_worker.py
index 72503ab9ab7..1474d7ecb7a 100644
--- a/vllm/v1/worker/cpu_worker.py
+++ b/vllm/v1/worker/cpu_worker.py
@@ -213,10 +213,13 @@ class CPUWorker(Worker):
         return kv_cache_size
 
     def compile_or_warm_up_model(self) -> CompilationTimes:
+        # Note: the model has been compiled in determine_available_memory(),
+        # Only compile here for models without kv cache
+        if len(self.model_runner.kv_caches) == 0:
+            self.model_runner.warming_up_model()
         # Reset the seed to ensure that the random state is not affected by
         # the model initialization and profiling.
         set_random_seed(self.model_config.seed)
-        # Note: the model has been compiled in determine_available_memory()
         return CompilationTimes(
             language_model=self.compilation_config.compilation_time,
             encoder=self.compilation_config.encoder_compilation_time,