From dc8df110bc8a5c80dbb8f1fef02e98deb8aa2b09 Mon Sep 17 00:00:00 2001 From: Albert Cheng Date: Tue, 14 Apr 2026 11:43:05 -0700 Subject: [PATCH] add warning when FP8 KV cache misses prefill query quantization (#39752) Signed-off-by: Michael Goin Co-authored-by: Albert Cheng (Engrg-Hardware 1) Co-authored-by: Michael Goin Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- .../layers/attention/mla_attention.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/vllm/model_executor/layers/attention/mla_attention.py b/vllm/model_executor/layers/attention/mla_attention.py index 9836a3198d8..cbbf5f3c3ca 100644 --- a/vllm/model_executor/layers/attention/mla_attention.py +++ b/vllm/model_executor/layers/attention/mla_attention.py @@ -1443,6 +1443,19 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]): scope="local", ) return model_dtype + elif ( + is_quantized_kv_cache(vllm_config.cache_config.cache_dtype) + and backend_supports_prefill_query_quantization() + ): + logger.warning_once( + "FP8 KV cache is enabled but prefill queries are not " + "quantized to FP8. For long-context workloads (ISL >= 4K), " + "enabling FP8 prefill attention can significantly optimize " + "prefill latency. To enable, add: " + '--attention-config \'{"use_prefill_query_quantization"' + ": true}'", + scope="local", + ) return model_dtype