diff --git a/vllm/model_executor/layers/quantization/turboquant/__init__.py b/vllm/model_executor/layers/quantization/turboquant/__init__.py index 333aef5f825..f9f4384e6e2 100644 --- a/vllm/model_executor/layers/quantization/turboquant/__init__.py +++ b/vllm/model_executor/layers/quantization/turboquant/__init__.py @@ -5,15 +5,20 @@ Hadamard rotation + per-coordinate Lloyd-Max scalar quantization for keys, uniform quantization for values. -The technique implemented here consists of the scalar case of the HIGGS +The core algorithmic pattern implemented for key quantization (Hadamard +rotation followed by deterministic scalar quantization and +re-normalization) was originally established in DRIVE (Vargaftik et al., +NeurIPS 2021) and EDEN (Vargaftik et al., ICML 2022). This formulation is +also mathematically equivalent to the scalar case of the HIGGS quantization method (Malinovskii et al., "Pushing the Limits of Large Language Model Quantization via the Linearity Theorem", NAACL 2025; -preprint arXiv:2411.17525): rotation + optimized grid + optional -re-normalization, applied to KV cache compression. A first application -of this approach to KV-cache compression is in "Cache Me If You Must: -Adaptive Key-Value Quantization for Large Language Models" (Shutova -et al., ICML 2025; preprint arXiv:2501.19392). Both these references -pre-date the TurboQuant paper (Zandieh et al., ICLR 2026). +preprint arXiv:2411.17525), which subsequently generalized these concepts. + +A first application of this approach to KV-cache compression is in "Cache +Me If You Must: Adaptive Key-Value Quantization for Large Language Models" +(Shutova et al., ICML 2025; preprint arXiv:2501.19392). All of these +foundational and application references pre-date the TurboQuant paper +(Zandieh et al., ICLR 2026). """ from vllm.model_executor.layers.quantization.turboquant.config import TurboQuantConfig diff --git a/vllm/model_executor/layers/quantization/turboquant/config.py b/vllm/model_executor/layers/quantization/turboquant/config.py index 50beb8d1d9b..84e3940d264 100644 --- a/vllm/model_executor/layers/quantization/turboquant/config.py +++ b/vllm/model_executor/layers/quantization/turboquant/config.py @@ -48,19 +48,25 @@ class TurboQuantConfig: Applies Hadamard rotation followed by per-coordinate Lloyd-Max scalar quantization for keys, and uniform quantization for values. - Historical note: this is the scalar case of the HIGGS quantization - method (Malinovskii et al., "Pushing the Limits of Large Language Model - Quantization via the Linearity Theorem", NAACL 2025; preprint - arXiv:2411.17525): rotation + optimized grid + optional re-normalization, - applied to KV cache compression. A first application of this approach to - KV-cache compression is in "Cache Me If You Must: Adaptive Key-Value - Quantization for Large Language Models" (Shutova et al., ICML 2025; - preprint arXiv:2501.19392). Both these references pre-date the - TurboQuant paper. + Historical note: the core algorithmic pattern implemented for key + quantization (Hadamard rotation followed by deterministic scalar + quantization and re-normalization) was originally established in DRIVE + (Vargaftik et al., NeurIPS 2021) and EDEN (Vargaftik et al., ICML + 2022). This formulation is also mathematically equivalent to the + scalar case of the HIGGS quantization method (Malinovskii et al., + "Pushing the Limits of Large Language Model Quantization via the + Linearity Theorem", NAACL 2025; preprint arXiv:2411.17525), which + subsequently generalized these concepts. - QJL is intentionally omitted — community consensus (5+ independent - groups) found it hurts attention quality by amplifying variance through - softmax. + A first application of this approach to KV-cache compression is in + "Cache Me If You Must: Adaptive Key-Value Quantization for Large + Language Models" (Shutova et al., ICML 2025; preprint + arXiv:2501.19392). All of these foundational and application + references pre-date the TurboQuant paper (Zandieh et al., ICLR 2026). + + QJL is intentionally omitted: community consensus (5+ independent + groups) found it hurts attention quality by amplifying variance + through softmax. Named presets (use via --kv-cache-dtype): turboquant_k8v4: FP8 keys + 4-bit values, 2.6x, +1.17% PPL