[Quant] Consolidate GPTQ: rename gptq_marlin.py to auto_gptq.py (#38288)

Signed-off-by: Chengyi Nie <cnie@roblox.com>
Co-authored-by: Chengyi Nie <cnie@roblox.com>
Co-authored-by: Cursor <cursoragent@cursor.com>

.buildkite/hardware_tests/cpu.yaml

@@ -61,7 +61,7 @@ steps:
   source_file_dependencies:
   - csrc/cpu/
   - vllm/model_executor/layers/quantization/cpu_wna16.py
-  - vllm/model_executor/layers/quantization/gptq_marlin.py
+  - vllm/model_executor/layers/quantization/auto_gptq.py
   - vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py
   - vllm/model_executor/layers/quantization/kernels/scaled_mm/cpu.py
   - vllm/model_executor/layers/quantization/kernels/mixed_precision/cpu.py

tests/models/quantization/test_gptq_marlin.py

@@ -1,10 +1,7 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""Compares the outputs of gptq vs gptq_marlin.
+"""Tests AutoGPTQ (GPTQ with Marlin kernels) output correctness.
 
-Note: GPTQ and Marlin do not have bitwise correctness.
-As a result, in this test, we just confirm that the top selected tokens of the
-Marlin/GPTQ models are in the top 5 selections of each other.
 Note: Marlin internally uses locks to synchronize the threads. This can
 result in very slight nondeterminism for Marlin. As a result, we re-run the test
 up to 3 times to see if we pass.
@@ -36,10 +33,10 @@ MODELS = [
 
 @pytest.mark.flaky(reruns=3)
 @pytest.mark.skipif(
-    not is_quant_method_supported("gptq_marlin")
+    not is_quant_method_supported("auto_gptq")
     or current_platform.is_rocm()
     or not current_platform.is_cuda(),
-    reason="gptq_marlin is not supported on this GPU type.",
+    reason="auto_gptq is not supported on this GPU type.",
 )
 @pytest.mark.parametrize("model", MODELS)
 @pytest.mark.parametrize("dtype", ["half", "bfloat16"])

tests/quantization/test_auto_gptq.py

@@ -0,0 +1,56 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Tests that the auto_gptq quantization method works correctly.
+
+Run `pytest tests/quantization/test_auto_gptq.py -v -s`.
+"""
+
+import pytest
+import torch
+
+from tests.quantization.utils import is_quant_method_supported
+from vllm.model_executor.layers.quantization.auto_gptq import (
+    AutoGPTQConfig,
+    AutoGPTQLinearMethod,
+)
+
+PROMPT = "On the surface of Mars, we found"
+
+MODELS = [
+    "TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ",
+]
+
+
+@pytest.mark.skipif(
+    not is_quant_method_supported("auto_gptq"),
+    reason="auto_gptq is not supported on this GPU type.",
+)
+@pytest.mark.parametrize("model_id", MODELS)
+def test_auto_gptq_quantization_method(vllm_runner, model_id: str, monkeypatch):
+    """Test that quantization='auto_gptq' loads and runs correctly."""
+    monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
+
+    with vllm_runner(
+        model_id,
+        dtype=torch.float16,
+        quantization="auto_gptq",
+        max_model_len=2048,
+        enforce_eager=True,
+    ) as llm:
+
+        def check_model(model):
+            for name, submodule in model.named_modules():
+                if name == "model.layers.0.self_attn.qkv_proj":
+                    assert isinstance(submodule.quant_method, AutoGPTQLinearMethod)
+                    break
+
+        llm.apply_model(check_model)
+
+        outputs = llm.generate_greedy([PROMPT], max_tokens=8)
+        assert outputs
+        assert len(outputs[0][1]) > 0
+
+
+def test_auto_gptq_config_get_name():
+    """Test that AutoGPTQConfig.get_name() returns 'auto_gptq'."""
+    assert AutoGPTQConfig.get_name() == "auto_gptq"

tests/quantization/test_configs.py

@@ -24,31 +24,23 @@ MODEL_ARG_EXPTYPES = [
     # AUTOGPTQ
     # compat: autogptq <=0.7.1 is_marlin_format: bool
     # Model Serialized in Exllama Format.
-    (
-        "TheBloke/Llama-2-7B-Chat-GPTQ",
-        None,
-        "gptq_marlin" if current_platform.is_cuda() else "gptq",
-    ),
+    ("TheBloke/Llama-2-7B-Chat-GPTQ", None, "auto_gptq"),
     (
         "TheBloke/Llama-2-7B-Chat-GPTQ",
         "marlin",
-        "gptq_marlin" if current_platform.is_cuda() else "ERROR",
+        "auto_gptq" if current_platform.is_cuda() else "ERROR",
     ),
-    ("TheBloke/Llama-2-7B-Chat-GPTQ", "gptq", "gptq"),
+    ("TheBloke/Llama-2-7B-Chat-GPTQ", "gptq", "auto_gptq"),
     ("TheBloke/Llama-2-7B-Chat-GPTQ", "awq", "ERROR"),
     # compat: autogptq >=0.8.0 use checkpoint_format: str
     # Model Serialized in Exllama Format.
-    (
-        "LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit",
-        None,
-        "gptq_marlin" if current_platform.is_cuda() else "gptq",
-    ),
+    ("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", None, "auto_gptq"),
     (
         "LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit",
         "marlin",
-        "gptq_marlin" if current_platform.is_cuda() else "ERROR",
+        "auto_gptq" if current_platform.is_cuda() else "ERROR",
     ),
-    ("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", "gptq", "gptq"),
+    ("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", "gptq", "auto_gptq"),
     ("LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit", "awq", "ERROR"),
     # AUTOAWQ
     (

tests/quantization/test_gptq_dynamic.py

@@ -3,46 +3,36 @@
 """Tests whether gptq models with dynamic quantized can be loaded.
 
 Run `pytest tests/quantization/test_gptq_dynamic.py --forked`.
+
+Note: Only symmetric GPTQ models are supported after consolidation to Marlin.
 """
 
 import pytest
 import torch
 
 from vllm.model_executor.layers.linear import UnquantizedLinearMethod
-from vllm.model_executor.layers.quantization.gptq import GPTQLinearMethod
-from vllm.model_executor.layers.quantization.gptq_marlin import GPTQMarlinLinearMethod
+from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQLinearMethod
 from vllm.model_executor.layers.quantization.utils.gptq_utils import (
     get_dynamic_override,
 )
-from vllm.platforms import current_platform
 
 PROMPT = "On the surface of Mars, we found"
 
 # The first layer is quantized using bits=4, group_size=128
 # The second layer is quantized using bits=8, group_size=32
 # All other layers (layer index >= 2) are not quantized
-MODEL_QUANT = [
-    (
-        "ModelCloud/Qwen1.5-1.8B-Chat-GPTQ-4bits-dynamic-cfg-with-lm_head-symTrue",
-        current_platform.is_cuda(),
-    ),
-    (
-        "ModelCloud/Qwen1.5-1.8B-Chat-GPTQ-4bits-dynamic-cfg-with-lm_head-symFalse",
-        False,
-    ),
+# Note: Only symmetric models are supported with Marlin kernels
+MODELS = [
+    "ModelCloud/Qwen1.5-1.8B-Chat-GPTQ-4bits-dynamic-cfg-with-lm_head-symTrue",
 ]
 
 
-@pytest.mark.parametrize("model_id, use_marlin_kernel", MODEL_QUANT)
-def test_gptq_with_dynamic(
-    vllm_runner, model_id: str, use_marlin_kernel: bool, monkeypatch
-):
+@pytest.mark.parametrize("model_id", MODELS)
+def test_gptq_with_dynamic(vllm_runner, model_id: str, monkeypatch):
     # `LLM.apply_model` requires pickling a function.
     monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
 
-    linear_method_cls = (
-        GPTQMarlinLinearMethod if use_marlin_kernel else (GPTQLinearMethod)
-    )
+    linear_method_cls = AutoGPTQLinearMethod
 
     with vllm_runner(
         model_id, dtype=torch.float16, max_model_len=2048, enforce_eager=True

tests/quantization/test_gptq_v2.py

@@ -3,6 +3,9 @@
 """Tests whether vllm correctly load and run gptq_v2 format checkpoints.
 
 Run `pytest tests/quantization/test_gptq_v2.py --forked`.
+
+Note: 2/3-bit GPTQ models are no longer supported after the consolidation
+to Marlin kernels. Only 4/8-bit symmetric GPTQ models are supported.
 """
 
 import pytest
@@ -10,9 +13,10 @@ import torch
 from transformers import AutoTokenizer
 
 from vllm import SamplingParams
-from vllm.model_executor.layers.quantization.gptq import GPTQLinearMethod
+from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQLinearMethod
 
 # A dummy small model quantized by GPTQModel, stored in GPTQ v2 format
+# Note: This is a 2-bit model which is no longer supported with Marlin kernels
 MODELS = ["XXXXyu/Qwen3-1.7B-w2g64-gptq_v2"]
 
 # Generate multiple sequences for testing, because an 1.7B 2-bit model
@@ -20,27 +24,19 @@ MODELS = ["XXXXyu/Qwen3-1.7B-w2g64-gptq_v2"]
 N_SEQ = 5
 
 
+@pytest.mark.skip(reason="2-bit GPTQ is no longer supported after Marlin consolidation")
 @pytest.mark.parametrize("model_id", MODELS)
 def test_model_load(vllm_runner, model_id, monkeypatch):
     # `LLM.apply_model` requires pickling a function.
     monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
 
-    # Only check the default GPTQ linear method (used for 2/3-bit models).
-    # 4/8-bit linear methods like Marlin already support gptq_v2.
-    linear_method_cls = GPTQLinearMethod
-
     with vllm_runner(model_id, dtype=torch.float16, max_model_len=512) as llm:
 
         def check_model(model_id):
             for name, submodule in model_id.named_modules():
                 # Could check more modules if necessary
                 if name == "model_id.layers.0.self_attn.qkv_proj":
-                    assert isinstance(submodule.quant_method, linear_method_cls)
-
-                    config = submodule.quant_method.quant_config
-                    assert config.checkpoint_format == "gptq_v2"
-                    assert submodule.quant_method.use_v2_format
-
+                    assert isinstance(submodule.quant_method, AutoGPTQLinearMethod)
                     # Just break since currently we only check 1 module
                     break
 
@@ -48,6 +44,7 @@ def test_model_load(vllm_runner, model_id, monkeypatch):
         llm.apply_model(check_model)
 
 
+@pytest.mark.skip(reason="2-bit GPTQ is no longer supported after Marlin consolidation")
 @pytest.mark.parametrize("model_id", MODELS)
 def test_model_inference(vllm_runner, model_id):
     # Prepare prompt to test the model's generation result.

tests/quantization/test_lm_head.py

@@ -8,8 +8,7 @@ Run `pytest tests/quantization/test_quant_lm_head_true.py --forked`.
 import pytest
 import torch
 
-from vllm.model_executor.layers.quantization.gptq import GPTQLinearMethod
-from vllm.model_executor.layers.quantization.gptq_marlin import GPTQMarlinLinearMethod
+from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQLinearMethod
 from vllm.model_executor.layers.vocab_parallel_embedding import (
     UnquantizedEmbeddingMethod,
 )
@@ -40,7 +39,7 @@ def test_lm_head(
             if lm_head_quantized:
                 assert isinstance(
                     lm_head_layer.quant_method,
-                    (GPTQLinearMethod, GPTQMarlinLinearMethod),
+                    AutoGPTQLinearMethod,
                 )
             else:
                 assert isinstance(

tests/weight_loading/models-large.txt

@@ -2,7 +2,7 @@ compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-quantized, main
 compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-channel-quantized, main
 compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W8A16-quantized, main
 compressed-tensors, nm-testing/test-w4a16-mixtral-actorder-group, main
-gptq_marlin, TheBloke/Mixtral-8x7B-v0.1-GPTQ, main
-gptq_marlin, TheBloke/Mixtral-8x7B-v0.1-GPTQ, gptq-8bit-128g-actorder_True
+gptq, TheBloke/Mixtral-8x7B-v0.1-GPTQ, main
+gptq, TheBloke/Mixtral-8x7B-v0.1-GPTQ, gptq-8bit-128g-actorder_True
 awq_marlin, casperhansen/deepseek-coder-v2-instruct-awq, main
 compressed-tensors, RedHatAI/Llama-4-Scout-17B-16E-Instruct-quantized.w4a16, main

tests/weight_loading/models.txt

@@ -1,9 +1,3 @@
-gptq_marlin, robertgshaw2/zephyr-7b-beta-channelwise-gptq, main
-gptq_marlin, TheBloke/Llama-2-7B-GPTQ, main
-gptq_marlin, TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ, main
-gptq_marlin, TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ, gptq-8bit--1g-actorder_True
-gptq_marlin, TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ, gptq-8bit-32g-actorder_True
-gptq_marlin, TechxGenus/gemma-1.1-2b-it-GPTQ, main
 gptq, robertgshaw2/zephyr-7b-beta-channelwise-gptq, main
 gptq, TheBloke/Llama-2-7B-GPTQ, main
 gptq, TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ, main

vllm/config/model.py

@@ -961,6 +961,8 @@ class ModelConfig:
             # `override_quantization_method` method) must be checked in order
             # of preference (this is particularly important for GPTQ).
             overrides = [
+                "auto_gptq",
+                "gptq",
                 "gptq_marlin",
                 "awq_marlin",
                 "inc",

vllm/model_executor/layers/fused_moe/layer.py

@@ -409,7 +409,7 @@ class FusedMoE(PluggableLayer):
         }
         # need full intermediate size pre-sharding for WNA16 act order
         if self.quant_method.__class__.__name__ in (
-            "GPTQMarlinMoEMethod",
+            "AutoGPTQMoEMethod",
             "CompressedTensorsWNA16MarlinMoEMethod",
             "CompressedTensorsWNA16MoEMethod",
         ):

vllm/model_executor/layers/fused_moe/oracle/int_wna16.py

@@ -27,7 +27,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
 )
 
 if TYPE_CHECKING:
-    from vllm.model_executor.layers.quantization.gptq_marlin import GPTQMarlinConfig
+    from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
 
 logger = init_logger(__name__)
 
@@ -212,7 +212,7 @@ def make_wna16_moe_kernel(
 
 def _process_weights_marlin(
     layer: torch.nn.Module,
-    quant_config: "GPTQMarlinConfig",
+    quant_config: "AutoGPTQConfig",
     input_dtype: torch.dtype | None,
     w13_qweight: torch.Tensor,
     w2_qweight: torch.Tensor,
@@ -417,13 +417,13 @@ def convert_to_wna16_moe_kernel_format(
         WNA16MoEBackend.MARLIN,
         WNA16MoEBackend.BATCHED_MARLIN,
     ):
-        from vllm.model_executor.layers.quantization.gptq_marlin import (
-            GPTQMarlinConfig,
+        from vllm.model_executor.layers.quantization.auto_gptq import (
+            AutoGPTQConfig,
         )
 
-        if not isinstance(quant_config, GPTQMarlinConfig):
+        if not isinstance(quant_config, AutoGPTQConfig):
             raise TypeError(
-                "Marlin WNA16 MoE backend requires GPTQMarlinConfig, got "
+                "Marlin WNA16 MoE backend requires AutoGPTQConfig, got "
                 f"{type(quant_config).__name__}."
             )
         return _process_weights_marlin(

vllm/model_executor/layers/linear.py

@@ -48,12 +48,11 @@ WEIGHT_LOADER_V2_SUPPORTED = [
     "CompressedTensorsLinearTransformMethod",
     "AWQMarlinLinearMethod",
     "AWQLinearMethod",
-    "GPTQMarlinLinearMethod",
+    "AutoGPTQLinearMethod",
     "Fp8LinearMethod",
     "MarlinLinearMethod",
     "GPTQMarlin24LinearMethod",
     "TPUInt8LinearMethod",
-    "GPTQLinearMethod",
     "FBGEMMFp8LinearMethod",
     "ModelOptFp8LinearMethod",
     "ModelOptFp8PcPtLinearMethod",

vllm/model_executor/layers/quantization/__init__.py

@@ -19,9 +19,10 @@ QuantizationMethods = Literal[
     "modelopt_mxfp8",
     "modelopt_mixed",
     "gguf",
+    "auto_gptq",
+    "gptq",
     "gptq_marlin",
     "awq_marlin",
-    "gptq",
     "humming",
     "compressed-tensors",
     "bitsandbytes",
@@ -114,6 +115,7 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
     from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
     from vllm.model_executor.models.deepseek_v4 import DeepseekV4FP8Config
 
+    from .auto_gptq import AutoGPTQConfig
     from .awq import AWQConfig
     from .awq_marlin import AWQMarlinConfig
     from .bitsandbytes import BitsAndBytesConfig
@@ -126,8 +128,6 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
     from .fp8 import Fp8Config
     from .fp_quant import FPQuantConfig
     from .gguf import GGUFConfig
-    from .gptq import GPTQConfig
-    from .gptq_marlin import GPTQMarlinConfig
     from .humming import HummingConfig
     from .inc import INCConfig
     from .modelopt import (
@@ -151,9 +151,10 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
         "modelopt_mxfp8": ModelOptMxFp8Config,
         "modelopt_mixed": ModelOptMixedPrecisionConfig,
         "gguf": GGUFConfig,
-        "gptq_marlin": GPTQMarlinConfig,
+        "auto_gptq": AutoGPTQConfig,
+        "gptq": AutoGPTQConfig,
+        "gptq_marlin": AutoGPTQConfig,
         "awq_marlin": AWQMarlinConfig,
-        "gptq": GPTQConfig,
         "compressed-tensors": CompressedTensorsConfig,
         "bitsandbytes": BitsAndBytesConfig,
         "experts_int8": ExpertsInt8Config,

vllm/model_executor/layers/quantization/auto_gptq.py

@@ -41,7 +41,6 @@ from vllm.model_executor.layers.quantization.utils.gptq_utils import (
     override_config,
 )
 from vllm.model_executor.layers.quantization.utils.marlin_utils import (
-    check_marlin_supported,
     check_moe_marlin_supports_layer,
     get_marlin_input_dtype,
     marlin_make_workspace_new,
@@ -60,7 +59,6 @@ from vllm.model_executor.parameter import (
     PackedvLLMParameter,
     RowvLLMParameter,
 )
-from vllm.platforms import current_platform
 from vllm.scalar_type import scalar_types
 from vllm.transformers_utils.config import get_safetensors_params_metadata
 from vllm.utils.collection_utils import is_list_of
@@ -69,7 +67,7 @@ logger = init_logger(__name__)
 
 
 def get_moe_quant_method(
-    config: "GPTQMarlinConfig",
+    config: "AutoGPTQConfig",
     layer: RoutedExperts,
     prefix: str,
     moe_method_cls: type,
@@ -94,8 +92,8 @@ def get_moe_quant_method(
     return moe_method_cls(cloned_config, layer.moe_config)
 
 
-class GPTQMarlinConfig(QuantizationConfig):
-    """Config class for GPTQ Marlin"""
+class AutoGPTQConfig(QuantizationConfig):
+    """Config class for AutoGPTQ quantization using Marlin kernels."""
 
     # (num_bits, is_sym) -> quant_type
     TYPE_MAP = {
@@ -167,7 +165,7 @@ class GPTQMarlinConfig(QuantizationConfig):
 
     def __repr__(self) -> str:
         return (
-            f"GPTQMarlinConfig(quant_type={self.quant_type}, "
+            f"AutoGPTQConfig(quant_type={self.quant_type}, "
             f"group_size={self.group_size}, "
             f"desc_act={self.desc_act}, "
             f"lm_head_quantized={self.lm_head_quantized}, "
@@ -177,7 +175,7 @@ class GPTQMarlinConfig(QuantizationConfig):
 
     @classmethod
     def get_name(cls) -> QuantizationMethods:
-        return "gptq_marlin"
+        return "auto_gptq"
 
     @classmethod
     def get_supported_act_dtypes(cls) -> list[torch.dtype]:
@@ -185,14 +183,14 @@ class GPTQMarlinConfig(QuantizationConfig):
 
     @classmethod
     def get_min_capability(cls) -> int:
-        return 75
+        return 60
 
     @classmethod
     def get_config_filenames(cls) -> list[str]:
         return ["quantize_config.json"]
 
     @classmethod
-    def from_config(cls, config: dict[str, Any]) -> "GPTQMarlinConfig":
+    def from_config(cls, config: dict[str, Any]) -> "AutoGPTQConfig":
         dynamic = cls.get_from_keys_or(config, ["dynamic"], default={})
         dynamic = {} if dynamic is None else dynamic
 
@@ -219,27 +217,22 @@ class GPTQMarlinConfig(QuantizationConfig):
     def override_quantization_method(
         cls, hf_quant_cfg, user_quant, hf_config=None
     ) -> QuantizationMethods | None:
-        can_convert = cls.is_gptq_marlin_compatible(hf_quant_cfg)
+        """Override to use AutoGPTQ for compatible GPTQ models."""
+        quant_method = hf_quant_cfg.get("quant_method", "").lower()
 
-        is_valid_user_quant = (
-            user_quant is None or user_quant == "marlin" or user_quant == "gptq_marlin"
+        if quant_method != "gptq":
+            return None
+
+        is_valid_user_quant = user_quant is None or user_quant in (
+            "gptq",
+            "gptq_marlin",
+            "auto_gptq",
+            "marlin",
         )
 
-        if can_convert and is_valid_user_quant:
-            msg = (
-                "The model is convertible to {} during runtime."
-                " Using {} kernel.".format(cls.get_name(), cls.get_name())
-            )
-            logger.info(msg)
+        if is_valid_user_quant:
             return cls.get_name()
 
-        if can_convert and user_quant == "gptq":
-            logger.info(
-                "Detected that the model can run with gptq_marlin"
-                ", however you specified quantization=gptq explicitly,"
-                " so forcing gptq. Use quantization=gptq_marlin for"
-                " faster inference"
-            )
         return None
 
     def get_quant_method(
@@ -257,7 +250,7 @@ class GPTQMarlinConfig(QuantizationConfig):
                     layer, prefix
                 )
             moe_quant_method = get_moe_quant_method(
-                self, layer, prefix, GPTQMarlinMoEMethod
+                self, layer, prefix, AutoGPTQMoEMethod
             )
             if moe_quant_method is None:
                 return None
@@ -265,38 +258,13 @@ class GPTQMarlinConfig(QuantizationConfig):
             return moe_quant_method
 
         quant_method = get_linear_quant_method(
-            self, layer, prefix, GPTQMarlinLinearMethod
+            self, layer, prefix, AutoGPTQLinearMethod
         )
         if quant_method is None:
             return None
         quant_method.input_dtype = get_marlin_input_dtype(prefix)
         return quant_method
 
-    @classmethod
-    def is_gptq_marlin_compatible(cls, quant_config: dict[str, Any]):
-        quant_method = quant_config.get("quant_method", "").lower()
-        num_bits = quant_config.get("bits")
-        group_size = quant_config.get("group_size")
-        sym = quant_config.get("sym")
-        desc_act = quant_config.get("desc_act")
-
-        if not (current_platform.is_cuda() or current_platform.is_cpu()):
-            return False
-
-        if quant_method != "gptq":
-            return False
-
-        # Marlin conversion is only valid if required properties are found
-        if num_bits is None or group_size is None or sym is None or desc_act is None:
-            return False
-
-        if (num_bits, sym) not in cls.TYPE_MAP:
-            return False
-
-        return check_marlin_supported(
-            quant_type=cls.TYPE_MAP[(num_bits, sym)], group_size=group_size
-        )
-
     def apply_vllm_mapper(self, hf_to_vllm_mapper):
         if self.modules_in_block_to_quantize is not None:
             self.modules_in_block_to_quantize = hf_to_vllm_mapper.apply_list(
@@ -331,16 +299,16 @@ class GPTQMarlinConfig(QuantizationConfig):
         self.modules_in_block_to_quantize = list(quant_layers)
 
 
-class GPTQMarlinLinearMethod(LinearMethodBase):
-    """Linear method for GPTQ Marlin.
+class AutoGPTQLinearMethod(LinearMethodBase):
+    """Linear method for AutoGPTQ using Marlin kernels.
 
     Args:
-        quant_config: The GPTQ Marlin quantization config.
+        quant_config: The AutoGPTQ quantization config.
     """
 
     _kernel_backends_being_used: set[str] = set()
 
-    def __init__(self, quant_config: GPTQMarlinConfig) -> None:
+    def __init__(self, quant_config: AutoGPTQConfig) -> None:
         self.quant_config = quant_config
         self.input_dtype = None
         self.quant_type = self.quant_config.quant_type
@@ -382,7 +350,7 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
         kernel_type = choose_mp_linear_kernel(mp_linear_kernel_config)
 
         if kernel_type.__name__ not in self._kernel_backends_being_used:
-            logger.info("Using %s for GPTQMarlinLinearMethod", kernel_type.__name__)
+            logger.info("Using %s for AutoGPTQLinearMethod", kernel_type.__name__)
             self._kernel_backends_being_used.add(kernel_type.__name__)
 
         # Normalize group_size
@@ -492,12 +460,12 @@ class GPTQMarlinLinearMethod(LinearMethodBase):
         return self.kernel.apply_weights(layer, x, bias)
 
 
-class GPTQMarlinMoEMethod(FusedMoEMethodBase):
+class AutoGPTQMoEMethod(FusedMoEMethodBase):
     """MoE Marlin method with quantization."""
 
     def __init__(
         self,
-        quant_config: GPTQMarlinConfig,
+        quant_config: AutoGPTQConfig,
         moe: FusedMoEConfig,
     ) -> None:
         super().__init__(moe)
@@ -509,7 +477,7 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
             quant_type = scalar_types.uint8b128
             scale = kInt8StaticGroupScale
         else:
-            raise ValueError("GPTQMarlinMoEMethod only supports int4 and int8 now.")
+            raise ValueError("AutoGPTQMoEMethod only supports int4 and int8 now.")
         self.input_dtype = None
         self.use_marlin = True
         weight_key = QuantKey(quant_type, scale)

vllm/model_executor/layers/quantization/gptq.py

@@ -1,399 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-import enum
-from enum import Enum
-from fractions import Fraction
-from typing import TYPE_CHECKING, Any, Union
-
-import torch
-from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE
-from torch.nn.parameter import Parameter
-from transformers import PretrainedConfig
-
-from vllm import _custom_ops as ops
-from vllm.logger import init_logger
-from vllm.model_executor.layers.fused_moe import RoutedExperts
-from vllm.model_executor.layers.linear import LinearMethodBase
-from vllm.model_executor.layers.quantization.base_config import (
-    QuantizationConfig,
-    QuantizeMethodBase,
-)
-from vllm.model_executor.layers.quantization.utils.gptq_utils import (
-    get_linear_quant_method,
-)
-from vllm.model_executor.parameter import (
-    ChannelQuantScaleParameter,
-    GroupQuantScaleParameter,
-    PackedColumnParameter,
-    PackedvLLMParameter,
-    RowvLLMParameter,
-)
-from vllm.transformers_utils.config import get_safetensors_params_metadata
-from vllm.utils.collection_utils import is_list_of
-
-if TYPE_CHECKING:
-    from vllm.model_executor.layers.quantization import QuantizationMethods
-    from vllm.model_executor.models.utils import WeightsMapper
-else:
-    QuantizationMethods = str
-
-logger = init_logger(__name__)
-
-
-class GPTQConfig(QuantizationConfig):
-    """Config class for GPTQ.
-
-    Reference: https://arxiv.org/abs/2210.17323
-    """
-
-    def __init__(
-        self,
-        weight_bits: int,
-        group_size: int,
-        desc_act: bool,
-        lm_head_quantized: bool,
-        dynamic: dict[str, dict[str, int | bool]],
-        autoround_version: str = "",
-        modules_in_block_to_quantize: list[str] | None = None,
-        checkpoint_format: str = "",
-    ) -> None:
-        # GPTQModel use `dynamic` config property to allow per module
-        # quantization config so each module can be individually optimized.
-        # Format is dict[str, dict] where key is a regex string that can
-        # perform both positive ("+:" prefixed) or negative ("-:" prefixed)
-        # matching of a module.
-        # Default to positive match, override base quant config mode, if no
-        # prefix is used. Value is in dict format of field key and override
-        # value.
-        # Negative matching will skip quantization init for this module
-        # entirely:
-        # non-quantized inference. More details and quantization examples can be
-        # found at: https://github.com/ModelCloud/GPTQModel
-        # Example:
-        #  # last 1/2 of the layers 10-21 has 8bit vs 4bit for 0-9
-        #  # last 1/4 of the layers 16-21 has 8bit and group_size 64
-        # dynamic = {
-        #  #`.*\.` matches the layers_node prefix
-        #  # positive match layer 10-15
-        #  r"+:.*\.(?:1[0-5])\..*": {"bits": 8,},
-        #  # positive match layer 16-21
-        #  r"+:.*\.(?:1[6-9]|20|21)\..*": {"bits": 8, "group_size": 64,},
-        #  r"-:.*\.moe\..*": {}, # negative match (skip) all `moe` layers
-        # }
-        super().__init__()
-        self.dynamic = dynamic
-
-        self.weight_bits = weight_bits
-        self.group_size = group_size
-        self.desc_act = desc_act
-        self.lm_head_quantized = lm_head_quantized
-        self.pack_factor = Fraction(32, self.weight_bits)
-        if self.weight_bits not in [2, 3, 4, 8]:
-            raise ValueError(
-                "Currently, only 2/3/4/8-bit weight quantization is "
-                f"supported for GPTQ, but got {self.weight_bits} bits."
-            )
-        # Somehow gptq_gemm 4-bit is buggy, maybe fix it in the future.
-        # For now, show a warning, since gptq_marlin will be used by default.
-        if self.weight_bits == 4:
-            logger.warning_once(
-                "Currently, the 4-bit gptq_gemm kernel for GPTQ is buggy. "
-                "Please switch to gptq_marlin."
-            )
-
-        self.modules_in_block_to_quantize = modules_in_block_to_quantize or []
-
-        # used to identify GPTQ model quantized by autoround
-        self.autoround_version = autoround_version
-
-        # GPTQ v1 and v2 format deals with zero points differently.
-        # Currently GPTQModel stores v1 format checkpoints by default,
-        # but provides the option to set `format="gptq_v2"` in `QuantizeConfig`.
-        self.checkpoint_format = checkpoint_format
-
-    def __repr__(self) -> str:
-        return (
-            f"GPTQConfig(weight_bits={self.weight_bits}, "
-            f"group_size={self.group_size}, "
-            f"desc_act={self.desc_act}), "
-            f"lm_head_quantized={self.lm_head_quantized}, "
-            f"dynamic={self.dynamic}, "
-            f"modules_in_block_to_quantize={self.modules_in_block_to_quantize}), "
-            f"checkpoint_format={self.checkpoint_format})"
-        )
-
-    @classmethod
-    def get_name(cls) -> QuantizationMethods:
-        return "gptq"
-
-    @classmethod
-    def get_supported_act_dtypes(cls) -> list[torch.dtype]:
-        return [torch.half]
-
-    @classmethod
-    # Need to figure it out
-    def get_min_capability(cls) -> int:
-        return 60
-
-    @classmethod
-    def get_config_filenames(cls) -> list[str]:
-        return ["quantize_config.json"]
-
-    @classmethod
-    def from_config(cls, config: dict[str, Any]) -> "GPTQConfig":
-        dynamic = cls.get_from_keys_or(config, ["dynamic"], default={})
-        dynamic = {} if dynamic is None else dynamic
-
-        weight_bits = cls.get_from_keys(config, ["bits"])
-        group_size = cls.get_from_keys(config, ["group_size"])
-        desc_act = cls.get_from_keys(config, ["desc_act"])
-        lm_head_quantized = cls.get_from_keys_or(config, ["lm_head"], default=False)
-        autoround_version = cls.get_from_keys_or(
-            config, ["autoround_version"], default=""
-        )
-        modules_in_block_to_quantize = cls.get_from_keys_or(
-            config, ["modules_in_block_to_quantize"], default=None
-        )
-        checkpoint_format = cls.get_from_keys_or(
-            config, ["checkpoint_format"], default=""
-        )
-        return cls(
-            weight_bits,
-            group_size,
-            desc_act,
-            lm_head_quantized,
-            dynamic,
-            autoround_version,
-            modules_in_block_to_quantize,
-            checkpoint_format,
-        )
-
-    def get_quant_method(
-        self, layer: torch.nn.Module, prefix: str
-    ) -> Union["GPTQLinearMethod", "QuantizeMethodBase"] | None:
-        if isinstance(layer, RoutedExperts):
-            # GPTQ MoE support: fall back to MoeWNA16 for broad compatibility
-            from .moe_wna16 import MoeWNA16Config
-
-            # TODO: maybe update this for GPTQv2 format checkpoints
-            config = {
-                "quant_method": "gptq",
-                "bits": self.weight_bits,
-                "group_size": self.group_size,
-                "sym": True,  # GPTQ typically uses symmetric quantization
-                "lm_head": False,
-            }
-            return MoeWNA16Config.from_config(config).get_quant_method(layer, prefix)
-
-        return get_linear_quant_method(self, layer, prefix, GPTQLinearMethod)
-
-    def apply_vllm_mapper(self, hf_to_vllm_mapper: "WeightsMapper"):
-        if self.modules_in_block_to_quantize is not None:
-            self.modules_in_block_to_quantize = hf_to_vllm_mapper.apply_list(
-                self.modules_in_block_to_quantize
-            )
-
-    def maybe_update_config(
-        self,
-        model_name: str,
-        hf_config: PretrainedConfig | None = None,
-        revision: str | None = None,
-    ):
-        if self.modules_in_block_to_quantize:
-            if is_list_of(self.modules_in_block_to_quantize, list):
-                # original modules_in_block_to_quantize: list[list[str]]
-                # flatten original modules_in_block_to_quantize
-                self.modules_in_block_to_quantize = [
-                    item
-                    for sublist in self.modules_in_block_to_quantize
-                    for item in sublist
-                ]
-            return
-
-        unquant_dtypes = [torch.float16, torch.bfloat16, torch.float32]
-        metadata = get_safetensors_params_metadata(model_name, revision=revision)
-        quant_layers: set[str] = {
-            param_name.rsplit(".", 1)[0]
-            for param_name, info in metadata.items()
-            if (dtype := info.get("dtype", None))
-            and _SAFETENSORS_TO_TORCH_DTYPE[dtype] not in unquant_dtypes
-        }
-        self.modules_in_block_to_quantize = list(quant_layers)
-
-
-class ExllamaState(Enum):
-    UNUSED = enum.auto()
-    UNINITIALIZED = enum.auto()
-    READY = enum.auto()
-
-
-class GPTQLinearMethod(LinearMethodBase):
-    """Linear method for GPTQ.
-
-    Args:
-        quant_config: The GPTQ quantization config.
-    """
-
-    def __init__(self, quant_config: GPTQConfig):
-        self.quant_config = quant_config
-
-        # GPTQ v1 and v2 format deals with zero points differently
-        self.use_v2_format = quant_config.checkpoint_format == "gptq_v2"
-
-    def create_weights(
-        self,
-        layer: torch.nn.Module,
-        input_size_per_partition: int,
-        output_partition_sizes: list[int],
-        input_size: int,
-        output_size: int,
-        params_dtype: torch.dtype,
-        **extra_weight_attrs,
-    ):
-        del output_size  # Unused.
-        weight_loader = extra_weight_attrs.get("weight_loader")
-        if input_size_per_partition % self.quant_config.group_size != 0:
-            raise ValueError(
-                "The input size is not aligned with the quantized "
-                "weight shape. This can be caused by too large "
-                "tensor parallel size."
-            )
-        output_size_per_partition = sum(output_partition_sizes)
-        if output_size_per_partition % self.quant_config.pack_factor.numerator != 0:
-            raise ValueError(
-                "The output size is not aligned with the quantized "
-                "weight shape. This can be caused by too large "
-                "tensor parallel size."
-            )
-
-        if self.quant_config.group_size != -1:
-            group_size = self.quant_config.group_size
-        else:
-            group_size = input_size
-        exllama_state = ExllamaState.UNINITIALIZED
-        scale_and_zero_size = input_size // group_size
-        scale_and_zero_input_dim = None
-        if (
-            input_size != input_size_per_partition
-            and self.quant_config.group_size != -1
-        ):
-            # For act-order models, we cannot use Exllama for row parallel layer
-            if self.quant_config.desc_act:
-                exllama_state = ExllamaState.UNUSED
-            else:
-                # we need to partition qzeros and scales for exllama kernel
-                scale_and_zero_size = input_size_per_partition // group_size
-                scale_and_zero_input_dim = 0
-
-        qweight = PackedvLLMParameter(
-            data=torch.empty(
-                input_size_per_partition // self.quant_config.pack_factor,
-                output_size_per_partition,
-                dtype=torch.int32,
-            ),
-            input_dim=0,
-            output_dim=1,
-            packed_dim=0,
-            packed_factor=self.quant_config.pack_factor,
-            weight_loader=weight_loader,
-        )
-
-        g_idx = RowvLLMParameter(
-            data=torch.tensor(
-                [
-                    i // self.quant_config.group_size
-                    for i in range(input_size_per_partition)
-                ],
-                dtype=torch.int32,
-            ),
-            input_dim=0,
-            weight_loader=weight_loader,
-        )
-        qzeros_args = {
-            "data": torch.empty(
-                scale_and_zero_size,
-                output_size_per_partition // self.quant_config.pack_factor,
-                dtype=torch.int32,
-            ),
-            "weight_loader": weight_loader,
-        }
-        weight_scale_args = {
-            "data": torch.empty(
-                scale_and_zero_size,
-                output_size_per_partition,
-                dtype=params_dtype,
-            ),
-            "weight_loader": weight_loader,
-        }
-        if scale_and_zero_input_dim is None:
-            scales = ChannelQuantScaleParameter(output_dim=1, **weight_scale_args)
-            qzeros = PackedColumnParameter(
-                output_dim=1,
-                packed_dim=1,
-                packed_factor=self.quant_config.pack_factor,
-                **qzeros_args,
-            )
-
-        else:
-            scales = GroupQuantScaleParameter(
-                output_dim=1, input_dim=0, **weight_scale_args
-            )
-            qzeros = PackedvLLMParameter(
-                input_dim=0,
-                output_dim=1,
-                packed_dim=1,
-                packed_factor=self.quant_config.pack_factor,
-                **qzeros_args,
-            )
-
-        layer.register_parameter("qweight", qweight)
-        layer.register_parameter("g_idx", g_idx)
-        layer.register_parameter("qzeros", qzeros)
-        layer.register_parameter("scales", scales)
-
-        layer.exllama_state = exllama_state
-
-    def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
-        # for torch.compile
-        layer.qzeros = Parameter(layer.qzeros.data, requires_grad=False)
-        layer.qweight = Parameter(layer.qweight.data, requires_grad=False)
-        layer.g_idx = Parameter(layer.g_idx.data, requires_grad=False)
-        layer.scales = Parameter(layer.scales.data, requires_grad=False)
-
-        # exllama needs to shuffle the weight after the weight is loaded
-        # here we do the shuffle on first forward pass
-        if layer.exllama_state == ExllamaState.UNINITIALIZED:
-            if self.quant_config.desc_act:
-                layer.g_idx.data = torch.argsort(layer.g_idx).to(torch.int)
-            else:
-                layer.g_idx.data = torch.empty(
-                    (0,), dtype=torch.int, device=layer.g_idx.device
-                )
-            layer.exllama_state = ExllamaState.READY
-            ops.gptq_shuffle(layer.qweight, layer.g_idx, self.quant_config.weight_bits)
-
-    def apply(
-        self,
-        layer: torch.nn.Module,
-        x: torch.Tensor,
-        bias: torch.Tensor | None = None,
-    ) -> torch.Tensor:
-        out_shape = x.shape[:-1] + (layer.qweight.shape[-1],)
-        reshaped_x = x.reshape(-1, x.shape[-1])
-
-        # GPTQ v1 and v2 format checkpoints deals with zero points differently,
-        # and require different gemm kernels.
-        output = ops.gptq_gemm(
-            reshaped_x,
-            layer.qweight,
-            layer.qzeros,
-            layer.scales,
-            layer.g_idx,
-            layer.exllama_state == ExllamaState.READY,
-            self.use_v2_format,
-            self.quant_config.weight_bits,
-        )
-        if bias is not None:
-            output.add_(bias)
-        return output.reshape(out_shape)

vllm/model_executor/layers/quantization/inc.py

@@ -356,13 +356,13 @@ class INCConfig(QuantizationConfig):
         else:
             use_marlin = False
         if use_marlin:
-            from vllm.model_executor.layers.quantization.gptq_marlin import (
-                GPTQMarlinConfig,
-                GPTQMarlinLinearMethod,
-                GPTQMarlinMoEMethod,
+            from vllm.model_executor.layers.quantization.auto_gptq import (
+                AutoGPTQConfig,
+                AutoGPTQLinearMethod,
+                AutoGPTQMoEMethod,
             )
 
-            quant_args_marlin = GPTQMarlinConfig(
+            quant_args_marlin = AutoGPTQConfig(
                 weight_bits=weight_bits,
                 group_size=group_size,
                 is_sym=sym,
@@ -371,23 +371,10 @@ class INCConfig(QuantizationConfig):
                 dynamic={},
                 full_config={},
             )
-        else:
-            from vllm.model_executor.layers.quantization.gptq import (
-                GPTQConfig,
-                GPTQLinearMethod,
-            )
-
-            quant_args = GPTQConfig(
-                weight_bits=weight_bits,
-                group_size=group_size,
-                lm_head_quantized=False,
-                desc_act=False,
-                dynamic={},
-            )
 
         if isinstance(layer, RoutedExperts):
             if use_marlin:
-                return GPTQMarlinMoEMethod(quant_args_marlin, layer.moe_config)
+                return AutoGPTQMoEMethod(quant_args_marlin, layer.moe_config)
             else:
                 from vllm.model_executor.layers.quantization.moe_wna16 import (
                     MoeWNA16Config,
@@ -406,9 +393,13 @@ class INCConfig(QuantizationConfig):
 
         if isinstance(layer, (LinearBase, ParallelLMHead)):
             if use_marlin:
-                return GPTQMarlinLinearMethod(quant_args_marlin)
+                return AutoGPTQLinearMethod(quant_args_marlin)
             else:
-                return GPTQLinearMethod(quant_args)
+                raise NotImplementedError(
+                    f"INC quantization with bits={weight_bits}, sym={sym} "
+                    "is not supported. Only 4-bit and 8-bit symmetric "
+                    "quantization is supported with Marlin kernels."
+                )
 
         return None
 

vllm/model_executor/layers/quantization/moe_wna16.py

@@ -60,10 +60,9 @@ class MoeWNA16Config(QuantizationConfig):
         # Avoid circular import
         from vllm.model_executor.layers.quantization.awq import AWQConfig
         from vllm.model_executor.layers.quantization.awq_marlin import AWQMarlinConfig
-        from vllm.model_executor.layers.quantization.gptq_marlin import GPTQMarlinConfig
 
         if self.linear_quant_method == "gptq":
-            self.use_marlin = GPTQMarlinConfig.is_gptq_marlin_compatible(full_config)
+            pass
         elif self.linear_quant_method in ("awq", "awq_marlin"):
             capability_tuple = current_platform.get_device_capability()
             device_capability = (
@@ -172,24 +171,18 @@ class MoeWNA16Config(QuantizationConfig):
             return UnquantizedLinearMethod()
         elif isinstance(layer, LinearBase):
             # Avoid circular import
+            from vllm.model_executor.layers.quantization.auto_gptq import (
+                AutoGPTQConfig,
+            )
             from vllm.model_executor.layers.quantization.awq import AWQConfig
             from vllm.model_executor.layers.quantization.awq_marlin import (
                 AWQMarlinConfig,
             )
-            from vllm.model_executor.layers.quantization.gptq import GPTQConfig
-            from vllm.model_executor.layers.quantization.gptq_marlin import (
-                GPTQMarlinConfig,
-            )
 
             if self.linear_quant_method == "gptq":
-                if self.use_marlin:
-                    return GPTQMarlinConfig.from_config(
-                        self.full_config
-                    ).get_quant_method(layer, prefix)
-                else:
-                    return GPTQConfig.from_config(self.full_config).get_quant_method(
-                        layer, prefix
-                    )
+                return AutoGPTQConfig.from_config(self.full_config).get_quant_method(
+                    layer, prefix
+                )
             elif self.linear_quant_method in ("awq", "awq_marlin"):
                 if self.use_marlin and check_marlin_supports_layer(
                     layer, self.group_size

vllm/model_executor/layers/quantization/utils/gptq_utils.py

@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 from collections.abc import Mapping
 from copy import deepcopy
-from fractions import Fraction
 from types import MappingProxyType
 from typing import TYPE_CHECKING
 
@@ -16,16 +15,14 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
 )
 
 if TYPE_CHECKING:
-    from ..gptq import GPTQConfig
-    from ..gptq_marlin import GPTQMarlinConfig
+    from ..auto_gptq import AutoGPTQConfig
 else:
-    GPTQConfig = object
-    GPTQMarlinConfig = object
+    AutoGPTQConfig = object
 
 
 # Match dynamic rules with module name (prefix) and override quantize
 # config if module (prefix) matches a rule
-def override_config(config: GPTQConfig | GPTQMarlinConfig, prefix: str):
+def override_config(config: AutoGPTQConfig, prefix: str):
     weight_bits = get_dynamic_override(config, prefix, "bits", config.weight_bits)
     if isinstance(weight_bits, int):
         config.weight_bits = weight_bits
@@ -36,31 +33,23 @@ def override_config(config: GPTQConfig | GPTQMarlinConfig, prefix: str):
     if isinstance(desc_act, bool):
         config.desc_act = desc_act
 
-    config.pack_factor = Fraction(32, config.weight_bits)  # packed into int32
-    if config.get_name() == "gptq_marlin":
-        assert isinstance(config, GPTQMarlinConfig)
-        is_sym = get_dynamic_override(config, prefix, "sym", config.is_sym)
-        if isinstance(is_sym, bool):
-            config.is_sym = is_sym
+    config.pack_factor = 32 // config.weight_bits  # packed into int32
+    assert isinstance(config, AutoGPTQConfig)
+    is_sym = get_dynamic_override(config, prefix, "sym", config.is_sym)
+    if isinstance(is_sym, bool):
+        config.is_sym = is_sym
 
-        if (config.weight_bits, config.is_sym) not in config.TYPE_MAP:
-            raise ValueError(
-                "Unsupported quantization config: "
-                f"bits={config.weight_bits}, sym={config.is_sym}"
-            )
+    if (config.weight_bits, config.is_sym) not in config.TYPE_MAP:
+        raise ValueError(
+            "Unsupported quantization config: "
+            f"bits={config.weight_bits}, sym={config.is_sym}"
+        )
 
-        config.quant_type = config.TYPE_MAP[(config.weight_bits, config.is_sym)]
-    elif config.get_name() == "gptq":
-        assert isinstance(config, GPTQConfig)
-        if config.weight_bits not in [2, 3, 4, 8]:
-            raise ValueError(
-                "Currently, only 2/3/4/8-bit weight quantization is "
-                f"supported for GPTQ, but got {config.weight_bits} bits."
-            )
+    config.quant_type = config.TYPE_MAP[(config.weight_bits, config.is_sym)]
 
 
 def get_dynamic_override(
-    config: GPTQConfig | GPTQMarlinConfig,
+    config: AutoGPTQConfig,
     layer_name: str,
     key: str | None = None,
     default_value: int | bool | None = None,
@@ -126,7 +115,7 @@ def is_layer_gptq_quantized(
 
 
 def get_linear_quant_method(
-    config: GPTQConfig | GPTQMarlinConfig,
+    config: AutoGPTQConfig,
     layer: torch.nn.Module,
     prefix: str,
     linear_method_cls: type,

vllm/model_executor/models/openpangu_vl.py

@@ -44,8 +44,7 @@ from vllm.model_executor.layers.linear import (
     RowParallelLinear,
 )
 from vllm.model_executor.layers.quantization import QuantizationConfig
-from vllm.model_executor.layers.quantization.gptq import GPTQConfig
-from vllm.model_executor.layers.quantization.gptq_marlin import GPTQMarlinConfig
+from vllm.model_executor.layers.quantization.auto_gptq import AutoGPTQConfig
 from vllm.model_executor.layers.rotary_embedding.common import ApplyRotaryEmb
 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
 from vllm.model_executor.models.interfaces import (
@@ -883,7 +882,7 @@ class OpenPanguVLForConditionalGeneration(
         self.image_std = tuple(image_processor.image_std)
 
     def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig):
-        if isinstance(quant_config, (GPTQConfig, GPTQMarlinConfig)):
+        if isinstance(quant_config, AutoGPTQConfig):
             return None
         return quant_config
 

vllm/platforms/rocm.py

@@ -407,7 +407,8 @@ class RocmPlatform(Platform):
         "awq",
         "awq_marlin",  # will be overwritten with awq
         "gptq",
-        "gptq_marlin",  # will be overwritten with gptq
+        "gptq_marlin",
+        "auto_gptq",
         "fp8",
         "deepseek_v4_fp8",
         "compressed-tensors",