Fix asym kernel issue by following autogptq's pr

README.md

@@ -42,6 +42,8 @@ image presents an overview of AutoRound.
 ```bash
 pip install -r requirements.txt
 python setup.py install
+or
+pip install -vvv --no-build-isolation -e .
 ```
 
 ### Install from pypi

auto_round/auto_quantizer.py

@@ -52,11 +52,17 @@
 else:
     import importlib.metadata as importlib_metadata
 
-AUTOGPTQ_MINIMUM_VERSION = version.parse("0.4.99")  # Allows 0.5.0.dev0
+AUTOROUND_MINIMUM_VERSION = version.parse("0.2")
 
 
 def _is_package_available(pkg_name: str, return_version: bool = False) -> Union[Tuple[bool, str], bool]:
     # Check we're not importing a "pkg_name" directory somewhere but the actual library by trying to grab the version
+    try:##TODO remove it later
+        import auto_round
+        return True, auto_round.__version__
+    except:
+        pass
+
     package_exists = importlib.util.find_spec(pkg_name) is not None
     package_version = "N/A"
     if package_exists:
@@ -71,26 +77,32 @@ def _is_package_available(pkg_name: str, return_version: bool = False) -> Union[
         return package_exists
 
 
-_auto_gptq_available = _is_package_available("auto_gptq")
+_auto_round_available = _is_package_available("auto_round")
 
 
-def is_auto_gptq_available():
-    if _auto_gptq_available:
-        version_autogptq = version.parse(importlib_metadata.version("auto_gptq"))
-        if AUTOGPTQ_MINIMUM_VERSION < version_autogptq:
+def is_auto_round_available():
+    if _auto_round_available:
+        version_autoround = version.parse(importlib_metadata.version("auto_round"))
+        if AUTOROUND_MINIMUM_VERSION < version_autoround:
             return True
         else:
             raise ImportError(
-                f"Found an incompatible version of auto-gptq. Found version {version_autogptq},"
-                f" but only version above {AUTOGPTQ_MINIMUM_VERSION} are supported"
+                f"Found an incompatible version of auto-round. Found version {version_autoround},"
+                f" but only version above {AUTOROUND_MINIMUM_VERSION} are supported"
             )
 
 
-if is_auto_gptq_available():
-    from auto_gptq import exllama_set_max_input_length
-    from auto_gptq.modeling._utils import autogptq_post_init
-    from auto_gptq.quantization import GPTQ
-    from auto_gptq.utils.import_utils import dynamically_import_QuantLinear
+def is_autoround_exllamav2_available():
+    res = True
+    try:
+        from autoround_exllamav2_kernels import gemm_half_q_half, make_q_matrix
+    except ImportError as e:
+        res = False
+    return res
+
+
+if is_auto_round_available():
+    from auto_round.export.export_to_autoround.post_init import autoround_post_init
 
 
 #
@@ -201,7 +213,7 @@ def __init__(
             dataset: str = None,
             group_size: int = 128,
             sym: bool = False,
-            backend="gptq:exllamav2",
+            backend="autoround:exllamav2",
             iters: int = 200,
             weight_config: dict = None,
             enable_quanted_input=True,
@@ -233,16 +245,12 @@ def __init__(
         self.post_init()
 
     def get_loading_attributes(self):
-        pass
-        # attibutes_dict = copy.deepcopy(self.__dict__)
-        # loading_attibutes = ["disable_exllama", "use_exllama", "exllama_config", "use_cuda_fp16", "max_input_length"]
-        # loading_attibutes_dict = {i: j for i, j in attibutes_dict.items() if i in loading_attibutes}
-        # return loading_attibutes_dict
+        return {}
 
     def post_init(self):
         r"""Safety checker that arguments are correct."""
-        if self.bits not in [2, 3, 4, 8]:
-            raise ValueError(f"Only support quantization to [2,3,4,8] bits but found {self.bits}")
+        if self.bits not in [2, 4, 8]:
+            raise ValueError(f"Only support quantization to [2,4,8] bits but found {self.bits}")
         if self.group_size != -1 and self.group_size <= 0:
             raise ValueError("group_size must be greater than 0 or equal to -1")
         ##TODO add more check
@@ -254,23 +262,21 @@ def to_dict(self):
 
 
 class AutoRoundQuantizer(HfQuantizer):
-    """Quantizer of the Autoround method, currently only gptq backend has been supported."""
+    """Quantizer of the AutoRound method, currently only triton and exllamav2 backend has been supported."""
 
     requires_calibration = False
-    required_packages = ["auto_gptq"]
+    required_packages = ["auto_round"]
     optimum_quantizer = None
 
     def __init__(self, quantization_config: QuantizationConfigMixin, **kwargs):
         super().__init__(quantization_config, **kwargs)
+        self.exllama2_available = not is_autoround_exllamav2_available
 
     def validate_environment(self, *args, **kwargs):
-        gptq_supports_cpu = version.parse(importlib.metadata.version("auto-gptq")) > version.parse("0.4.2")
-        if not gptq_supports_cpu and not torch.cuda.is_available():
-            raise RuntimeError("GPU is required to quantize or run quantize model.")
-        elif not is_auto_gptq_available():
-            raise ImportError("Loading a GPTQ quantized model requires auto-gptq library (`pip install auto-gptq`)")
-        elif version.parse(importlib.metadata.version("auto_gptq")) < version.parse("0.4.2"):
-            raise ImportError("You need a version of auto_gptq >= 0.4.2 to use GPTQ: `pip install --upgrade auto-gptq`")
+        if not is_auto_round_available():
+            raise ImportError("Loading a AutoRound quantized model requires auto-round library (`pip install auto-round`)")
+        elif version.parse(importlib.metadata.version("auto_round")) < version.parse("0.2.0"):
+            raise ImportError("You need a version of auto_round > 0.2.0 to use AutoRound: `pip install --upgrade auto-round`")
 
     def update_torch_dtype(self, torch_dtype: "torch.dtype") -> "torch.dtype":
         if torch_dtype is None:
@@ -280,7 +286,7 @@ def update_torch_dtype(self, torch_dtype: "torch.dtype") -> "torch.dtype":
         return torch_dtype
 
     def convert_model(self, model: nn.Module):
-        """Convert the model to a GPTQ model by getting and replacing the layers.
+        """Convert the model to an AutoRound model by getting and replacing the layers.
 
         Args:
             model (`nn.Module`):
@@ -308,15 +314,22 @@ def convert_model(self, model: nn.Module):
                 layer_configs[layer_name]["data_type"] = data_type
                 layer_configs[layer_name]["sym"] = sym
             else:
-                layer_configs[layer_name]["bits"] = extra_config.get("bits", bits)
-                layer_configs[layer_name]["group_size"] = extra_config.get("group_size", group_size)
-                layer_configs[layer_name]["data_type"] = extra_config.get("data_type", data_type)
-                layer_configs[layer_name]["sym"] = extra_config.get("sym", sym)
+                layer_configs[layer_name]["bits"] = extra_config[layer_name].get("bits", bits)
+                layer_configs[layer_name]["group_size"] = extra_config[layer_name].get("group_size", group_size)
+                layer_configs[layer_name]["data_type"] = extra_config[layer_name].get("data_type", data_type)
+                layer_configs[layer_name]["sym"] = extra_config[layer_name].get("sym", sym)
         backend = quantization_config.backend
 
         self._replace_by_quant_layers(model, layer_configs, backend)
         return model
 
+    def _dynamic_import_inference_linear(self, bits):
+        if bits == 4 and self.exllama2_available:
+            from auto_round.export.export_to_autoround.qliner_exllamav2 import QuantLinear
+        else:
+            from auto_round.export.export_to_autoround.qliner_triton import QuantLinear
+        return QuantLinear
+
     def _replace_by_quant_layers(self, module: nn.Module, layer_configs, backend):
         """Replaces linear layers in `module` by `QuantLinear`
 
@@ -335,21 +348,7 @@ def _replace_by_quant_layers(self, module: nn.Module, layer_configs, backend):
             data_type = config["data_type"]
             if not (bits <= 8 and data_type == "int"):
                 continue
-            from auto_round.export.export_to_autoround.export_to_autoround import get_autogptq_backend_config
-
-            use_triton, disable_exllama, disable_exllamav2, use_qigen, disable_marlin = get_autogptq_backend_config(
-                backend, bits
-            )
-            QuantLinear = dynamically_import_QuantLinear(
-                use_triton=False,
-                desc_act=False,
-                group_size=group_size,
-                bits=bits,
-                disable_exllama=True,
-                disable_exllamav2=False,
-                use_qigen=use_qigen,
-                disable_marlin=disable_marlin,
-            )
+            QuantLinear = self._dynamic_import_inference_linear(bits)
             layer = get_module(module, layer_name)
             device = get_device(layer)
             if isinstance(layer, nn.Linear):
@@ -381,24 +380,21 @@ def post_init_model(self, model):
             model (`nn.Module`):
                 The input model
         """
-
-        # if self.bits == 4 and not self.disable_exllama:
+        #
+        # if self.bits == 4:
         #     if get_device(model) == torch.device("cpu") or (
         #             hasattr(model, "hf_device_map") and any(d in model.hf_device_map for d in ["cpu", "disk"])
         #     ):
         #         raise ValueError(
-        #             "Found modules on cpu/disk. Using Exllama
-        #             or Exllamav2 backend requires all the modules to be on GPU."
-        #             "You can deactivate exllama backend by
-        #             setting `disable_exllama=True` in the quantization config object"
+        #             "Found modules on cpu/disk. Using Exllamav2 backend requires all the modules to be on GPU."
+        #             "You can deactivate exllama backend by setting `disable_exllama=True` in the quantization config object"
         #         )
 
         class StoreAttr(object):
             pass
 
         model.quantize_config = StoreAttr()
-        model.quantize_config.desc_act = False
-        model = autogptq_post_init(model, use_act_order=False)
+        model = autoround_post_init(model)
         return model
 
     def _process_model_before_weight_loading(self, model: "PreTrainedModel", **kwargs):

auto_round/export/__init__.py

@@ -15,6 +15,6 @@
 from .register import EXPORT_FORMAT
 from .export_to_autogptq import save_quantized_as_autogptq
 from .export_to_itrex import save_quantized_as_itrex, QuantConfig
-from .export_to_autoround.export_to_autoround import save_quantized_as_autoround
+from .export_to_autoround.export import save_quantized_as_autoround
 
 

auto_round/export/export_to_autoround/__init__.py

@@ -12,5 +12,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .export_to_autoround import save_quantized_as_autoround
+from .export import save_quantized_as_autoround
 

auto_round/export/export_to_autoround/export_to_autoround.py → auto_round/export/export_to_autoround/export.py

@@ -25,7 +25,6 @@
 from auto_round.utils import get_layer_names_in_block, get_block_names, get_module, logger, set_module
 
 
-
 def check_neq_config(config, data_type, bits, group_size, sym):
     res = []
     if data_type != config["data_type"]:
@@ -53,7 +52,7 @@ def get_autogptq_backend_config(backend, bits=4):
     if backend == "gptq:marlin":
         use_triton = False
         disable_marlin = True
-    if backend == "gptq:exllamav2":
+    if backend == "gptq:exllamav2":  ##need v1 code to export
         use_triton = False
         disable_marlin = True
     if backend == "gptq:exllamav1":
@@ -71,10 +70,33 @@ def get_autogptq_backend_config(backend, bits=4):
     return use_triton, disable_exllamav1, disable_exllamav2, use_qigen, disable_marlin
 
 
-@register_format("autoround")
-def save_quantized_as_autoround(output_dir, inplace=True, backend="gptq:exllamav2", **kwargs):
-    from auto_gptq.utils.import_utils import dynamically_import_QuantLinear
+def dynamic_QuantLienar_for_packing(backend, bits, group_size):
+    if "gptq" in backend:
+        use_triton, disable_exllamav1, disable_exllamav2, use_qigen, disable_marlin = get_autogptq_backend_config(
+            backend, bits
+        )
+        from auto_gptq.utils.import_utils import dynamically_import_QuantLinear
+        QuantLinear = dynamically_import_QuantLinear(
+            use_triton=use_triton,
+            desc_act=False,
+            group_size=group_size,
+            bits=bits,
+            disable_exllama=disable_exllamav1,
+            disable_exllamav2=disable_exllamav2,
+            use_qigen=use_qigen,
+            disable_marlin=disable_marlin,
+        )
+        return QuantLinear
+    elif "autoround" in backend or "auto-round" in backend or "auto_round" in backend:  ##export all use trition,inferce use exllamav2
+        from .qliner_triton import QuantLinear
+        return QuantLinear
+
+    else:
+        assert False, f"only support gptq and autoround backend"
 
+
+@register_format("autoround")
+def save_quantized_as_autoround(output_dir, inplace=True, backend="autoround:exllamav2", **kwargs):
     model = kwargs["model"]
     if not inplace:
         model = copy.deepcopy(model.to("cpu"))
@@ -90,22 +112,11 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="gptq:exllamav
 
         bits = config["bits"]
         group_size = config["group_size"]
-        use_triton, disable_exllamav1, disable_exllamav2, use_qigen, disable_marlin = get_autogptq_backend_config(
-            backend, bits
-        )
 
         layer = get_module(model, name)
-        device = "cpu"
-        QuantLinear = dynamically_import_QuantLinear(
-            use_triton=use_triton,
-            desc_act=False,
-            group_size=group_size,
-            bits=bits,
-            disable_exllama=disable_exllamav1,
-            disable_exllamav2=disable_exllamav2,
-            use_qigen=use_qigen,
-            disable_marlin=disable_marlin,
-        )
+        device = layer.weight.device
+
+        QuantLinear = dynamic_QuantLienar_for_packing(backend, bits, group_size)
 
         if isinstance(layer, nn.Linear):
             in_features = layer.in_features
@@ -138,7 +149,7 @@ def save_quantized_as_autoround(output_dir, inplace=True, backend="gptq:exllamav
     quantization_config["backend"] = backend
     extra_config = {}
     for layer_name in weight_config:
-        if weight_config[layer_name]["data_type"] != "int" and weight_config[layer_name]["bits"] >= 16:
+        if weight_config[layer_name]["bits"] >= 16:
             continue
         if layer_name not in layer_names_in_block:
             extra_config[layer_name] = {}
@@ -190,7 +201,7 @@ def save(model: nn.Module, save_dir: str, max_shard_size: str = "10GB", safe_ser
     """
     os.makedirs(save_dir, exist_ok=True)
     model.save_pretrained(save_dir, max_shard_size=max_shard_size, safe_serialization=safe_serialization)
-    config_file = "quantize_config.json"
-    if hasattr(model, "config") and hasattr(model.config, "quantize_config"):
+    config_file = "quantization_config.json"
+    if hasattr(model, "config") and hasattr(model.config, "quantization_config"):
         with open(os.path.join(save_dir, config_file), "w", encoding="utf-8") as f:
             json.dump(model.config.quantization_config, f, indent=2)