From d963b97a9b1b25c14097409df84a085f6270310a Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Mon, 24 Jul 2023 20:49:24 +0000 Subject: [PATCH 01/46] GTPQ integration --- src/transformers/modeling_utils.py | 30 +++++++++++++++++-- src/transformers/utils/quantization_config.py | 1 + 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index c7d31c09024e..91d96bc9d9f8 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -2344,15 +2344,37 @@ def from_pretrained( else: model_kwargs = kwargs + has_bnb_quantization_config = False + has_gptq_quantization_config = False + if hasattr(config, "quantization_config"): + # backward compatibility + has_bnb_quantization_config = ( + hasattr(config.quantization_config, "load_in_8bit") + or config.quantization_config.get("quant_method", None) == "bitsandbytes" + ) + has_gptq_quantization_config = config.quantization_config.get("quant_method", None) == "gptq" + + gtpq_quantizer = None + if has_gptq_quantization_config: + if not is_optimum_available(): + raise ImportError( + "Loading GTPQ quantized model requires optimum library : `pip install optimum` and auto_gptq library 'pip install auto_gptq'" + ) + else: + # Need to protect the import + from optimum.gptq import GPTQQuantizer + gtpq_quantizer = GPTQQuantizer.from_dict(config.quantization_config) + torch_dtype = config.torch_dtype + if is_8bit_serializable and quantization_config is not None and load_in_8bit: - if hasattr(config, "quantization_config"): + if has_bnb_quantization_config: logger.warning( "You passed `quantization_config` to `from_pretrained` but the model you're loading already has a" " `quantization_config` attribute. The `quantization_config` attribute will be overwritten with the" " one you passed to `from_pretrained`." ) config.quantization_config = quantization_config - elif is_8bit_serializable and not load_in_8bit and hasattr(config, "quantization_config"): + elif is_8bit_serializable and not load_in_8bit and has_bnb_quantization_config: quantization_config = config.quantization_config if isinstance(quantization_config, dict): quantization_config = BitsAndBytesConfig.from_dict(quantization_config, return_unused_kwargs=False) @@ -2382,7 +2404,7 @@ def from_pretrained( if low_cpu_mem_usage is None: low_cpu_mem_usage = True - elif not is_8bit_serializable and not load_in_8bit and hasattr(config, "quantization_config"): + elif not is_8bit_serializable and not load_in_8bit and has_bnb_quantization_config: logger.warning( "Detected the presence of a `quantization_config` attribute in the model's configuration but you don't have the correct" " `bitsandbytes` version to support int8 serialization. Please install the latest version of `bitsandbytes` with " @@ -2767,6 +2789,8 @@ def from_pretrained( "All non-linear modules will be loaded in full precision." " If you want to load the other modules in other precision, please specify a `torch_dtype` attribute." ) + if gtpq_quantizer is not None: + model = gtpq_quantizer.convert_model(model) if isinstance(device_map, str): special_dtypes = {} diff --git a/src/transformers/utils/quantization_config.py b/src/transformers/utils/quantization_config.py index cc7d195e1752..eeff3eac8846 100644 --- a/src/transformers/utils/quantization_config.py +++ b/src/transformers/utils/quantization_config.py @@ -97,6 +97,7 @@ def __init__( bnb_4bit_use_double_quant=False, **kwargs, ): + self.quant_method = "bitsandbytes" self.load_in_8bit = load_in_8bit self.load_in_4bit = load_in_4bit self.llm_int8_threshold = llm_int8_threshold From 93f0d84ac4c442ab9c8fb3b47378468d4ae707bb Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Mon, 24 Jul 2023 22:13:06 +0000 Subject: [PATCH 02/46] Add tests for gptq --- src/transformers/modeling_utils.py | 5 +- src/transformers/testing_utils.py | 8 ++ src/transformers/utils/__init__.py | 1 + src/transformers/utils/import_utils.py | 5 + tests/gptq/__init__.py | 0 tests/gptq/test_gptq.py | 145 +++++++++++++++++++++++++ 6 files changed, 162 insertions(+), 2 deletions(-) create mode 100644 tests/gptq/__init__.py create mode 100644 tests/gptq/test_gptq.py diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 91d96bc9d9f8..8af5b69d4832 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -64,6 +64,7 @@ download_url, has_file, is_accelerate_available, + is_auto_gptq_available, is_bitsandbytes_available, is_offline_mode, is_optimum_available, @@ -2356,9 +2357,9 @@ def from_pretrained( gtpq_quantizer = None if has_gptq_quantization_config: - if not is_optimum_available(): + if not (is_optimum_available() and is_auto_gptq_available()): raise ImportError( - "Loading GTPQ quantized model requires optimum library : `pip install optimum` and auto_gptq library 'pip install auto_gptq'" + "Loading GTPQ quantized model requires optimum library : `pip install optimum` and auto-gptq library 'pip install auto-gptq'" ) else: # Need to protect the import diff --git a/src/transformers/testing_utils.py b/src/transformers/testing_utils.py index a9ab304d2aa6..14aa69e7c67d 100644 --- a/src/transformers/testing_utils.py +++ b/src/transformers/testing_utils.py @@ -51,6 +51,7 @@ from .utils import ( is_accelerate_available, is_apex_available, + is_auto_gptq_available, is_bitsandbytes_available, is_bs4_available, is_cython_available, @@ -770,6 +771,13 @@ def require_optimum(test_case): return unittest.skipUnless(is_optimum_available(), "test requires optimum")(test_case) +def require_auto_gptq(test_case): + """ + Decorator for auto_gptq dependency + """ + return unittest.skipUnless(is_auto_gptq_available(), "test requires auto-gptq")(test_case) + + def require_phonemizer(test_case): """ Decorator marking a test that requires phonemizer diff --git a/src/transformers/utils/__init__.py b/src/transformers/utils/__init__.py index bca5440f8e02..4addab588134 100644 --- a/src/transformers/utils/__init__.py +++ b/src/transformers/utils/__init__.py @@ -103,6 +103,7 @@ get_torch_version, is_accelerate_available, is_apex_available, + is_auto_gptq_available, is_bitsandbytes_available, is_bs4_available, is_coloredlogs_available, diff --git a/src/transformers/utils/import_utils.py b/src/transformers/utils/import_utils.py index c0a8c80f0b09..54ed4030a2b6 100644 --- a/src/transformers/utils/import_utils.py +++ b/src/transformers/utils/import_utils.py @@ -98,6 +98,7 @@ def _is_package_available(pkg_name: str, return_version: bool = False) -> Union[ _onnx_available = _is_package_available("onnx") _openai_available = _is_package_available("openai") _optimum_available = _is_package_available("optimum") +_auto_gptq_available = _is_package_available("auto_gptq") _pandas_available = _is_package_available("pandas") _peft_available = _is_package_available("peft") _phonemizer_available = _is_package_available("phonemizer") @@ -554,6 +555,10 @@ def is_optimum_available(): return _optimum_available +def is_auto_gptq_available(): + return _auto_gptq_available + + def is_optimum_neuron_available(): return _optimum_available and _is_package_available("optimum.neuron") diff --git a/tests/gptq/__init__.py b/tests/gptq/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/gptq/test_gptq.py b/tests/gptq/test_gptq.py new file mode 100644 index 000000000000..0519be818d2c --- /dev/null +++ b/tests/gptq/test_gptq.py @@ -0,0 +1,145 @@ +# coding=utf-8 +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import tempfile +import unittest + +from transformers import AutoModelForCausalLM, AutoTokenizer +from transformers.testing_utils import ( + is_torch_available, + require_accelerate, + require_auto_gptq, + require_optimum, + require_torch_gpu, + require_torch_multi_gpu, + slow, +) + + +if is_torch_available(): + import torch + + +@slow +@require_optimum +@require_auto_gptq +@require_torch_gpu +class GTPQTest(unittest.TestCase): + model_name = "bigscience/bloom-560m" + + input_text = "Hello my name is" + EXPECTED_OUTPUT = "Hello my name is John and I am a professional photographer. I" + + # this seems a little small considering that we are doing 4bit quant but we have a small model and ww don't quantize the embeddings + EXPECTED_RELATIVE_DIFFERENCE = 1.664253062 + + bits = 4 + group_size = 128 + desc_act = False + + dataset = [ + "auto-gptq is an easy-to-use model quantization library with user-friendly apis, based on GPTQ algorithm." + ] + + device_map = None + + # called only once for all test in this class + @classmethod + def setUpClass(cls): + from optimum.gptq import GPTQQuantizer + + """ + Setup quantized model + """ + cls.model_fp16 = AutoModelForCausalLM.from_pretrained( + cls.model_name, torch_dtype=torch.float16, device_map=cls.device_map + ) + cls.mem_fp16 = cls.model_fp16.get_memory_footprint() + + cls.tokenizer = AutoTokenizer.from_pretrained(cls.model_name, use_fast=True) + cls.quantizer = GPTQQuantizer(bits=cls.bits, group_size=cls.group_size, desc_act=cls.desc_act) + + cls.quantized_model = cls.quantizer.quantize_model(cls.model_fp16, cls.tokenizer, cls.dataset) + + def test_memory_footprint(self): + r""" + A simple test to check if the model conversion has been done correctly by checking on the + memory footprint of the converted model + """ + + mem_quantized = self.quantized_model.get_memory_footprint() + + self.assertAlmostEqual(self.mem_fp16 / mem_quantized, self.EXPECTED_RELATIVE_DIFFERENCE) + + def test_quantized_layers_class(self): + """ + Simple test to check if the model conversion has been done correctly by checking on + the class type of the linear layers of the converted models + """ + from auto_gptq.utils.import_utils import dynamically_import_QuantLinear + + QuantLinear = dynamically_import_QuantLinear( + use_triton=False, desc_act=self.desc_act, group_size=self.group_size + ) + self.assertTrue(self.quantized_model.transformer.h[0].mlp.dense_4h_to_h.__class__ == QuantLinear) + + def check_inference_correctness(self, model): + r""" + Test the generation quality of the quantized model and see that we are matching the expected output. + Given that we are operating on small numbers + the testing model is relatively small, we might not get + the same output across GPUs. So we'll generate few tokens (5-10) and check their output. + """ + # Check that inference pass works on the model + encoded_input = self.tokenizer(self.input_text, return_tensors="pt") + + # Check the exactness of the results + output_parallel = model.generate(input_ids=encoded_input["input_ids"].to(0), max_new_tokens=10) + + # Get the generation + self.assertEqual(self.tokenizer.decode(output_parallel[0], skip_special_tokens=True), self.EXPECTED_OUTPUT) + + def test_generate_quality(self): + """ + Simple test to check the quality of the model by comapring the the generated tokens with the expected tokens + """ + if self.device_map is None: + self.check_inference_correctness(self.quantized_model.to(0)) + else: + self.check_inference_correctness(self.quantized_model) + + def test_serialization(self): + """ + Test the serialization of the model and the loading of the quantized weights works + """ + with tempfile.TemporaryDirectory() as tmpdirname: + self.quantized_model.to("cpu").save_pretrained(tmpdirname) + quantized_model_from_saved = AutoModelForCausalLM.from_pretrained(tmpdirname).to(0) + self.check_inference_correctness(quantized_model_from_saved) + + @require_accelerate + def test_serialization_big_model_inference(self): + """ + Test the serialization of the model and the loading of the quantized weights with big model inference + """ + with tempfile.TemporaryDirectory() as tmpdirname: + self.quantized_model.to("cpu").save_pretrained(tmpdirname) + quantized_model_from_saved = AutoModelForCausalLM.from_pretrained(tmpdirname, device_map="auto") + self.check_inference_correctness(quantized_model_from_saved) + + +@require_accelerate +@require_torch_multi_gpu +class GTPQTestDeviceMap(GTPQTest): + device_map = "auto" From 380baea0d1c6940d910b59745d8120980dd771ec Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Tue, 25 Jul 2023 14:33:49 +0000 Subject: [PATCH 03/46] support for more quantization model --- src/transformers/modeling_utils.py | 34 +++++++++---------- src/transformers/utils/quantization_config.py | 6 +++- tests/{ => quantization}/bnb/README.md | 0 tests/{ => quantization}/bnb/__init__.py | 0 tests/{ => quantization}/bnb/test_4bit.py | 0 .../{ => quantization}/bnb/test_mixed_int8.py | 0 tests/{ => quantization}/gptq/__init__.py | 0 tests/{ => quantization}/gptq/test_gptq.py | 0 8 files changed, 21 insertions(+), 19 deletions(-) rename tests/{ => quantization}/bnb/README.md (100%) rename tests/{ => quantization}/bnb/__init__.py (100%) rename tests/{ => quantization}/bnb/test_4bit.py (100%) rename tests/{ => quantization}/bnb/test_mixed_int8.py (100%) rename tests/{ => quantization}/gptq/__init__.py (100%) rename tests/{ => quantization}/gptq/test_gptq.py (100%) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 8af5b69d4832..54aeebf0bb95 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -76,7 +76,7 @@ ) from .utils.hub import convert_file_size_to_int, get_checkpoint_shard_files from .utils.import_utils import ENV_VARS_TRUE_VALUES, is_sagemaker_mp_enabled -from .utils.quantization_config import BitsAndBytesConfig +from .utils.quantization_config import BitsAndBytesConfig, QuantizationMethod from .utils.versions import require_version_core @@ -2345,18 +2345,16 @@ def from_pretrained( else: model_kwargs = kwargs - has_bnb_quantization_config = False - has_gptq_quantization_config = False + # get the quantization method inside the config of the model if it exist + quantization_method = None if hasattr(config, "quantization_config"): - # backward compatibility - has_bnb_quantization_config = ( - hasattr(config.quantization_config, "load_in_8bit") - or config.quantization_config.get("quant_method", None) == "bitsandbytes" - ) - has_gptq_quantization_config = config.quantization_config.get("quant_method", None) == "gptq" + if hasattr(config.quantization_config, "load_in_8bit"): + quantization_method = QuantizationMethod.BITS_AND_BYTES + else: + quantization_method = config.quantization_config.get("quant_method", None) - gtpq_quantizer = None - if has_gptq_quantization_config: + quantizer = None + if quantization_method == QuantizationMethod.GPTQ: if not (is_optimum_available() and is_auto_gptq_available()): raise ImportError( "Loading GTPQ quantized model requires optimum library : `pip install optimum` and auto-gptq library 'pip install auto-gptq'" @@ -2364,18 +2362,18 @@ def from_pretrained( else: # Need to protect the import from optimum.gptq import GPTQQuantizer - gtpq_quantizer = GPTQQuantizer.from_dict(config.quantization_config) + quantizer = GPTQQuantizer.from_dict(config.quantization_config) torch_dtype = config.torch_dtype - + if is_8bit_serializable and quantization_config is not None and load_in_8bit: - if has_bnb_quantization_config: + if quantization_method==QuantizationMethod.BITS_AND_BYTES: logger.warning( "You passed `quantization_config` to `from_pretrained` but the model you're loading already has a" " `quantization_config` attribute. The `quantization_config` attribute will be overwritten with the" " one you passed to `from_pretrained`." ) config.quantization_config = quantization_config - elif is_8bit_serializable and not load_in_8bit and has_bnb_quantization_config: + elif is_8bit_serializable and not load_in_8bit and quantization_method==QuantizationMethod.BITS_AND_BYTES: quantization_config = config.quantization_config if isinstance(quantization_config, dict): quantization_config = BitsAndBytesConfig.from_dict(quantization_config, return_unused_kwargs=False) @@ -2405,7 +2403,7 @@ def from_pretrained( if low_cpu_mem_usage is None: low_cpu_mem_usage = True - elif not is_8bit_serializable and not load_in_8bit and has_bnb_quantization_config: + elif not is_8bit_serializable and not load_in_8bit and quantization_method==QuantizationMethod.BITS_AND_BYTES: logger.warning( "Detected the presence of a `quantization_config` attribute in the model's configuration but you don't have the correct" " `bitsandbytes` version to support int8 serialization. Please install the latest version of `bitsandbytes` with " @@ -2790,8 +2788,8 @@ def from_pretrained( "All non-linear modules will be loaded in full precision." " If you want to load the other modules in other precision, please specify a `torch_dtype` attribute." ) - if gtpq_quantizer is not None: - model = gtpq_quantizer.convert_model(model) + if quantization_method == QuantizationMethod.GPTQ: + model = quantizer.convert_model(model) if isinstance(device_map, str): special_dtypes = {} diff --git a/src/transformers/utils/quantization_config.py b/src/transformers/utils/quantization_config.py index eeff3eac8846..72f92a2a28ab 100644 --- a/src/transformers/utils/quantization_config.py +++ b/src/transformers/utils/quantization_config.py @@ -24,6 +24,7 @@ from packaging import version from ..utils import is_torch_available, logging +from enum import Enum if is_torch_available(): @@ -32,6 +33,9 @@ logger = logging.get_logger(__name__) +class QuantizationMethod(Enum): + BITS_AND_BYTES = "bitsandbytes" + GPTQ = "gtpq" @dataclass class BitsAndBytesConfig: @@ -97,7 +101,7 @@ def __init__( bnb_4bit_use_double_quant=False, **kwargs, ): - self.quant_method = "bitsandbytes" + self.quant_method = QuantizationMethod.BITS_AND_BYTES self.load_in_8bit = load_in_8bit self.load_in_4bit = load_in_4bit self.llm_int8_threshold = llm_int8_threshold diff --git a/tests/bnb/README.md b/tests/quantization/bnb/README.md similarity index 100% rename from tests/bnb/README.md rename to tests/quantization/bnb/README.md diff --git a/tests/bnb/__init__.py b/tests/quantization/bnb/__init__.py similarity index 100% rename from tests/bnb/__init__.py rename to tests/quantization/bnb/__init__.py diff --git a/tests/bnb/test_4bit.py b/tests/quantization/bnb/test_4bit.py similarity index 100% rename from tests/bnb/test_4bit.py rename to tests/quantization/bnb/test_4bit.py diff --git a/tests/bnb/test_mixed_int8.py b/tests/quantization/bnb/test_mixed_int8.py similarity index 100% rename from tests/bnb/test_mixed_int8.py rename to tests/quantization/bnb/test_mixed_int8.py diff --git a/tests/gptq/__init__.py b/tests/quantization/gptq/__init__.py similarity index 100% rename from tests/gptq/__init__.py rename to tests/quantization/gptq/__init__.py diff --git a/tests/gptq/test_gptq.py b/tests/quantization/gptq/test_gptq.py similarity index 100% rename from tests/gptq/test_gptq.py rename to tests/quantization/gptq/test_gptq.py From 810d53797d9b36a930a85eaf938713754acd3074 Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Tue, 25 Jul 2023 14:34:30 +0000 Subject: [PATCH 04/46] fix style --- src/transformers/modeling_utils.py | 14 ++++++++------ src/transformers/utils/quantization_config.py | 4 +++- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 54aeebf0bb95..4a216f8b0824 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -2354,7 +2354,7 @@ def from_pretrained( quantization_method = config.quantization_config.get("quant_method", None) quantizer = None - if quantization_method == QuantizationMethod.GPTQ: + if quantization_method == QuantizationMethod.GPTQ: if not (is_optimum_available() and is_auto_gptq_available()): raise ImportError( "Loading GTPQ quantized model requires optimum library : `pip install optimum` and auto-gptq library 'pip install auto-gptq'" @@ -2364,16 +2364,16 @@ def from_pretrained( from optimum.gptq import GPTQQuantizer quantizer = GPTQQuantizer.from_dict(config.quantization_config) torch_dtype = config.torch_dtype - + if is_8bit_serializable and quantization_config is not None and load_in_8bit: - if quantization_method==QuantizationMethod.BITS_AND_BYTES: + if quantization_method == QuantizationMethod.BITS_AND_BYTES: logger.warning( "You passed `quantization_config` to `from_pretrained` but the model you're loading already has a" " `quantization_config` attribute. The `quantization_config` attribute will be overwritten with the" " one you passed to `from_pretrained`." ) config.quantization_config = quantization_config - elif is_8bit_serializable and not load_in_8bit and quantization_method==QuantizationMethod.BITS_AND_BYTES: + elif is_8bit_serializable and not load_in_8bit and quantization_method == QuantizationMethod.BITS_AND_BYTES: quantization_config = config.quantization_config if isinstance(quantization_config, dict): quantization_config = BitsAndBytesConfig.from_dict(quantization_config, return_unused_kwargs=False) @@ -2403,7 +2403,9 @@ def from_pretrained( if low_cpu_mem_usage is None: low_cpu_mem_usage = True - elif not is_8bit_serializable and not load_in_8bit and quantization_method==QuantizationMethod.BITS_AND_BYTES: + elif ( + not is_8bit_serializable and not load_in_8bit and quantization_method == QuantizationMethod.BITS_AND_BYTES + ): logger.warning( "Detected the presence of a `quantization_config` attribute in the model's configuration but you don't have the correct" " `bitsandbytes` version to support int8 serialization. Please install the latest version of `bitsandbytes` with " @@ -2788,7 +2790,7 @@ def from_pretrained( "All non-linear modules will be loaded in full precision." " If you want to load the other modules in other precision, please specify a `torch_dtype` attribute." ) - if quantization_method == QuantizationMethod.GPTQ: + if quantization_method == QuantizationMethod.GPTQ: model = quantizer.convert_model(model) if isinstance(device_map, str): diff --git a/src/transformers/utils/quantization_config.py b/src/transformers/utils/quantization_config.py index 72f92a2a28ab..8e66fbf36b49 100644 --- a/src/transformers/utils/quantization_config.py +++ b/src/transformers/utils/quantization_config.py @@ -19,12 +19,12 @@ import json import os from dataclasses import dataclass +from enum import Enum from typing import Any, Dict, Union from packaging import version from ..utils import is_torch_available, logging -from enum import Enum if is_torch_available(): @@ -33,10 +33,12 @@ logger = logging.get_logger(__name__) + class QuantizationMethod(Enum): BITS_AND_BYTES = "bitsandbytes" GPTQ = "gtpq" + @dataclass class BitsAndBytesConfig: """ From c3f52483b199ccf3f3b061df3dfdbcbdff149126 Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Tue, 25 Jul 2023 14:34:57 +0000 Subject: [PATCH 05/46] typo --- src/transformers/modeling_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 4a216f8b0824..7cc7fcf1bb3a 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -2345,7 +2345,7 @@ def from_pretrained( else: model_kwargs = kwargs - # get the quantization method inside the config of the model if it exist + # get the quantization method inside the config of the model if it exists quantization_method = None if hasattr(config, "quantization_config"): if hasattr(config.quantization_config, "load_in_8bit"): From fc70ef46b26324bb680cc7ddb3d16519c9515e07 Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Tue, 25 Jul 2023 14:52:06 +0000 Subject: [PATCH 06/46] fix method --- src/transformers/modeling_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 7cc7fcf1bb3a..2017ee69cd29 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -2348,7 +2348,7 @@ def from_pretrained( # get the quantization method inside the config of the model if it exists quantization_method = None if hasattr(config, "quantization_config"): - if hasattr(config.quantization_config, "load_in_8bit"): + if config.quantization_config.get("load_in_8bit", False): quantization_method = QuantizationMethod.BITS_AND_BYTES else: quantization_method = config.quantization_config.get("quant_method", None) From 6a04bb849f706401d91c2bfbea10fb79b61f2418 Mon Sep 17 00:00:00 2001 From: Marc Sun <57196510+SunMarc@users.noreply.github.com> Date: Tue, 25 Jul 2023 11:52:17 -0400 Subject: [PATCH 07/46] Update src/transformers/modeling_utils.py Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> --- src/transformers/modeling_utils.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 2017ee69cd29..e479135a0370 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -2348,10 +2348,7 @@ def from_pretrained( # get the quantization method inside the config of the model if it exists quantization_method = None if hasattr(config, "quantization_config"): - if config.quantization_config.get("load_in_8bit", False): - quantization_method = QuantizationMethod.BITS_AND_BYTES - else: - quantization_method = config.quantization_config.get("quant_method", None) + quantization_method = config.quantization_config.get("quant_method", QuantizationMethod.BITS_AND_BYTES) quantizer = None if quantization_method == QuantizationMethod.GPTQ: From 271dab67eed2f32a3e07137b97c2ab2a538da348 Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Tue, 25 Jul 2023 23:00:48 +0000 Subject: [PATCH 08/46] add dataclass and fix quantization_method --- src/transformers/modeling_utils.py | 65 +++-- src/transformers/utils/quantization_config.py | 250 ++++++++++++------ 2 files changed, 223 insertions(+), 92 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index e479135a0370..a7452e70ce85 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -33,6 +33,7 @@ from torch import Tensor, nn from torch.nn import CrossEntropyLoss +from . import AutoTokenizer from .activations import get_activation from .configuration_utils import PretrainedConfig from .deepspeed import deepspeed_config, is_deepspeed_zero3_enabled @@ -76,7 +77,7 @@ ) from .utils.hub import convert_file_size_to_int, get_checkpoint_shard_files from .utils.import_utils import ENV_VARS_TRUE_VALUES, is_sagemaker_mp_enabled -from .utils.quantization_config import BitsAndBytesConfig, QuantizationMethod +from .utils.quantization_config import AutoGPTQConfig, BitsAndBytesConfig, QuantizationMethod from .utils.versions import require_version_core @@ -2257,13 +2258,17 @@ def from_pretrained( "Using `low_cpu_mem_usage=True` or a `device_map` requires Accelerate: `pip install accelerate`" ) - if quantization_config is None: + quantization_method_from_args = None + if quantization_config is not None: + quantization_method_from_args = quantization_config.get("quant_method", QuantizationMethod.BITS_AND_BYTES) + + if quantization_config is None and (load_in_8bit or load_in_4bit): quantization_config, kwargs = BitsAndBytesConfig.from_dict( config_dict={"load_in_8bit": load_in_8bit, "load_in_4bit": load_in_4bit}, return_unused_kwargs=True, **kwargs, ) - elif quantization_config is not None: + elif quantization_method_from_args == QuantizationMethod.BITS_AND_BYTES: load_in_8bit = quantization_config.load_in_8bit load_in_4bit = quantization_config.load_in_4bit @@ -2345,13 +2350,24 @@ def from_pretrained( else: model_kwargs = kwargs - # get the quantization method inside the config of the model if it exists - quantization_method = None + quantizer = None + quantization_method_from_config = None if hasattr(config, "quantization_config"): - quantization_method = config.quantization_config.get("quant_method", QuantizationMethod.BITS_AND_BYTES) + quantization_method_from_config = config.quantization_config.get( + "quant_method", QuantizationMethod.BITS_AND_BYTES + ) - quantizer = None - if quantization_method == QuantizationMethod.GPTQ: + if quantization_method_from_config == QuantizationMethod.GPTQ and quantization_method_from_args is not None: + quantization_method_from_args = None + logger.warning( + "You passed `quantization_config` to `from_pretrained` but the model you're loading already has a" + " `quantization_config` attribute and has already quantized weights. We will not perform quantization" + "with the given `quantization config` that you have passed." + ) + if ( + quantization_method_from_args == QuantizationMethod.GPTQ + or quantization_method_from_config == QuantizationMethod.GPTQ + ): if not (is_optimum_available() and is_auto_gptq_available()): raise ImportError( "Loading GTPQ quantized model requires optimum library : `pip install optimum` and auto-gptq library 'pip install auto-gptq'" @@ -2359,18 +2375,28 @@ def from_pretrained( else: # Need to protect the import from optimum.gptq import GPTQQuantizer - quantizer = GPTQQuantizer.from_dict(config.quantization_config) - torch_dtype = config.torch_dtype - - if is_8bit_serializable and quantization_config is not None and load_in_8bit: - if quantization_method == QuantizationMethod.BITS_AND_BYTES: + if quantization_method_from_config == QuantizationMethod.GPTQ: + quantization_config = AutoGPTQConfig.from_dict(config.quantization_config) + torch_dtype = config.torch_dtype + quantizer = GPTQQuantizer.from_dict(quantization_config.to_dict()) + + if ( + is_8bit_serializable + and quantization_method_from_args == QuantizationMethod.BITS_AND_BYTES + and load_in_8bit + ): + if quantization_method_from_config == QuantizationMethod.BITS_AND_BYTES: logger.warning( "You passed `quantization_config` to `from_pretrained` but the model you're loading already has a" " `quantization_config` attribute. The `quantization_config` attribute will be overwritten with the" " one you passed to `from_pretrained`." ) config.quantization_config = quantization_config - elif is_8bit_serializable and not load_in_8bit and quantization_method == QuantizationMethod.BITS_AND_BYTES: + elif ( + is_8bit_serializable + and not load_in_8bit + and quantization_method_from_config == QuantizationMethod.BITS_AND_BYTES + ): quantization_config = config.quantization_config if isinstance(quantization_config, dict): quantization_config = BitsAndBytesConfig.from_dict(quantization_config, return_unused_kwargs=False) @@ -2401,7 +2427,9 @@ def from_pretrained( low_cpu_mem_usage = True elif ( - not is_8bit_serializable and not load_in_8bit and quantization_method == QuantizationMethod.BITS_AND_BYTES + not is_8bit_serializable + and not load_in_8bit + and quantization_method_from_config == QuantizationMethod.BITS_AND_BYTES ): logger.warning( "Detected the presence of a `quantization_config` attribute in the model's configuration but you don't have the correct" @@ -2787,7 +2815,7 @@ def from_pretrained( "All non-linear modules will be loaded in full precision." " If you want to load the other modules in other precision, please specify a `torch_dtype` attribute." ) - if quantization_method == QuantizationMethod.GPTQ: + if quantization_method_from_config == QuantizationMethod.GPTQ: model = quantizer.convert_model(model) if isinstance(device_map, str): @@ -2984,6 +3012,11 @@ def from_pretrained( kwargs["skip_keys"] = model._skip_keys_device_placement dispatch_model(model, **kwargs) + if quantization_method_from_args == QuantizationMethod.GPTQ: + tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path, use_fast=True) + quantizer.quantize_model(model, tokenizer) + model.config.quantization_config = AutoGPTQConfig.from_dict(quantizer.to_dict()) + if output_loading_info: if loading_info is None: loading_info = { diff --git a/src/transformers/utils/quantization_config.py b/src/transformers/utils/quantization_config.py index 8e66fbf36b49..c25e289c93a6 100644 --- a/src/transformers/utils/quantization_config.py +++ b/src/transformers/utils/quantization_config.py @@ -18,9 +18,9 @@ import importlib.metadata import json import os -from dataclasses import dataclass +from dataclasses import dataclass, field from enum import Enum -from typing import Any, Dict, Union +from typing import Any, Dict, List, Union from packaging import version @@ -34,13 +34,96 @@ logger = logging.get_logger(__name__) -class QuantizationMethod(Enum): +class QuantizationMethod(str, Enum): BITS_AND_BYTES = "bitsandbytes" GPTQ = "gtpq" @dataclass -class BitsAndBytesConfig: +class QuantizationConfig: + quant_method: QuantizationMethod + + @classmethod + def from_dict(cls, config_dict, return_unused_kwargs, **kwargs): + """ + Instantiates a [`QuantizationConfig`] from a Python dictionary of parameters. + + Args: + config_dict (`Dict[str, Any]`): + Dictionary that will be used to instantiate the configuration object. + return_unused_kwargs (`bool`): + Whether or not to return a list of unused keyword arguments. Used for `from_pretrained` method in + `PreTrainedModel`. + kwargs (`Dict[str, Any]`): + Additional parameters from which to initialize the configuration object. + + Returns: + [`QuantizationConfig`]: The configuration object instantiated from those parameters. + """ + + config = cls(**config_dict) + + to_remove = [] + for key, value in kwargs.items(): + if hasattr(config, key): + setattr(config, key, value) + to_remove.append(key) + for key in to_remove: + kwargs.pop(key, None) + + if return_unused_kwargs: + return config, kwargs + else: + return config + + def to_json_file(self, json_file_path: Union[str, os.PathLike]): + """ + Save this instance to a JSON file. + + Args: + json_file_path (`str` or `os.PathLike`): + Path to the JSON file in which this configuration instance's parameters will be saved. + use_diff (`bool`, *optional*, defaults to `True`): + If set to `True`, only the difference between the config instance and the default + `QuantizationConfig()` is serialized to JSON file. + """ + with open(json_file_path, "w", encoding="utf-8") as writer: + config_dict = self.to_dict() + json_string = json.dumps(config_dict, indent=2, sort_keys=True) + "\n" + + writer.write(json_string) + + def to_dict(self) -> Dict[str, Any]: + """ + Serializes this instance to a Python dictionary. Returns: + `Dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance. + """ + return copy.deepcopy(self.__dict__) + + def __repr__(self): + return f"{self.__class__.__name__} {self.to_json_string()}" + + def to_json_string(self, use_diff: bool = True) -> str: + """ + Serializes this instance to a JSON string. + + Args: + use_diff (`bool`, *optional*, defaults to `True`): + If set to `True`, only the difference between the config instance and the default `PretrainedConfig()` + is serialized to JSON string. + + Returns: + `str`: String containing all the attributes that make up this configuration instance in JSON format. + """ + if use_diff is True: + config_dict = self.to_diff_dict() + else: + config_dict = self.to_dict() + return json.dumps(config_dict, indent=2, sort_keys=True) + "\n" + + +@dataclass +class BitsAndBytesConfig(QuantizationConfig): """ This is a wrapper class about all possible attributes and features that you can play with a model that has been loaded using `bitsandbytes`. @@ -175,88 +258,16 @@ def quantization_method(self): else: return None - @classmethod - def from_dict(cls, config_dict, return_unused_kwargs, **kwargs): - """ - Instantiates a [`BitsAndBytesConfig`] from a Python dictionary of parameters. - - Args: - config_dict (`Dict[str, Any]`): - Dictionary that will be used to instantiate the configuration object. - return_unused_kwargs (`bool`): - Whether or not to return a list of unused keyword arguments. Used for `from_pretrained` method in - `PreTrainedModel`. - kwargs (`Dict[str, Any]`): - Additional parameters from which to initialize the configuration object. - - Returns: - [`BitsAndBytesConfig`]: The configuration object instantiated from those parameters. - """ - - config = cls(**config_dict) - - to_remove = [] - for key, value in kwargs.items(): - if hasattr(config, key): - setattr(config, key, value) - to_remove.append(key) - for key in to_remove: - kwargs.pop(key, None) - - if return_unused_kwargs: - return config, kwargs - else: - return config - - def to_json_file(self, json_file_path: Union[str, os.PathLike]): - """ - Save this instance to a JSON file. - - Args: - json_file_path (`str` or `os.PathLike`): - Path to the JSON file in which this configuration instance's parameters will be saved. - use_diff (`bool`, *optional*, defaults to `True`): - If set to `True`, only the difference between the config instance and the default - `BitsAndBytesConfig()` is serialized to JSON file. - """ - with open(json_file_path, "w", encoding="utf-8") as writer: - config_dict = self.to_dict() - json_string = json.dumps(config_dict, indent=2, sort_keys=True) + "\n" - - writer.write(json_string) - def to_dict(self) -> Dict[str, Any]: """ Serializes this instance to a Python dictionary. Returns: `Dict[str, Any]`: Dictionary of all the attributes that make up this configuration instance. """ - output = copy.deepcopy(self.__dict__) output["bnb_4bit_compute_dtype"] = str(output["bnb_4bit_compute_dtype"]).split(".")[1] return output - def __repr__(self): - return f"{self.__class__.__name__} {self.to_json_string()}" - - def to_json_string(self, use_diff: bool = True) -> str: - """ - Serializes this instance to a JSON string. - - Args: - use_diff (`bool`, *optional*, defaults to `True`): - If set to `True`, only the difference between the config instance and the default `PretrainedConfig()` - is serialized to JSON string. - - Returns: - `str`: String containing all the attributes that make up this configuration instance in JSON format. - """ - if use_diff is True: - config_dict = self.to_diff_dict() - else: - config_dict = self.to_dict() - return json.dumps(config_dict, indent=2, sort_keys=True) + "\n" - def to_diff_dict(self) -> Dict[str, Any]: """ Removes all attributes from config which correspond to the default config attributes for better readability and @@ -278,3 +289,90 @@ def to_diff_dict(self) -> Dict[str, Any]: serializable_config_dict[key] = value return serializable_config_dict + + +@dataclass +class AutoGPTQConfig(QuantizationConfig): + """ + This is a wrapper class about all possible attributes and features that you can play with a model that has been + loaded using `optimum` api for gptq quantization relying on auto_gptq backend. + """ + + quant_method: QuantizationMethod = QuantizationMethod.GPTQ + bits: int = field( + default=None, metadata={"help": "The number of bits to quantize to, supported numbers are (2, 3, 4, 8)."} + ) + dataset: Union[List[str], str] = field( + default=None, + metadata={ + "help": "The dataset used for quantization. You can provide your own dataset in a list of string" + "or just use the original datasets used in GPTQ paper ['wikitext2','c4','c4-new','ptb','ptb-new']" + }, + ) + group_size: int = field( + default=128, + metadata={ + "help": "The group size to use for quantization. Recommended value is 128 and -1 uses per-column quantization." + }, + ) + damp_percent: float = field( + default=0.01, + metadata={ + "help": "The percent of the average Hessian diagonal to use for dampening. Recommended value is 0.01." + }, + ) + dataset: Union[str, List[str]] = field( + default=None, + metadata={ + "help": "The percent of the average Hessian diagonal to use for dampening. Recommended value is 0.01." + }, + ) + desc_act: bool = field( + default=True, + metadata={ + "help": "Whether to quantize columns in order of decreasing activation size." + "Setting it to False can significantly speed up inference but the perplexity may become slightly worse." + "Also known as act-order." + }, + ) + sym: bool = field(default=True, metadata={"help": " Whether to use symetric quantization."}) + true_sequential: bool = field( + default=True, + metadata={ + "help": "Whether to perform sequential quantization even within a single Transformer block." + "Instead of quantizing the entire block at once, we perform layer-wise quantization." + "As a result, each layer undergoes quantization using inputs that have passed through the previously quantized layers." + }, + ) + pack_sequentially: bool = field( + default=False, + metadata={ + "help": "Whether to pack the layer just after it is quantized. If False, we will pack the model at the end." + }, + ) + use_cuda_fp16: bool = field( + default=False, + metadata={"help": "Whether or not to use optimized cuda kernel for fp16 model. Need to have model in fp16."}, + ) + model_seqlen: int = field(default=None, metadata={"help": "The maximum sequence length that the model can take."}) + block_name_to_quantize: str = field(default=None, metadata={"help": "The transformers block name to quantize."}) + module_name_preceding_first_block: List[str] = field( + default=None, metadata={"help": "The layers that are preceding the first Transformer block."} + ) + + batch_size: int = field(default=1, metadata={"help": " The batch size of the dataset"}) + + pad_token_id: int = field( + default=None, metadata={"help": " The pad token id. Needed to prepare the dataset when `batch_size` > 1."} + ) + + def __post_init__(self): + r""" + Safety checker that arguments are correct + """ + if self.bits not in [2, 4, 6, 8]: + raise ValueError(f"Only support quantization to [2,4,6,8] bits but found {self.bits}") + if self.group_size != -1 and self.group_size <= 0: + raise ValueError("group_size must be greater than 0 or equal to -1") + if not (0 < self.damp_percent < 1): + raise ValueError("damp_percent must between 0 and 1.") From 992881ef3c8f055ae597d3c6ed20bab7a628ee2e Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Wed, 26 Jul 2023 13:59:34 +0000 Subject: [PATCH 09/46] fix doc --- src/transformers/modeling_utils.py | 9 +++--- src/transformers/utils/quantization_config.py | 28 ++++++++++--------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index a7452e70ce85..fb2df4386304 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -33,7 +33,6 @@ from torch import Tensor, nn from torch.nn import CrossEntropyLoss -from . import AutoTokenizer from .activations import get_activation from .configuration_utils import PretrainedConfig from .deepspeed import deepspeed_config, is_deepspeed_zero3_enabled @@ -77,7 +76,7 @@ ) from .utils.hub import convert_file_size_to_int, get_checkpoint_shard_files from .utils.import_utils import ENV_VARS_TRUE_VALUES, is_sagemaker_mp_enabled -from .utils.quantization_config import AutoGPTQConfig, BitsAndBytesConfig, QuantizationMethod +from .utils.quantization_config import BitsAndBytesConfig, GPTQConfig, QuantizationMethod from .utils.versions import require_version_core @@ -2376,7 +2375,7 @@ def from_pretrained( # Need to protect the import from optimum.gptq import GPTQQuantizer if quantization_method_from_config == QuantizationMethod.GPTQ: - quantization_config = AutoGPTQConfig.from_dict(config.quantization_config) + quantization_config = GPTQConfig.from_dict(config.quantization_config) torch_dtype = config.torch_dtype quantizer = GPTQQuantizer.from_dict(quantization_config.to_dict()) @@ -3013,9 +3012,11 @@ def from_pretrained( dispatch_model(model, **kwargs) if quantization_method_from_args == QuantizationMethod.GPTQ: + from . import AutoTokenizer + tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path, use_fast=True) quantizer.quantize_model(model, tokenizer) - model.config.quantization_config = AutoGPTQConfig.from_dict(quantizer.to_dict()) + model.config.quantization_config = GPTQConfig.from_dict(quantizer.to_dict()) if output_loading_info: if loading_info is None: diff --git a/src/transformers/utils/quantization_config.py b/src/transformers/utils/quantization_config.py index c25e289c93a6..70cb6c760095 100644 --- a/src/transformers/utils/quantization_config.py +++ b/src/transformers/utils/quantization_config.py @@ -40,7 +40,7 @@ class QuantizationMethod(str, Enum): @dataclass -class QuantizationConfig: +class QuantizationConfigMixin: quant_method: QuantizationMethod @classmethod @@ -123,7 +123,7 @@ def to_json_string(self, use_diff: bool = True) -> str: @dataclass -class BitsAndBytesConfig(QuantizationConfig): +class BitsAndBytesConfig(QuantizationConfigMixin): """ This is a wrapper class about all possible attributes and features that you can play with a model that has been loaded using `bitsandbytes`. @@ -292,7 +292,7 @@ def to_diff_dict(self) -> Dict[str, Any]: @dataclass -class AutoGPTQConfig(QuantizationConfig): +class GPTQConfig(QuantizationConfigMixin): """ This is a wrapper class about all possible attributes and features that you can play with a model that has been loaded using `optimum` api for gptq quantization relying on auto_gptq backend. @@ -321,12 +321,6 @@ class AutoGPTQConfig(QuantizationConfig): "help": "The percent of the average Hessian diagonal to use for dampening. Recommended value is 0.01." }, ) - dataset: Union[str, List[str]] = field( - default=None, - metadata={ - "help": "The percent of the average Hessian diagonal to use for dampening. Recommended value is 0.01." - }, - ) desc_act: bool = field( default=True, metadata={ @@ -359,11 +353,9 @@ class AutoGPTQConfig(QuantizationConfig): module_name_preceding_first_block: List[str] = field( default=None, metadata={"help": "The layers that are preceding the first Transformer block."} ) - - batch_size: int = field(default=1, metadata={"help": " The batch size of the dataset"}) - + batch_size: int = field(default=1, metadata={"help": "The batch size used when processing the dataset"}) pad_token_id: int = field( - default=None, metadata={"help": " The pad token id. Needed to prepare the dataset when `batch_size` > 1."} + default=None, metadata={"help": "The pad token id. Needed to prepare the dataset when `batch_size` > 1."} ) def __post_init__(self): @@ -376,3 +368,13 @@ def __post_init__(self): raise ValueError("group_size must be greater than 0 or equal to -1") if not (0 < self.damp_percent < 1): raise ValueError("damp_percent must between 0 and 1.") + if isinstance(self.dataset, str) and self.dataset not in ["wikitext2", "c4", "c4-new", "ptb", "ptb-new"]: + raise ValueError( + f"""You have entered a string value for dataset. You can only choose between + ['wikitext2','c4','c4-new','ptb','ptb-new'], but we found {self.dataset}""" + ) + elif not isinstance(self.dataset, list): + raise ValueError( + f"""dataset needs to be either a list of string or a value in + ['wikitext2','c4','c4-new','ptb','ptb-new'], but we found {self.dataset}""" + ) From 3c2d940fe46bb40482c07fdb8cb898f13fe52443 Mon Sep 17 00:00:00 2001 From: Marc Sun <57196510+SunMarc@users.noreply.github.com> Date: Wed, 26 Jul 2023 09:59:57 -0400 Subject: [PATCH 10/46] Update tests/quantization/gptq/test_gptq.py Co-authored-by: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> --- tests/quantization/gptq/test_gptq.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/quantization/gptq/test_gptq.py b/tests/quantization/gptq/test_gptq.py index 0519be818d2c..d1c316db77e0 100644 --- a/tests/quantization/gptq/test_gptq.py +++ b/tests/quantization/gptq/test_gptq.py @@ -141,5 +141,5 @@ def test_serialization_big_model_inference(self): @require_accelerate @require_torch_multi_gpu -class GTPQTestDeviceMap(GTPQTest): +class GPTQTestDeviceMap(GPTQTest): device_map = "auto" From 9bbb336f11db81aff451951c96993e7c9f359aed Mon Sep 17 00:00:00 2001 From: Marc Sun <57196510+SunMarc@users.noreply.github.com> Date: Wed, 26 Jul 2023 10:02:33 -0400 Subject: [PATCH 11/46] Apply suggestions from code review Co-authored-by: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> --- src/transformers/modeling_utils.py | 2 +- src/transformers/utils/quantization_config.py | 2 +- tests/quantization/gptq/test_gptq.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index fb2df4386304..1d787a89c53d 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -2369,7 +2369,7 @@ def from_pretrained( ): if not (is_optimum_available() and is_auto_gptq_available()): raise ImportError( - "Loading GTPQ quantized model requires optimum library : `pip install optimum` and auto-gptq library 'pip install auto-gptq'" + "Loading GPTQ quantized model requires optimum library : `pip install optimum` and auto-gptq library 'pip install auto-gptq'" ) else: # Need to protect the import diff --git a/src/transformers/utils/quantization_config.py b/src/transformers/utils/quantization_config.py index 70cb6c760095..59a5329d343c 100644 --- a/src/transformers/utils/quantization_config.py +++ b/src/transformers/utils/quantization_config.py @@ -36,7 +36,7 @@ class QuantizationMethod(str, Enum): BITS_AND_BYTES = "bitsandbytes" - GPTQ = "gtpq" + GPTQ = "gptq" @dataclass diff --git a/tests/quantization/gptq/test_gptq.py b/tests/quantization/gptq/test_gptq.py index d1c316db77e0..bcd6d489e64e 100644 --- a/tests/quantization/gptq/test_gptq.py +++ b/tests/quantization/gptq/test_gptq.py @@ -36,7 +36,7 @@ @require_optimum @require_auto_gptq @require_torch_gpu -class GTPQTest(unittest.TestCase): +class GPTQTest(unittest.TestCase): model_name = "bigscience/bloom-560m" input_text = "Hello my name is" From 0134c79fe8aa420d499d6c3c5a1248de8341086c Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Wed, 26 Jul 2023 14:45:33 +0000 Subject: [PATCH 12/46] modify dataclass --- src/transformers/utils/quantization_config.py | 134 ++++++++++-------- 1 file changed, 72 insertions(+), 62 deletions(-) diff --git a/src/transformers/utils/quantization_config.py b/src/transformers/utils/quantization_config.py index 59a5329d343c..92db4a81490e 100644 --- a/src/transformers/utils/quantization_config.py +++ b/src/transformers/utils/quantization_config.py @@ -18,9 +18,9 @@ import importlib.metadata import json import os -from dataclasses import dataclass, field +from dataclasses import dataclass from enum import Enum -from typing import Any, Dict, List, Union +from typing import Any, Dict, List, Optional, Union from packaging import version @@ -296,69 +296,79 @@ class GPTQConfig(QuantizationConfigMixin): """ This is a wrapper class about all possible attributes and features that you can play with a model that has been loaded using `optimum` api for gptq quantization relying on auto_gptq backend. - """ - quant_method: QuantizationMethod = QuantizationMethod.GPTQ - bits: int = field( - default=None, metadata={"help": "The number of bits to quantize to, supported numbers are (2, 3, 4, 8)."} - ) - dataset: Union[List[str], str] = field( - default=None, - metadata={ - "help": "The dataset used for quantization. You can provide your own dataset in a list of string" - "or just use the original datasets used in GPTQ paper ['wikitext2','c4','c4-new','ptb','ptb-new']" - }, - ) - group_size: int = field( - default=128, - metadata={ - "help": "The group size to use for quantization. Recommended value is 128 and -1 uses per-column quantization." - }, - ) - damp_percent: float = field( - default=0.01, - metadata={ - "help": "The percent of the average Hessian diagonal to use for dampening. Recommended value is 0.01." - }, - ) - desc_act: bool = field( - default=True, - metadata={ - "help": "Whether to quantize columns in order of decreasing activation size." + Args: + bits (`int`): + The number of bits to quantize to, supported numbers are (2, 3, 4, 8). + dataset (`Union[List[str], str]`): + "The dataset used for quantization. You can provide your own dataset in a list of string" "or just use the + original datasets used in GPTQ paper ['wikitext2','c4','c4-new','ptb','ptb-new']" + group_size (`int`, *optional*, defaults to `128`): + "The group size to use for quantization. Recommended value is 128 and -1 uses per-column quantization. + damp_percent (`float`, *optional*, defaults to `0.01`): + The percent of the average Hessian diagonal to use for dampening. Recommended value is 0.01. + desc_act (`bool`, *optional*, defaults to `True`): + "Whether to quantize columns in order of decreasing activation size." "Setting it to False can significantly speed up inference but the perplexity may become slightly worse." "Also known as act-order." - }, - ) - sym: bool = field(default=True, metadata={"help": " Whether to use symetric quantization."}) - true_sequential: bool = field( - default=True, - metadata={ - "help": "Whether to perform sequential quantization even within a single Transformer block." - "Instead of quantizing the entire block at once, we perform layer-wise quantization." - "As a result, each layer undergoes quantization using inputs that have passed through the previously quantized layers." - }, - ) - pack_sequentially: bool = field( - default=False, - metadata={ - "help": "Whether to pack the layer just after it is quantized. If False, we will pack the model at the end." - }, - ) - use_cuda_fp16: bool = field( - default=False, - metadata={"help": "Whether or not to use optimized cuda kernel for fp16 model. Need to have model in fp16."}, - ) - model_seqlen: int = field(default=None, metadata={"help": "The maximum sequence length that the model can take."}) - block_name_to_quantize: str = field(default=None, metadata={"help": "The transformers block name to quantize."}) - module_name_preceding_first_block: List[str] = field( - default=None, metadata={"help": "The layers that are preceding the first Transformer block."} - ) - batch_size: int = field(default=1, metadata={"help": "The batch size used when processing the dataset"}) - pad_token_id: int = field( - default=None, metadata={"help": "The pad token id. Needed to prepare the dataset when `batch_size` > 1."} - ) - - def __post_init__(self): + sym (`bool`, *optional*, defaults to `True`): + Whether to use symetric quantization. + true_sequential (`bool`, *optional*, defaults to `True`): + "Whether to perform sequential quantization even within a single Transformer block." "Instead of quantizing + the entire block at once, we perform layer-wise quantization." "As a result, each layer undergoes + quantization using inputs that have passed through the previously quantized layers." + pack_sequentially (`bool`, *optional*, defaults to `False`): + Whether to pack the layer just after it is quantized. If False, we will pack the model at the end. + use_cuda_fp16 (`bool`, *optional*, defaults to `False`): + "Whether or not to use optimized cuda kernel for fp16 model. Need to have model in fp16. + model_seqlen (`Optional[int]`, *optional*, defaults to `None`): + The maximum sequence length that the model can take. + block_name_to_quantize (`Optional[str]`, *optional*, defaults to `None`): + The transformers block name to quantize. + module_name_preceding_first_block (`Optional[List[str]]`, *optional*, defaults to `None`): + The layers that are preceding the first Transformer block. + batch_size (`int`, *optional*, defaults to `1`): + The batch size used when processing the dataset + pad_token_id (`Optional[int]`, *optional*, defaults to `None`): + The pad token id. Needed to prepare the dataset when `batch_size` > 1. + """ + + def __init__( + self, + bits: int, + dataset: Union[List[str], str], + group_size: int = 128, + damp_percent: float = 0.01, + desc_act: bool = True, + sym: bool = True, + true_sequential: bool = True, + pack_sequentially: bool = False, + use_cuda_fp16: bool = False, + model_seqlen: Optional[int] = None, + block_name_to_quantize: Optional[str] = None, + module_name_preceding_first_block: Optional[List[str]] = None, + batch_size: int = 1, + pad_token_id: Optional[int] = None, + **kwargs, + ): + self.quant_method: QuantizationMethod.GPTQ + self.bits = bits + self.dataset = dataset + self.group_size = group_size + self.damp_percent = damp_percent + self.desc_act = desc_act + self.sym = sym + self.true_sequential = true_sequential + self.pack_sequentially = pack_sequentially + self.use_cuda_fp16 = use_cuda_fp16 + self.model_seqlen = model_seqlen + self.block_name_to_quantize = block_name_to_quantize + self.module_name_preceding_first_block = module_name_preceding_first_block + self.batch_size = batch_size + self.pad_token_id = pad_token_id + self.post_init() + + def post_init(self): r""" Safety checker that arguments are correct """ From a2a7f5d39c388e351c1cd03e8c10bf59a6e6be6d Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Wed, 26 Jul 2023 14:59:19 +0000 Subject: [PATCH 13/46] add gtpqconfig import --- src/transformers/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index 9eaba9cd1d10..d513d88f8851 100644 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -731,6 +731,7 @@ ], "utils.bitsandbytes": [], "utils.quantization_config": ["BitsAndBytesConfig"], + "uils.quantization_config": ["GPTQConfig"] } # sentencepiece-backed objects @@ -4674,6 +4675,7 @@ # bitsandbytes config from .utils.quantization_config import BitsAndBytesConfig + from .utils.quantization_config import GPTQConfig try: if not is_sentencepiece_available(): From 70e14160e8bee7dc6216a107b1e509e5fafb8244 Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Wed, 26 Jul 2023 15:02:57 +0000 Subject: [PATCH 14/46] fix typo --- src/transformers/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index d513d88f8851..5b19a8594c5d 100644 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -731,7 +731,7 @@ ], "utils.bitsandbytes": [], "utils.quantization_config": ["BitsAndBytesConfig"], - "uils.quantization_config": ["GPTQConfig"] + "utils.quantization_config": ["GPTQConfig"] } # sentencepiece-backed objects From 0e2014b5e96d3ba63f30ad1c26275c7f8e1ac16a Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Wed, 26 Jul 2023 15:35:05 +0000 Subject: [PATCH 15/46] fix tests --- src/transformers/__init__.py | 6 ++---- src/transformers/modeling_utils.py | 4 +++- src/transformers/utils/quantization_config.py | 10 +++++++--- tests/quantization/gptq/test_gptq.py | 16 +++++++++++----- 4 files changed, 23 insertions(+), 13 deletions(-) diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index 5b19a8594c5d..d731731ee843 100644 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -730,8 +730,7 @@ "logging", ], "utils.bitsandbytes": [], - "utils.quantization_config": ["BitsAndBytesConfig"], - "utils.quantization_config": ["GPTQConfig"] + "utils.quantization_config": ["BitsAndBytesConfig", "GPTQConfig"], } # sentencepiece-backed objects @@ -4674,8 +4673,7 @@ ) # bitsandbytes config - from .utils.quantization_config import BitsAndBytesConfig - from .utils.quantization_config import GPTQConfig + from .utils.quantization_config import BitsAndBytesConfig, GPTQConfig try: if not is_sentencepiece_available(): diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 1d787a89c53d..5870d5459de8 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -2259,7 +2259,9 @@ def from_pretrained( quantization_method_from_args = None if quantization_config is not None: - quantization_method_from_args = quantization_config.get("quant_method", QuantizationMethod.BITS_AND_BYTES) + quantization_method_from_args = getattr( + quantization_config, "quant_method", QuantizationMethod.BITS_AND_BYTES + ) if quantization_config is None and (load_in_8bit or load_in_4bit): quantization_config, kwargs = BitsAndBytesConfig.from_dict( diff --git a/src/transformers/utils/quantization_config.py b/src/transformers/utils/quantization_config.py index 92db4a81490e..c6f8b062d44b 100644 --- a/src/transformers/utils/quantization_config.py +++ b/src/transformers/utils/quantization_config.py @@ -41,17 +41,21 @@ class QuantizationMethod(str, Enum): @dataclass class QuantizationConfigMixin: + """ + Mixin class for quantization config + """ + quant_method: QuantizationMethod @classmethod - def from_dict(cls, config_dict, return_unused_kwargs, **kwargs): + def from_dict(cls, config_dict, return_unused_kwargs=False, **kwargs): """ Instantiates a [`QuantizationConfig`] from a Python dictionary of parameters. Args: config_dict (`Dict[str, Any]`): Dictionary that will be used to instantiate the configuration object. - return_unused_kwargs (`bool`): + return_unused_kwargs (`bool`,*optional*, defaults to `False`): Whether or not to return a list of unused keyword arguments. Used for `from_pretrained` method in `PreTrainedModel`. kwargs (`Dict[str, Any]`): @@ -351,7 +355,7 @@ def __init__( pad_token_id: Optional[int] = None, **kwargs, ): - self.quant_method: QuantizationMethod.GPTQ + self.quant_method = QuantizationMethod.GPTQ self.bits = bits self.dataset = dataset self.group_size = group_size diff --git a/tests/quantization/gptq/test_gptq.py b/tests/quantization/gptq/test_gptq.py index bcd6d489e64e..3b5962fcf68d 100644 --- a/tests/quantization/gptq/test_gptq.py +++ b/tests/quantization/gptq/test_gptq.py @@ -16,7 +16,7 @@ import tempfile import unittest -from transformers import AutoModelForCausalLM, AutoTokenizer +from transformers import AutoModelForCausalLM, AutoTokenizer, GPTQConfig from transformers.testing_utils import ( is_torch_available, require_accelerate, @@ -58,8 +58,6 @@ class GPTQTest(unittest.TestCase): # called only once for all test in this class @classmethod def setUpClass(cls): - from optimum.gptq import GPTQQuantizer - """ Setup quantized model """ @@ -69,9 +67,17 @@ def setUpClass(cls): cls.mem_fp16 = cls.model_fp16.get_memory_footprint() cls.tokenizer = AutoTokenizer.from_pretrained(cls.model_name, use_fast=True) - cls.quantizer = GPTQQuantizer(bits=cls.bits, group_size=cls.group_size, desc_act=cls.desc_act) - cls.quantized_model = cls.quantizer.quantize_model(cls.model_fp16, cls.tokenizer, cls.dataset) + quantization_config = GPTQConfig( + bits=cls.bits, dataset=cls.dataset, group_size=cls.group_size, desc_act=cls.desc_act + ) + + cls.quantized_model = AutoModelForCausalLM.from_pretrained( + cls.model_name, + torch_dtype=torch.float16, + device_map=cls.device_map, + quantization_config=quantization_config, + ) def test_memory_footprint(self): r""" From 69e3c8854964e159b364243bfab7e9128a3ad475 Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Wed, 26 Jul 2023 15:53:13 +0000 Subject: [PATCH 16/46] remove dataset as req arg --- src/transformers/utils/quantization_config.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/transformers/utils/quantization_config.py b/src/transformers/utils/quantization_config.py index c6f8b062d44b..65740acf7cab 100644 --- a/src/transformers/utils/quantization_config.py +++ b/src/transformers/utils/quantization_config.py @@ -304,7 +304,7 @@ class GPTQConfig(QuantizationConfigMixin): Args: bits (`int`): The number of bits to quantize to, supported numbers are (2, 3, 4, 8). - dataset (`Union[List[str], str]`): + dataset (`Union[List[str]]`, *optional*, defaults to `None`): "The dataset used for quantization. You can provide your own dataset in a list of string" "or just use the original datasets used in GPTQ paper ['wikitext2','c4','c4-new','ptb','ptb-new']" group_size (`int`, *optional*, defaults to `128`): @@ -325,22 +325,22 @@ class GPTQConfig(QuantizationConfigMixin): Whether to pack the layer just after it is quantized. If False, we will pack the model at the end. use_cuda_fp16 (`bool`, *optional*, defaults to `False`): "Whether or not to use optimized cuda kernel for fp16 model. Need to have model in fp16. - model_seqlen (`Optional[int]`, *optional*, defaults to `None`): + model_seqlen (`int`, *optional*, defaults to `None`): The maximum sequence length that the model can take. block_name_to_quantize (`Optional[str]`, *optional*, defaults to `None`): The transformers block name to quantize. - module_name_preceding_first_block (`Optional[List[str]]`, *optional*, defaults to `None`): + module_name_preceding_first_block (`List[str]`, *optional*, defaults to `None`): The layers that are preceding the first Transformer block. batch_size (`int`, *optional*, defaults to `1`): The batch size used when processing the dataset - pad_token_id (`Optional[int]`, *optional*, defaults to `None`): + pad_token_id (`int`, *optional*, defaults to `None`): The pad token id. Needed to prepare the dataset when `batch_size` > 1. """ def __init__( self, bits: int, - dataset: Union[List[str], str], + dataset: Optional[Union[List[str], str]] = None, group_size: int = 128, damp_percent: float = 0.01, desc_act: bool = True, From cb46d756e4752a6a12052f8150a2d86926c0b68a Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Wed, 26 Jul 2023 19:09:12 +0000 Subject: [PATCH 17/46] remove tokenizer import --- src/transformers/modeling_utils.py | 7 ++--- src/transformers/utils/quantization_config.py | 31 +++++++++++++------ tests/quantization/gptq/test_gptq.py | 15 ++++++--- 3 files changed, 35 insertions(+), 18 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 5870d5459de8..736f4dbc06d5 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -3014,10 +3014,9 @@ def from_pretrained( dispatch_model(model, **kwargs) if quantization_method_from_args == QuantizationMethod.GPTQ: - from . import AutoTokenizer - - tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path, use_fast=True) - quantizer.quantize_model(model, tokenizer) + if quantization_config.tokenizer is None: + quantization_config.tokenizer = pretrained_model_name_or_path + quantizer.quantize_model(model, quantization_config.tokenizer) model.config.quantization_config = GPTQConfig.from_dict(quantizer.to_dict()) if output_loading_info: diff --git a/src/transformers/utils/quantization_config.py b/src/transformers/utils/quantization_config.py index 65740acf7cab..467cf6370481 100644 --- a/src/transformers/utils/quantization_config.py +++ b/src/transformers/utils/quantization_config.py @@ -304,6 +304,14 @@ class GPTQConfig(QuantizationConfigMixin): Args: bits (`int`): The number of bits to quantize to, supported numbers are (2, 3, 4, 8). + tokenizer(`Any`): + The tokenizer used to process the dataset. You can pass either: + - A custom tokenizer object. + - A string, the *model id* of a predefined tokenizer hosted inside a model repo on huggingface.co. + Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a + user or organization name, like `dbmdz/bert-base-german-cased`. + - A path to a *directory* containing vocabulary files required by the tokenizer, for instance saved + using the [`~PreTrainedTokenizer.save_pretrained`] method, e.g., `./my_model_directory/`. dataset (`Union[List[str]]`, *optional*, defaults to `None`): "The dataset used for quantization. You can provide your own dataset in a list of string" "or just use the original datasets used in GPTQ paper ['wikitext2','c4','c4-new','ptb','ptb-new']" @@ -340,6 +348,7 @@ class GPTQConfig(QuantizationConfigMixin): def __init__( self, bits: int, + tokenizer: Any = None, dataset: Optional[Union[List[str], str]] = None, group_size: int = 128, damp_percent: float = 0.01, @@ -357,6 +366,7 @@ def __init__( ): self.quant_method = QuantizationMethod.GPTQ self.bits = bits + self.tokenizer = tokenizer self.dataset = dataset self.group_size = group_size self.damp_percent = damp_percent @@ -382,13 +392,14 @@ def post_init(self): raise ValueError("group_size must be greater than 0 or equal to -1") if not (0 < self.damp_percent < 1): raise ValueError("damp_percent must between 0 and 1.") - if isinstance(self.dataset, str) and self.dataset not in ["wikitext2", "c4", "c4-new", "ptb", "ptb-new"]: - raise ValueError( - f"""You have entered a string value for dataset. You can only choose between - ['wikitext2','c4','c4-new','ptb','ptb-new'], but we found {self.dataset}""" - ) - elif not isinstance(self.dataset, list): - raise ValueError( - f"""dataset needs to be either a list of string or a value in - ['wikitext2','c4','c4-new','ptb','ptb-new'], but we found {self.dataset}""" - ) + if self.dataset is not None: + if isinstance(self.dataset, str) and self.dataset not in ["wikitext2", "c4", "c4-new", "ptb", "ptb-new"]: + raise ValueError( + f"""You have entered a string value for dataset. You can only choose between + ['wikitext2','c4','c4-new','ptb','ptb-new'], but we found {self.dataset}""" + ) + elif not isinstance(self.dataset, list): + raise ValueError( + f"""dataset needs to be either a list of string or a value in + ['wikitext2','c4','c4-new','ptb','ptb-new'], but we found {self.dataset}""" + ) diff --git a/tests/quantization/gptq/test_gptq.py b/tests/quantization/gptq/test_gptq.py index 3b5962fcf68d..8bba58db25cf 100644 --- a/tests/quantization/gptq/test_gptq.py +++ b/tests/quantization/gptq/test_gptq.py @@ -40,7 +40,10 @@ class GPTQTest(unittest.TestCase): model_name = "bigscience/bloom-560m" input_text = "Hello my name is" - EXPECTED_OUTPUT = "Hello my name is John and I am a professional photographer. I" + + EXPECTED_OUTPUTS = set() + EXPECTED_OUTPUTS.add("Hello my name is John and I am a professional photographer. I") + EXPECTED_OUTPUTS.add("Hello my name is John and I am a very good looking man.") # this seems a little small considering that we are doing 4bit quant but we have a small model and ww don't quantize the embeddings EXPECTED_RELATIVE_DIFFERENCE = 1.664253062 @@ -69,7 +72,11 @@ def setUpClass(cls): cls.tokenizer = AutoTokenizer.from_pretrained(cls.model_name, use_fast=True) quantization_config = GPTQConfig( - bits=cls.bits, dataset=cls.dataset, group_size=cls.group_size, desc_act=cls.desc_act + bits=cls.bits, + dataset=cls.dataset, + tokenizer=cls.tokenizer, + group_size=cls.group_size, + desc_act=cls.desc_act, ) cls.quantized_model = AutoModelForCausalLM.from_pretrained( @@ -111,10 +118,10 @@ def check_inference_correctness(self, model): encoded_input = self.tokenizer(self.input_text, return_tensors="pt") # Check the exactness of the results - output_parallel = model.generate(input_ids=encoded_input["input_ids"].to(0), max_new_tokens=10) + output_sequences = model.generate(input_ids=encoded_input["input_ids"].to(0), max_new_tokens=10) # Get the generation - self.assertEqual(self.tokenizer.decode(output_parallel[0], skip_special_tokens=True), self.EXPECTED_OUTPUT) + self.assertIn(self.tokenizer.decode(output_sequences[0], skip_special_tokens=True), self.EXPECTED_OUTPUTS) def test_generate_quality(self): """ From 9a3cafd0955f116b759df7abd21d1eb0fbb04259 Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Wed, 26 Jul 2023 20:42:49 +0000 Subject: [PATCH 18/46] add offload cpu quantization test --- tests/quantization/gptq/test_gptq.py | 35 ++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/tests/quantization/gptq/test_gptq.py b/tests/quantization/gptq/test_gptq.py index 8bba58db25cf..e59cf2745a7b 100644 --- a/tests/quantization/gptq/test_gptq.py +++ b/tests/quantization/gptq/test_gptq.py @@ -156,3 +156,38 @@ def test_serialization_big_model_inference(self): @require_torch_multi_gpu class GPTQTestDeviceMap(GPTQTest): device_map = "auto" + + +@require_accelerate +@require_torch_multi_gpu +class GPTQTestDeviceMapCPUOffload(GPTQTest): + device_map = { + "transformer.word_embeddings": 0, + "transformer.word_embeddings_layernorm": 0, + "lm_head": 0, + "transformer.h.0": 0, + "transformer.h.1": 0, + "transformer.h.2": 0, + "transformer.h.3": 0, + "transformer.h.4": 0, + "transformer.h.5": 0, + "transformer.h.6": 0, + "transformer.h.7": 0, + "transformer.h.8": 0, + "transformer.h.9": 0, + "transformer.h.10": 1, + "transformer.h.11": 1, + "transformer.h.12": 1, + "transformer.h.13": 1, + "transformer.h.14": 1, + "transformer.h.15": 1, + "transformer.h.16": 1, + "transformer.h.17": 0, + "transformer.h.18": "cpu", + "transformer.h.19": "cpu", + "transformer.h.20": "cpu", + "transformer.h.21": "cpu", + "transformer.h.22": "cpu", + "transformer.h.23": 1, + "transformer.ln_f": 0, + } From 27e9b799343708e9bd225d9ae481bf85f8f21758 Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Wed, 26 Jul 2023 20:52:54 +0000 Subject: [PATCH 19/46] fix check dataset --- src/transformers/utils/quantization_config.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/transformers/utils/quantization_config.py b/src/transformers/utils/quantization_config.py index 467cf6370481..3c3e5097301b 100644 --- a/src/transformers/utils/quantization_config.py +++ b/src/transformers/utils/quantization_config.py @@ -393,11 +393,12 @@ def post_init(self): if not (0 < self.damp_percent < 1): raise ValueError("damp_percent must between 0 and 1.") if self.dataset is not None: - if isinstance(self.dataset, str) and self.dataset not in ["wikitext2", "c4", "c4-new", "ptb", "ptb-new"]: - raise ValueError( - f"""You have entered a string value for dataset. You can only choose between - ['wikitext2','c4','c4-new','ptb','ptb-new'], but we found {self.dataset}""" - ) + if isinstance(self.dataset, str): + if self.dataset not in ["wikitext2", "c4", "c4-new", "ptb", "ptb-new"]: + raise ValueError( + f"""You have entered a string value for dataset. You can only choose between + ['wikitext2','c4','c4-new','ptb','ptb-new'], but we found {self.dataset}""" + ) elif not isinstance(self.dataset, list): raise ValueError( f"""dataset needs to be either a list of string or a value in From f47ecb40fb2b0fe5d1f3e3c6d577a17faf7a9d17 Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Wed, 26 Jul 2023 21:25:06 +0000 Subject: [PATCH 20/46] modify dockerfile --- docker/transformers-all-latest-gpu/Dockerfile | 7 +++++-- src/transformers/utils/quantization_config.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docker/transformers-all-latest-gpu/Dockerfile b/docker/transformers-all-latest-gpu/Dockerfile index c5ada9209bca..10f8e08328b5 100644 --- a/docker/transformers-all-latest-gpu/Dockerfile +++ b/docker/transformers-all-latest-gpu/Dockerfile @@ -45,8 +45,11 @@ RUN python3 -m pip install --no-cache-dir git+https://github.com/huggingface/acc # Add bitsandbytes for mixed int8 testing RUN python3 -m pip install --no-cache-dir bitsandbytes -# For bettertransformer -RUN python3 -m pip install --no-cache-dir optimum +# Add auto-gptq for gtpq quantization testing +RUN python3 -m pip install --no-cache-dir auto-gptq + +# For bettertransformer + gptq +RUN python3 -m pip install --no-cache-dir git+https://github.com/huggingface/optimum@main#egg=optimum # For video model testing RUN python3 -m pip install --no-cache-dir decord av==9.2.0 diff --git a/src/transformers/utils/quantization_config.py b/src/transformers/utils/quantization_config.py index 3c3e5097301b..0a392250de78 100644 --- a/src/transformers/utils/quantization_config.py +++ b/src/transformers/utils/quantization_config.py @@ -403,4 +403,4 @@ def post_init(self): raise ValueError( f"""dataset needs to be either a list of string or a value in ['wikitext2','c4','c4-new','ptb','ptb-new'], but we found {self.dataset}""" - ) + ) \ No newline at end of file From 19d05d36779f015a7d00cbf09df78e7b319362a7 Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Wed, 26 Jul 2023 21:34:09 +0000 Subject: [PATCH 21/46] protect trainer --- src/transformers/modeling_utils.py | 1 + src/transformers/trainer.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 736f4dbc06d5..24dde8d86b1d 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -3017,6 +3017,7 @@ def from_pretrained( if quantization_config.tokenizer is None: quantization_config.tokenizer = pretrained_model_name_or_path quantizer.quantize_model(model, quantization_config.tokenizer) + model._is_gptq_quantized = True model.config.quantization_config = GPTQConfig.from_dict(quantizer.to_dict()) if output_loading_info: diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index 4ccad5b276d2..47b38b0ca58f 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -402,6 +402,10 @@ def __init__( "The model you want to train is loaded in 8-bit precision. if you want to fine-tune an 8-bit" " model, please make sure that you have installed `bitsandbytes>=0.37.0`. " ) + if getattr(model, "_is_gptq_quantized", False): + raise ValueError( + "Training GTPQ quantized model is not possible yet." + ) # Setup Sharded DDP training self.sharded_ddp = None From 76dffe212b8c30413d7ee878f1287f5a6a57d0bb Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Wed, 26 Jul 2023 21:35:15 +0000 Subject: [PATCH 22/46] style --- src/transformers/trainer.py | 4 +--- src/transformers/utils/quantization_config.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index 47b38b0ca58f..cdbf55f489ed 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -403,9 +403,7 @@ def __init__( " model, please make sure that you have installed `bitsandbytes>=0.37.0`. " ) if getattr(model, "_is_gptq_quantized", False): - raise ValueError( - "Training GTPQ quantized model is not possible yet." - ) + raise ValueError("Training GTPQ quantized model is not possible yet.") # Setup Sharded DDP training self.sharded_ddp = None diff --git a/src/transformers/utils/quantization_config.py b/src/transformers/utils/quantization_config.py index 0a392250de78..3c3e5097301b 100644 --- a/src/transformers/utils/quantization_config.py +++ b/src/transformers/utils/quantization_config.py @@ -403,4 +403,4 @@ def post_init(self): raise ValueError( f"""dataset needs to be either a list of string or a value in ['wikitext2','c4','c4-new','ptb','ptb-new'], but we found {self.dataset}""" - ) \ No newline at end of file + ) From 0f61037c74e34a7d933599d9d54b760585e9f4b3 Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Wed, 26 Jul 2023 22:14:05 +0000 Subject: [PATCH 23/46] test for config --- tests/quantization/gptq/test_gptq.py | 41 ++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/quantization/gptq/test_gptq.py b/tests/quantization/gptq/test_gptq.py index e59cf2745a7b..77798ffee4e8 100644 --- a/tests/quantization/gptq/test_gptq.py +++ b/tests/quantization/gptq/test_gptq.py @@ -32,6 +32,47 @@ import torch +class GPTQConfigTest(unittest.TestCase): + def test_bits(self): + with self.assertRaises(ValueError): + GPTQConfig(bits="") + GPTQConfig(bits=1) + GPTQConfig(bits=2) + GPTQConfig(bits=4) + + def test_dataset(self): + with self.assertRaises(ValueError): + GPTQConfig(bits=2, dataset="auto_gpt") + GPTQConfig(bits=2, dataset="c4") + GPTQConfig(bits=2, dataset="ptb-new") + + def test_damp_percent(self): + with self.assertRaises(ValueError): + GPTQConfig(bits=2, damp_percent=10) + GPTQConfig(bits=2, damp_percent=-1) + GPTQConfig(bits=2, damp_percent="0") + GPTQConfig(bits=2, damp_percent=0.01) + + def test_to_dict(self): + quantization_config = GPTQConfig(bits=2) + quantization_config.to_dict() + + def test_from_dict(self): + dict = {"bits": 2} + quantization_config = GPTQConfig.from_dict(dict) + self.assertEqual(dict["bits"], quantization_config.bits) + + @require_optimum + def test_optimum_config(self): + from optimum.gptq import GPTQQuantizer + + config = GPTQConfig(bits=2) + optimum_config = GPTQQuantizer.from_dict(config.to_dict()) + self.assertEqual(optimum_config.bits, config.bits) + new_config = GPTQConfig.from_dict(optimum_config.to_dict()) + self.assertEqual(optimum_config.bits, new_config.bits) + + @slow @require_optimum @require_auto_gptq From b0eccd52193dd3f0d0db1e85377680ab48e7d9f1 Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Thu, 27 Jul 2023 13:46:04 +0000 Subject: [PATCH 24/46] add more log --- src/transformers/modeling_utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 24dde8d86b1d..6b7084863e3d 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -2369,7 +2369,9 @@ def from_pretrained( quantization_method_from_args == QuantizationMethod.GPTQ or quantization_method_from_config == QuantizationMethod.GPTQ ): - if not (is_optimum_available() and is_auto_gptq_available()): + if not torch.cuda.is_available(): + raise RuntimeError("GPU is required to quantize or run quantize model.") + elif not (is_optimum_available() and is_auto_gptq_available()): raise ImportError( "Loading GPTQ quantized model requires optimum library : `pip install optimum` and auto-gptq library 'pip install auto-gptq'" ) @@ -3016,7 +3018,7 @@ def from_pretrained( if quantization_method_from_args == QuantizationMethod.GPTQ: if quantization_config.tokenizer is None: quantization_config.tokenizer = pretrained_model_name_or_path - quantizer.quantize_model(model, quantization_config.tokenizer) + quantizer.quantize_model(model, quantization_config.tokenizer, verbose=True) model._is_gptq_quantized = True model.config.quantization_config = GPTQConfig.from_dict(quantizer.to_dict()) From 2e7a025c5cf02aee961b0c0a2719243136ac6686 Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Thu, 27 Jul 2023 15:56:50 +0000 Subject: [PATCH 25/46] overwrite torch_dtype --- src/transformers/modeling_utils.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 6b7084863e3d..33749964abaa 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -2381,6 +2381,12 @@ def from_pretrained( if quantization_method_from_config == QuantizationMethod.GPTQ: quantization_config = GPTQConfig.from_dict(config.quantization_config) torch_dtype = config.torch_dtype + else: + logger.info( + f"Overriding torch_dtype={torch_dtype} with `torch_dtype=torch.float16` due to " + "requirements of `auto-gptq` to enable model quantization " + ) + torch_dtype = torch.float16 quantizer = GPTQQuantizer.from_dict(quantization_config.to_dict()) if ( From a07126a079762b101220a84e314ba7be4b66463d Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Thu, 27 Jul 2023 19:37:08 +0000 Subject: [PATCH 26/46] draft doc --- docs/source/en/main_classes/quantization.md | 111 +++++++++++++++++++- 1 file changed, 107 insertions(+), 4 deletions(-) diff --git a/docs/source/en/main_classes/quantization.md b/docs/source/en/main_classes/quantization.md index c8547ab0c714..c15bdee47d1d 100644 --- a/docs/source/en/main_classes/quantization.md +++ b/docs/source/en/main_classes/quantization.md @@ -16,6 +16,112 @@ rendered properly in your Markdown viewer. # Quantize 🤗 Transformers models +## `AutoGPTQ` Integration + +🤗 Transformers has integrated `optimum` API to perform GPTQ quantization on language models. You can load and quantize your model in 8,6,4 or even 2 bits without a big drop of performance and faster inference speed! This is supported by most GPU hardwares. + +To learn more about the the quantization model, check out: +- the [GPTQ](https://arxiv.org/pdf/2210.17323.pdf) paper + +- the [`AutoGPTQ`](https://github.com/PanQiWei/AutoGPTQ) library used as the backend + +### Requirements + +You need to have the following requirements installed to run the code below: + +- Install latest `AutoGPTQ` library +`pip install auto-gptq` + +- Install latest `optimum` from source +`pip install --upgrade optimum` + +- Install latest `transformers` from source +`pip install --upgrade transformers` + +- Install latest `accelerate` from source +`pip install --upgrade accelerate` + +### Load and quantize a model + +#### GPTQ Configuration + +In order to load and quantize a model, you need to create a [`GPTQConfig`]. You need to pass the number of `bits`, a `dataset` in order to calibrate the quantization and the `tokenizer` of the model in order prepare the dataset. + +```python +model_id = "facebook/opt-125m" +tokenizer = AutoTokenizer.from_pretrained(model_id) +gptq_config = GPTQConfig(bits=4, dataset = "c4", tokenizer=tokenizer) +``` + +Note that you can pass your own dataset as a list of string. However, it is highly recommended to use the dataset from the GPTQ paper. +```python +dataset = ["auto-gptq is an easy-to-use model quantization library with user-friendly apis, based on GPTQ algorithm."] +quantization = GPTQConfig(bits=4, dataset = dataset, tokenizer=tokenizer) +``` + +#### Quantization + +You can quantize a model by using `from_pretrained` and setting the `quantization_config`. + +```python +from transformers import AutoModelForCausalLM +model = AutoModelForCausalLM.from_pretrained(model_id, quantization_config=gptq_config) +``` +Note that you will need a GPU to quantize a model. We will put the model in the cpu and move the modules back and forth to the gpu in order to quantize them. + +If you want to maximize your gpus usage while using cpu offload, you can set `device_map = "auto"`. +```python +from transformers import AutoModelForCausalLM +model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto", quantization_config=gptq_config) +``` +Note that disk offload is not supported. Furthermore, if you are out of memory because of the dataset, you may have to pass `max_memory` in `from_pretained`. Checkout this [guide](https://huggingface.co/docs/accelerate/usage_guides/big_modeling#designing-a-device-map) to learn more about `device_map` and `max_memory`. + + +GPTQ quantization only works for text model for now. Futhermore, the quantization process can a lot of time depending on one's hardware (175B model = 4 gpu hours using NVIDIA A100). Please check on the hub if there is not a GPTQ quantized version of the model. If not, you can submit a demand on github. + + +### Push quantized model to 🤗 Hub + +You can push the quantized model like any 🤗 model to Hub with `push_to_hub`: + +```python +quantized_model.push_to_hub("opt-125m-gptq") +tokenizer.push_to_hub("opt-125m-gptq") +``` + +If you want to save your quantized model on your local machine, you can also do it with `save_pretrained`: +```python +quantized_model.save_pretrained("opt-125m-gptq") +tokenizer.save_pretrained("opt-125m-gptq") +``` + +Note that if you have quantized your model with a `device_map`, make sure to move the entire model to one of your gpus or the `cpu` before saving it. +```python +quantized_model.to("cpu") +quantized_model.save_pretrained("opt-125m-gptq") +``` + +### Load a quantized model from the 🤗 Hub + +You can load a quantized model from the Hub by using `from_pretrained`. +Make sure that the pushed weights are quantized, by checking that the attribute `quantization_config` is present in the model configuration object. + +```python +from transformers import AutoModelForCausalLM +model = AutoModelForCausalLM.from_pretrained("{your_username}/opt-125m-gptq") +``` +Note that in this case, you don't need to specify the `quantization_config`. It will look for the `quantization_config` and prepare the model +before loading the quantized weights. However, you need to make sure that `optimum` and `auto-gptq` are installed. + +If you want to load a model faster and without allocating more memory than needed, the `device_map` argument also works with quantized model. Make sure that you have `accelerate` library installed. +```python +from transformers import AutoModelForCausalLM +model = AutoModelForCausalLM.from_pretrained("{your_username}/opt-125m-gptq", device_map="auto") +``` + +### GPTQConfig +[[autodoc]] GPTQConfig + ## `bitsandbytes` Integration 🤗 Transformers is closely integrated with most used modules on `bitsandbytes`. You can load your model in 8-bit precision with few lines of code. @@ -192,7 +298,7 @@ This section is intended to advanced users, that want to explore what it is poss One of the advanced usecase of this is being able to load a model and dispatch the weights between `CPU` and `GPU`. Note that the weights that will be dispatched on CPU **will not** be converted in 8-bit, thus kept in `float32`. This feature is intended for users that want to fit a very large model and dispatch the model between GPU and CPU. -First, load a `BitsAndBytesConfig` from `transformers` and set the attribute `llm_int8_enable_fp32_cpu_offload` to `True`: +First, load a [`BitsAndBytesConfig`] from `transformers` and set the attribute `llm_int8_enable_fp32_cpu_offload` to `True`: ```python from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig @@ -274,10 +380,7 @@ This enables fine-tuning large models such as `flan-t5-large` or `facebook/opt-6 Note that you don't need to pass `device_map` when loading the model for training. It will automatically load your model on your GPU. You can also set the device map to a specific device if needed (e.g. `cuda:0`, `0`, `torch.device('cuda:0')`). Please note that `device_map=auto` should be used for inference only. ### BitsAndBytesConfig - [[autodoc]] BitsAndBytesConfig - - ## Quantization with 🤗 `optimum` Please have a look at [Optimum documentation](https://huggingface.co/docs/optimum/index) to learn more about quantization methods that are supported by `optimum` and see if these are applicable for your usecase. From c9d3f267e96db56833277ade907555406f12c785 Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Mon, 31 Jul 2023 13:36:54 +0000 Subject: [PATCH 27/46] modify quantization_config docstring --- src/transformers/modeling_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 33749964abaa..1270bc9690e2 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -2100,9 +2100,9 @@ def from_pretrained( https://test.pypi.org/simple/ bitsandbytes-cudaXXX` where XXX is your CUDA version (e.g. 11.6 = 116). Make also sure that you have enough GPU RAM to store half of the model size since the 8bit modules are not compiled and adapted for CPUs. - quantization_config (`Dict`, *optional*): - A dictionary of configuration parameters for the `bitsandbytes` library and loading the model using - advanced features such as offloading in fp32 on CPU or on disk. + quantization_config (`Union[QuantizationConfigMixin,Dict]`, *optional*): + A dictionary of configuration parameters or a QuantizationConfigMixin object for quantization (e.g + bitsandbytes, gptq) subfolder (`str`, *optional*, defaults to `""`): In case the relevant files are located inside a subfolder of the model repo on huggingface.co, you can specify the folder name here. From ecce1dae0f474556796b19dd1fec50752378dd6a Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Mon, 31 Jul 2023 13:43:21 +0000 Subject: [PATCH 28/46] fix class name in docstring --- src/transformers/utils/quantization_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/utils/quantization_config.py b/src/transformers/utils/quantization_config.py index 3c3e5097301b..9b7e6de881ae 100644 --- a/src/transformers/utils/quantization_config.py +++ b/src/transformers/utils/quantization_config.py @@ -50,7 +50,7 @@ class QuantizationConfigMixin: @classmethod def from_dict(cls, config_dict, return_unused_kwargs=False, **kwargs): """ - Instantiates a [`QuantizationConfig`] from a Python dictionary of parameters. + Instantiates a [`QuantizationConfigMixin`] from a Python dictionary of parameters. Args: config_dict (`Dict[str, Any]`): @@ -62,7 +62,7 @@ def from_dict(cls, config_dict, return_unused_kwargs=False, **kwargs): Additional parameters from which to initialize the configuration object. Returns: - [`QuantizationConfig`]: The configuration object instantiated from those parameters. + [`QuantizationConfigMixin`]: The configuration object instantiated from those parameters. """ config = cls(**config_dict) From 222618437743d3d8d2314eba98438d857d24555a Mon Sep 17 00:00:00 2001 From: Marc Sun <57196510+SunMarc@users.noreply.github.com> Date: Mon, 31 Jul 2023 10:05:37 -0400 Subject: [PATCH 29/46] Apply suggestions from code review Co-authored-by: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> --- docs/source/en/main_classes/quantization.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/source/en/main_classes/quantization.md b/docs/source/en/main_classes/quantization.md index c15bdee47d1d..164ac2c221e5 100644 --- a/docs/source/en/main_classes/quantization.md +++ b/docs/source/en/main_classes/quantization.md @@ -33,16 +33,20 @@ You need to have the following requirements installed to run the code below: `pip install auto-gptq` - Install latest `optimum` from source -`pip install --upgrade optimum` +`pip install git+https://github.com/huggingface/optimum.git` - Install latest `transformers` from source -`pip install --upgrade transformers` +`pip install git+https://github.com/huggingface/transformers.git` -- Install latest `accelerate` from source +- Install latest `accelerate` library `pip install --upgrade accelerate` +GPTQ integration supports for now only text models and you may encounter unexpected behaviour for vision, speech or multi-modal models. ### Load and quantize a model +GPTQ is a quantization method that requires weights calibration before using the quantized models. If you want to quantize transformers model from scratch, it might take some time before producing the quantized model (~10 min on a Google colab for `facebook/opt-350m` model. + +Hence, there are two different scenarios where you want to use GPTQ-quantized models. The first use case would be to load models that has been already quantized by other users that are available on the Hub, the second use case would be to quantize your model from scratch and save it or push it on the Hub so that other users can also use it. #### GPTQ Configuration In order to load and quantize a model, you need to create a [`GPTQConfig`]. You need to pass the number of `bits`, a `dataset` in order to calibrate the quantization and the `tokenizer` of the model in order prepare the dataset. From eff99cbb60cb9d8faef0a7a4cbb07a2786520a49 Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Mon, 31 Jul 2023 14:31:14 +0000 Subject: [PATCH 30/46] more warning --- src/transformers/modeling_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 1270bc9690e2..c5f9944c8868 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -3024,6 +3024,8 @@ def from_pretrained( if quantization_method_from_args == QuantizationMethod.GPTQ: if quantization_config.tokenizer is None: quantization_config.tokenizer = pretrained_model_name_or_path + if cls.main_input_name != "input_ids": + raise RuntimeError("We can only quantize pure text model.") quantizer.quantize_model(model, quantization_config.tokenizer, verbose=True) model._is_gptq_quantized = True model.config.quantization_config = GPTQConfig.from_dict(quantizer.to_dict()) From 159cf878b06ced7875ddef8590a536b6397d0f80 Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Mon, 31 Jul 2023 15:21:38 +0000 Subject: [PATCH 31/46] fix 8bit kwargs tests --- tests/quantization/bnb/test_mixed_int8.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/quantization/bnb/test_mixed_int8.py b/tests/quantization/bnb/test_mixed_int8.py index f905b26e3f71..fba4b05c8b84 100644 --- a/tests/quantization/bnb/test_mixed_int8.py +++ b/tests/quantization/bnb/test_mixed_int8.py @@ -706,6 +706,7 @@ def test_cpu_gpu_disk_loading_custom_device_map_kwargs(self): model_8bit = AutoModelForCausalLM.from_pretrained( self.model_name, device_map=device_map, + load_in_8bit=True, llm_int8_enable_fp32_cpu_offload=True, offload_folder=tmpdirname, ) From 98db723e6a14b059c49b973f97a312ae374aae20 Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Mon, 31 Jul 2023 22:48:34 +0000 Subject: [PATCH 32/46] peft compatibility --- src/transformers/modeling_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index c5f9944c8868..770d22b0adf1 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -3029,6 +3029,8 @@ def from_pretrained( quantizer.quantize_model(model, quantization_config.tokenizer, verbose=True) model._is_gptq_quantized = True model.config.quantization_config = GPTQConfig.from_dict(quantizer.to_dict()) + if quantization_method_from_config == QuantizationMethod.GPTQ: + model = quantizer.get_compatible_with_peft(model) if output_loading_info: if loading_info is None: From 0144760f32eef5632b00ad973f7d6653413bcc9f Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Tue, 1 Aug 2023 14:54:55 +0000 Subject: [PATCH 33/46] remove var --- src/transformers/modeling_utils.py | 2 +- src/transformers/utils/quantization_config.py | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 770d22b0adf1..9734b07d58de 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -3026,7 +3026,7 @@ def from_pretrained( quantization_config.tokenizer = pretrained_model_name_or_path if cls.main_input_name != "input_ids": raise RuntimeError("We can only quantize pure text model.") - quantizer.quantize_model(model, quantization_config.tokenizer, verbose=True) + quantizer.quantize_model(model, quantization_config.tokenizer) model._is_gptq_quantized = True model.config.quantization_config = GPTQConfig.from_dict(quantizer.to_dict()) if quantization_method_from_config == QuantizationMethod.GPTQ: diff --git a/src/transformers/utils/quantization_config.py b/src/transformers/utils/quantization_config.py index 9b7e6de881ae..c24ae5ad698e 100644 --- a/src/transformers/utils/quantization_config.py +++ b/src/transformers/utils/quantization_config.py @@ -329,8 +329,6 @@ class GPTQConfig(QuantizationConfigMixin): "Whether to perform sequential quantization even within a single Transformer block." "Instead of quantizing the entire block at once, we perform layer-wise quantization." "As a result, each layer undergoes quantization using inputs that have passed through the previously quantized layers." - pack_sequentially (`bool`, *optional*, defaults to `False`): - Whether to pack the layer just after it is quantized. If False, we will pack the model at the end. use_cuda_fp16 (`bool`, *optional*, defaults to `False`): "Whether or not to use optimized cuda kernel for fp16 model. Need to have model in fp16. model_seqlen (`int`, *optional*, defaults to `None`): @@ -355,7 +353,6 @@ def __init__( desc_act: bool = True, sym: bool = True, true_sequential: bool = True, - pack_sequentially: bool = False, use_cuda_fp16: bool = False, model_seqlen: Optional[int] = None, block_name_to_quantize: Optional[str] = None, @@ -373,7 +370,6 @@ def __init__( self.desc_act = desc_act self.sym = sym self.true_sequential = true_sequential - self.pack_sequentially = pack_sequentially self.use_cuda_fp16 = use_cuda_fp16 self.model_seqlen = model_seqlen self.block_name_to_quantize = block_name_to_quantize From fd8d70c264b4207b720eb9a1755410bcb6b5b068 Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Tue, 1 Aug 2023 21:25:41 +0000 Subject: [PATCH 34/46] fix is_gptq_quantized --- src/transformers/modeling_utils.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 9734b07d58de..984cd38c8ebb 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -2380,8 +2380,7 @@ def from_pretrained( from optimum.gptq import GPTQQuantizer if quantization_method_from_config == QuantizationMethod.GPTQ: quantization_config = GPTQConfig.from_dict(config.quantization_config) - torch_dtype = config.torch_dtype - else: + config.quantization_config = quantization_config logger.info( f"Overriding torch_dtype={torch_dtype} with `torch_dtype=torch.float16` due to " "requirements of `auto-gptq` to enable model quantization " @@ -2826,6 +2825,7 @@ def from_pretrained( ) if quantization_method_from_config == QuantizationMethod.GPTQ: model = quantizer.convert_model(model) + model.is_gptq_quantized = True if isinstance(device_map, str): special_dtypes = {} @@ -3027,10 +3027,8 @@ def from_pretrained( if cls.main_input_name != "input_ids": raise RuntimeError("We can only quantize pure text model.") quantizer.quantize_model(model, quantization_config.tokenizer) - model._is_gptq_quantized = True model.config.quantization_config = GPTQConfig.from_dict(quantizer.to_dict()) - if quantization_method_from_config == QuantizationMethod.GPTQ: - model = quantizer.get_compatible_with_peft(model) + model.is_gptq_quantized = True if output_loading_info: if loading_info is None: From be19916c03744166ec7514797f642cce7fae7041 Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Wed, 2 Aug 2023 17:41:44 +0000 Subject: [PATCH 35/46] remove is_gptq_quantized --- src/transformers/modeling_utils.py | 25 ++++++++++++++++--------- src/transformers/trainer.py | 15 ++++++++------- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index ae2ae46e63e4..9f25b563f3fc 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -1927,7 +1927,7 @@ def cuda(self, *args, **kwargs): @wraps(torch.nn.Module.to) def to(self, *args, **kwargs): # Checks if the model has been loaded in 8-bit - if getattr(self, "is_quantized", False): + if getattr(self, "quantization_method", None) == QuantizationMethod.BITS_AND_BYTES: raise ValueError( "`.to` is not supported for `4-bit` or `8-bit` models. Please use the model as it is, since the" " model has already been set to the correct devices and casted to the correct `dtype`." @@ -1936,8 +1936,8 @@ def to(self, *args, **kwargs): return super().to(*args, **kwargs) def half(self, *args): - # Checks if the model has been loaded in 8-bit - if getattr(self, "is_quantized", False): + # Checks if the model is quantized with bitsandbytes + if getattr(self, "quantization_method", None) == QuantizationMethod.BITS_AND_BYTES: raise ValueError( "`.half()` is not supported for `4-bit` or `8-bit` models. Please use the model as it is, since the" " model has already been casted to the correct `dtype`." @@ -1946,8 +1946,8 @@ def half(self, *args): return super().half(*args) def float(self, *args): - # Checks if the model has been loaded in 8-bit - if getattr(self, "is_quantized", False): + # Checks if the model is quantized with bitsandbytes + if getattr(self, "quantization_method", None) == QuantizationMethod.BITS_AND_BYTES: raise ValueError( "`.float()` is not supported for `4-bit` or `8-bit` models. Please use the model as it is, since the" " model has already been casted to the correct `dtype`." @@ -2295,6 +2295,7 @@ def from_pretrained( ) if quantization_config is None and (load_in_8bit or load_in_4bit): + quantization_method_from_args = QuantizationMethod.BITS_AND_BYTES quantization_config, kwargs = BitsAndBytesConfig.from_dict( config_dict={"load_in_8bit": load_in_8bit, "load_in_4bit": load_in_4bit}, return_unused_kwargs=True, @@ -2858,7 +2859,14 @@ def from_pretrained( ) if quantization_method_from_config == QuantizationMethod.GPTQ: model = quantizer.convert_model(model) - model.is_gptq_quantized = True + model._is_quantized_training_enabled = True + + if quantization_method_from_config is not None: + model.quantization_method = quantization_method_from_config + elif quantization_method_from_args is not None: + model.quantization_method = quantization_method_from_args + if hasattr(model, "quantization_method"): + model.is_quantized = True if isinstance(device_map, str): special_dtypes = {} @@ -3010,13 +3018,12 @@ def from_pretrained( offload_folder=offload_folder, offload_state_dict=offload_state_dict, dtype=torch_dtype, - is_quantized=(load_in_8bit or load_in_4bit), + is_quantized=(getattr(model, "quantization_method", None) == QuantizationMethod.BITS_AND_BYTES), keep_in_fp32_modules=keep_in_fp32_modules, ) model.is_loaded_in_4bit = load_in_4bit model.is_loaded_in_8bit = load_in_8bit - model.is_quantized = load_in_8bit or load_in_4bit # make sure token embedding weights are still tied if needed model.tie_weights() @@ -3061,7 +3068,7 @@ def from_pretrained( raise RuntimeError("We can only quantize pure text model.") quantizer.quantize_model(model, quantization_config.tokenizer) model.config.quantization_config = GPTQConfig.from_dict(quantizer.to_dict()) - model.is_gptq_quantized = True + model._is_quantized_training_enabled = True if output_loading_info: if loading_info is None: diff --git a/src/transformers/trainer.py b/src/transformers/trainer.py index 59a0d579a1aa..ff473babe9f7 100755 --- a/src/transformers/trainer.py +++ b/src/transformers/trainer.py @@ -144,6 +144,7 @@ logging, strtobool, ) +from .utils.quantization_config import QuantizationMethod DEFAULT_CALLBACKS = [DefaultFlowCallback] @@ -391,18 +392,15 @@ def __init__( if getattr(model, "is_quantized", False): if getattr(model, "_is_quantized_training_enabled", False): logger.info( - "The model is loaded in 8-bit precision. To train this model you need to add additional modules" + "The model is quantized. To train this model you need to add additional modules" " inside the model such as adapters using `peft` library and freeze the model weights. Please" - " check " - " the examples in https://github.com/huggingface/peft for more details." + " check the examples in https://github.com/huggingface/peft for more details." ) else: raise ValueError( "The model you want to train is loaded in 8-bit precision. if you want to fine-tune an 8-bit" " model, please make sure that you have installed `bitsandbytes>=0.37.0`. " ) - if getattr(model, "_is_gptq_quantized", False): - raise ValueError("Training GTPQ quantized model is not possible yet.") # Setup Sharded DDP training self.sharded_ddp = None @@ -495,8 +493,11 @@ def __init__( self.eval_dataset = eval_dataset self.tokenizer = tokenizer - # Quantized models doesn't support `.to` operation. - if self.place_model_on_device and not getattr(model, "is_quantized", False): + # Bnb Quantized models doesn't support `.to` operation. + if ( + self.place_model_on_device + and not getattr(model, "quantization_method", None) == QuantizationMethod.BITS_AND_BYTES + ): self._move_model_to_device(model, args.device) # Force n_gpu to 1 to avoid DataParallel as MP will manage the GPUs From 4b4336e7ab0cc9959fe7f61f230a791f4548dd6b Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Wed, 2 Aug 2023 20:46:28 +0000 Subject: [PATCH 36/46] fix wrap --- src/transformers/modeling_utils.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 9197894b21be..424576ce4e8f 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -1916,7 +1916,7 @@ def get_memory_footprint(self, return_buffers=True): @wraps(torch.nn.Module.cuda) def cuda(self, *args, **kwargs): # Checks if the model has been loaded in 8-bit - if getattr(self, "is_quantized", False): + if getattr(self, "quantization_method", None) == QuantizationMethod.BITS_AND_BYTES: raise ValueError( "Calling `cuda()` is not supported for `4-bit` or `8-bit` quantized models. Please use the model as it is, since the" " model has already been set to the correct devices and casted to the correct `dtype`." @@ -1936,20 +1936,20 @@ def to(self, *args, **kwargs): return super().to(*args, **kwargs) def half(self, *args): - # Checks if the model is quantized with bitsandbytes - if getattr(self, "quantization_method", None) == QuantizationMethod.BITS_AND_BYTES: + # Checks if the model is quantized + if getattr(self, "is_quantized", False): raise ValueError( - "`.half()` is not supported for `4-bit` or `8-bit` models. Please use the model as it is, since the" + "`.half()` is not supported for quantized model. Please use the model as it is, since the" " model has already been casted to the correct `dtype`." ) else: return super().half(*args) def float(self, *args): - # Checks if the model is quantized with bitsandbytes - if getattr(self, "quantization_method", None) == QuantizationMethod.BITS_AND_BYTES: + # Checks if the model is quantized + if getattr(self, "is_quantized", False): raise ValueError( - "`.float()` is not supported for `4-bit` or `8-bit` models. Please use the model as it is, since the" + "`.float()` is not supported for quantized model. Please use the model as it is, since the" " model has already been casted to the correct `dtype`." ) else: From 42d0049c4f2eb75192d0e60da144d40c1044574e Mon Sep 17 00:00:00 2001 From: Marc Sun <57196510+SunMarc@users.noreply.github.com> Date: Tue, 8 Aug 2023 10:41:42 -0400 Subject: [PATCH 37/46] Update src/transformers/modeling_utils.py Co-authored-by: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> --- src/transformers/modeling_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 424576ce4e8f..fd93ec9a23ce 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -1929,7 +1929,7 @@ def to(self, *args, **kwargs): # Checks if the model has been loaded in 8-bit if getattr(self, "quantization_method", None) == QuantizationMethod.BITS_AND_BYTES: raise ValueError( - "`.to` is not supported for `4-bit` or `8-bit` models. Please use the model as it is, since the" + "`.to` is not supported for `4-bit` or `8-bit` bitsandbytes models. Please use the model as it is, since the" " model has already been set to the correct devices and casted to the correct `dtype`." ) else: From 62aa2936229c5d1720ebf91fa7824075bb60e700 Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Wed, 9 Aug 2023 21:35:58 +0000 Subject: [PATCH 38/46] add exllama --- docs/source/en/main_classes/quantization.md | 12 ++++ src/transformers/modeling_utils.py | 11 +++- src/transformers/models/auto/auto_factory.py | 5 ++ src/transformers/utils/quantization_config.py | 10 ++++ tests/quantization/gptq/test_gptq.py | 57 +++++++++++++++---- 5 files changed, 80 insertions(+), 15 deletions(-) diff --git a/docs/source/en/main_classes/quantization.md b/docs/source/en/main_classes/quantization.md index bde011dbea88..a0bee2861197 100644 --- a/docs/source/en/main_classes/quantization.md +++ b/docs/source/en/main_classes/quantization.md @@ -123,6 +123,18 @@ from transformers import AutoModelForCausalLM model = AutoModelForCausalLM.from_pretrained("{your_username}/opt-125m-gptq", device_map="auto") ``` +### Exllama kernels for faster inference + +For 4-bit model, you can use the exllama kernels in order to a faster inference speed. You just need to pass `disable_exllama=False` in [`GPTQConfig`]. This will overwrite the quantization config stored in the config. Note that you will only be able to overwrite the attributes related to the kernel. Furthermore, you need to have the entire model on gpus. + +```py +import torch +gptq_config = GPTQConfig(bits=4, disable_exllama=False) +model = AutoModelForCausalLM.from_pretrained("{your_username}/opt-125m-gptq", device_map="auto", quantization_config = gptq_config) +``` + +Note that only 4-bit models are supported for now + ### GPTQConfig [[autodoc]] GPTQConfig diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index d3be04267788..6d69fdd72985 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -2391,11 +2391,14 @@ def from_pretrained( ) if quantization_method_from_config == QuantizationMethod.GPTQ and quantization_method_from_args is not None: + loading_attr_dict = quantization_config.get_loading_attributes() + for attr, val in loading_attr_dict.items(): + config.quantization_config[attr] = val quantization_method_from_args = None logger.warning( - "You passed `quantization_config` to `from_pretrained` but the model you're loading already has a" - " `quantization_config` attribute and has already quantized weights. We will not perform quantization" - "with the given `quantization config` that you have passed." + "You passed `quantization_config` to `from_pretrained` but the model you're loading already has a " + "`quantization_config` attribute and has already quantized weights. However, loading attributes" + " (e.g. disable_exllama, use_cuda_fp16) will be overwritten with the one you passed to `from_pretrained`. The rest will be ignored." ) if ( quantization_method_from_args == QuantizationMethod.GPTQ @@ -3069,6 +3072,8 @@ def from_pretrained( quantizer.quantize_model(model, quantization_config.tokenizer) model.config.quantization_config = GPTQConfig.from_dict(quantizer.to_dict()) model._is_quantized_training_enabled = True + if quantization_method_from_config == QuantizationMethod.GPTQ: + model = quantizer.post_init_model(model) if output_loading_info: if loading_info is None: diff --git a/src/transformers/models/auto/auto_factory.py b/src/transformers/models/auto/auto_factory.py index 36d3435eb7b5..a5e4fcbb90ee 100644 --- a/src/transformers/models/auto/auto_factory.py +++ b/src/transformers/models/auto/auto_factory.py @@ -475,6 +475,9 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): # meaningless in the context of the config object - torch.dtype values are acceptable if kwargs.get("torch_dtype", None) == "auto": _ = kwargs.pop("torch_dtype") + # to not overwrite the quantization_config if config has a quantization_config + if kwargs.get("quantization_config", None) is not None: + _ = kwargs.pop("quantization_config") config, kwargs = AutoConfig.from_pretrained( pretrained_model_name_or_path, @@ -487,6 +490,8 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): # if torch_dtype=auto was passed here, ensure to pass it on if kwargs_orig.get("torch_dtype", None) == "auto": kwargs["torch_dtype"] = "auto" + if kwargs_orig.get("quantization_config", None) is not None: + kwargs["quantization_config"] = kwargs_orig["quantization_config"] has_remote_code = hasattr(config, "auto_map") and cls.__name__ in config.auto_map has_local_code = type(config) in cls._model_mapping.keys() diff --git a/src/transformers/utils/quantization_config.py b/src/transformers/utils/quantization_config.py index c24ae5ad698e..202f1dc0f33d 100644 --- a/src/transformers/utils/quantization_config.py +++ b/src/transformers/utils/quantization_config.py @@ -341,6 +341,8 @@ class GPTQConfig(QuantizationConfigMixin): The batch size used when processing the dataset pad_token_id (`int`, *optional*, defaults to `None`): The pad token id. Needed to prepare the dataset when `batch_size` > 1. + disable_exllama (`bool`, defaults to `False`): + Whether to use exllama backend. Only works with `bits` = 4. """ def __init__( @@ -359,6 +361,7 @@ def __init__( module_name_preceding_first_block: Optional[List[str]] = None, batch_size: int = 1, pad_token_id: Optional[int] = None, + disable_exllama: bool = False, **kwargs, ): self.quant_method = QuantizationMethod.GPTQ @@ -376,8 +379,15 @@ def __init__( self.module_name_preceding_first_block = module_name_preceding_first_block self.batch_size = batch_size self.pad_token_id = pad_token_id + self.disable_exllama = disable_exllama self.post_init() + def get_loading_attributes(self): + attibutes_dict = copy.deepcopy(self.__dict__) + loading_attibutes = ["disable_exllama", "use_cuda_fp16"] + loading_attibutes_dict = {i: j for i, j in attibutes_dict.items() if i in loading_attibutes} + return loading_attibutes_dict + def post_init(self): r""" Safety checker that arguments are correct diff --git a/tests/quantization/gptq/test_gptq.py b/tests/quantization/gptq/test_gptq.py index 77798ffee4e8..d183292eac73 100644 --- a/tests/quantization/gptq/test_gptq.py +++ b/tests/quantization/gptq/test_gptq.py @@ -92,6 +92,7 @@ class GPTQTest(unittest.TestCase): bits = 4 group_size = 128 desc_act = False + disable_exllama = True dataset = [ "auto-gptq is an easy-to-use model quantization library with user-friendly apis, based on GPTQ algorithm." @@ -118,6 +119,7 @@ def setUpClass(cls): tokenizer=cls.tokenizer, group_size=cls.group_size, desc_act=cls.desc_act, + disable_exllama=cls.disable_exllama, ) cls.quantized_model = AutoModelForCausalLM.from_pretrained( @@ -145,7 +147,11 @@ def test_quantized_layers_class(self): from auto_gptq.utils.import_utils import dynamically_import_QuantLinear QuantLinear = dynamically_import_QuantLinear( - use_triton=False, desc_act=self.desc_act, group_size=self.group_size + use_triton=False, + desc_act=self.desc_act, + group_size=self.group_size, + bits=self.bits, + disable_exllama=self.disable_exllama, ) self.assertTrue(self.quantized_model.transformer.h[0].mlp.dense_4h_to_h.__class__ == QuantLinear) @@ -178,8 +184,12 @@ def test_serialization(self): Test the serialization of the model and the loading of the quantized weights works """ with tempfile.TemporaryDirectory() as tmpdirname: - self.quantized_model.to("cpu").save_pretrained(tmpdirname) - quantized_model_from_saved = AutoModelForCausalLM.from_pretrained(tmpdirname).to(0) + self.quantized_model.save_pretrained(tmpdirname) + if self.disable_exllama: + quantized_model_from_saved = AutoModelForCausalLM.from_pretrained(tmpdirname).to(0) + else: + # we need to put it directly to the gpu. Otherwise, we won't be able to initialize the exllama kernel + quantized_model_from_saved = AutoModelForCausalLM.from_pretrained(tmpdirname, device_map={"": 0}) self.check_inference_correctness(quantized_model_from_saved) @require_accelerate @@ -188,10 +198,26 @@ def test_serialization_big_model_inference(self): Test the serialization of the model and the loading of the quantized weights with big model inference """ with tempfile.TemporaryDirectory() as tmpdirname: - self.quantized_model.to("cpu").save_pretrained(tmpdirname) + self.quantized_model.save_pretrained(tmpdirname) quantized_model_from_saved = AutoModelForCausalLM.from_pretrained(tmpdirname, device_map="auto") self.check_inference_correctness(quantized_model_from_saved) + def test_change_loading_attributes(self): + """ + Test the serialization of the model and the loading of the quantized weights works with another config file + """ + with tempfile.TemporaryDirectory() as tmpdirname: + self.quantized_model.save_pretrained(tmpdirname) + if self.disable_exllama: + self.assertEqual(self.quantized_model.config.quantization_config.disable_exllama, True) + # we need to put it directly to the gpu. Otherwise, we won't be able to initialize the exllama kernel + quantized_model_from_saved = AutoModelForCausalLM.from_pretrained( + tmpdirname, quantization_config=GPTQConfig(disable_exllama=False, bits=6), device_map={"": 0} + ) + self.assertEqual(quantized_model_from_saved.config.quantization_config.disable_exllama, False) + self.assertEqual(quantized_model_from_saved.config.quantization_config.bits, self.bits) + self.check_inference_correctness(quantized_model_from_saved) + @require_accelerate @require_torch_multi_gpu @@ -199,6 +225,13 @@ class GPTQTestDeviceMap(GPTQTest): device_map = "auto" +@require_accelerate +@require_torch_multi_gpu +class GPTQTestDeviceMapExllama(GPTQTest): + device_map = "auto" + disable_exllama = False + + @require_accelerate @require_torch_multi_gpu class GPTQTestDeviceMapCPUOffload(GPTQTest): @@ -216,19 +249,19 @@ class GPTQTestDeviceMapCPUOffload(GPTQTest): "transformer.h.7": 0, "transformer.h.8": 0, "transformer.h.9": 0, - "transformer.h.10": 1, - "transformer.h.11": 1, - "transformer.h.12": 1, - "transformer.h.13": 1, - "transformer.h.14": 1, - "transformer.h.15": 1, - "transformer.h.16": 1, + "transformer.h.10": 0, + "transformer.h.11": 0, + "transformer.h.12": 0, + "transformer.h.13": 0, + "transformer.h.14": 0, + "transformer.h.15": 0, + "transformer.h.16": 0, "transformer.h.17": 0, "transformer.h.18": "cpu", "transformer.h.19": "cpu", "transformer.h.20": "cpu", "transformer.h.21": "cpu", "transformer.h.22": "cpu", - "transformer.h.23": 1, + "transformer.h.23": 0, "transformer.ln_f": 0, } From 39137eb906dbdde72d29534b1e729846710b3a23 Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Wed, 9 Aug 2023 22:01:06 +0000 Subject: [PATCH 39/46] skip test --- tests/quantization/gptq/test_gptq.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/tests/quantization/gptq/test_gptq.py b/tests/quantization/gptq/test_gptq.py index d183292eac73..adde75076be7 100644 --- a/tests/quantization/gptq/test_gptq.py +++ b/tests/quantization/gptq/test_gptq.py @@ -25,6 +25,7 @@ require_torch_gpu, require_torch_multi_gpu, slow, + skip ) @@ -231,7 +232,7 @@ class GPTQTestDeviceMapExllama(GPTQTest): device_map = "auto" disable_exllama = False - +@skip("fail when run all together") @require_accelerate @require_torch_multi_gpu class GPTQTestDeviceMapCPUOffload(GPTQTest): @@ -249,19 +250,19 @@ class GPTQTestDeviceMapCPUOffload(GPTQTest): "transformer.h.7": 0, "transformer.h.8": 0, "transformer.h.9": 0, - "transformer.h.10": 0, - "transformer.h.11": 0, - "transformer.h.12": 0, - "transformer.h.13": 0, - "transformer.h.14": 0, - "transformer.h.15": 0, - "transformer.h.16": 0, + "transformer.h.10": 1, + "transformer.h.11": 1, + "transformer.h.12": 1, + "transformer.h.13": 1, + "transformer.h.14": 1, + "transformer.h.15": 1, + "transformer.h.16": 1, "transformer.h.17": 0, "transformer.h.18": "cpu", "transformer.h.19": "cpu", "transformer.h.20": "cpu", "transformer.h.21": "cpu", "transformer.h.22": "cpu", - "transformer.h.23": 0, + "transformer.h.23": 1, "transformer.ln_f": 0, } From 0b0633bc325ef7b0e393bbe47e57f5bed86562c7 Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Wed, 9 Aug 2023 23:08:45 +0000 Subject: [PATCH 40/46] overwrite float16 --- src/transformers/modeling_utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/transformers/modeling_utils.py b/src/transformers/modeling_utils.py index 6d69fdd72985..b1edfb72503f 100644 --- a/src/transformers/modeling_utils.py +++ b/src/transformers/modeling_utils.py @@ -2416,11 +2416,11 @@ def from_pretrained( if quantization_method_from_config == QuantizationMethod.GPTQ: quantization_config = GPTQConfig.from_dict(config.quantization_config) config.quantization_config = quantization_config - logger.info( - f"Overriding torch_dtype={torch_dtype} with `torch_dtype=torch.float16` due to " - "requirements of `auto-gptq` to enable model quantization " - ) - torch_dtype = torch.float16 + logger.info( + f"Overriding torch_dtype={torch_dtype} with `torch_dtype=torch.float16` due to " + "requirements of `auto-gptq` to enable model quantization " + ) + torch_dtype = torch.float16 quantizer = GPTQQuantizer.from_dict(quantization_config.to_dict()) if ( From c3c4a160d061f6953e399899529bc7a22df45ac5 Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Wed, 9 Aug 2023 23:10:28 +0000 Subject: [PATCH 41/46] style --- tests/quantization/gptq/test_gptq.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/quantization/gptq/test_gptq.py b/tests/quantization/gptq/test_gptq.py index adde75076be7..5f8c97ee8176 100644 --- a/tests/quantization/gptq/test_gptq.py +++ b/tests/quantization/gptq/test_gptq.py @@ -24,8 +24,8 @@ require_optimum, require_torch_gpu, require_torch_multi_gpu, + skip, slow, - skip ) @@ -232,6 +232,7 @@ class GPTQTestDeviceMapExllama(GPTQTest): device_map = "auto" disable_exllama = False + @skip("fail when run all together") @require_accelerate @require_torch_multi_gpu From a45b5b09246d69d6784b281e2ceeda1394dcebb4 Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Wed, 9 Aug 2023 23:19:18 +0000 Subject: [PATCH 42/46] fix skip test --- tests/quantization/gptq/test_gptq.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/quantization/gptq/test_gptq.py b/tests/quantization/gptq/test_gptq.py index 5f8c97ee8176..9c6d37f3028c 100644 --- a/tests/quantization/gptq/test_gptq.py +++ b/tests/quantization/gptq/test_gptq.py @@ -16,6 +16,8 @@ import tempfile import unittest +import pytest + from transformers import AutoModelForCausalLM, AutoTokenizer, GPTQConfig from transformers.testing_utils import ( is_torch_available, @@ -24,7 +26,6 @@ require_optimum, require_torch_gpu, require_torch_multi_gpu, - skip, slow, ) @@ -233,7 +234,8 @@ class GPTQTestDeviceMapExllama(GPTQTest): disable_exllama = False -@skip("fail when run all together") +# fail when run all together +@pytest.mark.skip @require_accelerate @require_torch_multi_gpu class GPTQTestDeviceMapCPUOffload(GPTQTest): From 69c8fce8fa161dab746d6f4e59cc84b430b5796e Mon Sep 17 00:00:00 2001 From: Marc Sun <57196510+SunMarc@users.noreply.github.com> Date: Thu, 10 Aug 2023 09:27:34 -0400 Subject: [PATCH 43/46] Apply suggestions from code review Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> --- src/transformers/utils/quantization_config.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/transformers/utils/quantization_config.py b/src/transformers/utils/quantization_config.py index 202f1dc0f33d..15b9309bf295 100644 --- a/src/transformers/utils/quantization_config.py +++ b/src/transformers/utils/quantization_config.py @@ -304,7 +304,7 @@ class GPTQConfig(QuantizationConfigMixin): Args: bits (`int`): The number of bits to quantize to, supported numbers are (2, 3, 4, 8). - tokenizer(`Any`): + tokenizer (`str` or `PreTrainedTokenizerBase`): The tokenizer used to process the dataset. You can pass either: - A custom tokenizer object. - A string, the *model id* of a predefined tokenizer hosted inside a model repo on huggingface.co. @@ -312,12 +312,12 @@ class GPTQConfig(QuantizationConfigMixin): user or organization name, like `dbmdz/bert-base-german-cased`. - A path to a *directory* containing vocabulary files required by the tokenizer, for instance saved using the [`~PreTrainedTokenizer.save_pretrained`] method, e.g., `./my_model_directory/`. - dataset (`Union[List[str]]`, *optional*, defaults to `None`): + dataset (`Union[List[str]]`, *optional*): "The dataset used for quantization. You can provide your own dataset in a list of string" "or just use the original datasets used in GPTQ paper ['wikitext2','c4','c4-new','ptb','ptb-new']" - group_size (`int`, *optional*, defaults to `128`): + group_size (`int`, *optional*, defaults to 128): "The group size to use for quantization. Recommended value is 128 and -1 uses per-column quantization. - damp_percent (`float`, *optional*, defaults to `0.01`): + damp_percent (`float`, *optional*, defaults to 0.01): The percent of the average Hessian diagonal to use for dampening. Recommended value is 0.01. desc_act (`bool`, *optional*, defaults to `True`): "Whether to quantize columns in order of decreasing activation size." @@ -330,18 +330,18 @@ class GPTQConfig(QuantizationConfigMixin): the entire block at once, we perform layer-wise quantization." "As a result, each layer undergoes quantization using inputs that have passed through the previously quantized layers." use_cuda_fp16 (`bool`, *optional*, defaults to `False`): - "Whether or not to use optimized cuda kernel for fp16 model. Need to have model in fp16. + Whether or not to use optimized cuda kernel for fp16 model. Need to have model in fp16. model_seqlen (`int`, *optional*, defaults to `None`): The maximum sequence length that the model can take. - block_name_to_quantize (`Optional[str]`, *optional*, defaults to `None`): + block_name_to_quantize (`Optional[str]`, *optional*): The transformers block name to quantize. - module_name_preceding_first_block (`List[str]`, *optional*, defaults to `None`): + module_name_preceding_first_block (`List[str]`, *optional*): The layers that are preceding the first Transformer block. - batch_size (`int`, *optional*, defaults to `1`): + batch_size (`int`, *optional*, defaults to 1): The batch size used when processing the dataset - pad_token_id (`int`, *optional*, defaults to `None`): + pad_token_id (`int`, *optional*): The pad token id. Needed to prepare the dataset when `batch_size` > 1. - disable_exllama (`bool`, defaults to `False`): + disable_exllama (`bool`, *optional*, defaults to `False`): Whether to use exllama backend. Only works with `bits` = 4. """ From bf987994dfc091787c89a3b123304182d2a27fce Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Thu, 10 Aug 2023 13:44:52 +0000 Subject: [PATCH 44/46] fix docsting formatting --- docs/source/en/main_classes/quantization.md | 9 +++++--- src/transformers/utils/quantization_config.py | 23 +++++++++---------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/docs/source/en/main_classes/quantization.md b/docs/source/en/main_classes/quantization.md index a0bee2861197..1949fe351cb9 100644 --- a/docs/source/en/main_classes/quantization.md +++ b/docs/source/en/main_classes/quantization.md @@ -86,7 +86,7 @@ GPTQ quantization only works for text model for now. Futhermore, the quantizatio ### Push quantized model to 🤗 Hub -You can push the quantized model like any 🤗 model to Hub with `push_to_hub`: +You can push the quantized model like any 🤗 model to Hub with `push_to_hub`. The quantization config will be saved and pushed along the model. ```python quantized_model.push_to_hub("opt-125m-gptq") @@ -114,8 +114,6 @@ Make sure that the pushed weights are quantized, by checking that the attribute from transformers import AutoModelForCausalLM model = AutoModelForCausalLM.from_pretrained("{your_username}/opt-125m-gptq") ``` -Note that in this case, you don't need to specify the `quantization_config`. It will look for the `quantization_config` and prepare the model -before loading the quantized weights. However, you need to make sure that `optimum` and `auto-gptq` are installed. If you want to load a model faster and without allocating more memory than needed, the `device_map` argument also works with quantized model. Make sure that you have `accelerate` library installed. ```python @@ -136,8 +134,10 @@ model = AutoModelForCausalLM.from_pretrained("{your_username}/opt-125m-gptq", de Note that only 4-bit models are supported for now ### GPTQConfig + [[autodoc]] GPTQConfig + ## `bitsandbytes` Integration 🤗 Transformers is closely integrated with most used modules on `bitsandbytes`. You can load your model in 8-bit precision with few lines of code. @@ -419,7 +419,10 @@ This enables fine-tuning large models such as `flan-t5-large` or `facebook/opt-6 Note that you don't need to pass `device_map` when loading the model for training. It will automatically load your model on your GPU. You can also set the device map to a specific device if needed (e.g. `cuda:0`, `0`, `torch.device('cuda:0')`). Please note that `device_map=auto` should be used for inference only. ### BitsAndBytesConfig + [[autodoc]] BitsAndBytesConfig + + ## Quantization with 🤗 `optimum` Please have a look at [Optimum documentation](https://huggingface.co/docs/optimum/index) to learn more about quantization methods that are supported by `optimum` and see if these are applicable for your use case. diff --git a/src/transformers/utils/quantization_config.py b/src/transformers/utils/quantization_config.py index 15b9309bf295..f0c82602f02e 100644 --- a/src/transformers/utils/quantization_config.py +++ b/src/transformers/utils/quantization_config.py @@ -304,7 +304,7 @@ class GPTQConfig(QuantizationConfigMixin): Args: bits (`int`): The number of bits to quantize to, supported numbers are (2, 3, 4, 8). - tokenizer (`str` or `PreTrainedTokenizerBase`): + tokenizer (`str` or `PreTrainedTokenizerBase`, *optional*): The tokenizer used to process the dataset. You can pass either: - A custom tokenizer object. - A string, the *model id* of a predefined tokenizer hosted inside a model repo on huggingface.co. @@ -313,27 +313,26 @@ class GPTQConfig(QuantizationConfigMixin): - A path to a *directory* containing vocabulary files required by the tokenizer, for instance saved using the [`~PreTrainedTokenizer.save_pretrained`] method, e.g., `./my_model_directory/`. dataset (`Union[List[str]]`, *optional*): - "The dataset used for quantization. You can provide your own dataset in a list of string" "or just use the - original datasets used in GPTQ paper ['wikitext2','c4','c4-new','ptb','ptb-new']" + The dataset used for quantization. You can provide your own dataset in a list of string or just use the + original datasets used in GPTQ paper ['wikitext2','c4','c4-new','ptb','ptb-new'] group_size (`int`, *optional*, defaults to 128): - "The group size to use for quantization. Recommended value is 128 and -1 uses per-column quantization. + The group size to use for quantization. Recommended value is 128 and -1 uses per-column quantization. damp_percent (`float`, *optional*, defaults to 0.01): The percent of the average Hessian diagonal to use for dampening. Recommended value is 0.01. desc_act (`bool`, *optional*, defaults to `True`): - "Whether to quantize columns in order of decreasing activation size." - "Setting it to False can significantly speed up inference but the perplexity may become slightly worse." - "Also known as act-order." + Whether to quantize columns in order of decreasing activation size. Setting it to False can significantly + speed up inference but the perplexity may become slightly worse. Also known as act-order. sym (`bool`, *optional*, defaults to `True`): Whether to use symetric quantization. true_sequential (`bool`, *optional*, defaults to `True`): - "Whether to perform sequential quantization even within a single Transformer block." "Instead of quantizing - the entire block at once, we perform layer-wise quantization." "As a result, each layer undergoes - quantization using inputs that have passed through the previously quantized layers." + Whether to perform sequential quantization even within a single Transformer block. Instead of quantizing + the entire block at once, we perform layer-wise quantization. As a result, each layer undergoes + quantization using inputs that have passed through the previously quantized layers. use_cuda_fp16 (`bool`, *optional*, defaults to `False`): Whether or not to use optimized cuda kernel for fp16 model. Need to have model in fp16. - model_seqlen (`int`, *optional*, defaults to `None`): + model_seqlen (`int`, *optional*): The maximum sequence length that the model can take. - block_name_to_quantize (`Optional[str]`, *optional*): + block_name_to_quantize (`str`, *optional*): The transformers block name to quantize. module_name_preceding_first_block (`List[str]`, *optional*): The layers that are preceding the first Transformer block. From 7adf9cb9efdeb81b034fed9094616741d1d5da84 Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Thu, 10 Aug 2023 18:52:29 +0000 Subject: [PATCH 45/46] add doc --- docs/source/en/main_classes/quantization.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/source/en/main_classes/quantization.md b/docs/source/en/main_classes/quantization.md index 1949fe351cb9..eae71c9cc9e4 100644 --- a/docs/source/en/main_classes/quantization.md +++ b/docs/source/en/main_classes/quantization.md @@ -123,7 +123,7 @@ model = AutoModelForCausalLM.from_pretrained("{your_username}/opt-125m-gptq", de ### Exllama kernels for faster inference -For 4-bit model, you can use the exllama kernels in order to a faster inference speed. You just need to pass `disable_exllama=False` in [`GPTQConfig`]. This will overwrite the quantization config stored in the config. Note that you will only be able to overwrite the attributes related to the kernel. Furthermore, you need to have the entire model on gpus. +For 4-bit model, you can use the exllama kernels in order to a faster inference speed. It is activated by default. You can change that behavior by passing `disable_exllama` in [`GPTQConfig`]. This will overwrite the quantization config stored in the config. Note that you will only be able to overwrite the attributes related to the kernels. Furthermore, you need to have the entire model on gpus if you want to use exllama kernels. ```py import torch @@ -131,7 +131,16 @@ gptq_config = GPTQConfig(bits=4, disable_exllama=False) model = AutoModelForCausalLM.from_pretrained("{your_username}/opt-125m-gptq", device_map="auto", quantization_config = gptq_config) ``` -Note that only 4-bit models are supported for now +Note that only 4-bit models are supported for now. Furthermore, it is recommended to deactivate the exllama kernels if you are finetuning a quantized model with peft. + +#### Fine-tune a quantized model + +With the official support of adapters in the Hugging Face ecosystem, you can fine-tune models that have been quantized with GPTQ. +Please have a look at [`peft`](https://github.com/huggingface/peft) library for more details. + +### Example demo + +Check out the Google Colab [notebook](https://colab.research.google.com/drive/1_TIrmuKOFhuRRiTWN94iLKUFu6ZX4ceb?usp=sharing) to learn how to quantize your model with GPTQ and how finetune the quantized model with peft. ### GPTQConfig From c93d1d00e23e3237f3590a4a8f4128966da6e1b5 Mon Sep 17 00:00:00 2001 From: Marc Sun Date: Thu, 10 Aug 2023 19:41:01 +0000 Subject: [PATCH 46/46] better test --- tests/quantization/gptq/test_gptq.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/quantization/gptq/test_gptq.py b/tests/quantization/gptq/test_gptq.py index 9c6d37f3028c..257c6f020dd3 100644 --- a/tests/quantization/gptq/test_gptq.py +++ b/tests/quantization/gptq/test_gptq.py @@ -87,6 +87,7 @@ class GPTQTest(unittest.TestCase): EXPECTED_OUTPUTS = set() EXPECTED_OUTPUTS.add("Hello my name is John and I am a professional photographer. I") EXPECTED_OUTPUTS.add("Hello my name is John and I am a very good looking man.") + EXPECTED_OUTPUTS.add("Hello my name is Alyson and I am a professional photographer") # this seems a little small considering that we are doing 4bit quant but we have a small model and ww don't quantize the embeddings EXPECTED_RELATIVE_DIFFERENCE = 1.664253062