diff --git a/src/transformers/quantizers/base.py b/src/transformers/quantizers/base.py
index 015c0015cf7..e3236ab8f0f 100755
--- a/src/transformers/quantizers/base.py
+++ b/src/transformers/quantizers/base.py
@@ -215,6 +215,9 @@ def dequantize(self, model):
 
         # Delete quantizer and quantization config
         del model.hf_quantizer
+        del model.config.quantization_config
+        del model.config._pre_quantization_dtype
+        model.is_quantized = False
 
         return model