fix convert_quantized model bug (#158)

Signed-off-by: Yu, Zhentao <[email protected]>
intel · Mar 8, 2024 · 37d01f3 · 37d01f3
1 parent a129213
commit 37d01f3
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 3 deletions.
diff --git a/neural_speed/convert/convert_quantized_llama.py b/neural_speed/convert/convert_quantized_llama.py
@@ -71,7 +71,7 @@ def convert_to_q4_bestla_tensor(src_name, dst_name, model, fout, q_config, n_hea
         int_weight = (int_weight - 8) * 16
         gptq_scales = gptq_scales / 16
         gptq_zeros = (gptq_zeros - 8) * 16
-        weight_dtype == "int4"
+        weight_dtype = "int4"
 
     dst = np.zeros((int_weight.shape[0], int_weight.shape[1] * 4), dtype=np.int8)
     int_weight = np.ascontiguousarray(int_weight.numpy())

diff --git a/neural_speed/convert/convert_quantized_mistral.py b/neural_speed/convert/convert_quantized_mistral.py
@@ -74,7 +74,7 @@ def convert_to_q4_bestla_tensor(src_name, dst_name, model, fout, q_config, n_hea
         int_weight = (int_weight - 8) * 16
         gptq_scales = gptq_scales / 16
         gptq_zeros = (gptq_zeros - 8) * 16
-        weight_dtype == "int4"
+        weight_dtype = "int4"
 
     dst = np.zeros((int_weight.shape[0], int_weight.shape[1] * 4), dtype=np.int8)
     int_weight = np.ascontiguousarray(int_weight.numpy())

diff --git a/neural_speed/convert/convert_quantized_mixtral.py b/neural_speed/convert/convert_quantized_mixtral.py
@@ -80,7 +80,7 @@ def convert_to_q4_bestla_tensor(src_name, dst_name, model, fout, q_config, n_hea
         int_weight = (int_weight - 8) * 16
         gptq_scales = gptq_scales / 16
         gptq_zeros = (gptq_zeros - 8) * 16
-        weight_dtype == "int4"
+        weight_dtype = "int4"
 
     dst = np.zeros((int_weight.shape[0], int_weight.shape[1] * 4), dtype=np.int8)
     int_weight = np.ascontiguousarray(int_weight.numpy())