Skip to content
This repository has been archived by the owner on Oct 25, 2024. It is now read-only.

Improve SQ model restored from json #1600

Merged
merged 5 commits into from
Jun 11, 2024
Merged

Improve SQ model restored from json #1600

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
050de5c
improve SQ model restore from json
changwangss Jun 7, 2024
50dfdfe
improve line length
changwangss Jun 7, 2024
5bd4555
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Jun 7, 2024
43a3087
Update utility.py
changwangss Jun 7, 2024
31878e9
Merge branch 'main' into changwangss-patch-5
changwangss Jun 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions intel_extension_for_transformers/transformers/utils/utility.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -462,6 +462,8 @@ def recover_model_from_json(fp32_model_name_or_path, json_file_path, trust_remot
(object): quantized model
"""
from transformers import AutoModelForCausalLM

# ipex recovered int8 model from configure.json requests float32 model input and on cpu device.
user_model = AutoModelForCausalLM.from_pretrained(fp32_model_name_or_path,
trust_remote_code=trust_remote_code).float()
if user_model.config.model_type in IPEX_OPT_LLM_SUPPORTED:
Expand Down Expand Up @@ -496,5 +498,9 @@ def recover_model_from_json(fp32_model_name_or_path, json_file_path, trust_remot
from intel_extension_for_transformers.transformers.llm.evaluation.models import (
TSModelCausalLMForITREX,
)
origin_model_type = config.model_type
if origin_model_type in ["chatglm", "qwen", "baichuan"]:
config.model_type = "qwen2"
user_model = TSModelCausalLMForITREX(user_model, config=config)
user_model.config.model_type = origin_model_type
return user_model
Loading