Skip to content
This repository has been archived by the owner on Oct 25, 2024. It is now read-only.

improve lm_eval get chatglm2 tokenizer from local saved files #1598

Merged
merged 1 commit into from
Jun 7, 2024
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -838,12 +838,20 @@ def _create_tokenizer(
else:
# get the HF hub name via accessor on model
model_name = self.model.name_or_path
self.tokenizer = transformers.AutoTokenizer.from_pretrained(
model_name,
revision=revision,
trust_remote_code=trust_remote_code,
use_fast=use_fast_tokenizer,
)

# chatglm2 tokenizer doesn't support loading from local.
if hasattr(self.model, "config") and hasattr(self.model.config, "auto_map") and \
"chatglm2" in self.model.config.auto_map["AutoConfig"]:
self.tokenizer = transformers.AutoTokenizer.from_pretrained(
"THUDM/chatglm2-6b", trust_remote_code=True
)
else:
self.tokenizer = transformers.AutoTokenizer.from_pretrained(
model_name,
revision=revision,
trust_remote_code=trust_remote_code,
use_fast=use_fast_tokenizer,
)
return None

def _detect_batch_size(self, requests=None, pos: int = 0):
Expand Down
Loading