From 72deb518af2de97d539f5c438af222be713dfeec Mon Sep 17 00:00:00 2001 From: Xu Song Date: Fri, 18 Aug 2023 17:20:15 +0800 Subject: [PATCH 1/3] Add default value for tokenizer path --- applications/DeepSpeed-Chat/training/utils/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/applications/DeepSpeed-Chat/training/utils/utils.py b/applications/DeepSpeed-Chat/training/utils/utils.py index b1117854d..b9b740e6c 100644 --- a/applications/DeepSpeed-Chat/training/utils/utils.py +++ b/applications/DeepSpeed-Chat/training/utils/utils.py @@ -68,7 +68,7 @@ def load_hf_tokenizer(model_name_or_path, fast_tokenizer=True): model_json = os.path.join(model_name_or_path, "config.json") if os.path.exists(model_json): model_json_file = json.load(open(model_json)) - model_name = model_json_file["_name_or_path"] + model_name = model_json_file.get("_name_or_path", model_name_or_path) tokenizer = get_tokenizer(model_name, fast_tokenizer=fast_tokenizer) else: From c25a7cc1acd857694e8ec3b55418d6290d1a33d4 Mon Sep 17 00:00:00 2001 From: Xu Song Date: Sat, 26 Aug 2023 17:55:55 +0800 Subject: [PATCH 2/3] Fix yapf check --- applications/DeepSpeed-Chat/training/utils/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/applications/DeepSpeed-Chat/training/utils/utils.py b/applications/DeepSpeed-Chat/training/utils/utils.py index b9b740e6c..1d273f5cc 100644 --- a/applications/DeepSpeed-Chat/training/utils/utils.py +++ b/applications/DeepSpeed-Chat/training/utils/utils.py @@ -68,7 +68,8 @@ def load_hf_tokenizer(model_name_or_path, fast_tokenizer=True): model_json = os.path.join(model_name_or_path, "config.json") if os.path.exists(model_json): model_json_file = json.load(open(model_json)) - model_name = model_json_file.get("_name_or_path", model_name_or_path) + model_name = model_json_file.get("_name_or_path", + model_name_or_path) tokenizer = get_tokenizer(model_name, fast_tokenizer=fast_tokenizer) else: From 2dcb76ef002a534abf95279af1405c7eb20c2d8b Mon Sep 17 00:00:00 2001 From: Michael Wyatt Date: Tue, 3 Oct 2023 13:31:40 -0700 Subject: [PATCH 3/3] Update applications/DeepSpeed-Chat/training/utils/utils.py --- applications/DeepSpeed-Chat/training/utils/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/applications/DeepSpeed-Chat/training/utils/utils.py b/applications/DeepSpeed-Chat/training/utils/utils.py index ab4ff0137..b5cfb8d6d 100644 --- a/applications/DeepSpeed-Chat/training/utils/utils.py +++ b/applications/DeepSpeed-Chat/training/utils/utils.py @@ -70,7 +70,7 @@ def load_hf_tokenizer(model_name_or_path, fast_tokenizer=True): if os.path.exists(model_json): model_json_file = json.load(open(model_json)) model_name = model_json_file.get("_name_or_path", - model_name_or_path) + model_name_or_path) tokenizer = get_tokenizer(model_name, fast_tokenizer=fast_tokenizer) else: