Update pretraining.py

fix eval file path
shibing624 · Aug 16, 2023 · 0210776 · 0210776
1 parent cfd2bf5
commit 0210776
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/pretraining.py b/pretraining.py
@@ -457,7 +457,7 @@ def group_texts(examples):
                 raise ValueError(f"train files must be same type, e.g. all txt or all jsonl, but got {types}")
             data_files["train"] = train_data_files
         if data_args.validation_file_dir is not None and os.path.exists(data_args.validation_file_dir):
-            eval_data_files = glob(f'{data_args.train_file_dir}/**/*.txt', recursive=True) + glob(
+            eval_data_files = glob(f'{data_args.validation_file_dir}/**/*.txt', recursive=True) + glob(
                 f'{data_args.train_file_dir}/**/*.json', recursive=True) + glob(
                 f'{data_args.train_file_dir}/**/*.jsonl', recursive=True)
             logger.info(f"eval files: {eval_data_files}")