Add tokenizer to MLM Trainer (#432)

huggingface · Dec 28, 2022 · 1d4e07f · 1d4e07f
1 parent af0c221
commit 1d4e07f
Show file tree

Hide file tree

Showing 6 changed files with 6 additions and 1 deletion.
diff --git a/chapters/en/chapter7/3.mdx b/chapters/en/chapter7/3.mdx
@@ -723,6 +723,7 @@ trainer = Trainer(
     train_dataset=downsampled_dataset["train"],
     eval_dataset=downsampled_dataset["test"],
     data_collator=data_collator,
+    tokenizer=tokenizer,
 )
 ```
 

diff --git a/chapters/fr/chapter7/3.mdx b/chapters/fr/chapter7/3.mdx
@@ -728,6 +728,7 @@ trainer = Trainer(
     train_dataset=downsampled_dataset["train"],
     eval_dataset=downsampled_dataset["test"],
     data_collator=data_collator,
+    tokenizer=tokenizer,
 )
 ```
 

diff --git a/chapters/ja/chapter7/3.mdx b/chapters/ja/chapter7/3.mdx
@@ -738,6 +738,7 @@ trainer = Trainer(
     train_dataset=downsampled_dataset["train"],
     eval_dataset=downsampled_dataset["test"],
     data_collator=data_collator,
+    tokenizer=tokenizer,
 )
 ```
 

diff --git a/chapters/vi/chapter7/3.mdx b/chapters/vi/chapter7/3.mdx
@@ -723,6 +723,7 @@ trainer = Trainer(
     train_dataset=downsampled_dataset["train"],
     eval_dataset=downsampled_dataset["test"],
     data_collator=data_collator,
+    tokenizer=tokenizer,
 )
 ```
 

diff --git a/chapters/zh-CN/chapter7/3.mdx b/chapters/zh-CN/chapter7/3.mdx
@@ -724,6 +724,7 @@ trainer = Trainer(
     train_dataset=downsampled_dataset["train"],
     eval_dataset=downsampled_dataset["test"],
     data_collator=data_collator,
+    tokenizer=tokenizer,
 )
 ```
 

diff --git a/utils/generate_notebooks.py b/utils/generate_notebooks.py
@@ -201,7 +201,7 @@ def build_notebook(fname, title, output_dir="."):
         installs = ["!pip install datasets evaluate transformers[sentencepiece]"]
         if section_name in sections_with_accelerate:
             installs.append("!pip install accelerate")
-            installs.append("# To run the training on TPU, you will need to uncomment the followin line:")
+            installs.append("# To run the training on TPU, you will need to uncomment the following line:")
             installs.append(
                 "# !pip install cloud-tpu-client==0.10 torch==1.9.0 https://storage.googleapis.com/tpu-pytorch/wheels/torch_xla-1.9-cp37-cp37m-linux_x86_64.whl"
             )