🚨 🚨 🚨 Fix Issue 15003: SentencePiece Tokenizers Not Adding Special Tokens in convert_tokens_to_string

src/transformers/models/albert/tokenization_albert.py

@@ -22,7 +22,7 @@
 
 import sentencepiece as spm
 
-from ...tokenization_utils import AddedToken, PreTrainedTokenizer
+from ...tokenization_utils import AddedToken, PreTrainedTokenizer, SentencePieceStringConversionMixin
 from ...utils import logging
 
 
@@ -56,7 +56,7 @@
 SPIECE_UNDERLINE = "▁"
 
 
-class AlbertTokenizer(PreTrainedTokenizer):
+class AlbertTokenizer(SentencePieceStringConversionMixin, PreTrainedTokenizer):
     """
     Construct an ALBERT tokenizer. Based on [SentencePiece](https://github.com/google/sentencepiece).
 
@@ -249,9 +249,6 @@ def _convert_id_to_token(self, index):
         """Converts an index (integer) in a token (str) using the vocab."""
         return self.sp_model.IdToPiece(index)
 
-    def convert_tokens_to_string(self, tokens):
-        return self.sp_model.decode(tokens)
-
     def build_inputs_with_special_tokens(
         self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
     ) -> List[int]:

src/transformers/models/barthez/tokenization_barthez.py

@@ -21,7 +21,7 @@
 
 import sentencepiece as spm
 
-from ...tokenization_utils import AddedToken, PreTrainedTokenizer
+from ...tokenization_utils import AddedToken, PreTrainedTokenizer, SentencePieceStringConversionMixin
 from ...utils import logging
 
 
@@ -46,7 +46,7 @@
 SPIECE_UNDERLINE = "▁"
 
 
-class BarthezTokenizer(PreTrainedTokenizer):
+class BarthezTokenizer(SentencePieceStringConversionMixin, PreTrainedTokenizer):
     """
     Adapted from [`CamembertTokenizer`] and [`BartTokenizer`]. Construct a BARThez tokenizer. Based on
     [SentencePiece](https://github.com/google/sentencepiece).
@@ -276,10 +276,6 @@ def __setstate__(self, d):
         self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
         self.sp_model.Load(self.vocab_file)
 
-    def convert_tokens_to_string(self, tokens):
-        """Converts a sequence of tokens (strings for sub-words) in a single string."""
-        return self.sp_model.decode(tokens)
-
     def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
         if not os.path.isdir(save_directory):
             logger.error(f"Vocabulary path ({save_directory}) should be a directory")

src/transformers/models/camembert/tokenization_camembert.py

@@ -21,7 +21,7 @@
 
 import sentencepiece as spm
 
-from ...tokenization_utils import AddedToken, PreTrainedTokenizer
+from ...tokenization_utils import AddedToken, PreTrainedTokenizer, SentencePieceStringConversionMixin
 from ...utils import logging
 
 
@@ -42,7 +42,7 @@
 SPIECE_UNDERLINE = "▁"
 
 
-class CamembertTokenizer(PreTrainedTokenizer):
+class CamembertTokenizer(SentencePieceStringConversionMixin, PreTrainedTokenizer):
     """
     Adapted from [`RobertaTokenizer`] and [`XLNetTokenizer`]. Construct a CamemBERT tokenizer. Based on
     [SentencePiece](https://github.com/google/sentencepiece).
@@ -276,10 +276,6 @@ def __setstate__(self, d):
         self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
         self.sp_model.Load(self.vocab_file)
 
-    def convert_tokens_to_string(self, tokens):
-        """Converts a sequence of tokens (strings for sub-words) in a single string."""
-        return self.sp_model.decode(tokens)
-
     def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
         if not os.path.isdir(save_directory):
             logger.error(f"Vocabulary path ({save_directory}) should be a directory")

src/transformers/models/deberta_v2/tokenization_deberta_v2.py

@@ -142,7 +142,9 @@ def __init__(
             )
         self.do_lower_case = do_lower_case
         self.split_by_punct = split_by_punct
-        self._tokenizer = SPMTokenizer(vocab_file, split_by_punct=split_by_punct, sp_model_kwargs=self.sp_model_kwargs)
+        self._tokenizer = SPMTokenizer(
+            vocab_file, self.all_special_tokens, split_by_punct=split_by_punct, sp_model_kwargs=self.sp_model_kwargs
+        )
 
     @property
     def vocab_size(self):
@@ -287,7 +289,9 @@ class SPMTokenizer:
               BPE-dropout.
     """
 
-    def __init__(self, vocab_file, split_by_punct=False, sp_model_kwargs: Optional[Dict[str, Any]] = None):
+    def __init__(
+        self, vocab_file, special_tokens, split_by_punct=False, sp_model_kwargs: Optional[Dict[str, Any]] = None
+    ):
         self.split_by_punct = split_by_punct
         self.vocab_file = vocab_file
         self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
@@ -307,6 +311,7 @@ def __init__(self, vocab_file, split_by_punct=False, sp_model_kwargs: Optional[D
         # self.vocab['[UNK]'] = 3
 
         self.spm = spm
+        self.special_tokens = special_tokens
 
     def __getstate__(self):
         state = self.__dict__.copy()
@@ -343,7 +348,17 @@ def convert_ids_to_tokens(self, ids):
 
     def decode(self, tokens, start=-1, end=-1, raw_text=None):
         if raw_text is None:
-            return self.spm.decode_pieces([t for t in tokens])
+            current_sub_tokens = []
+            out_string = ""
+            for token in tokens:
+                # make sure that special tokens are not decoded using sentencepiece model
+                if token in self.special_tokens:
+                    out_string += self.spm.decode_pieces(current_sub_tokens) + token + " "
+                    current_sub_tokens = []
+                else:
+                    current_sub_tokens.append(token)
+            out_string += self.spm.decode_pieces(current_sub_tokens)
+            return out_string.strip()
         else:
             words = self.split_to_words(raw_text)
             word_tokens = [self.tokenize(w) for w in words]

src/transformers/models/fnet/tokenization_fnet.py

@@ -21,7 +21,7 @@
 
 import sentencepiece as spm
 
-from ...tokenization_utils import AddedToken, PreTrainedTokenizer
+from ...tokenization_utils import AddedToken, PreTrainedTokenizer, SentencePieceStringConversionMixin
 from ...utils import logging
 
 
@@ -43,7 +43,7 @@
 SPIECE_UNDERLINE = "▁"
 
 
-class FNetTokenizer(PreTrainedTokenizer):
+class FNetTokenizer(SentencePieceStringConversionMixin, PreTrainedTokenizer):
     """
     Construct an FNet tokenizer. Adapted from [`AlbertTokenizer`]. Based on
     [SentencePiece](https://github.com/google/sentencepiece). This tokenizer inherits from [`PreTrainedTokenizer`]
@@ -212,9 +212,6 @@ def _convert_id_to_token(self, index):
         """Converts an index (integer) in a token (str) using the vocab."""
         return self.sp_model.IdToPiece(index)
 
-    def convert_tokens_to_string(self, tokens):
-        return self.sp_model.decode(tokens)
-
     def build_inputs_with_special_tokens(
         self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
     ) -> List[int]:

src/transformers/models/m2m_100/tokenization_m2m_100.py

@@ -21,7 +21,7 @@
 
 import sentencepiece
 
-from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
+from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer, SentencePieceStringConversionMixin
 from ...utils import logging
 
 
@@ -62,7 +62,7 @@
 # fmt: on
 
 
-class M2M100Tokenizer(PreTrainedTokenizer):
+class M2M100Tokenizer(SentencePieceStringConversionMixin, PreTrainedTokenizer):
     """
     Construct an M2M100 tokenizer. Based on [SentencePiece](https://github.com/google/sentencepiece).
 
@@ -221,10 +221,6 @@ def _convert_id_to_token(self, index: int) -> str:
             return self.id_to_lang_token[index]
         return self.decoder.get(index, self.unk_token)
 
-    def convert_tokens_to_string(self, tokens: List[str]) -> str:
-        """Converts a sequence of tokens (strings for sub-words) in a single string."""
-        return self.sp_model.decode(tokens)
-
     def get_special_tokens_mask(
         self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
     ) -> List[int]:

src/transformers/models/marian/tokenization_marian.py

@@ -253,10 +253,18 @@ def decode(self, token_ids, **kwargs):
 
     def convert_tokens_to_string(self, tokens: List[str]) -> str:
         """Uses source spm if _decode_use_source_tokenizer is True, and target spm otherwise"""
-        if self._decode_use_source_tokenizer:
-            return self.spm_source.DecodePieces(tokens)
-        else:
-            return self.spm_target.DecodePieces(tokens)
+        sp_model = self.spm_source if self._decode_use_source_tokenizer else self.spm_target
+        current_sub_tokens = []
+        out_string = ""
+        for token in tokens:
+            # make sure that special tokens are not decoded using sentencepiece model
+            if token in self.all_special_tokens:
+                out_string += sp_model.decode_pieces(current_sub_tokens) + token + " "
+                current_sub_tokens = []
+            else:
+                current_sub_tokens.append(token)
+        out_string += sp_model.decode_pieces(current_sub_tokens)
+        return out_string.strip()
 
     def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None) -> List[int]:
         """Build model inputs from a sequence by appending eos_token_id."""

src/transformers/models/mbart50/tokenization_mbart50.py

@@ -20,7 +20,7 @@
 
 import sentencepiece as spm
 
-from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
+from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer, SentencePieceStringConversionMixin
 from ...utils import logging
 
 
@@ -45,7 +45,7 @@
 # fmt: on
 
 
-class MBart50Tokenizer(PreTrainedTokenizer):
+class MBart50Tokenizer(SentencePieceStringConversionMixin, PreTrainedTokenizer):
     """
     Construct a MBart50 tokenizer. Based on [SentencePiece](https://github.com/google/sentencepiece).
 
@@ -233,9 +233,9 @@ def _convert_id_to_token(self, index: int) -> str:
             return self.fairseq_ids_to_tokens[index]
         return self.sp_model.IdToPiece(index - self.fairseq_offset)
 
-    def convert_tokens_to_string(self, tokens: List[str]) -> str:
-        """Converts a sequence of tokens (strings for sub-words) in a single string."""
-        return self.sp_model.decode(tokens)
+    # def convert_tokens_to_string(self, tokens: List[str]) -> str:
+    #     """Converts a sequence of tokens (strings for sub-words) in a single string."""
+    #     return self.sp_model.decode(tokens)
 
     def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
         if not os.path.isdir(save_directory):

src/transformers/models/pegasus/tokenization_pegasus.py

@@ -18,7 +18,7 @@
 
 import sentencepiece as spm
 
-from ...tokenization_utils import PreTrainedTokenizer
+from ...tokenization_utils import PreTrainedTokenizer, SentencePieceStringConversionMixin
 from ...utils import logging
 
 
@@ -38,7 +38,7 @@
 logger = logging.get_logger(__name__)
 
 
-class PegasusTokenizer(PreTrainedTokenizer):
+class PegasusTokenizer(SentencePieceStringConversionMixin, PreTrainedTokenizer):
     r"""
     Construct a PEGASUS tokenizer. Based on [SentencePiece](https://github.com/google/sentencepiece).
 
@@ -226,10 +226,10 @@ def _convert_id_to_token(self, index: int) -> str:
             token = self.sp_model.IdToPiece(index - self.offset)
         return token
 
-    def convert_tokens_to_string(self, tokens):
-        """Converts a sequence of tokens (string) in a single string."""
-        out_string = self.sp_model.decode_pieces(tokens)
-        return out_string
+    # def convert_tokens_to_string(self, tokens):
+    #     """Converts a sequence of tokens (string) in a single string."""
+    #     out_string = self.sp_model.decode_pieces(tokens)
+    #     return out_string
 
     def num_special_tokens_to_add(self, pair=False):
         """Just EOS"""

src/transformers/models/speech_to_text/tokenization_speech_to_text.py

@@ -186,11 +186,19 @@ def _convert_id_to_token(self, index: int) -> str:
 
     def convert_tokens_to_string(self, tokens: List[str]) -> str:
         """Converts a sequence of tokens (strings for sub-words) in a single string."""
-        out_string = self.sp_model.decode(tokens)
-
-        if self.do_upper_case:
-            out_string = out_string.upper()
-        return out_string
+        current_sub_tokens = []
+        out_string = ""
+        for token in tokens:
+            # make sure that special tokens are not decoded using sentencepiece model
+            if token in self.all_special_tokens:
+                decoded = self.sp_model.decode(current_sub_tokens)
+                out_string += (decoded.upper() if self.do_upper_case else decoded) + token + " "
+                current_sub_tokens = []
+            else:
+                current_sub_tokens.append(token)
+        decoded = self.sp_model.decode(current_sub_tokens)
+        out_string += decoded.upper() if self.do_upper_case else decoded
+        return out_string.strip()
 
     def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None) -> List[int]:
         """Build model inputs from a sequence by appending eos_token_id."""

src/transformers/tokenization_utils.py

@@ -956,3 +956,19 @@ def _decode(
             return clean_text
         else:
             return text
+
+
+class SentencePieceStringConversionMixin:
+    def convert_tokens_to_string(self, tokens):
+        """Converts a sequence of tokens (string) in a single string."""
+        current_sub_tokens = []
+        out_string = ""
+        for token in tokens:
+            # make sure that special tokens are not decoded using sentencepiece model
+            if token in self.all_special_tokens:
+                out_string += self.sp_model.decode(current_sub_tokens) + token + " "
+                current_sub_tokens = []
+            else:
+                current_sub_tokens.append(token)
+        out_string += self.sp_model.decode(current_sub_tokens)
+        return out_string.strip()

tests/test_tokenization_common.py

@@ -37,6 +37,7 @@
     AutoTokenizer,
     BertTokenizer,
     BertTokenizerFast,
+    LayoutXLMTokenizer,
     PreTrainedTokenizer,
     PreTrainedTokenizerBase,
     PreTrainedTokenizerFast,
@@ -356,7 +357,7 @@ def test_sentencepiece_tokenize_and_convert_tokens_to_string(self):
             return
 
         tokenizer = self.get_tokenizer()
-        text = "This is text to test the tokenizer."
+        text = "This is a test"
 
         if self.test_sentencepiece_ignore_case:
             text = text.lower()
@@ -368,11 +369,21 @@ def test_sentencepiece_tokenize_and_convert_tokens_to_string(self):
         # check if converting back to original text works
         reverse_text = tokenizer.convert_tokens_to_string(tokens)
 
-        if self.test_sentencepiece_ignore_case:
-            reverse_text = reverse_text.lower()
+        # All tokenizers pass this test without the below commented out code.
+        # if self.test_sentencepiece_ignore_case:
+        #     reverse_text = reverse_text.lower()
 
         self.assertEqual(reverse_text, text)
 
+        if isinstance(tokenizer, LayoutXLMTokenizer):
+            return
+
+        input_ids = tokenizer(text).input_ids
+        tokens_including_special = tokenizer.convert_ids_to_tokens(input_ids)
+        reverse_text = tokenizer.convert_tokens_to_string(tokens_including_special)
+
+        self.assertEqual(len(tokenizer.tokenize(reverse_text)), len(input_ids))
+
     def test_subword_regularization_tokenizer(self) -> None:
         if not self.test_sentencepiece:
             return