refactor(decoder): tweak comments

hyperonym · Mar 5, 2023 · 0291fa5 · 0291fa5
1 parent 581a34b
commit 0291fa5
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/basaran/decoder.py b/basaran/decoder.py
@@ -21,8 +21,8 @@ def decode(self, token):
         # <unk>, <pad> and other special tokens will be decoded into ''.
         text = self.tokenizer.decode(token, skip_special_tokens=True)
 
-        # Handle replacement characters caused by Unicode surrogates or
-        # multi-code-point graphemes like emojis.
+        # Handle replacement characters caused by multi-byte-pair-encoding or
+        # Unicode surrogates or multi-code-point graphemes like emojis.
         if self.replacement in text:
             n = -self.surrogates if self.surrogates > 0 else len(self.buffer)
             tokens = self.buffer[n:] + [token]