Revert oups (I though copilot hallucinated smt)

rustformers · May 28, 2023 · ec197c1 · ec197c1
1 parent 7b06bb9
commit ec197c1
Showing 1 changed file with 4 additions and 0 deletions.
diff --git a/crates/llm-base/src/vocabulary.rs b/crates/llm-base/src/vocabulary.rs
@@ -311,6 +311,10 @@ impl ModelVocabulary {
         let mut vec = vec![];
 
         for token in tokens {
+            if skip_special_tokens && token == 1 {
+                continue;
+            }
+
             vec.append(&mut self.id_to_token[token as usize].to_vec());
         }