remove torchtext vectors and torchtext dependency (refs #829)

tests/methods/test_lime_text.py

@@ -69,12 +69,15 @@ class TestLimeOnTextSpecialCharacters:
 
     def test_lime_text_special_chars_regression_test(self, text):
         """Just don't raise an error on this input with special characters."""
-        _ = dianna.explain_text(self.runner,
-                                text,
-                                tokenizer=self.runner.tokenizer,
-                                labels=[0],
-                                method='LIME',
-                                random_state=0)
+        _ = dianna.explain_text(
+            self.runner,
+            text,
+            tokenizer=self.runner.tokenizer,
+            labels=[0],
+            method='LIME',
+            random_state=0,
+            num_samples=10,
+        )
 
 
 @pytest.fixture

tests/utils.py

@@ -103,16 +103,7 @@ def __call__(self, sentences):
         output = []
         for sentence in sentences:
             # tokenize and pad to minimum length
-            tokens = self.tokenizer.tokenize(sentence.lower())
-            if len(tokens) < self.max_filter_size:
-                tokens += ['<pad>'] * (self.max_filter_size - len(tokens))
-
-            # numericalize the tokens
-            tokens_numerical = [
-                self.vocab.stoi[token]
-                if token in self.vocab.stoi else self.vocab.stoi['<unk>']
-                for token in tokens
-            ]
+            tokens_numerical = self.tokenize(sentence)
 
             # run the model, applying a sigmoid because the model outputs logits, remove any remaining batch axis
             onnx_input = {input_name: [tokens_numerical]}
@@ -128,7 +119,7 @@ def __call__(self, sentences):
     def tokenize(self, sentence: str):
         """Tokenize sentence."""
         # tokenize and pad to minimum length
-        tokens = self.tokenizer.tokenize(sentence)
+        tokens = self.tokenizer.tokenize(sentence.lower())
         if len(tokens) < self.max_filter_size:
             tokens += ['<pad>'] * (self.max_filter_size - len(tokens))