Skip to content

Commit

Permalink
remove download check function from test_tokenize
Browse files Browse the repository at this point in the history
  • Loading branch information
christinestraub committed Jan 7, 2025
1 parent 26854e5 commit f340d65
Show file tree
Hide file tree
Showing 2 changed files with 1 addition and 8 deletions.
7 changes: 0 additions & 7 deletions test_unstructured/nlp/test_tokenize.py
Original file line number Diff line number Diff line change
@@ -1,16 +1,9 @@
from typing import List, Tuple
from unittest.mock import patch

from test_unstructured.nlp.mock_nltk import mock_sent_tokenize, mock_word_tokenize
from unstructured.nlp import tokenize


def test_nltk_assets_validation():
with patch("unstructured.nlp.tokenize._ensure_nltk_packages_available") as mock_validate:
tokenize._ensure_nltk_packages_available()
mock_validate.assert_called_once()


def mock_pos_tag(tokens: List[str]) -> List[Tuple[str, str]]:
pos_tags: List[Tuple[str, str]] = []
for token in tokens:
Expand Down
2 changes: 1 addition & 1 deletion unstructured/nlp/tokenize.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ def download_nltk_packages():


def check_for_nltk_package(package_name: str, package_category: str) -> bool:
"""Checks to see if the specified NLTK package exists on the file system."""
"""Checks to see if the specified NLTK package exists on the image."""
paths: list[str] = []
for path in nltk.data.path:
if not path.endswith("nltk_data"):
Expand Down

0 comments on commit f340d65

Please sign in to comment.