Skip to content

Commit

Permalink
GH-1711: Rename id method to name and provide a default implementation
Browse files Browse the repository at this point in the history
  • Loading branch information
Mario Sänger committed Jun 25, 2020
1 parent fce2d19 commit dc95088
Showing 1 changed file with 7 additions and 9 deletions.
16 changes: 7 additions & 9 deletions flair/data.py
Original file line number Diff line number Diff line change
Expand Up @@ -1440,18 +1440,18 @@ class Tokenizer(ABC):
Tokenizers are used to represent algorithms and models to split plain text into
individual tokens / words. All subclasses should overwrite :meth:`tokenize`, which
splits the given plain text into tokens, and :meth:`id`, returning a unique identifier
representing the tokenizer's configuration.
splits the given plain text into tokens. Moreover, subclasses may overwrite
:meth:`name`, returning a unique identifier representing the tokenizer's
configuration.
"""

@abstractmethod
def tokenize(self, text: str) -> List[Token]:
raise NotImplementedError()

@property
@abstractmethod
def id(self) -> str:
raise NotImplementedError()
def name(self) -> str:
return self.__class__.__name__


class SpacyTokenizer(Tokenizer):
Expand Down Expand Up @@ -1494,7 +1494,8 @@ def tokenize(self, text: str) -> List[Token]:

return tokens

def id(self) -> str:
@property
def name(self) -> str:
return (
self.__class__.__name__
+ "_"
Expand Down Expand Up @@ -1551,6 +1552,3 @@ def tokenize(self, text: str) -> List[Token]:
previous_token = token

return tokens

def id(self) -> str:
return self.__class__.__name__

0 comments on commit dc95088

Please sign in to comment.