Add pandas ExtensionArray for storing homogeneous ragged arrays #687
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -123,17 +123,26 @@ class RaggedDtype(ExtensionDtype): | |
|
|
||
| @property | ||
| def name(self): | ||
| """ | ||
| See docstring for ExtensionDtype.name | ||
| """ | ||
| return 'Ragged[{subtype}]'.format(subtype=self.subtype) | ||
|
|
||
| def __repr__(self): | ||
| return self.name | ||
|
|
||
| @classmethod | ||
| def construct_array_type(cls): | ||
| """ | ||
| See docstring for ExtensionDtype.construct_array_type | ||
| """ | ||
| return RaggedArray | ||
|
|
||
| @classmethod | ||
| def construct_from_string(cls, string): | ||
| """ | ||
| See docstring for ExtensionDtype.construct_from_string | ||
| """ | ||
| # lowercase string | ||
| string = string.lower() | ||
|
|
||
|
|
@@ -372,25 +381,13 @@ def start_indices(self): | |
|
|
||
| def __len__(self): | ||
| """ | ||
| See docstring for ExtensionArray.__len__ | ||
| """ | ||
| return len(self._start_indices) | ||
|
|
||
| def __getitem__(self, item): | ||
jonmmease marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| """ | ||
| See docstring for ExtensionArray.__getitem__ | ||
| """ | ||
| if isinstance(item, Integral): | ||
| if item < -len(self) or item >= len(self): | ||
|
|
@@ -434,41 +431,14 @@ def __getitem__(self, item): | |
| @classmethod | ||
| def _from_sequence(cls, scalars, dtype=None, copy=False): | ||
| """ | ||
| See docstring for ExtensionArray._from_sequence | ||
| """ | ||
| return RaggedArray(scalars, dtype=dtype) | ||
|
|
||
| @classmethod | ||
| def _from_factorized(cls, values, original): | ||
| """ | ||
| See docstring for ExtensionArray._from_factorized | ||
| """ | ||
| return RaggedArray( | ||
| [_RaggedElement.array_or_nan(v) for v in values], | ||
|
|
@@ -479,18 +449,20 @@ def _as_ragged_element_array(self): | |
| for i in range(len(self))]) | ||
|
|
||
| def _values_for_factorize(self): | ||
| """ | ||
| See docstring for ExtensionArray._values_for_factorize | ||
| """ | ||
| return self._as_ragged_element_array(), np.nan | ||
|
|
||
| def _values_for_argsort(self): | ||
| """ | ||
| See docstring for ExtensionArray._values_for_argsort | ||
| """ | ||
| return self._as_ragged_element_array() | ||
|
|
||
| def unique(self): | ||
| """ | ||
| See docstring for ExtensionArray.unique | ||
| """ | ||
|
There was a problem hiding this comment. I would prefer not to copy docstrings unmodified from the parent class, whether trivial (as here) or complicated (below). Basically, if the parent class defines the semantics, I want the reader to refer to the parent class, not to this possibly outdated copy of the docstring; that way people know to go find it in the parent, rather than thinking this actually covers everything. Conversely, if there is a docstring here, I think it should be customized to just be about RaggedArray. There was a problem hiding this comment. Updated in 1538909 |
||
| from pandas import unique | ||
|
|
||
|
|
@@ -501,29 +473,7 @@ def unique(self): | |
|
|
||
| def fillna(self, value=None, method=None, limit=None): | ||
| """ | ||
| See docstring for ExtensionArray.fillna | ||
| """ | ||
|
There was a problem hiding this comment. This docstring seems just copied from the parent class, but if there are differences in behavior from ExtensionArray, please describe those here and refer to the parent class for anything else. There was a problem hiding this comment. Updated in 1538909 |
||
| # Override in RaggedArray to handle ndarray fill values | ||
| from pandas.util._validators import validate_fillna_kwargs | ||
|
|
@@ -560,37 +510,7 @@ def fillna(self, value=None, method=None, limit=None): | |
| def shift(self, periods=1, fill_value=None): | ||
| # type: (int, object) -> ExtensionArray | ||
| """ | ||
| See docstring for ExtensionArray.shift | ||
| """ | ||
| # Override in RaggedArray to handle ndarray fill values | ||
|
|
||
|
|
@@ -616,49 +536,8 @@ def shift(self, periods=1, fill_value=None): | |
|
|
||
| def searchsorted(self, value, side="left", sorter=None): | ||
| """ | ||
| See docstring for ExtensionArray.searchsorted | ||
| """ | ||
| arr = self._as_ragged_element_array() | ||
| if isinstance(value, RaggedArray): | ||
| search_value = value._as_ragged_element_array() | ||
|
|
@@ -668,13 +547,7 @@ def searchsorted(self, value, side="left", sorter=None): | |
|
|
||
| def isna(self): | ||
| """ | ||
| See docstring for ExtensionArray.isna | ||
| """ | ||
| stop_indices = np.hstack([self.start_indices[1:], | ||
| [len(self.flat_array)]]) | ||
|
|
@@ -684,34 +557,7 @@ def isna(self): | |
|
|
||
| def take(self, indices, allow_fill=False, fill_value=None): | ||
| """ | ||
| See docstring for ExtensionArray.take | ||
| """ | ||
| if allow_fill: | ||
| invalid_inds = [i for i in indices if i < -1] | ||
|
|
@@ -731,16 +577,7 @@ def take(self, indices, allow_fill=False, fill_value=None): | |
|
|
||
| def copy(self, deep=False): | ||
| """ | ||
| See docstring for ExtensionArray.copy | ||
| """ | ||
| data = dict( | ||
| flat_array=self.flat_array, | ||
|
|
@@ -751,15 +588,7 @@ def copy(self, deep=False): | |
| @classmethod | ||
| def _concat_same_type(cls, to_concat): | ||
| """ | ||
| See docstring for ExtensionArray._concat_same_type | ||
| """ | ||
| # concat flat_arrays | ||
| flat_array = np.hstack(ra.flat_array for ra in to_concat) | ||
|
|
@@ -778,18 +607,23 @@ def _concat_same_type(cls, to_concat): | |
|
|
||
| @property | ||
| def dtype(self): | ||
| """ | ||
| See docstring for ExtensionArray.dtype | ||
| """ | ||
| return self._dtype | ||
|
|
||
| @property | ||
| def nbytes(self): | ||
| """ | ||
| See docstring for ExtensionArray.nbytes | ||
| """ | ||
| return (self._flat_array.nbytes + | ||
| self._start_indices.nbytes) | ||
|
|
||
| def astype(self, dtype, copy=True): | ||
| """ | ||
| See docstring for ExtensionArray.astype | ||
| """ | ||
| dtype = pandas_dtype(dtype) | ||
| if isinstance(dtype, RaggedDtype): | ||
jonmmease marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| if copy: | ||
|
|
||
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
With most docs/API tools, docstrings will simply be inherited as-is if you don't specify one here, so please remove these altogether unless they need to say something explicitly about how this method relates to that of the parent class. You can mention the parent class explicitly in the class docstring, once, with something like "Methods not otherwise documented here are inherited from ExtensionDtype; please see the corresponding method on that class for the docstring".