CFTimeIndex

xarray/__init__.py

@@ -14,7 +14,7 @@
 
 from .backends.api import (open_dataset, open_dataarray, open_mfdataset,
                            save_mfdataset)
-from .conventions import decode_cf
+from .conventions.coding import decode_cf
 
 try:
     from .version import version as __version__

xarray/backends/api.py

@@ -10,8 +10,9 @@
 
 import numpy as np
 
-from .. import backends, conventions
+from .. import backends
 from .common import ArrayWriter, GLOBAL_LOCK
+from ..conventions import coding
 from ..core import indexing
 from ..core.combine import auto_combine
 from ..core.utils import close_on_error, is_remote_uri
@@ -217,7 +218,7 @@ def open_dataset(filename_or_obj, group=None, decode_cf=True,
         cache = chunks is None
 
     def maybe_decode_store(store, lock=False):
-        ds = conventions.decode_cf(
+        ds = coding.decode_cf(
             store, mask_and_scale=mask_and_scale, decode_times=decode_times,
             concat_characters=concat_characters, decode_coords=decode_coords,
             drop_variables=drop_variables)

xarray/backends/common.py

@@ -8,7 +8,7 @@
 from collections import Mapping
 from distutils.version import StrictVersion
 
-from ..conventions import cf_encoder
+from ..conventions.coding import cf_encoder
 from ..core.utils import FrozenOrderedDict
 from ..core.pycompat import iteritems, dask_array_type
 

xarray/backends/netCDF4_.py

@@ -7,7 +7,7 @@
 import numpy as np
 
 from .. import Variable
-from ..conventions import pop_to
+from ..conventions.coding import pop_to
 from ..core import indexing
 from ..core.utils import (FrozenOrderedDict, NDArrayMixin,
                           close_on_error, is_remote_uri)

xarray/backends/netcdf3.py

@@ -5,7 +5,8 @@
 
 import numpy as np
 
-from .. import conventions, Variable
+from .. import Variable
+from ..conventions import coding
 from ..core import ops
 from ..core.pycompat import basestring, unicode_type, OrderedDict
 
@@ -56,7 +57,7 @@ def coerce_nc3_dtype(arr):
 
 def maybe_convert_to_char_array(data, dims):
     if data.dtype.kind == 'S' and data.dtype.itemsize > 1:
-        data = conventions.string_to_char(data)
+        data = coding.string_to_char(data)
         dims = dims + ('string%s' % data.shape[-1],)
     return data, dims
 

xarray/conventions.py → xarray/conventions/coding.py

@@ -11,10 +11,10 @@
 from collections import defaultdict
 from pandas.tslib import OutOfBoundsDatetime
 
-from .core import indexing, ops, utils
-from .core.formatting import format_timestamp, first_n_items, last_item
-from .core.variable import as_variable, Variable
-from .core.pycompat import iteritems, OrderedDict, PY3, basestring
+from ..core import indexing, ops, utils
+from ..core.formatting import format_timestamp, first_n_items, last_item
+from ..core.variable import as_variable, Variable
+from ..core.pycompat import iteritems, OrderedDict, PY3, basestring
 
 
 # standard calendars recognized by netcdftime
@@ -929,8 +929,8 @@ def decode_cf(obj, concat_characters=True, mask_and_scale=True,
     -------
     decoded : Dataset
     """
-    from .core.dataset import Dataset
-    from .backends.common import AbstractDataStore
+    from ..core.dataset import Dataset
+    from ..backends.common import AbstractDataStore
 
     if isinstance(obj, Dataset):
         vars = obj._variables

xarray/conventions/netcdftimeindex.py

@@ -0,0 +1,208 @@
+import re
+from datetime import timedelta
+
+import numpy as np
+import pandas as pd
+
+from xarray.core import pycompat
+from xarray.core.utils import is_scalar
+
+
+def named(name, pattern):
+    return '(?P<' + name + '>' + pattern + ')'
+
+
+def optional(x):
+    return '(?:' + x + ')?'
+
+
+def trailing_optional(xs):
+    if not xs:
+        return ''
+    return xs[0] + optional(trailing_optional(xs[1:]))
+
+
+def build_pattern(date_sep='\-', datetime_sep='T', time_sep='\:'):
+    pieces = [(None, 'year', '\d{4}'),
+              (date_sep, 'month', '\d{2}'),
+              (date_sep, 'day', '\d{2}'),
+              (datetime_sep, 'hour', '\d{2}'),
+              (time_sep, 'minute', '\d{2}'),
+              (time_sep, 'second', '\d{2}' + optional('\.\d+'))]
+    pattern_list = []
+    for sep, name, sub_pattern in pieces:
+        pattern_list.append((sep if sep else '') + named(name, sub_pattern))
+        # TODO: allow timezone offsets?
+    return '^' + trailing_optional(pattern_list) + '$'
+
+
+basic_pattern = build_pattern(date_sep='', time_sep='')
+extended_pattern = build_pattern()
+patterns = [basic_pattern, extended_pattern]
+
+
+def parse_iso8601(datetime_string):
+    for pattern in patterns:
+        match = re.match(pattern, datetime_string)
+        if match:
+            return match.groupdict()
+    raise ValueError('no ISO-8601 match for string: %s' % datetime_string)
+
+
+def _parse_iso8601_with_reso(date_type, timestr):
+    default = date_type(1, 1, 1)
+    result = parse_iso8601(timestr)
+    replace = {}
+
+    for attr in ['year', 'month', 'day', 'hour', 'minute', 'second']:
+        value = result.get(attr, None)
+        if value is not None:
+            # Note ISO8601 conventions allow for fractional seconds; casting
+            # to an int means all seconds values get rounded down to the
+            # nearest integer.  TODO: Consider adding support for sub-second
+            # resolution?
+            replace[attr] = int(value)
+            resolution = attr
+
+    return default.replace(**replace), resolution
+
+
+def _parsed_string_to_bounds(date_type, resolution, parsed):
+    """Generalization of
+    pandas.tseries.index.DatetimeIndex._parsed_string_to_bounds
+    for use with non-standard calendars and netcdftime._netcdftime.datetime
+    objects.
+    """
+    if resolution == 'year':
+        return (date_type(parsed.year, 1, 1),
+                date_type(parsed.year + 1, 1, 1) - timedelta(microseconds=1))
+    if resolution == 'month':
+        if parsed.month == 12:
+            end = date_type(parsed.year + 1, 1, 1) - timedelta(microseconds=1)
+        else:
+            end = (date_type(parsed.year, parsed.month + 1, 1) -
+                   timedelta(microseconds=1))
+        return date_type(parsed.year, parsed.month, 1), end
+    if resolution == 'day':
+        start = date_type(parsed.year, parsed.month, parsed.day)
+        return start, start + timedelta(days=1, microseconds=-1)
+    if resolution == 'hour':
+        start = date_type(parsed.year, parsed.month, parsed.day, parsed.hour)
+        return start, start + timedelta(hours=1, microseconds=-1)
+    if resolution == 'minute':
+        start = date_type(parsed.year, parsed.month, parsed.day, parsed.hour,
+                          parsed.minute)
+        return start, start + timedelta(minutes=1, microseconds=-1)
+    if resolution == 'second':
+        start = date_type(parsed.year, parsed.month, parsed.day, parsed.hour,
+                          parsed.minute, parsed.second)
+        return start, start + timedelta(seconds=1, microseconds=-1)
+    else:
+        raise KeyError
+
+
+def get_date_field(datetimes, field):
+    """Adapted from pandas.tslib.get_date_field"""
+    return [getattr(date, field) for date in datetimes]
+
+
+def _field_accessor(name, docstring=None):
+    """Adapted from pandas.tseries.index._field_accessor"""
+    def f(self):
+        return get_date_field(self._data, name)
+
+    f.__name__ = name
+    f.__doc__ = docstring
+    return property(f)
+
+
+def get_date_type(self):
+    return type(self._data[0])
+
+
+def assert_all_same_netcdftime_datetimes(data):
+    from netcdftime._netcdftime import datetime
+
+    if not isinstance(data[0], datetime):
+        raise TypeError(
+            'NetCDFTimeIndex requires netcdftime._netcdftime.datetime'
+            ' objects.')
+    if not all(isinstance(value, type(data[0])) for value in data):
+        raise TypeError(
+            'NetCDFTimeIndex requires using netcdftime._netcdftime.datetime'
+            ' objects of all the same type.')
+
+
+class NetCDFTimeIndex(pd.Index):
+    def __new__(cls, data):
+        result = object.__new__(cls)
+        assert_all_same_netcdftime_datetimes(data)
+        result._data = np.array(data)
+        return result
+
+    year = _field_accessor('year', 'The year of the datetime')
+    month = _field_accessor('month', 'The month of the datetime')
+    day = _field_accessor('day', 'The days of the datetime')
+    hour = _field_accessor('hour', 'The hours of the datetime')
+    minute = _field_accessor('minute', 'The minutes of the datetime')
+    second = _field_accessor('second', 'The seconds of the datetime')
+    microsecond = _field_accessor('microsecond',
+                                  'The microseconds of the datetime')
+    date_type = property(get_date_type)
+
+    def _partial_date_slice(self, resolution, parsed):
+        """Adapted from
+        pandas.tseries.index.DatetimeIndex._partial_date_slice"""
+        start, end = _parsed_string_to_bounds(self.date_type, resolution,
+                                              parsed)
+        lhs_mask = (self._data >= start)
+        rhs_mask = (self._data <= end)
+        return (lhs_mask & rhs_mask).nonzero()[0]
+
+    def _get_string_slice(self, key):
+        """Adapted from pandas.tseries.index.DatetimeIndex._get_string_slice"""
+        parsed, resolution = _parse_iso8601_with_reso(self.date_type, key)
+        loc = self._partial_date_slice(resolution, parsed)
+        return loc
+
+    def get_loc(self, key, method=None, tolerance=None):
+        """Adapted from pandas.tseries.index.DatetimeIndex.get_loc"""
+        if isinstance(key, pycompat.basestring):
+            return self._get_string_slice(key)
+        else:
+            return pd.Index.get_loc(self, key, method=method,
+                                    tolerance=tolerance)
+
+    def _maybe_cast_slice_bound(self, label, side, kind):
+        """Adapted from
+        pandas.tseries.index.DatetimeIndex._maybe_cast_slice_bound"""
+        if isinstance(label, pycompat.basestring):
+            parsed, resolution = _parse_iso8601_with_reso(self.date_type,
+                                                          label)
+            start, end = _parsed_string_to_bounds(self.date_type, resolution,
+                                                  parsed)
+            if self.is_monotonic_decreasing and len(self):
+                return end if side == 'left' else start
+            return start if side == 'left' else end
+        else:
+            return label
+
+    # TODO: Add ability to use integer range outside of iloc?
+    # e.g. series[1:5].
+    def get_value(self, series, key):
+        """Adapted from pandas.tseries.index.DatetimeIndex.get_value"""
+        if not isinstance(key, slice):
+            return series.iloc[self.get_loc(key)]
+        else:
+            return series.iloc[self.slice_indexer(
+                key.start, key.stop, key.step)]
+
+    def __contains__(self, key):
+        """Adapted from
+        pandas.tseries.base.DatetimeIndexOpsMixin.__contains__"""
+        try:
+            result = self.get_loc(key)
+            return (is_scalar(result) or type(result) == slice or
+                    (isinstance(result, np.ndarray) and result.size))
+        except (KeyError, TypeError, ValueError):
+            return False

xarray/convert.py

@@ -7,7 +7,7 @@
 import numpy as np
 
 from .core.dataarray import DataArray
-from .conventions import (
+from .conventions.coding import (
     maybe_encode_timedelta, maybe_encode_datetime, decode_cf)
 
 ignored_attrs = set(['name', 'tileIndex'])

xarray/core/dataset.py

@@ -16,8 +16,8 @@
 from . import indexing
 from . import alignment
 from . import formatting
-from .. import conventions
 from .alignment import align
+from ..conventions import coding
 from .coordinates import DatasetCoordinates, LevelCoordinatesSource, Indexes
 from .common import ImplementsDatasetReduce, BaseDataObject
 from .merge import (dataset_update_method, dataset_merge_method,
@@ -875,7 +875,7 @@ def dump_to_store(self, store, encoder=None, sync=True, encoding=None,
         """Store dataset contents to a backends.*DataStore object."""
         if encoding is None:
             encoding = {}
-        variables, attrs = conventions.encode_dataset_coordinates(self)
+        variables, attrs = coding.encode_dataset_coordinates(self)
 
         check_encoding = set()
         for k, enc in encoding.items():