diff --git a/.travis.yml b/.travis.yml index ea9ee7adcf4..155c0271b30 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,6 @@ matrix: fast_finish: true include: - env: CONDA_ENV=py35-min - - env: CONDA_ENV=py35 - env: CONDA_ENV=py36 - env: CONDA_ENV=py37 - env: @@ -65,7 +64,7 @@ script: elif [[ "$CONDA_ENV" == "py36-hypothesis" ]]; then pytest properties ; else - py.test xarray --cov=xarray --cov-config ci/.coveragerc --cov-report term-missing --verbose $EXTRA_FLAGS; + py.test xarray --cov=xarray --cov-config ci/.coveragerc --cov-report term-missing $EXTRA_FLAGS; fi after_success: diff --git a/README.rst b/README.rst index 6dbf774549d..83382f87ed5 100644 --- a/README.rst +++ b/README.rst @@ -8,9 +8,9 @@ xarray: N-D labeled arrays and datasets .. image:: https://coveralls.io/repos/pydata/xarray/badge.svg :target: https://coveralls.io/r/pydata/xarray .. image:: https://readthedocs.org/projects/xray/badge/?version=latest - :target: http://xarray.pydata.org/ -.. image:: http://img.shields.io/badge/benchmarked%20by-asv-green.svg?style=flat - :target: http://pandas.pydata.org/speed/xarray/ + :target: https://xarray.pydata.org/ +.. image:: https://img.shields.io/badge/benchmarked%20by-asv-green.svg?style=flat + :target: https://pandas.pydata.org/speed/xarray/ .. image:: https://img.shields.io/pypi/v/xarray.svg :target: https://pypi.python.org/pypi/xarray/ @@ -30,10 +30,10 @@ It is particularly tailored to working with netCDF_ files, which were the source of xarray's data model, and integrates tightly with dask_ for parallel computing. -.. _NumPy: http://www.numpy.org -.. _pandas: http://pandas.pydata.org -.. _dask: http://dask.org -.. _netCDF: http://www.unidata.ucar.edu/software/netcdf +.. _NumPy: https://www.numpy.org +.. _pandas: https://pandas.pydata.org +.. _dask: https://dask.org +.. _netCDF: https://www.unidata.ucar.edu/software/netcdf Why xarray? ----------- @@ -66,12 +66,12 @@ powerful and concise interface. For example: Documentation ------------- -Learn more about xarray in its official documentation at http://xarray.pydata.org/ +Learn more about xarray in its official documentation at https://xarray.pydata.org/ Contributing ------------ -You can find information about contributing to xarray at our `Contributing page `_. +You can find information about contributing to xarray at our `Contributing page `_. Get in touch ------------ @@ -81,9 +81,9 @@ Get in touch - For less well defined questions or ideas, or to announce other projects of interest to xarray users, use the `mailing list`_. -.. _StackOverFlow: http://stackoverflow.com/questions/tagged/python-xarray +.. _StackOverFlow: https://stackoverflow.com/questions/tagged/python-xarray .. _mailing list: https://groups.google.com/forum/#!forum/xarray -.. _on GitHub: http://github.com/pydata/xarray +.. _on GitHub: https://github.com/pydata/xarray NumFOCUS -------- @@ -120,7 +120,7 @@ Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, diff --git a/appveyor.yml b/appveyor.yml index 347883b96cb..ffa8a2f6e8d 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -35,4 +35,4 @@ install: build: false test_script: - - "py.test xarray --verbose" + - "py.test xarray" diff --git a/asv_bench/benchmarks/dataset_io.py b/asv_bench/benchmarks/dataset_io.py index 3e070e1355b..07bcc6d71b4 100644 --- a/asv_bench/benchmarks/dataset_io.py +++ b/asv_bench/benchmarks/dataset_io.py @@ -19,7 +19,7 @@ os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE' -class IOSingleNetCDF(object): +class IOSingleNetCDF: """ A few examples that benchmark reading/writing a single netCDF file with xarray @@ -214,7 +214,7 @@ def time_load_dataset_scipy_with_time_chunks(self): chunks=self.time_chunks).load() -class IOMultipleNetCDF(object): +class IOMultipleNetCDF: """ A few examples that benchmark reading/writing multiple netCDF files with xarray @@ -419,7 +419,7 @@ def create_delayed_write(): return ds.to_netcdf('file.nc', engine='netcdf4', compute=False) -class IOWriteNetCDFDask(object): +class IOWriteNetCDFDask: timeout = 60 repeat = 1 number = 5 @@ -432,7 +432,7 @@ def time_write(self): self.write.compute() -class IOWriteNetCDFDaskDistributed(object): +class IOWriteNetCDFDaskDistributed: def setup(self): try: import distributed diff --git a/asv_bench/benchmarks/indexing.py b/asv_bench/benchmarks/indexing.py index 54262b12a19..84e7fea33af 100644 --- a/asv_bench/benchmarks/indexing.py +++ b/asv_bench/benchmarks/indexing.py @@ -58,7 +58,7 @@ } -class Base(object): +class Base: def setup(self, key): self.ds = xr.Dataset( {'var1': (('x', 'y'), randn((nx, ny), frac_nan=0.1)), diff --git a/asv_bench/benchmarks/interp.py b/asv_bench/benchmarks/interp.py index edec6df34dd..c3c1b7c533b 100644 --- a/asv_bench/benchmarks/interp.py +++ b/asv_bench/benchmarks/interp.py @@ -24,7 +24,7 @@ new_y_long = np.linspace(0.1, 0.9, 1000) -class Interpolation(object): +class Interpolation: def setup(self, *args, **kwargs): self.ds = xr.Dataset( {'var1': (('x', 'y'), randn_xy), diff --git a/asv_bench/benchmarks/reindexing.py b/asv_bench/benchmarks/reindexing.py index 28e14d52e89..42529f2cfe6 100644 --- a/asv_bench/benchmarks/reindexing.py +++ b/asv_bench/benchmarks/reindexing.py @@ -7,7 +7,7 @@ from . import requires_dask -class Reindex(object): +class Reindex: def setup(self): data = np.random.RandomState(0).randn(1000, 100, 100) self.ds = xr.Dataset({'temperature': (('time', 'x', 'y'), data)}, diff --git a/asv_bench/benchmarks/rolling.py b/asv_bench/benchmarks/rolling.py index 5ba7406f6e0..7eb9f8a2358 100644 --- a/asv_bench/benchmarks/rolling.py +++ b/asv_bench/benchmarks/rolling.py @@ -19,7 +19,7 @@ randn_long = randn((long_nx, ), frac_nan=0.1) -class Rolling(object): +class Rolling: def setup(self, *args, **kwargs): self.ds = xr.Dataset( {'var1': (('x', 'y'), randn_xy), diff --git a/asv_bench/benchmarks/unstacking.py b/asv_bench/benchmarks/unstacking.py index 54436b422e9..d050e864e86 100644 --- a/asv_bench/benchmarks/unstacking.py +++ b/asv_bench/benchmarks/unstacking.py @@ -7,7 +7,7 @@ from . import requires_dask -class Unstacking(object): +class Unstacking: def setup(self): data = np.random.RandomState(0).randn(1, 1000, 500) self.ds = xr.DataArray(data).stack(flat_dim=['dim_1', 'dim_2']) diff --git a/ci/requirements-py35.yml b/ci/requirements-py35.yml deleted file mode 100644 index a71434865cc..00000000000 --- a/ci/requirements-py35.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: test_env -channels: - - conda-forge -dependencies: - - python=3.5 - - cftime - - dask=0.16 - - h5py - - h5netcdf - - matplotlib=1.5 - - netcdf4 - - pytest - - pytest-cov - - pytest-env - - coveralls - - flake8 - - numpy - - pandas - - scipy - - seaborn - - toolz - - rasterio - - zarr diff --git a/ci/requirements-py36-dask-dev.yml b/ci/requirements-py36-dask-dev.yml index 32d01765439..29603a59f7e 100644 --- a/ci/requirements-py36-dask-dev.yml +++ b/ci/requirements-py36-dask-dev.yml @@ -13,8 +13,8 @@ dependencies: - pytest-env - coveralls - flake8 - - numpy - - pandas + - numpy>=1.12 + - pandas>=0.19 - scipy - seaborn - toolz diff --git a/ci/requirements-py36-hypothesis.yml b/ci/requirements-py36-hypothesis.yml index 8066a53b6bc..495f81c9d3a 100644 --- a/ci/requirements-py36-hypothesis.yml +++ b/ci/requirements-py36-hypothesis.yml @@ -15,8 +15,8 @@ dependencies: - coveralls - hypothesis - flake8 - - numpy - - pandas + - numpy>=1.12 + - pandas>=0.19 - scipy - seaborn - toolz diff --git a/ci/requirements-py36-pandas-dev.yml b/ci/requirements-py36-pandas-dev.yml index bc0e5d0de09..05d2c11486c 100644 --- a/ci/requirements-py36-pandas-dev.yml +++ b/ci/requirements-py36-pandas-dev.yml @@ -16,7 +16,7 @@ dependencies: - pytest-env - coveralls - flake8 - - numpy + - numpy>=1.12 - scipy - toolz - pip: diff --git a/ci/requirements-py36-rasterio.yml b/ci/requirements-py36-rasterio.yml index e5ef1d29777..7307ed60d9a 100644 --- a/ci/requirements-py36-rasterio.yml +++ b/ci/requirements-py36-rasterio.yml @@ -14,8 +14,8 @@ dependencies: - pytest-cov - pytest-env - coveralls - - numpy - - pandas + - numpy>=1.12 + - pandas>=0.19 - scipy - seaborn - toolz diff --git a/ci/requirements-py36-windows.yml b/ci/requirements-py36-windows.yml index b139d5c78ca..22d917e332c 100644 --- a/ci/requirements-py36-windows.yml +++ b/ci/requirements-py36-windows.yml @@ -12,11 +12,10 @@ dependencies: - netcdf4 - pytest - pytest-env - - numpy - - pandas + - numpy>=1.12 + - pandas>=0.19 - scipy - seaborn - toolz - rasterio - zarr - diff --git a/ci/requirements-py36-zarr-dev.yml b/ci/requirements-py36-zarr-dev.yml index 94bdc50fbfe..2dbdf172b6c 100644 --- a/ci/requirements-py36-zarr-dev.yml +++ b/ci/requirements-py36-zarr-dev.yml @@ -12,8 +12,8 @@ dependencies: - pytest-env - coveralls - flake8 - - numpy - - pandas + - numpy>=1.12 + - pandas>=0.19 - scipy - seaborn - toolz diff --git a/ci/requirements-py36.yml b/ci/requirements-py36.yml index 7a3f0f53223..03242426a36 100644 --- a/ci/requirements-py36.yml +++ b/ci/requirements-py36.yml @@ -15,8 +15,8 @@ dependencies: - pytest-env - coveralls - pycodestyle - - numpy - - pandas + - numpy>=1.12 + - pandas>=0.19 - scipy - seaborn - toolz diff --git a/ci/requirements-py37-windows.yml b/ci/requirements-py37-windows.yml index fb4b97cde7c..1ad310a12e0 100644 --- a/ci/requirements-py37-windows.yml +++ b/ci/requirements-py37-windows.yml @@ -13,8 +13,8 @@ dependencies: - netcdf4 - pytest - pytest-env - - numpy - - pandas + - numpy>=1.12 + - pandas>=0.19 - scipy - seaborn - toolz diff --git a/ci/requirements-py37.yml b/ci/requirements-py37.yml index 4f4d2b1728b..0cece4ed6dd 100644 --- a/ci/requirements-py37.yml +++ b/ci/requirements-py37.yml @@ -16,8 +16,8 @@ dependencies: - pytest-env - coveralls - pycodestyle - - numpy - - pandas + - numpy>=1.12 + - pandas>=0.19 - scipy - seaborn - toolz diff --git a/doc/api.rst b/doc/api.rst index 00b33959eed..0e766f2cf9a 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -460,6 +460,7 @@ Dataset methods :toctree: generated/ open_dataset + load_dataset open_mfdataset open_rasterio open_zarr @@ -487,6 +488,7 @@ DataArray methods :toctree: generated/ open_dataarray + load_dataarray DataArray.to_dataset DataArray.to_netcdf DataArray.to_pandas diff --git a/doc/contributing.rst b/doc/contributing.rst index da9c89234a3..fba09497abe 100644 --- a/doc/contributing.rst +++ b/doc/contributing.rst @@ -285,17 +285,23 @@ How to build the *xarray* documentation Requirements ~~~~~~~~~~~~ +Make sure to follow the instructions on :ref:`creating a development environment above `, but +to build the docs you need to use the environment file ``doc/environment.yml``. -First, you need to have a development environment to be able to build xarray -(see the docs on :ref:`creating a development environment above `). +.. code-block:: none -Building the documentation -~~~~~~~~~~~~~~~~~~~~~~~~~~ + # Create and activate the docs environment + conda env create -f doc/environment.yml + conda activate xarray-docs -In your development environment, install ``sphinx``, ``sphinx_rtd_theme``, -``sphinx-gallery`` and ``numpydoc``:: + # or with older versions of Anaconda: + source activate xarray-docs - conda install -c conda-forge sphinx sphinx_rtd_theme sphinx-gallery numpydoc + # Build and install xarray + pip install -e . + +Building the documentation +~~~~~~~~~~~~~~~~~~~~~~~~~~ Navigate to your local ``xarray/doc/`` directory in the console and run:: @@ -451,7 +457,7 @@ typically find tests wrapped in a class. .. code-block:: python - class TestReallyCoolFeature(object): + class TestReallyCoolFeature: .... Going forward, we are moving to a more *functional* style using the diff --git a/doc/data-structures.rst b/doc/data-structures.rst index 5be1f7b4262..3f481f225f7 100644 --- a/doc/data-structures.rst +++ b/doc/data-structures.rst @@ -363,6 +363,7 @@ example, to create this example dataset from scratch, we could have written: ds = xr.Dataset() ds['temperature'] = (('x', 'y', 'time'), temp) + ds['temperature_double'] = (('x', 'y', 'time'), temp * 2 ) ds['precipitation'] = (('x', 'y', 'time'), precip) ds.coords['lat'] = (('x', 'y'), lat) ds.coords['lon'] = (('x', 'y'), lon) @@ -397,9 +398,9 @@ operations keep around coordinates: .. ipython:: python - list(ds[['temperature']]) - list(ds[['x']]) - list(ds.drop('temperature')) + ds[['temperature']] + ds[['temperature', 'temperature_double']] + ds.drop('temperature') To remove a dimension, you can use :py:meth:`~xarray.Dataset.drop_dims` method. Any variables using that dimension are dropped: diff --git a/doc/examples.rst b/doc/examples.rst index ee0cd2b2ed0..4d726d494e8 100644 --- a/doc/examples.rst +++ b/doc/examples.rst @@ -4,7 +4,6 @@ Examples .. toctree:: :maxdepth: 2 - examples/quick-overview examples/weather-data examples/monthly-means examples/multidimensional-coords diff --git a/doc/examples/_code/accessor_example.py b/doc/examples/_code/accessor_example.py index a11ebf9329b..d179d38fba9 100644 --- a/doc/examples/_code/accessor_example.py +++ b/doc/examples/_code/accessor_example.py @@ -2,7 +2,7 @@ @xr.register_dataset_accessor('geo') -class GeoAccessor(object): +class GeoAccessor: def __init__(self, xarray_obj): self._obj = xarray_obj self._center = None diff --git a/doc/examples/quick-overview.rst b/doc/examples/quick-overview.rst deleted file mode 100644 index aa0381444e1..00000000000 --- a/doc/examples/quick-overview.rst +++ /dev/null @@ -1,183 +0,0 @@ -############## -Quick overview -############## - -Here are some quick examples of what you can do with :py:class:`xarray.DataArray` -objects. Everything is explained in much more detail in the rest of the -documentation. - -To begin, import numpy, pandas and xarray using their customary abbreviations: - -.. ipython:: python - - import numpy as np - import pandas as pd - import xarray as xr - -Create a DataArray ------------------- - -You can make a DataArray from scratch by supplying data in the form of a numpy -array or list, with optional *dimensions* and *coordinates*: - -.. ipython:: python - - xr.DataArray(np.random.randn(2, 3)) - data = xr.DataArray(np.random.randn(2, 3), coords={'x': ['a', 'b']}, dims=('x', 'y')) - data - -If you supply a pandas :py:class:`~pandas.Series` or -:py:class:`~pandas.DataFrame`, metadata is copied directly: - -.. ipython:: python - - xr.DataArray(pd.Series(range(3), index=list('abc'), name='foo')) - -Here are the key properties for a ``DataArray``: - -.. ipython:: python - - # like in pandas, values is a numpy array that you can modify in-place - data.values - data.dims - data.coords - # you can use this dictionary to store arbitrary metadata - data.attrs - -Indexing --------- - -xarray supports four kind of indexing. These operations are just as fast as in -pandas, because we borrow pandas' indexing machinery. - -.. ipython:: python - - # positional and by integer label, like numpy - data[[0, 1]] - - # positional and by coordinate label, like pandas - data.loc['a':'b'] - - # by dimension name and integer label - data.isel(x=slice(2)) - - # by dimension name and coordinate label - data.sel(x=['a', 'b']) - -Computation ------------ - -Data arrays work very similarly to numpy ndarrays: - -.. ipython:: python - - data + 10 - np.sin(data) - data.T - data.sum() - -However, aggregation operations can use dimension names instead of axis -numbers: - -.. ipython:: python - - data.mean(dim='x') - -Arithmetic operations broadcast based on dimension name. This means you don't -need to insert dummy dimensions for alignment: - -.. ipython:: python - - a = xr.DataArray(np.random.randn(3), [data.coords['y']]) - b = xr.DataArray(np.random.randn(4), dims='z') - - a - b - - a + b - -It also means that in most cases you do not need to worry about the order of -dimensions: - -.. ipython:: python - - data - data.T - -Operations also align based on index labels: - -.. ipython:: python - - data[:-1] - data[:1] - -GroupBy -------- - -xarray supports grouped operations using a very similar API to pandas: - -.. ipython:: python - - labels = xr.DataArray(['E', 'F', 'E'], [data.coords['y']], name='labels') - labels - data.groupby(labels).mean('y') - data.groupby(labels).apply(lambda x: x - x.min()) - -pandas ------- - -Xarray objects can be easily converted to and from pandas objects: - -.. ipython:: python - - series = data.to_series() - series - - # convert back - series.to_xarray() - -Datasets --------- - -:py:class:`xarray.Dataset` is a dict-like container of aligned ``DataArray`` -objects. You can think of it as a multi-dimensional generalization of the -:py:class:`pandas.DataFrame`: - -.. ipython:: python - - ds = xr.Dataset({'foo': data, 'bar': ('x', [1, 2]), 'baz': np.pi}) - ds - -Use dictionary indexing to pull out ``Dataset`` variables as ``DataArray`` -objects: - -.. ipython:: python - - ds['foo'] - -Variables in datasets can have different ``dtype`` and even different -dimensions, but all dimensions are assumed to refer to points in the same shared -coordinate system. - -You can do almost everything you can do with ``DataArray`` objects with -``Dataset`` objects (including indexing and arithmetic) if you prefer to work -with multiple variables at once. - -NetCDF ------- - -NetCDF is the recommended binary serialization format for xarray objects. Users -from the geosciences will recognize that the :py:class:`~xarray.Dataset` data -model looks very similar to a netCDF file (which, in fact, inspired it). - -You can directly read and write xarray objects to disk using :py:meth:`~xarray.Dataset.to_netcdf`, :py:func:`~xarray.open_dataset` and -:py:func:`~xarray.open_dataarray`: - -.. ipython:: python - - ds.to_netcdf('example.nc') - xr.open_dataset('example.nc') - -.. ipython:: python - :suppress: - - import os - os.remove('example.nc') diff --git a/doc/gallery/plot_rasterio.py b/doc/gallery/plot_rasterio.py index 98801990af3..82d5ce61284 100644 --- a/doc/gallery/plot_rasterio.py +++ b/doc/gallery/plot_rasterio.py @@ -16,9 +16,6 @@ original map projection (see :ref:`recipes.rasterio_rgb`). """ -import os -import urllib.request - import cartopy.crs as ccrs import matplotlib.pyplot as plt import numpy as np @@ -26,12 +23,9 @@ import xarray as xr -# Download the file from rasterio's repository -url = 'https://github.com/mapbox/rasterio/raw/master/tests/data/RGB.byte.tif' -urllib.request.urlretrieve(url, 'RGB.byte.tif') - # Read the data -da = xr.open_rasterio('RGB.byte.tif') +url = 'https://github.com/mapbox/rasterio/raw/master/tests/data/RGB.byte.tif' +da = xr.open_rasterio(url) # Compute the lon/lat coordinates with rasterio.warp.transform ny, nx = len(da['y']), len(da['x']) @@ -54,6 +48,3 @@ cmap='Greys_r', add_colorbar=False) ax.coastlines('10m', color='r') plt.show() - -# Delete the file -os.remove('RGB.byte.tif') diff --git a/doc/gallery/plot_rasterio_rgb.py b/doc/gallery/plot_rasterio_rgb.py index 2733bf149e5..23a56d5a291 100644 --- a/doc/gallery/plot_rasterio_rgb.py +++ b/doc/gallery/plot_rasterio_rgb.py @@ -13,20 +13,14 @@ transformation. """ -import os -import urllib.request - import cartopy.crs as ccrs import matplotlib.pyplot as plt import xarray as xr -# Download the file from rasterio's repository -url = 'https://github.com/mapbox/rasterio/raw/master/tests/data/RGB.byte.tif' -urllib.request.urlretrieve(url, 'RGB.byte.tif') - # Read the data -da = xr.open_rasterio('RGB.byte.tif') +url = 'https://github.com/mapbox/rasterio/raw/master/tests/data/RGB.byte.tif' +da = xr.open_rasterio(url) # The data is in UTM projection. We have to set it manually until # https://github.com/SciTools/cartopy/issues/813 is implemented @@ -37,6 +31,3 @@ da.plot.imshow(ax=ax, rgb='band', transform=crs) ax.coastlines('10m', color='r') plt.show() - -# Delete the file -os.remove('RGB.byte.tif') diff --git a/doc/index.rst b/doc/index.rst index 002bd102e12..4d0105f350a 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -29,6 +29,7 @@ Documentation * :doc:`why-xarray` * :doc:`faq` +* :doc:`quick-overview` * :doc:`examples` * :doc:`installing` @@ -39,6 +40,7 @@ Documentation why-xarray faq + quick-overview examples installing diff --git a/doc/io.rst b/doc/io.rst index 51c747189da..b470284f071 100644 --- a/doc/io.rst +++ b/doc/io.rst @@ -302,16 +302,23 @@ to using encoded character arrays. Character arrays can be selected even for netCDF4 files by setting the ``dtype`` field in ``encoding`` to ``S1`` (corresponding to NumPy's single-character bytes dtype). -If character arrays are used, the string encoding that was used is stored on -disk in the ``_Encoding`` attribute, which matches an ad-hoc convention -`adopted by the netCDF4-Python library `_. -At the time of this writing (October 2017), a standard convention for indicating -string encoding for character arrays in netCDF files was -`still under discussion `_. -Technically, you can use -`any string encoding recognized by Python `_ if you feel the need to deviate from UTF-8, -by setting the ``_Encoding`` field in ``encoding``. But -`we don't recommend it `_. +If character arrays are used: + +- The string encoding that was used is stored on + disk in the ``_Encoding`` attribute, which matches an ad-hoc convention + `adopted by the netCDF4-Python library `_. + At the time of this writing (October 2017), a standard convention for indicating + string encoding for character arrays in netCDF files was + `still under discussion `_. + Technically, you can use + `any string encoding recognized by Python `_ if you feel the need to deviate from UTF-8, + by setting the ``_Encoding`` field in ``encoding``. But + `we don't recommend it `_. +- The character dimension name can be specifed by the ``char_dim_name`` field of a variable's + ``encoding``. If this is not specified the default name for the character dimension is + ``'string%s' % data.shape[-1]``. When decoding character arrays from existing files, the + ``char_dim_name`` is added to the variables ``encoding`` to preserve if encoding happens, but + the field can be edited by the user. .. warning:: diff --git a/doc/quick-overview.rst b/doc/quick-overview.rst new file mode 100644 index 00000000000..1224f59515b --- /dev/null +++ b/doc/quick-overview.rst @@ -0,0 +1,229 @@ +############## +Quick overview +############## + +Here are some quick examples of what you can do with :py:class:`xarray.DataArray` +objects. Everything is explained in much more detail in the rest of the +documentation. + +To begin, import numpy, pandas and xarray using their customary abbreviations: + +.. ipython:: python + + import numpy as np + import pandas as pd + import xarray as xr + +Create a DataArray +------------------ + +You can make a DataArray from scratch by supplying data in the form of a numpy +array or list, with optional *dimensions* and *coordinates*: + +.. ipython:: python + + data = xr.DataArray(np.random.randn(2, 3), + dims=('x', 'y'), + coords={'x': [10, 20]}) + data + +In this case, we have generated a 2D array, assigned the names *x* and *y* to the two dimensions respectively and associated two *coordinate labels* '10' and '20' with the two locations along the x dimension. If you supply a pandas :py:class:`~pandas.Series` or :py:class:`~pandas.DataFrame`, metadata is copied directly: + +.. ipython:: python + + xr.DataArray(pd.Series(range(3), index=list('abc'), name='foo')) + +Here are the key properties for a ``DataArray``: + +.. ipython:: python + + # like in pandas, values is a numpy array that you can modify in-place + data.values + data.dims + data.coords + # you can use this dictionary to store arbitrary metadata + data.attrs + + +Indexing +-------- + +xarray supports four kind of indexing. Since we have assigned coordinate labels to the x dimension we can use label-based indexing along that dimension just like pandas. The four examples below all yield the same result but at varying levels of convenience and intuitiveness. + +.. ipython:: python + + # positional and by integer label, like numpy + data[[0, 1]] + + # positional and by coordinate label, like pandas + data.loc[10:20] + + # by dimension name and integer label + data.isel(x=slice(2)) + + # by dimension name and coordinate label + data.sel(x=[10, 20]) + + +Unlike positional indexing, label-based indexing frees us from having to know how our array is organized. All we need to know are the dimension name and the label we wish to index i.e. ``data.sel(x=10)`` works regardless of whether ``x`` is the first or second dimension of the array and regardless of whether ``10`` is the first or second element of ``x``. We have already told xarray that x is the first dimension when we created ``data``: xarray keeps track of this so we don't have to. For more, see :ref:`indexing`. + + +Attributes +---------- + +While you're setting up your DataArray, it's often a good idea to set metadata attributes. A useful choice is to set ``data.attrs['long_name']`` and ``data.attrs['units']`` since xarray will use these, if present, to automatically label your plots. These special names were chosen following the `NetCDF Climate and Forecast (CF) Metadata Conventions `_. ``attrs`` is just a Python dictionary, so you can assign anything you wish. + +.. ipython:: python + + data.attrs['long_name'] = 'random velocity' + data.attrs['units'] = 'metres/sec' + data.attrs['description'] = 'A random variable created as an example.' + data.attrs['random_attribute'] = 123 + data.attrs + # you can add metadata to coordinates too + data.x.attrs['units'] = 'x units' + + +Computation +----------- + +Data arrays work very similarly to numpy ndarrays: + +.. ipython:: python + + data + 10 + np.sin(data) + # transpose + data.T + data.sum() + +However, aggregation operations can use dimension names instead of axis +numbers: + +.. ipython:: python + + data.mean(dim='x') + +Arithmetic operations broadcast based on dimension name. This means you don't +need to insert dummy dimensions for alignment: + +.. ipython:: python + + a = xr.DataArray(np.random.randn(3), [data.coords['y']]) + b = xr.DataArray(np.random.randn(4), dims='z') + + a + b + + a + b + +It also means that in most cases you do not need to worry about the order of +dimensions: + +.. ipython:: python + + data - data.T + +Operations also align based on index labels: + +.. ipython:: python + + data[:-1] - data[:1] + +For more, see :ref:`comput`. + +GroupBy +------- + +xarray supports grouped operations using a very similar API to pandas (see :ref:`groupby`): + +.. ipython:: python + + labels = xr.DataArray(['E', 'F', 'E'], [data.coords['y']], name='labels') + labels + data.groupby(labels).mean('y') + data.groupby(labels).apply(lambda x: x - x.min()) + +Plotting +-------- + +Visualizing your datasets is quick and convenient: + +.. ipython:: python + + @savefig plotting_quick_overview.png + data.plot() + +Note the automatic labeling with names and units. Our effort in adding metadata attributes has paid off! Many aspects of these figures are customizable: see :ref:`plotting`. + +pandas +------ + +Xarray objects can be easily converted to and from pandas objects using the :py:meth:`~xarray.DataArray.to_series`, :py:meth:`~xarray.DataArray.to_dataframe` and :py:meth:`~pandas.DataFrame.to_xarray` methods: + +.. ipython:: python + + series = data.to_series() + series + + # convert back + series.to_xarray() + +Datasets +-------- + +:py:class:`xarray.Dataset` is a dict-like container of aligned ``DataArray`` +objects. You can think of it as a multi-dimensional generalization of the +:py:class:`pandas.DataFrame`: + +.. ipython:: python + + ds = xr.Dataset({'foo': data, 'bar': ('x', [1, 2]), 'baz': np.pi}) + ds + + +This creates a dataset with three DataArrays named ``foo``, ``bar`` and ``baz``. Use dictionary or dot indexing to pull out ``Dataset`` variables as ``DataArray`` objects but note that assignment only works with dictionary indexing: + +.. ipython:: python + + ds['foo'] + ds.foo + + +When creating ``ds``, we specified that ``foo`` is identical to ``data`` created earlier, ``bar`` is one-dimensional with single dimension ``x`` and associated values '1' and '2', and ``baz`` is a scalar not associated with any dimension in ``ds``. Variables in datasets can have different ``dtype`` and even different dimensions, but all dimensions are assumed to refer to points in the same shared coordinate system i.e. if two variables have dimension ``x``, that dimension must be identical in both variables. + +For example, when creating ``ds`` xarray automatically *aligns* ``bar`` with ``DataArray`` ``foo``, i.e., they share the same coordinate system so that ``ds.bar['x'] == ds.foo['x'] == ds['x']``. Consequently, the following works without explicitly specifying the coordinate ``x`` when creating ``ds['bar']``: + +.. ipython:: python + + ds.bar.sel(x=10) + + + +You can do almost everything you can do with ``DataArray`` objects with +``Dataset`` objects (including indexing and arithmetic) if you prefer to work +with multiple variables at once. + +Read & write netCDF files +------------------------- + +NetCDF is the recommended file format for xarray objects. Users +from the geosciences will recognize that the :py:class:`~xarray.Dataset` data +model looks very similar to a netCDF file (which, in fact, inspired it). + +You can directly read and write xarray objects to disk using :py:meth:`~xarray.Dataset.to_netcdf`, :py:func:`~xarray.open_dataset` and +:py:func:`~xarray.open_dataarray`: + +.. ipython:: python + + ds.to_netcdf('example.nc') + xr.open_dataset('example.nc') + +.. ipython:: python + :suppress: + + import os + os.remove('example.nc') + + +It is common for datasets to be distributed across multiple files (commonly one file per timestep). xarray supports this use-case by providing the :py:meth:`~xarray.open_mfdataset` and the :py:meth:`~xarray.save_mfdataset` methods. For more, see :ref:`io`. diff --git a/doc/related-projects.rst b/doc/related-projects.rst index e899022e5d4..3e6a745daa0 100644 --- a/doc/related-projects.rst +++ b/doc/related-projects.rst @@ -33,6 +33,7 @@ Geosciences - `xarray-simlab `_: xarray extension for computer model simulations. - `xarray-topo `_: xarray extension for topographic analysis and modelling. - `xbpch `_: xarray interface for bpch files. +- `xclim `_: A library for calculating climate science indices with unit handling built from xarray and dask. - `xESMF `_: Universal Regridder for Geospatial Data. - `xgcm `_: Extends the xarray data model to understand finite volume grid cells (common in General Circulation Models) and provides interpolation and difference operations for such grids. - `xmitgcm `_: a python package for reading `MITgcm `_ binary MDS files into xarray data structures. diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 6cf2720a033..d904a3814f1 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -13,21 +13,64 @@ What's New import xarray as xr np.random.seed(123456) -.. _whats-new.0.12.1: +.. _whats-new.0.12.2: -v0.12.1 (unreleased) +v0.12.2 (unreleased) -------------------- Enhancements ~~~~~~~~~~~~ +- Add ``fill_value`` argument for reindex, align, and merge operations + to enable custom fill values. (:issue:`2876`) + By `Zach Griffith `_. +- Character arrays' character dimension name decoding and encoding handled by + ``var.encoding['char_dim_name']`` (:issue:`2895`) + By `James McCreight `_. +- Clean up Python 2 compatibility in code (:issue:`2950`) + By `Guido Imperiale `_. +- Implement ``load_dataset()`` and ``load_dataarray()`` as alternatives to + ``open_dataset()`` and ``open_dataarray()`` to open, load into memory, + and close files, returning the Dataset or DataArray. These functions are + helpful for avoiding file-lock errors when trying to write to files opened + using ``open_dataset()`` or ``open_dataarray()``. (:issue:`2887`) + By `Dan Nowacki `_. Bug fixes ~~~~~~~~~ -- ``swap_dims`` would create incorrect ``indexes`` (:issue:`2842`). - By `Stephan Hoyer `_. +- indexing with an empty list creates an object with zero-length axis (:issue:`2882`) + By `Mayeul d'Avezac `_. +- Return correct count for scalar datetime64 arrays (:issue:`2770`) + By `Dan Nowacki `_. +- Fix facetgrid colormap bug when ``extend=True``. (:issue:`2932`) + By `Deepak Cherian `_. + +.. _whats-new.0.12.1: + +v0.12.1 (4 April 2019) +---------------------- +Enhancements +~~~~~~~~~~~~ + +- Allow ``expand_dims`` method to support inserting/broadcasting dimensions + with size > 1. (:issue:`2710`) + By `Martin Pletcher `_. + +Bug fixes +~~~~~~~~~ + +- Dataset.copy(deep=True) now creates a deep copy of the attrs (:issue:`2835`). + By `Andras Gefferth `_. +- Fix incorrect ``indexes`` resulting from various ``Dataset`` operations + (e.g., ``swap_dims``, ``isel``, ``reindex``, ``[]``) (:issue:`2842`, + :issue:`2856`). + By `Stephan Hoyer `_. +- open_rasterio() now supports rasterio.vrt.WarpedVRT with custom transform, width and height (:issue:`2864`). + By `Julien Michel `_. .. _whats-new.0.12.0: @@ -116,6 +159,11 @@ Other enhancements By `Keisuke Fujii `_. - Added :py:meth:`~xarray.Dataset.drop_dims` (:issue:`1949`). By `Kevin Squire `_. +- ``xr.open_zarr`` now accepts manually specified chunks with the ``chunks=`` + parameter. ``auto_chunk=True`` is equivalent to ``chunks='auto'`` for + backwards compatibility. The ``overwrite_encoded_chunks`` parameter is + added to remove the original zarr chunk encoding. + By `Lily Wang `_. Bug fixes ~~~~~~~~~ diff --git a/xarray/__init__.py b/xarray/__init__.py index 773dfe19d01..506cb46de26 100644 --- a/xarray/__init__.py +++ b/xarray/__init__.py @@ -17,7 +17,7 @@ from .core.options import set_options from .backends.api import (open_dataset, open_dataarray, open_mfdataset, - save_mfdataset) + save_mfdataset, load_dataset, load_dataarray) from .backends.rasterio_ import open_rasterio from .backends.zarr import open_zarr diff --git a/xarray/backends/api.py b/xarray/backends/api.py index afb69f6e9e9..01188e92752 100644 --- a/xarray/backends/api.py +++ b/xarray/backends/api.py @@ -185,12 +185,64 @@ def _finalize_store(write, store): store.close() +def load_dataset(filename_or_obj, **kwargs): + """Open, load into memory, and close a Dataset from a file or file-like + object. + + This is a thin wrapper around :py:meth:`~xarray.open_dataset`. It differs + from `open_dataset` in that it loads the Dataset into memory, closes the + file, and returns the Dataset. In contrast, `open_dataset` keeps the file + handle open and lazy loads its contents. All parameters are passed directly + to `open_dataset`. See that documentation for further details. + + Returns + ------- + dataset : Dataset + The newly created Dataset. + + See Also + -------- + open_dataset + """ + if 'cache' in kwargs: + raise TypeError('cache has no effect in this context') + + with open_dataset(filename_or_obj, **kwargs) as ds: + return ds.load() + + +def load_dataarray(filename_or_obj, **kwargs): + """Open, load into memory, and close a DataArray from a file or file-like + object containing a single data variable. + + This is a thin wrapper around :py:meth:`~xarray.open_dataarray`. It differs + from `open_dataarray` in that it loads the Dataset into memory, closes the + file, and returns the Dataset. In contrast, `open_dataarray` keeps the file + handle open and lazy loads its contents. All parameters are passed directly + to `open_dataarray`. See that documentation for further details. + + Returns + ------- + datarray : DataArray + The newly created DataArray. + + See Also + -------- + open_dataarray + """ + if 'cache' in kwargs: + raise TypeError('cache has no effect in this context') + + with open_dataarray(filename_or_obj, **kwargs) as da: + return da.load() + + def open_dataset(filename_or_obj, group=None, decode_cf=True, mask_and_scale=None, decode_times=True, autoclose=None, concat_characters=True, decode_coords=True, engine=None, chunks=None, lock=None, cache=None, drop_variables=None, backend_kwargs=None, use_cftime=None): - """Load and decode a dataset from a file or file-like object. + """Open and decode a dataset from a file or file-like object. Parameters ---------- @@ -406,7 +458,8 @@ def open_dataarray(filename_or_obj, group=None, decode_cf=True, concat_characters=True, decode_coords=True, engine=None, chunks=None, lock=None, cache=None, drop_variables=None, backend_kwargs=None, use_cftime=None): - """Open an DataArray from a netCDF file containing a single data variable. + """Open an DataArray from a file or file-like object containing a single + data variable. This is designed to read netCDF files with only one data variable. If multiple variables are present then a ValueError is raised. @@ -529,7 +582,7 @@ def open_dataarray(filename_or_obj, group=None, decode_cf=True, return data_array -class _MultiFileCloser(object): +class _MultiFileCloser: def __init__(self, file_objs): self.file_objs = file_objs diff --git a/xarray/backends/common.py b/xarray/backends/common.py index a52daaaa65c..5819292bb8a 100644 --- a/xarray/backends/common.py +++ b/xarray/backends/common.py @@ -154,7 +154,7 @@ def __exit__(self, exception_type, exception_value, traceback): self.close() -class ArrayWriter(object): +class ArrayWriter: def __init__(self, lock=None): self.sources = [] self.targets = [] diff --git a/xarray/backends/file_manager.py b/xarray/backends/file_manager.py index d0efba86bf9..5955ef54d6e 100644 --- a/xarray/backends/file_manager.py +++ b/xarray/backends/file_manager.py @@ -19,7 +19,7 @@ _DEFAULT_MODE = utils.ReprObject('') -class FileManager(object): +class FileManager: """Manager for acquiring and closing a file object. Use FileManager subclasses (CachingFileManager in particular) on backend @@ -237,7 +237,7 @@ def __repr__(self): type(self).__name__, self._opener, args_string, self._kwargs) -class _RefCounter(object): +class _RefCounter: """Class for keeping track of reference counts.""" def __init__(self, counts): self._counts = counts diff --git a/xarray/backends/locks.py b/xarray/backends/locks.py index bca27a0bbc1..65150562538 100644 --- a/xarray/backends/locks.py +++ b/xarray/backends/locks.py @@ -135,7 +135,7 @@ def acquire(lock, blocking=True): return lock.acquire(blocking) -class CombinedLock(object): +class CombinedLock: """A combination of multiple locks. Like a locked door, a CombinedLock is locked if any of its constituent @@ -167,7 +167,7 @@ def __repr__(self): return "CombinedLock(%r)" % list(self.locks) -class DummyLock(object): +class DummyLock: """DummyLock provides the lock API without any actual locking.""" def acquire(self, blocking=True): diff --git a/xarray/backends/netCDF4_.py b/xarray/backends/netCDF4_.py index b26d5575d23..b3bab9617ee 100644 --- a/xarray/backends/netCDF4_.py +++ b/xarray/backends/netCDF4_.py @@ -228,7 +228,7 @@ def _extract_nc4_variable_encoding(variable, raise_on_invalid=False, return encoding -class GroupWrapper(object): +class GroupWrapper: """Wrap netCDF4.Group objects so closing them closes the root group.""" def __init__(self, value): self.value = value diff --git a/xarray/backends/rasterio_.py b/xarray/backends/rasterio_.py index b629eb51241..2f00702f854 100644 --- a/xarray/backends/rasterio_.py +++ b/xarray/backends/rasterio_.py @@ -224,6 +224,9 @@ def open_rasterio(filename, parse_coordinates=None, chunks=None, cache=None, src_nodata=vrt.src_nodata, dst_nodata=vrt.dst_nodata, tolerance=vrt.tolerance, + transform=vrt.transform, + width=vrt.width, + height=vrt.height, warp_extras=vrt.warp_extras) if lock is None: diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index ee77e0833c4..f5364314af8 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -1,3 +1,4 @@ +import warnings from collections import OrderedDict from distutils.version import LooseVersion @@ -352,10 +353,11 @@ def close(self): zarr.consolidate_metadata(self.ds.store) -def open_zarr(store, group=None, synchronizer=None, auto_chunk=True, +def open_zarr(store, group=None, synchronizer=None, chunks='auto', decode_cf=True, mask_and_scale=True, decode_times=True, concat_characters=True, decode_coords=True, - drop_variables=None, consolidated=False): + drop_variables=None, consolidated=False, + overwrite_encoded_chunks=False, **kwargs): """Load and decode a dataset from a Zarr store. .. note:: Experimental @@ -375,10 +377,15 @@ def open_zarr(store, group=None, synchronizer=None, auto_chunk=True, Array synchronizer provided to zarr group : str, obtional Group path. (a.k.a. `path` in zarr terminology.) - auto_chunk : bool, optional - Whether to automatically create dask chunks corresponding to each - variable's zarr chunks. If False, zarr array data will lazily convert - to numpy arrays upon access. + chunks : int or dict or tuple or {None, 'auto'}, optional + Chunk sizes along each dimension, e.g., ``5`` or + ``{'x': 5, 'y': 5}``. If `chunks='auto'`, dask chunks are created + based on the variable's zarr chunks. If `chunks=None`, zarr array + data will lazily convert to numpy arrays upon access. This accepts + all the chunk specifications as Dask does. + overwrite_encoded_chunks: bool, optional + Whether to drop the zarr chunks encoded for each variable when a + dataset is loaded with specified chunk sizes (default: False) decode_cf : bool, optional Whether to decode these variables, assuming they were saved according to CF conventions. @@ -422,6 +429,24 @@ def open_zarr(store, group=None, synchronizer=None, auto_chunk=True, ---------- http://zarr.readthedocs.io/ """ + if 'auto_chunk' in kwargs: + auto_chunk = kwargs.pop('auto_chunk') + if auto_chunk: + chunks = 'auto' # maintain backwards compatibility + else: + chunks = None + + warnings.warn("auto_chunk is deprecated. Use chunks='auto' instead.", + FutureWarning, stacklevel=2) + + if kwargs: + raise TypeError("open_zarr() got unexpected keyword arguments " + + ",".join(kwargs.keys())) + + if not isinstance(chunks, (int, dict)): + if chunks != 'auto' and chunks is not None: + raise ValueError("chunks must be an int, dict, 'auto', or None. " + "Instead found %s. " % chunks) if not decode_cf: mask_and_scale = False @@ -449,21 +474,60 @@ def maybe_decode_store(store, lock=False): # auto chunking needs to be here and not in ZarrStore because variable # chunks do not survive decode_cf - if auto_chunk: - # adapted from Dataset.Chunk() - def maybe_chunk(name, var): - from dask.base import tokenize - chunks = var.encoding.get('chunks') - if (var.ndim > 0) and (chunks is not None): - # does this cause any data to be read? - token2 = tokenize(name, var._data) - name2 = 'zarr-%s' % token2 - return var.chunk(chunks, name=name2, lock=None) - else: - return var - - variables = OrderedDict([(k, maybe_chunk(k, v)) - for k, v in ds.variables.items()]) - return ds._replace_vars_and_dims(variables) - else: + # return trivial case + if not chunks: return ds + + # adapted from Dataset.Chunk() + if isinstance(chunks, int): + chunks = dict.fromkeys(ds.dims, chunks) + + if isinstance(chunks, tuple) and len(chunks) == len(ds.dims): + chunks = dict(zip(ds.dims, chunks)) + + def get_chunk(name, var, chunks): + chunk_spec = dict(zip(var.dims, var.encoding.get('chunks'))) + + # Coordinate labels aren't chunked + if var.ndim == 1 and var.dims[0] == name: + return chunk_spec + + if chunks == 'auto': + return chunk_spec + + for dim in var.dims: + if dim in chunks: + spec = chunks[dim] + if isinstance(spec, int): + spec = (spec,) + if isinstance(spec, (tuple, list)) and chunk_spec[dim]: + if any(s % chunk_spec[dim] for s in spec): + warnings.warn("Specified Dask chunks %r would " + "separate Zarr chunk shape %r for " + "dimension %r. This significantly " + "degrades performance. Consider " + "rechunking after loading instead." + % (chunks[dim], chunk_spec[dim], dim), + stacklevel=2) + chunk_spec[dim] = chunks[dim] + return chunk_spec + + def maybe_chunk(name, var, chunks): + from dask.base import tokenize + + chunk_spec = get_chunk(name, var, chunks) + + if (var.ndim > 0) and (chunk_spec is not None): + # does this cause any data to be read? + token2 = tokenize(name, var._data) + name2 = 'zarr-%s' % token2 + var = var.chunk(chunk_spec, name=name2, lock=None) + if overwrite_encoded_chunks and var.chunks is not None: + var.encoding['chunks'] = tuple(x[0] for x in var.chunks) + return var + else: + return var + + variables = OrderedDict([(k, maybe_chunk(k, v, chunks)) + for k, v in ds.variables.items()]) + return ds._replace_vars_and_dims(variables) diff --git a/xarray/coding/cftime_offsets.py b/xarray/coding/cftime_offsets.py index d724554b458..e89118e8be4 100644 --- a/xarray/coding/cftime_offsets.py +++ b/xarray/coding/cftime_offsets.py @@ -77,9 +77,9 @@ def get_date_type(calendar): return calendars[calendar] -class BaseCFTimeOffset(object): +class BaseCFTimeOffset: _freq = None # type: ClassVar[str] - _day_option = None + _day_option = None # type: ClassVar[str] def __init__(self, n=1): if not isinstance(n, int): diff --git a/xarray/coding/strings.py b/xarray/coding/strings.py index 205d285cd81..007bcb8a502 100644 --- a/xarray/coding/strings.py +++ b/xarray/coding/strings.py @@ -103,16 +103,20 @@ def encode(self, variable, name=None): dims, data, attrs, encoding = unpack_for_encoding(variable) if data.dtype.kind == 'S' and encoding.get('dtype') is not str: data = bytes_to_char(data) - dims = dims + ('string%s' % data.shape[-1],) + if 'char_dim_name' in encoding.keys(): + char_dim_name = encoding.pop('char_dim_name') + else: + char_dim_name = 'string%s' % data.shape[-1] + dims = dims + (char_dim_name,) return Variable(dims, data, attrs, encoding) def decode(self, variable, name=None): dims, data, attrs, encoding = unpack_for_decoding(variable) if data.dtype == 'S1' and dims: + encoding['char_dim_name'] = dims[-1] dims = dims[:-1] data = char_to_bytes(data) - return Variable(dims, data, attrs, encoding) diff --git a/xarray/coding/variables.py b/xarray/coding/variables.py index 1f74181f3b3..ae8b97c7352 100644 --- a/xarray/coding/variables.py +++ b/xarray/coding/variables.py @@ -15,7 +15,7 @@ class SerializationWarning(RuntimeWarning): """Warnings about encoding/decoding issues in serialization.""" -class VariableCoder(object): +class VariableCoder: """Base class for encoding and decoding transformations on variables. We use coders for transforming variables between xarray's data model and diff --git a/xarray/core/accessors.py b/xarray/core/accessors.py index 10c900c4ad1..640060fafe5 100644 --- a/xarray/core/accessors.py +++ b/xarray/core/accessors.py @@ -110,7 +110,7 @@ def _round_field(values, name, freq): return _round_series(values, name, freq) -class DatetimeAccessor(object): +class DatetimeAccessor: """Access datetime fields for DataArrays with datetime-like dtypes. Similar to pandas, fields can be accessed through the `.dt` attribute diff --git a/xarray/core/alignment.py b/xarray/core/alignment.py index 71cdfdebb61..295f69a2afc 100644 --- a/xarray/core/alignment.py +++ b/xarray/core/alignment.py @@ -8,7 +8,7 @@ import numpy as np import pandas as pd -from . import utils +from . import utils, dtypes from .indexing import get_indexer_nd from .utils import is_dict_like, is_full_slice from .variable import IndexVariable, Variable @@ -31,20 +31,17 @@ def _get_joiner(join): raise ValueError('invalid value for join: %s' % join) -_DEFAULT_EXCLUDE = frozenset() # type: frozenset - - -def align(*objects, **kwargs): - """align(*objects, join='inner', copy=True, indexes=None, - exclude=frozenset()) - +def align(*objects, join='inner', copy=True, indexes=None, exclude=frozenset(), + fill_value=dtypes.NA): + """ Given any number of Dataset and/or DataArray objects, returns new objects with aligned indexes and dimension sizes. Array from the aligned objects are suitable as input to mathematical operators, because along each dimension they have the same index and size. - Missing values (if ``join != 'inner'``) are filled with NaN. + Missing values (if ``join != 'inner'``) are filled with ``fill_value``. + The default fill value is NaN. Parameters ---------- @@ -65,11 +62,13 @@ def align(*objects, **kwargs): ``copy=False`` and reindexing is unnecessary, or can be performed with only slice operations, then the output may share memory with the input. In either case, new xarray objects are always returned. - exclude : sequence of str, optional - Dimensions that must be excluded from alignment indexes : dict-like, optional Any indexes explicitly provided with the `indexes` argument should be used in preference to the aligned indexes. + exclude : sequence of str, optional + Dimensions that must be excluded from alignment + fill_value : scalar, optional + Value to use for newly missing values Returns ------- @@ -82,15 +81,8 @@ def align(*objects, **kwargs): If any dimensions without labels on the arguments have different sizes, or a different size than the size of the aligned dimension labels. """ - join = kwargs.pop('join', 'inner') - copy = kwargs.pop('copy', True) - indexes = kwargs.pop('indexes', None) - exclude = kwargs.pop('exclude', _DEFAULT_EXCLUDE) if indexes is None: indexes = {} - if kwargs: - raise TypeError('align() got unexpected keyword arguments: %s' - % list(kwargs)) if not indexes and len(objects) == 1: # fast path for the trivial case @@ -162,7 +154,8 @@ def align(*objects, **kwargs): # fast path for no reindexing necessary new_obj = obj.copy(deep=copy) else: - new_obj = obj.reindex(copy=copy, **valid_indexers) + new_obj = obj.reindex(copy=copy, fill_value=fill_value, + **valid_indexers) new_obj.encoding = obj.encoding result.append(new_obj) @@ -170,7 +163,8 @@ def align(*objects, **kwargs): def deep_align(objects, join='inner', copy=True, indexes=None, - exclude=frozenset(), raise_on_invalid=True): + exclude=frozenset(), raise_on_invalid=True, + fill_value=dtypes.NA): """Align objects for merging, recursing into dictionary values. This function is not public API. @@ -214,7 +208,7 @@ def is_alignable(obj): out.append(variables) aligned = align(*targets, join=join, copy=copy, indexes=indexes, - exclude=exclude) + exclude=exclude, fill_value=fill_value) for position, key, aligned_obj in zip(positions, keys, aligned): if key is no_key: @@ -270,6 +264,7 @@ def reindex_variables( method: Optional[str] = None, tolerance: Any = None, copy: bool = True, + fill_value: Optional[Any] = dtypes.NA, ) -> 'Tuple[OrderedDict[Any, Variable], OrderedDict[Any, pd.Index]]': """Conform a dictionary of aligned variables onto a new set of variables, filling in missing values with NaN. @@ -298,13 +293,15 @@ def reindex_variables( * nearest: use nearest valid index value tolerance : optional Maximum distance between original and new labels for inexact matches. - The values of the index at the matching locations most satisfy the + The values of the index at the matching locations must satisfy the equation ``abs(index[indexer] - target) <= tolerance``. copy : bool, optional If ``copy=True``, data in the return values is always copied. If ``copy=False`` and reindexing is unnecessary, or can be performed with only slice operations, then the output may share memory with the input. In either case, new xarray objects are always returned. + fill_value : scalar, optional + Value to use for newly missing values Returns ------- @@ -315,36 +312,51 @@ def reindex_variables( """ from .dataarray import DataArray + # create variables for the new dataset + reindexed = OrderedDict() # type: OrderedDict[Any, Variable] + # build up indexers for assignment along each dimension int_indexers = {} - targets = OrderedDict() # type: OrderedDict[Any, pd.Index] + new_indexes = OrderedDict(indexes) masked_dims = set() unchanged_dims = set() - # size of reindexed dimensions - new_sizes = {} + for dim, indexer in indexers.items(): + if isinstance(indexer, DataArray) and indexer.dims != (dim,): + warnings.warn( + "Indexer has dimensions {0:s} that are different " + "from that to be indexed along {1:s}. " + "This will behave differently in the future.".format( + str(indexer.dims), dim), + FutureWarning, stacklevel=3) + + target = new_indexes[dim] = utils.safe_cast_to_index(indexers[dim]) + + if dim in indexes: + index = indexes[dim] - for name, index in indexes.items(): - if name in indexers: if not index.is_unique: raise ValueError( 'cannot reindex or align along dimension %r because the ' - 'index has duplicate values' % name) - - target = utils.safe_cast_to_index(indexers[name]) - new_sizes[name] = len(target) + 'index has duplicate values' % dim) int_indexer = get_indexer_nd(index, target, method, tolerance) # We uses negative values from get_indexer_nd to signify # values that are missing in the index. if (int_indexer < 0).any(): - masked_dims.add(name) + masked_dims.add(dim) elif np.array_equal(int_indexer, np.arange(len(index))): - unchanged_dims.add(name) + unchanged_dims.add(dim) - int_indexers[name] = int_indexer - targets[name] = target + int_indexers[dim] = int_indexer + + if dim in variables: + var = variables[dim] + args = (var.attrs, var.encoding) # type: tuple + else: + args = () + reindexed[dim] = IndexVariable((dim,), target, *args) for dim in sizes: if dim not in indexes and dim in indexers: @@ -356,25 +368,6 @@ def reindex_variables( 'index because its size %r is different from the size of ' 'the new index %r' % (dim, existing_size, new_size)) - # create variables for the new dataset - reindexed = OrderedDict() # type: OrderedDict[Any, Variable] - - for dim, indexer in indexers.items(): - if isinstance(indexer, DataArray) and indexer.dims != (dim,): - warnings.warn( - "Indexer has dimensions {0:s} that are different " - "from that to be indexed along {1:s}. " - "This will behave differently in the future.".format( - str(indexer.dims), dim), - FutureWarning, stacklevel=3) - - if dim in variables: - var = variables[dim] - args = (var.attrs, var.encoding) # type: tuple - else: - args = () - reindexed[dim] = IndexVariable((dim,), indexers[dim], *args) - for name, var in variables.items(): if name not in indexers: key = tuple(slice(None) @@ -384,7 +377,7 @@ def reindex_variables( needs_masking = any(d in masked_dims for d in var.dims) if needs_masking: - new_var = var._getitem_with_mask(key) + new_var = var._getitem_with_mask(key, fill_value=fill_value) elif all(is_full_slice(k) for k in key): # no reindexing necessary # here we need to manually deal with copying data, since @@ -395,9 +388,6 @@ def reindex_variables( reindexed[name] = new_var - new_indexes = OrderedDict(indexes) - new_indexes.update(targets) - return reindexed, new_indexes diff --git a/xarray/core/arithmetic.py b/xarray/core/arithmetic.py index 39901f0befd..9da4c84697e 100644 --- a/xarray/core/arithmetic.py +++ b/xarray/core/arithmetic.py @@ -8,7 +8,7 @@ from .utils import not_implemented -class SupportsArithmetic(object): +class SupportsArithmetic: """Base class for xarray types that support arithmetic. Used by Dataset, DataArray, Variable and GroupBy. diff --git a/xarray/core/common.py b/xarray/core/common.py index 6ec07156160..b518e8431fd 100644 --- a/xarray/core/common.py +++ b/xarray/core/common.py @@ -1,6 +1,8 @@ from collections import OrderedDict from contextlib import suppress from textwrap import dedent +from typing import (Any, Callable, Hashable, Iterable, Iterator, List, Mapping, + MutableMapping, Optional, Tuple, TypeVar, Union) import numpy as np import pandas as pd @@ -11,13 +13,18 @@ from .pycompat import dask_array_type from .utils import Frozen, ReprObject, SortedKeysDict, either_dict_or_kwargs + # Used as a sentinel value to indicate a all dimensions ALL_DIMS = ReprObject('') -class ImplementsArrayReduce(object): +T = TypeVar('T') + + +class ImplementsArrayReduce: @classmethod - def _reduce_method(cls, func, include_skipna, numeric_only): + def _reduce_method(cls, func: Callable, include_skipna: bool, + numeric_only: bool): if include_skipna: def wrapped_func(self, dim=None, axis=None, skipna=None, **kwargs): @@ -46,9 +53,10 @@ def wrapped_func(self, dim=None, axis=None, # type: ignore and 'axis' arguments can be supplied.""") -class ImplementsDatasetReduce(object): +class ImplementsDatasetReduce: @classmethod - def _reduce_method(cls, func, include_skipna, numeric_only): + def _reduce_method(cls, func: Callable, include_skipna: bool, + numeric_only: bool): if include_skipna: def wrapped_func(self, dim=None, skipna=None, **kwargs): @@ -76,46 +84,38 @@ def wrapped_func(self, dim=None, **kwargs): # type: ignore class AbstractArray(ImplementsArrayReduce): - """Shared base class for DataArray and Variable.""" - - def __bool__(self): + """Shared base class for DataArray and Variable. + """ + def __bool__(self: Any) -> bool: return bool(self.values) - # Python 3 uses __bool__, Python 2 uses __nonzero__ - __nonzero__ = __bool__ - - def __float__(self): + def __float__(self: Any) -> float: return float(self.values) - def __int__(self): + def __int__(self: Any) -> int: return int(self.values) - def __complex__(self): + def __complex__(self: Any) -> complex: return complex(self.values) - def __long__(self): - return long(self.values) # noqa - - def __array__(self, dtype=None): + def __array__(self: Any, dtype: Union[str, np.dtype, None] = None + ) -> np.ndarray: return np.asarray(self.values, dtype=dtype) - def __repr__(self): + def __repr__(self) -> str: return formatting.array_repr(self) - def _iter(self): + def _iter(self: Any) -> Iterator[Any]: for n in range(len(self)): yield self[n] - def __iter__(self): + def __iter__(self: Any) -> Iterator[Any]: if self.ndim == 0: raise TypeError('iteration over a 0-d array') return self._iter() - @property - def T(self): - return self.transpose() - - def get_axis_num(self, dim): + def get_axis_num(self, dim: Union[Hashable, Iterable[Hashable]] + ) -> Union[int, Tuple[int, ...]]: """Return axis number(s) corresponding to dimension(s) in this array. Parameters @@ -128,12 +128,12 @@ def get_axis_num(self, dim): int or tuple of int Axis number or numbers corresponding to the given dimensions. """ - if isinstance(dim, str): - return self._get_axis_num(dim) - else: + if isinstance(dim, Iterable) and not isinstance(dim, str): return tuple(self._get_axis_num(d) for d in dim) + else: + return self._get_axis_num(dim) - def _get_axis_num(self, dim): + def _get_axis_num(self: Any, dim: Hashable) -> int: try: return self.dims.index(dim) except ValueError: @@ -141,7 +141,7 @@ def _get_axis_num(self, dim): (dim, self.dims)) @property - def sizes(self): + def sizes(self: Any) -> Mapping[Hashable, int]: """Ordered mapping from dimension names to lengths. Immutable. @@ -153,7 +153,7 @@ def sizes(self): return Frozen(OrderedDict(zip(self.dims, self.shape))) -class AttrAccessMixin(object): +class AttrAccessMixin: """Mixin class that allows getting keys with attribute access """ _initialized = False @@ -168,7 +168,7 @@ def _item_sources(self): """List of places to look-up items for key-autocompletion """ return [] - def __getattr__(self, name): + def __getattr__(self, name: str) -> Any: if name != '__setstate__': # this avoids an infinite loop when pickle looks for the # __setstate__ attribute before the xarray object is initialized @@ -178,7 +178,7 @@ def __getattr__(self, name): raise AttributeError("%r object has no attribute %r" % (type(self).__name__, name)) - def __setattr__(self, name, value): + def __setattr__(self, name: str, value: Any) -> None: if self._initialized: try: # Allow setting instance variables if they already exist @@ -192,7 +192,7 @@ def __setattr__(self, name, value): "assign variables." % (name, type(self).__name__)) object.__setattr__(self, name, value) - def __dir__(self): + def __dir__(self) -> List[str]: """Provide method name lookup and completion. Only provide 'public' methods. """ @@ -202,7 +202,7 @@ def __dir__(self): if isinstance(item, str)] return sorted(set(dir(type(self)) + extra_attrs)) - def _ipython_key_completions_(self): + def _ipython_key_completions_(self) -> List[str]: """Provide method for the key-autocompletions in IPython. See http://ipython.readthedocs.io/en/stable/config/integrating.html#tab-completion For the details. @@ -214,49 +214,57 @@ def _ipython_key_completions_(self): return list(set(item_lists)) -def get_squeeze_dims(xarray_obj, dim, axis=None): +def get_squeeze_dims(xarray_obj, + dim: Union[Hashable, Iterable[Hashable], None] = None, + axis: Union[int, Iterable[int], None] = None + ) -> List[Hashable]: """Get a list of dimensions to squeeze out. """ if dim is not None and axis is not None: raise ValueError('cannot use both parameters `axis` and `dim`') - if dim is None and axis is None: - dim = [d for d, s in xarray_obj.sizes.items() if s == 1] + return [d for d, s in xarray_obj.sizes.items() if s == 1] + + if isinstance(dim, Iterable) and not isinstance(dim, str): + dim = list(dim) + elif dim is not None: + dim = [dim] else: - if isinstance(dim, str): - dim = [dim] + assert axis is not None if isinstance(axis, int): - axis = (axis, ) - if isinstance(axis, tuple): - for a in axis: - if not isinstance(a, int): - raise ValueError( - 'parameter `axis` must be int or tuple of int.') - alldims = list(xarray_obj.sizes.keys()) - dim = [alldims[a] for a in axis] - if any(xarray_obj.sizes[k] > 1 for k in dim): - raise ValueError('cannot select a dimension to squeeze out ' - 'which has length greater than one') + axis = [axis] + axis = list(axis) + if any(not isinstance(a, int) for a in axis): + raise TypeError( + 'parameter `axis` must be int or iterable of int.') + alldims = list(xarray_obj.sizes.keys()) + dim = [alldims[a] for a in axis] + + if any(xarray_obj.sizes[k] > 1 for k in dim): + raise ValueError('cannot select a dimension to squeeze out ' + 'which has length greater than one') return dim class DataWithCoords(SupportsArithmetic, AttrAccessMixin): """Shared base class for Dataset and DataArray.""" - def squeeze(self, dim=None, drop=False, axis=None): + def squeeze(self, dim: Union[Hashable, Iterable[Hashable], None] = None, + drop: bool = False, + axis: Union[int, Iterable[int], None] = None): """Return a new object with squeezed data. Parameters ---------- - dim : None or str or tuple of str, optional + dim : None or Hashable or iterable of Hashable, optional Selects a subset of the length one dimensions. If a dimension is selected with length greater than one, an error is raised. If None, all length one dimensions are squeezed. drop : bool, optional If ``drop=True``, drop squeezed coordinates instead of making them scalar. - axis : int, optional - Select the dimension to squeeze. Added for compatibility reasons. + axis : None or int or iterable of int, optional + Like dim, but positional. Returns ------- @@ -271,7 +279,7 @@ def squeeze(self, dim=None, drop=False, axis=None): dims = get_squeeze_dims(self, dim, axis) return self.isel(drop=drop, **{d: 0 for d in dims}) - def get_index(self, key): + def get_index(self, key: Hashable) -> pd.Index: """Get an index for a dimension, with fall-back to a default RangeIndex """ if key not in self.dims: @@ -283,8 +291,9 @@ def get_index(self, key): # need to ensure dtype=int64 in case range is empty on Python 2 return pd.Index(range(self.sizes[key]), name=key, dtype=np.int64) - def _calc_assign_results(self, kwargs): - results = SortedKeysDict() + def _calc_assign_results(self, kwargs: Mapping[str, T] + ) -> MutableMapping[str, T]: + results = SortedKeysDict() # type: SortedKeysDict[str, T] for k, v in kwargs.items(): if callable(v): results[k] = v(self) @@ -372,7 +381,8 @@ def assign_attrs(self, *args, **kwargs): out.attrs.update(*args, **kwargs) return out - def pipe(self, func, *args, **kwargs): + def pipe(self, func: Union[Callable[..., T], Tuple[Callable[..., T], str]], + *args, **kwargs) -> T: """ Apply func(self, *args, **kwargs) @@ -424,15 +434,14 @@ def pipe(self, func, *args, **kwargs): if isinstance(func, tuple): func, target = func if target in kwargs: - msg = ('%s is both the pipe target and a keyword argument' - % target) - raise ValueError(msg) + raise ValueError('%s is both the pipe target and a keyword ' + 'argument' % target) kwargs[target] = self return func(*args, **kwargs) else: return func(self, *args, **kwargs) - def groupby(self, group, squeeze=True): + def groupby(self, group, squeeze: bool = True): """Returns a GroupBy object for performing grouped operations. Parameters @@ -478,8 +487,9 @@ def groupby(self, group, squeeze=True): """ # noqa return self._groupby_cls(self, group, squeeze=squeeze) - def groupby_bins(self, group, bins, right=True, labels=None, precision=3, - include_lowest=False, squeeze=True): + def groupby_bins(self, group, bins, right: bool = True, labels=None, + precision: int = 3, include_lowest: bool = False, + squeeze: bool = True): """Returns a GroupBy object for performing grouped operations. Rather than using all unique values of `group`, the values are discretized @@ -530,7 +540,9 @@ def groupby_bins(self, group, bins, right=True, labels=None, precision=3, 'precision': precision, 'include_lowest': include_lowest}) - def rolling(self, dim=None, min_periods=None, center=False, **dim_kwargs): + def rolling(self, dim: Optional[Mapping[Hashable, int]] = None, + min_periods: Optional[int] = None, center: bool = False, + **dim_kwargs: int): """ Rolling window object. @@ -590,8 +602,11 @@ def rolling(self, dim=None, min_periods=None, center=False, **dim_kwargs): return self._rolling_cls(self, dim, min_periods=min_periods, center=center) - def coarsen(self, dim=None, boundary='exact', side='left', - coord_func='mean', **dim_kwargs): + def coarsen(self, dim: Optional[Mapping[Hashable, int]] = None, + boundary: str = 'exact', + side: Union[str, Mapping[Hashable, str]] = 'left', + coord_func: str = 'mean', + **dim_kwargs: int): """ Coarsen object. @@ -650,8 +665,12 @@ def coarsen(self, dim=None, boundary='exact', side='left', self, dim, boundary=boundary, side=side, coord_func=coord_func) - def resample(self, indexer=None, skipna=None, closed=None, label=None, - base=0, keep_attrs=None, loffset=None, **indexer_kwargs): + def resample(self, indexer: Optional[Mapping[Hashable, str]] = None, + skipna=None, closed: Optional[str] = None, + label: Optional[str] = None, + base: int = 0, keep_attrs: Optional[bool] = None, + loffset=None, + **indexer_kwargs: str): """Returns a Resample object for performing resampling operations. Handles both downsampling and upsampling. If any intervals contain no @@ -745,12 +764,11 @@ def resample(self, indexer=None, skipna=None, closed=None, label=None, "objects, e.g., data.resample(time='1D').mean()") indexer = either_dict_or_kwargs(indexer, indexer_kwargs, 'resample') - if len(indexer) != 1: raise ValueError( "Resampling only supported along single dimensions." ) - dim, freq = indexer.popitem() + dim, freq = next(iter(indexer.items())) dim_name = dim dim_coord = self[dim] @@ -772,7 +790,7 @@ def resample(self, indexer=None, skipna=None, closed=None, label=None, return resampler - def where(self, cond, other=dtypes.NA, drop=False): + def where(self, cond, other=dtypes.NA, drop: bool = False): """Filter elements from this object according to a condition. This operation follows the normal broadcasting and alignment rules that @@ -858,7 +876,7 @@ def where(self, cond, other=dtypes.NA, drop=False): return ops.where_method(self, cond, other) - def close(self): + def close(self: Any) -> None: """Close any files linked to this object """ if self._file_obj is not None: @@ -914,14 +932,14 @@ def isin(self, test_elements): dask='allowed', ) - def __enter__(self): + def __enter__(self: T) -> T: return self - def __exit__(self, exc_type, exc_value, traceback): + def __exit__(self, exc_type, exc_value, traceback) -> None: self.close() -def full_like(other, fill_value, dtype=None): +def full_like(other, fill_value, dtype: Union[str, np.dtype, None] = None): """Return a new object with the same shape and type as a given object. Parameters @@ -961,7 +979,8 @@ def full_like(other, fill_value, dtype=None): raise TypeError("Expected DataArray, Dataset, or Variable") -def _full_like_variable(other, fill_value, dtype=None): +def _full_like_variable(other, fill_value, + dtype: Union[str, np.dtype, None] = None): """Inner function of full_like, where other must be a variable """ from .variable import Variable @@ -978,27 +997,28 @@ def _full_like_variable(other, fill_value, dtype=None): return Variable(dims=other.dims, data=data, attrs=other.attrs) -def zeros_like(other, dtype=None): +def zeros_like(other, dtype: Union[str, np.dtype, None] = None): """Shorthand for full_like(other, 0, dtype) """ return full_like(other, 0, dtype) -def ones_like(other, dtype=None): +def ones_like(other, dtype: Union[str, np.dtype, None] = None): """Shorthand for full_like(other, 1, dtype) """ return full_like(other, 1, dtype) -def is_np_datetime_like(dtype): +def is_np_datetime_like(dtype: Union[str, np.dtype]) -> bool: """Check if a dtype is a subclass of the numpy datetime types """ return (np.issubdtype(dtype, np.datetime64) or np.issubdtype(dtype, np.timedelta64)) -def _contains_cftime_datetimes(array): - """Check if an array contains cftime.datetime objects""" +def _contains_cftime_datetimes(array) -> bool: + """Check if an array contains cftime.datetime objects + """ try: from cftime import datetime as cftime_datetime except ImportError: @@ -1015,12 +1035,14 @@ def _contains_cftime_datetimes(array): return False -def contains_cftime_datetimes(var): - """Check if an xarray.Variable contains cftime.datetime objects""" +def contains_cftime_datetimes(var) -> bool: + """Check if an xarray.Variable contains cftime.datetime objects + """ return _contains_cftime_datetimes(var.data) -def _contains_datetime_like_objects(var): +def _contains_datetime_like_objects(var) -> bool: """Check if a variable contains datetime like objects (either - np.datetime64, np.timedelta64, or cftime.datetime)""" + np.datetime64, np.timedelta64, or cftime.datetime) + """ return is_np_datetime_like(var.dtype) or contains_cftime_datetimes(var) diff --git a/xarray/core/computation.py b/xarray/core/computation.py index 451d95ee542..6a6fa74c858 100644 --- a/xarray/core/computation.py +++ b/xarray/core/computation.py @@ -29,7 +29,7 @@ _JOINS_WITHOUT_FILL_VALUES = frozenset({'inner', 'exact'}) -class _UFuncSignature(object): +class _UFuncSignature: """Core dimensions signature for a given function. Based on the signature provided by generalized ufuncs in NumPy. diff --git a/xarray/core/coordinates.py b/xarray/core/coordinates.py index 9347ba6b6db..ea3eaa0f4f2 100644 --- a/xarray/core/coordinates.py +++ b/xarray/core/coordinates.py @@ -258,7 +258,7 @@ def _ipython_key_completions_(self): return self._data._ipython_key_completions_() -class LevelCoordinatesSource(object): +class LevelCoordinatesSource: """Iterator for MultiIndex level coordinates. Used for attribute style lookup with AttrAccessMixin. Not returned directly diff --git a/xarray/core/dataarray.py b/xarray/core/dataarray.py index 7cd856db5b4..8d3836f5d8c 100644 --- a/xarray/core/dataarray.py +++ b/xarray/core/dataarray.py @@ -1,4 +1,5 @@ import functools +import sys import warnings from collections import OrderedDict @@ -91,7 +92,7 @@ def _infer_coords_and_dims(shape, coords, dims): return new_coords, dims -class _LocIndexer(object): +class _LocIndexer: def __init__(self, data_array): self.data_array = data_array @@ -230,9 +231,6 @@ def __init__(self, data, coords=None, dims=None, name=None, coords, dims = _infer_coords_and_dims(data.shape, coords, dims) variable = Variable(dims, data, attrs, encoding, fastpath=True) - # uncomment for a useful consistency check: - # assert all(isinstance(v, Variable) for v in coords.values()) - # These fully describe a DataArray self._variable = variable self._coords = coords @@ -881,9 +879,10 @@ def sel_points(self, dim='points', method=None, tolerance=None, dim=dim, method=method, tolerance=tolerance, **indexers) return self._from_temp_dataset(ds) - def reindex_like(self, other, method=None, tolerance=None, copy=True): - """Conform this object onto the indexes of another object, filling - in missing values with NaN. + def reindex_like(self, other, method=None, tolerance=None, copy=True, + fill_value=dtypes.NA): + """Conform this object onto the indexes of another object, filling in + missing values with ``fill_value``. The default fill value is NaN. Parameters ---------- @@ -904,7 +903,7 @@ def reindex_like(self, other, method=None, tolerance=None, copy=True): * nearest: use nearest valid index value (requires pandas>=0.16) tolerance : optional Maximum distance between original and new labels for inexact - matches. The values of the index at the matching locations most + matches. The values of the index at the matching locations must satisfy the equation ``abs(index[indexer] - target) <= tolerance``. Requires pandas>=0.17. copy : bool, optional @@ -912,6 +911,8 @@ def reindex_like(self, other, method=None, tolerance=None, copy=True): ``copy=False`` and reindexing is unnecessary, or can be performed with only slice operations, then the output may share memory with the input. In either case, a new xarray object is always returned. + fill_value : scalar, optional + Value to use for newly missing values Returns ------- @@ -926,12 +927,12 @@ def reindex_like(self, other, method=None, tolerance=None, copy=True): """ indexers = reindex_like_indexers(self, other) return self.reindex(method=method, tolerance=tolerance, copy=copy, - **indexers) + fill_value=fill_value, **indexers) def reindex(self, indexers=None, method=None, tolerance=None, copy=True, - **indexers_kwargs): - """Conform this object onto a new set of indexes, filling in - missing values with NaN. + fill_value=dtypes.NA, **indexers_kwargs): + """Conform this object onto the indexes of another object, filling in + missing values with ``fill_value``. The default fill value is NaN. Parameters ---------- @@ -956,8 +957,10 @@ def reindex(self, indexers=None, method=None, tolerance=None, copy=True, * nearest: use nearest valid index value (requires pandas>=0.16) tolerance : optional Maximum distance between original and new labels for inexact - matches. The values of the index at the matching locations most + matches. The values of the index at the matching locations must satisfy the equation ``abs(index[indexer] - target) <= tolerance``. + fill_value : scalar, optional + Value to use for newly missing values **indexers_kwarg : {dim: indexer, ...}, optional The keyword arguments form of ``indexers``. One of indexers or indexers_kwargs must be provided. @@ -976,7 +979,8 @@ def reindex(self, indexers=None, method=None, tolerance=None, copy=True, indexers = either_dict_or_kwargs( indexers, indexers_kwargs, 'reindex') ds = self._to_temp_dataset().reindex( - indexers=indexers, method=method, tolerance=tolerance, copy=copy) + indexers=indexers, method=method, tolerance=tolerance, copy=copy, + fill_value=fill_value) return self._from_temp_dataset(ds) def interp(self, coords=None, method='linear', assume_sorted=False, @@ -1138,7 +1142,7 @@ def swap_dims(self, dims_dict): ds = self._to_temp_dataset().swap_dims(dims_dict) return self._from_temp_dataset(ds) - def expand_dims(self, dim, axis=None): + def expand_dims(self, dim=None, axis=None, **dim_kwargs): """Return a new object with an additional axis (or axes) inserted at the corresponding position in the array shape. @@ -1147,21 +1151,53 @@ def expand_dims(self, dim, axis=None): Parameters ---------- - dim : str or sequence of str. + dim : str, sequence of str, dict, or None Dimensions to include on the new variable. - dimensions are inserted with length 1. + If provided as str or sequence of str, then dimensions are inserted + with length 1. If provided as a dict, then the keys are the new + dimensions and the values are either integers (giving the length of + the new dimensions) or sequence/ndarray (giving the coordinates of + the new dimensions). **WARNING** for python 3.5, if ``dim`` is + dict-like, then it must be an ``OrderedDict``. This is to ensure + that the order in which the dims are given is maintained. axis : integer, list (or tuple) of integers, or None Axis position(s) where new axis is to be inserted (position(s) on the result array). If a list (or tuple) of integers is passed, multiple axes are inserted. In this case, dim arguments should be same length list. If axis=None is passed, all the axes will be inserted to the start of the result array. + **dim_kwargs : int or sequence/ndarray + The keywords are arbitrary dimensions being inserted and the values + are either the lengths of the new dims (if int is given), or their + coordinates. Note, this is an alternative to passing a dict to the + dim kwarg and will only be used if dim is None. **WARNING** for + python 3.5 ``dim_kwargs`` is not available. Returns ------- expanded : same type as caller This object, but with an additional dimension(s). """ + if isinstance(dim, int): + raise TypeError('dim should be str or sequence of strs or dict') + elif isinstance(dim, str): + dim = OrderedDict(((dim, 1),)) + elif isinstance(dim, (list, tuple)): + if len(dim) != len(set(dim)): + raise ValueError('dims should not contain duplicate values.') + dim = OrderedDict(((d, 1) for d in dim)) + + # TODO: get rid of the below code block when python 3.5 is no longer + # supported. + python36_plus = sys.version_info[0] == 3 and sys.version_info[1] > 5 + not_ordereddict = dim is not None and not isinstance(dim, OrderedDict) + if not python36_plus and not_ordereddict: + raise TypeError("dim must be an OrderedDict for python <3.6") + elif not python36_plus and dim_kwargs: + raise ValueError("dim_kwargs isn't available for python <3.6") + dim_kwargs = OrderedDict(dim_kwargs) + + dim = either_dict_or_kwargs(dim, dim_kwargs, 'expand_dims') ds = self._to_temp_dataset().expand_dims(dim, axis) return self._from_temp_dataset(ds) @@ -1369,7 +1405,7 @@ def unstack(self, dim=None): ds = self._to_temp_dataset().unstack(dim) return self._from_temp_dataset(ds) - def transpose(self, *dims): + def transpose(self, *dims) -> 'DataArray': """Return a new DataArray object with transposed dimensions. Parameters @@ -1397,6 +1433,10 @@ def transpose(self, *dims): variable = self.variable.transpose(*dims) return self._replace(variable) + @property + def T(self) -> 'DataArray': + return self.transpose() + def drop(self, labels, dim=None): """Drop coordinates or index labels from this DataArray. diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 3bb54e80456..e9ec1445dd4 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -8,7 +8,7 @@ from distutils.version import LooseVersion from numbers import Number from typing import ( - Any, Callable, Dict, List, Optional, Set, Tuple, TypeVar, Union) + Any, Callable, Dict, List, Optional, Set, Tuple, TypeVar, Union, Sequence) import numpy as np import pandas as pd @@ -303,7 +303,7 @@ def _ipython_key_completions_(self): if key not in self._dataset._coord_names] -class _LocIndexer(object): +class _LocIndexer: def __init__(self, dataset): self.dataset = dataset @@ -343,7 +343,7 @@ def __init__(self, data_vars=None, coords=None, attrs=None, ---------- data_vars : dict-like, optional A mapping from variable names to :py:class:`~xarray.DataArray` - objects, :py:class:`~xarray.Variable` objects or tuples of the + objects, :py:class:`~xarray.Variable` objects or to tuples of the form ``(dims, data[, attrs])`` which can be used as arguments to create a new ``Variable``. Each dimension must have the same length in all variables in which it appears. @@ -915,7 +915,9 @@ def copy(self: T, deep: bool = False, data: Mapping = None) -> T: variables = OrderedDict((k, v.copy(deep=deep, data=data.get(k))) for k, v in self._variables.items()) - return self._replace(variables) + attrs = copy.deepcopy(self._attrs) if deep else copy.copy(self._attrs) + + return self._replace(variables, attrs=attrs) @property def _level_coords(self): @@ -936,6 +938,7 @@ def _copy_listed(self: T, names) -> T: """ variables = OrderedDict() # type: OrderedDict[Any, Variable] coord_names = set() + indexes = OrderedDict() # type: OrderedDict[Any, pd.Index] for name in names: try: @@ -946,6 +949,8 @@ def _copy_listed(self: T, names) -> T: variables[var_name] = var if ref_name in self._coord_names or ref_name in self.dims: coord_names.add(var_name) + if (var_name,) == var.dims: + indexes[var_name] = var.to_index() needed_dims = set() # type: set for v in variables.values(): @@ -957,12 +962,8 @@ def _copy_listed(self: T, names) -> T: if set(self.variables[k].dims) <= needed_dims: variables[k] = self._variables[k] coord_names.add(k) - - if self._indexes is None: - indexes = None - else: - indexes = OrderedDict((k, v) for k, v in self._indexes.items() - if k in coord_names) + if k in self.indexes: + indexes[k] = self.indexes[k] return self._replace(variables, coord_names, dims, indexes=indexes) @@ -1380,7 +1381,7 @@ def info(self, buf=None): See Also -------- pandas.DataFrame.assign - netCDF's ncdump + ncdump: netCDF's ncdump """ if buf is None: # pragma: no cover @@ -1501,9 +1502,13 @@ def _validate_indexers( raise ValueError("dimensions %r do not exist" % invalid) # all indexers should be int, slice, np.ndarrays, or Variable - indexers_list = [] + indexers_list = [] # type: List[Tuple[Any, Union[slice, Variable]]] for k, v in indexers.items(): - if isinstance(v, (slice, Variable)): + if isinstance(v, slice): + indexers_list.append((k, v)) + continue + + if isinstance(v, Variable): pass elif isinstance(v, DataArray): v = v.variable @@ -1511,6 +1516,8 @@ def _validate_indexers( v = as_variable(v) elif isinstance(v, Dataset): raise TypeError('cannot use a Dataset as an indexer') + elif isinstance(v, Sequence) and len(v) == 0: + v = IndexVariable((k, ), np.zeros((0,), dtype='int64')) else: v = np.asarray(v) @@ -1522,14 +1529,19 @@ def _validate_indexers( v = _parse_array_of_cftime_strings(v, index.date_type) if v.ndim == 0: - v = as_variable(v) + v = Variable((), v) elif v.ndim == 1: - v = as_variable((k, v)) + v = IndexVariable((k,), v) else: raise IndexError( "Unlabeled multi-dimensional array cannot be " "used for indexing: {}".format(k)) + + if v.ndim == 1: + v = v.to_index_variable() + indexers_list.append((k, v)) + return indexers_list def _get_indexers_coords_and_indexes(self, indexers): @@ -1629,7 +1641,7 @@ def isel(self, indexers=None, drop=False, **indexers_kwargs): if name in self.indexes: new_var, new_index = isel_variable_and_index( - var, self.indexes[name], var_indexers) + name, var, self.indexes[name], var_indexers) if new_index is not None: indexes[name] = new_index else: @@ -1689,7 +1701,7 @@ def sel(self, indexers=None, method=None, tolerance=None, drop=False, * nearest: use nearest valid index value tolerance : optional Maximum distance between original and new labels for inexact - matches. The values of the index at the matching locations most + matches. The values of the index at the matching locations must satisfy the equation ``abs(index[indexer] - target) <= tolerance``. Requires pandas>=0.17. drop : bool, optional @@ -1889,7 +1901,7 @@ def sel_points(self, dim='points', method=None, tolerance=None, * nearest: use nearest valid index value tolerance : optional Maximum distance between original and new labels for inexact - matches. The values of the index at the matching locations most + matches. The values of the index at the matching locations must satisfy the equation ``abs(index[indexer] - target) <= tolerance``. Requires pandas>=0.17. **indexers : {dim: indexer, ...} @@ -1920,9 +1932,10 @@ def sel_points(self, dim='points', method=None, tolerance=None, ) return self.isel_points(dim=dim, **pos_indexers) - def reindex_like(self, other, method=None, tolerance=None, copy=True): - """Conform this object onto the indexes of another object, filling - in missing values with NaN. + def reindex_like(self, other, method=None, tolerance=None, copy=True, + fill_value=dtypes.NA): + """Conform this object onto the indexes of another object, filling in + missing values with ``fill_value``. The default fill value is NaN. Parameters ---------- @@ -1943,7 +1956,7 @@ def reindex_like(self, other, method=None, tolerance=None, copy=True): * nearest: use nearest valid index value (requires pandas>=0.16) tolerance : optional Maximum distance between original and new labels for inexact - matches. The values of the index at the matching locations most + matches. The values of the index at the matching locations must satisfy the equation ``abs(index[indexer] - target) <= tolerance``. Requires pandas>=0.17. copy : bool, optional @@ -1951,6 +1964,8 @@ def reindex_like(self, other, method=None, tolerance=None, copy=True): ``copy=False`` and reindexing is unnecessary, or can be performed with only slice operations, then the output may share memory with the input. In either case, a new xarray object is always returned. + fill_value : scalar, optional + Value to use for newly missing values Returns ------- @@ -1965,12 +1980,12 @@ def reindex_like(self, other, method=None, tolerance=None, copy=True): """ indexers = alignment.reindex_like_indexers(self, other) return self.reindex(indexers=indexers, method=method, copy=copy, - tolerance=tolerance) + fill_value=fill_value, tolerance=tolerance) def reindex(self, indexers=None, method=None, tolerance=None, copy=True, - **indexers_kwargs): + fill_value=dtypes.NA, **indexers_kwargs): """Conform this object onto a new set of indexes, filling in - missing values with NaN. + missing values with ``fill_value``. The default fill value is NaN. Parameters ---------- @@ -1990,7 +2005,7 @@ def reindex(self, indexers=None, method=None, tolerance=None, copy=True, * nearest: use nearest valid index value (requires pandas>=0.16) tolerance : optional Maximum distance between original and new labels for inexact - matches. The values of the index at the matching locations most + matches. The values of the index at the matching locations must satisfy the equation ``abs(index[indexer] - target) <= tolerance``. Requires pandas>=0.17. copy : bool, optional @@ -1998,6 +2013,8 @@ def reindex(self, indexers=None, method=None, tolerance=None, copy=True, ``copy=False`` and reindexing is unnecessary, or can be performed with only slice operations, then the output may share memory with the input. In either case, a new xarray object is always returned. + fill_value : scalar, optional + Value to use for newly missing values **indexers_kwarg : {dim: indexer, ...}, optional Keyword arguments in the same form as ``indexers``. One of indexers or indexers_kwargs must be provided. @@ -2022,7 +2039,7 @@ def reindex(self, indexers=None, method=None, tolerance=None, copy=True, variables, indexes = alignment.reindex_variables( self.variables, self.sizes, self.indexes, indexers, method, - tolerance, copy=copy) + tolerance, copy=copy, fill_value=fill_value) coord_names = set(self._coord_names) coord_names.update(indexers) return self._replace_with_new_dims( @@ -2115,15 +2132,20 @@ def _validate_interp_indexer(x, new_x): indexes = OrderedDict( (k, v) for k, v in obj.indexes.items() if k not in indexers) selected = self._replace_with_new_dims( - variables, coord_names, indexes=indexes) + variables.copy(), coord_names, indexes=indexes) # attach indexer as coordinate variables.update(indexers) + indexes.update( + (k, v.to_index()) for k, v in indexers.items() if v.dims == (k,) + ) + # Extract coordinates from indexers coord_vars, new_indexes = ( selected._get_indexers_coords_and_indexes(coords)) variables.update(coord_vars) indexes.update(new_indexes) + coord_names = (set(variables) .intersection(obj._coord_names) .union(coord_vars)) @@ -2324,7 +2346,7 @@ def swap_dims(self, dims_dict, inplace=None): return self._replace_with_new_dims(variables, coord_names, indexes=indexes, inplace=inplace) - def expand_dims(self, dim, axis=None): + def expand_dims(self, dim=None, axis=None, **dim_kwargs): """Return a new object with an additional axis (or axes) inserted at the corresponding position in the array shape. @@ -2333,15 +2355,27 @@ def expand_dims(self, dim, axis=None): Parameters ---------- - dim : str or sequence of str. + dim : str, sequence of str, dict, or None Dimensions to include on the new variable. - dimensions are inserted with length 1. + If provided as str or sequence of str, then dimensions are inserted + with length 1. If provided as a dict, then the keys are the new + dimensions and the values are either integers (giving the length of + the new dimensions) or sequence/ndarray (giving the coordinates of + the new dimensions). **WARNING** for python 3.5, if ``dim`` is + dict-like, then it must be an ``OrderedDict``. This is to ensure + that the order in which the dims are given is maintained. axis : integer, list (or tuple) of integers, or None Axis position(s) where new axis is to be inserted (position(s) on the result array). If a list (or tuple) of integers is passed, multiple axes are inserted. In this case, dim arguments should be - the same length list. If axis=None is passed, all the axes will - be inserted to the start of the result array. + same length list. If axis=None is passed, all the axes will be + inserted to the start of the result array. + **dim_kwargs : int or sequence/ndarray + The keywords are arbitrary dimensions being inserted and the values + are either the lengths of the new dims (if int is given), or their + coordinates. Note, this is an alternative to passing a dict to the + dim kwarg and will only be used if dim is None. **WARNING** for + python 3.5 ``dim_kwargs`` is not available. Returns ------- @@ -2349,10 +2383,25 @@ def expand_dims(self, dim, axis=None): This object, but with an additional dimension(s). """ if isinstance(dim, int): - raise ValueError('dim should be str or sequence of strs or dict') + raise TypeError('dim should be str or sequence of strs or dict') + elif isinstance(dim, str): + dim = OrderedDict(((dim, 1),)) + elif isinstance(dim, (list, tuple)): + if len(dim) != len(set(dim)): + raise ValueError('dims should not contain duplicate values.') + dim = OrderedDict(((d, 1) for d in dim)) + + # TODO: get rid of the below code block when python 3.5 is no longer + # supported. + python36_plus = sys.version_info[0] == 3 and sys.version_info[1] > 5 + not_ordereddict = dim is not None and not isinstance(dim, OrderedDict) + if not python36_plus and not_ordereddict: + raise TypeError("dim must be an OrderedDict for python <3.6") + elif not python36_plus and dim_kwargs: + raise ValueError("dim_kwargs isn't available for python <3.6") + + dim = either_dict_or_kwargs(dim, dim_kwargs, 'expand_dims') - if isinstance(dim, str): - dim = [dim] if axis is not None and not isinstance(axis, (list, tuple)): axis = [axis] @@ -2371,13 +2420,28 @@ def expand_dims(self, dim, axis=None): '{dim} already exists as coordinate or' ' variable name.'.format(dim=d)) - if len(dim) != len(set(dim)): - raise ValueError('dims should not contain duplicate values.') - variables = OrderedDict() + coord_names = self._coord_names.copy() + # If dim is a dict, then ensure that the values are either integers + # or iterables. + for k, v in dim.items(): + if hasattr(v, "__iter__"): + # If the value for the new dimension is an iterable, then + # save the coordinates to the variables dict, and set the + # value within the dim dict to the length of the iterable + # for later use. + variables[k] = xr.IndexVariable((k,), v) + coord_names.add(k) + dim[k] = variables[k].size + elif isinstance(v, int): + pass # Do nothing if the dimensions value is just an int + else: + raise TypeError('The value of new dimension {k} must be ' + 'an iterable or an int'.format(k=k)) + for k, v in self._variables.items(): if k not in dim: - if k in self._coord_names: # Do not change coordinates + if k in coord_names: # Do not change coordinates variables[k] = v else: result_ndim = len(v.dims) + len(axis) @@ -2395,11 +2459,13 @@ def expand_dims(self, dim, axis=None): ' values.') # We need to sort them to make sure `axis` equals to the # axis positions of the result array. - zip_axis_dim = sorted(zip(axis_pos, dim)) + zip_axis_dim = sorted(zip(axis_pos, dim.items())) + + all_dims = list(zip(v.dims, v.shape)) + for d, c in zip_axis_dim: + all_dims.insert(d, c) + all_dims = OrderedDict(all_dims) - all_dims = list(v.dims) - for a, d in zip_axis_dim: - all_dims.insert(a, d) variables[k] = v.set_dims(all_dims) else: # If dims includes a label of a non-dimension coordinate, @@ -2407,10 +2473,10 @@ def expand_dims(self, dim, axis=None): variables[k] = v.set_dims(k) new_dims = self._dims.copy() - for d in dim: - new_dims[d] = 1 + new_dims.update(dim) - return self._replace(variables, dims=new_dims) + return self._replace_vars_and_dims( + variables, dims=new_dims, coord_names=coord_names) def set_index(self, indexes=None, append=False, inplace=None, **indexes_kwargs): @@ -2691,7 +2757,7 @@ def update(self, other, inplace=None): inplace=inplace) def merge(self, other, inplace=None, overwrite_vars=frozenset(), - compat='no_conflicts', join='outer'): + compat='no_conflicts', join='outer', fill_value=dtypes.NA): """Merge the arrays of two datasets into a single dataset. This method generally not allow for overriding data, with the exception @@ -2729,6 +2795,8 @@ def merge(self, other, inplace=None, overwrite_vars=frozenset(), - 'left': use indexes from ``self`` - 'right': use indexes from ``other`` - 'exact': error instead of aligning non-equal indexes + fill_value: scalar, optional + Value to use for newly missing values Returns ------- @@ -2743,7 +2811,7 @@ def merge(self, other, inplace=None, overwrite_vars=frozenset(), inplace = _check_inplace(inplace) variables, coord_names, dims = dataset_merge_method( self, other, overwrite_vars=overwrite_vars, compat=compat, - join=join) + join=join, fill_value=fill_value) return self._replace_vars_and_dims(variables, coord_names, dims, inplace=inplace) diff --git a/xarray/core/dtypes.py b/xarray/core/dtypes.py index 00ff7958183..794f2b62183 100644 --- a/xarray/core/dtypes.py +++ b/xarray/core/dtypes.py @@ -9,7 +9,7 @@ @functools.total_ordering -class AlwaysGreaterThan(object): +class AlwaysGreaterThan: def __gt__(self, other): return True @@ -18,7 +18,7 @@ def __eq__(self, other): @functools.total_ordering -class AlwaysLessThan(object): +class AlwaysLessThan: def __lt__(self, other): return True diff --git a/xarray/core/duck_array_ops.py b/xarray/core/duck_array_ops.py index b67a220ed4c..b37e01cb7af 100644 --- a/xarray/core/duck_array_ops.py +++ b/xarray/core/duck_array_ops.py @@ -185,7 +185,7 @@ def array_notnull_equiv(arr1, arr2): def count(data, axis=None): """Count the number of non-NA in this array along the given axis or axes """ - return np.sum(~isnull(data), axis=axis) + return np.sum(np.logical_not(isnull(data)), axis=axis) def where(condition, x, y): diff --git a/xarray/core/extensions.py b/xarray/core/extensions.py index 574c05f1a6b..cb34e87f88d 100644 --- a/xarray/core/extensions.py +++ b/xarray/core/extensions.py @@ -8,7 +8,7 @@ class AccessorRegistrationWarning(Warning): """Warning for conflicts in accessor registration.""" -class _CachedAccessor(object): +class _CachedAccessor: """Custom property-like object (descriptor) for caching accessors.""" def __init__(self, name, accessor): @@ -81,7 +81,7 @@ def register_dataset_accessor(name): import xarray as xr @xr.register_dataset_accessor('geo') - class GeoAccessor(object): + class GeoAccessor: def __init__(self, xarray_obj): self._obj = xarray_obj diff --git a/xarray/core/formatting.py b/xarray/core/formatting.py index f3fcc1ecb37..e190c9c2670 100644 --- a/xarray/core/formatting.py +++ b/xarray/core/formatting.py @@ -116,7 +116,7 @@ def format_timestamp(t): if time_str == '00:00:00': return date_str else: - return '%sT%s' % (date_str, time_str) + return '{}T{}'.format(date_str, time_str) def format_timedelta(t, timedelta_format=None): @@ -212,12 +212,12 @@ def summarize_variable(name, var, col_width, show_values=True, marker=' ', max_width=None): if max_width is None: max_width = OPTIONS['display_width'] - first_col = pretty_print(' %s %s ' % (marker, name), col_width) + first_col = pretty_print(' {} {} '.format(marker, name), col_width) if var.dims: - dims_str = '(%s) ' % ', '.join(map(str, var.dims)) + dims_str = '({}) '.format(', '.join(map(str, var.dims))) else: dims_str = '' - front_str = '%s%s%s ' % (first_col, dims_str, var.dtype) + front_str = '{}{}{} '.format(first_col, dims_str, var.dtype) if show_values: values_str = format_array_flat(var, max_width - len(front_str)) elif isinstance(var._data, dask_array_type): @@ -229,8 +229,9 @@ def summarize_variable(name, var, col_width, show_values=True, def _summarize_coord_multiindex(coord, col_width, marker): - first_col = pretty_print(' %s %s ' % (marker, coord.name), col_width) - return '%s(%s) MultiIndex' % (first_col, str(coord.dims[0])) + first_col = pretty_print(' {} {} '.format( + marker, coord.name), col_width) + return '{}({}) MultiIndex'.format(first_col, str(coord.dims[0])) def _summarize_coord_levels(coord, col_width, marker='-'): @@ -264,13 +265,14 @@ def summarize_coord(name, var, col_width): def summarize_attr(key, value, col_width=None): """Summary for __repr__ - use ``X.attrs[key]`` for full value.""" # Indent key and add ':', then right-pad if col_width is not None - k_str = ' %s:' % key + k_str = ' {}:'.format(key) if col_width is not None: k_str = pretty_print(k_str, col_width) # Replace tabs and newlines, so we print on one line in known width v_str = str(value).replace('\t', '\\t').replace('\n', '\\n') # Finally, truncate to the desired display width - return maybe_truncate('%s %s' % (k_str, v_str), OPTIONS['display_width']) + return maybe_truncate('{} {}'.format(k_str, v_str), + OPTIONS['display_width']) EMPTY_REPR = ' *empty*' @@ -303,7 +305,7 @@ def _calculate_col_width(col_items): def _mapping_repr(mapping, title, summarizer, col_width=None): if col_width is None: col_width = _calculate_col_width(mapping) - summary = ['%s:' % title] + summary = ['{}:'.format(title)] if mapping: summary += [summarizer(k, v, col_width) for k, v in mapping.items()] else: @@ -329,19 +331,19 @@ def coords_repr(coords, col_width=None): def indexes_repr(indexes): summary = [] for k, v in indexes.items(): - summary.append(wrap_indent(repr(v), '%s: ' % k)) + summary.append(wrap_indent(repr(v), '{}: '.format(k))) return '\n'.join(summary) def dim_summary(obj): - elements = ['%s: %s' % (k, v) for k, v in obj.sizes.items()] + elements = ['{}: {}'.format(k, v) for k, v in obj.sizes.items()] return ', '.join(elements) def unindexed_dims_repr(dims, coords): unindexed_dims = [d for d in dims if d not in coords] if unindexed_dims: - dims_str = ', '.join('%s' % d for d in unindexed_dims) + dims_str = ', '.join('{}'.format(d) for d in unindexed_dims) return 'Dimensions without coordinates: ' + dims_str else: return None @@ -382,10 +384,11 @@ def short_dask_repr(array, show_dtype=True): """ chunksize = tuple(c[0] for c in array.chunks) if show_dtype: - return 'dask.array' % ( + return 'dask.array'.format( array.shape, array.dtype, chunksize) else: - return 'dask.array' % (array.shape, chunksize) + return 'dask.array'.format( + array.shape, chunksize) def short_data_repr(array): @@ -394,18 +397,18 @@ def short_data_repr(array): elif array._in_memory or array.size < 1e5: return short_array_repr(array.values) else: - return u'[%s values with dtype=%s]' % (array.size, array.dtype) + return u'[{} values with dtype={}]'.format(array.size, array.dtype) def array_repr(arr): # used for DataArray, Variable and IndexVariable if hasattr(arr, 'name') and arr.name is not None: - name_str = '%r ' % arr.name + name_str = '{!r} '.format(arr.name) else: name_str = '' - summary = ['' - % (type(arr).__name__, name_str, dim_summary(arr))] + summary = [''.format( + type(arr).__name__, name_str, dim_summary(arr))] summary.append(short_data_repr(arr)) @@ -424,12 +427,12 @@ def array_repr(arr): def dataset_repr(ds): - summary = ['' % type(ds).__name__] + summary = [''.format(type(ds).__name__)] col_width = _calculate_col_width(_get_col_items(ds.variables)) dims_start = pretty_print('Dimensions:', col_width) - summary.append('%s(%s)' % (dims_start, dim_summary(ds))) + summary.append('{}({})'.format(dims_start, dim_summary(ds))) if ds.coords: summary.append(coords_repr(ds.coords, col_width=col_width)) diff --git a/xarray/core/groupby.py b/xarray/core/groupby.py index e8e2f1b08d4..82a92044caf 100644 --- a/xarray/core/groupby.py +++ b/xarray/core/groupby.py @@ -113,7 +113,7 @@ def _inverse_permutation_indices(positions): return indices -class _DummyGroup(object): +class _DummyGroup: """Class for keeping track of grouped dimensions without coordinates. Should not be user visible. @@ -519,6 +519,7 @@ def apply(self, func, shortcut=False, args=(), **kwargs): Apply uses heuristics (like `pandas.GroupBy.apply`) to figure out how to stack together the array. The rule is: + 1. If the dimension along which the group coordinate is defined is still in the first grouped array after applying `func`, then stack over this dimension. @@ -661,6 +662,7 @@ def apply(self, func, args=(), **kwargs): Apply uses heuristics (like `pandas.GroupBy.apply`) to figure out how to stack together the datasets. The rule is: + 1. If the dimension along which the group coordinate is defined is still in the first grouped item after applying `func`, then stack over this dimension. diff --git a/xarray/core/indexes.py b/xarray/core/indexes.py index 6d8b553036a..eccb72b6a58 100644 --- a/xarray/core/indexes.py +++ b/xarray/core/indexes.py @@ -1,6 +1,6 @@ import collections.abc from collections import OrderedDict -from typing import Any, Iterable, Mapping, Optional, Tuple, Union +from typing import Any, Hashable, Iterable, Mapping, Optional, Tuple, Union import pandas as pd @@ -59,6 +59,7 @@ def default_indexes( def isel_variable_and_index( + name: Hashable, variable: Variable, index: pd.Index, indexers: Mapping[Any, Union[slice, Variable]], @@ -75,8 +76,8 @@ def isel_variable_and_index( new_variable = variable.isel(indexers) - if new_variable.ndim != 1: - # can't preserve a index if result is not 0D + if new_variable.dims != (name,): + # can't preserve a index if result has new dimensions return new_variable, None # we need to compute the new index diff --git a/xarray/core/indexing.py b/xarray/core/indexing.py index 65a123c3319..1effb9347dd 100644 --- a/xarray/core/indexing.py +++ b/xarray/core/indexing.py @@ -294,7 +294,7 @@ def _index_indexer_1d(old_indexer, applied_indexer, size): return indexer -class ExplicitIndexer(object): +class ExplicitIndexer: """Base class for explicit indexer objects. ExplicitIndexer objects wrap a tuple of values given by their ``tuple`` @@ -430,7 +430,7 @@ def __init__(self, key): super(VectorizedIndexer, self).__init__(new_key) -class ExplicitlyIndexed(object): +class ExplicitlyIndexed: """Mixin to mark support for Indexer subclasses in indexing.""" @@ -740,7 +740,7 @@ def _combine_indexers(old_key, shape, new_key): np.broadcast_arrays(*old_key.tuple))) -class IndexingSupport(object): # could inherit from enum.Enum on Python 3 +class IndexingSupport: # could inherit from enum.Enum on Python 3 # for backends that support only basic indexer BASIC = 'BASIC' # for backends that support basic / outer indexer diff --git a/xarray/core/merge.py b/xarray/core/merge.py index 363fdfc2337..421ac39ebd8 100644 --- a/xarray/core/merge.py +++ b/xarray/core/merge.py @@ -4,6 +4,7 @@ import pandas as pd +from . import dtypes from .alignment import deep_align from .pycompat import TYPE_CHECKING from .utils import Frozen @@ -349,7 +350,7 @@ def expand_and_merge_variables(objs, priority_arg=None): def merge_coords(objs, compat='minimal', join='outer', priority_arg=None, - indexes=None): + indexes=None, fill_value=dtypes.NA): """Merge coordinate variables. See merge_core below for argument descriptions. This works similarly to @@ -358,7 +359,8 @@ def merge_coords(objs, compat='minimal', join='outer', priority_arg=None, """ _assert_compat_valid(compat) coerced = coerce_pandas_values(objs) - aligned = deep_align(coerced, join=join, copy=False, indexes=indexes) + aligned = deep_align(coerced, join=join, copy=False, indexes=indexes, + fill_value=fill_value) expanded = expand_variable_dicts(aligned) priority_vars = _get_priority_vars(aligned, priority_arg, compat=compat) variables = merge_variables(expanded, priority_vars, compat=compat) @@ -404,7 +406,8 @@ def merge_core(objs, join='outer', priority_arg=None, explicit_coords=None, - indexes=None): + indexes=None, + fill_value=dtypes.NA): """Core logic for merging labeled objects. This is not public API. @@ -423,6 +426,8 @@ def merge_core(objs, An explicit list of variables from `objs` that are coordinates. indexes : dict, optional Dictionary with values given by pandas.Index objects. + fill_value : scalar, optional + Value to use for newly missing values Returns ------- @@ -442,7 +447,8 @@ def merge_core(objs, _assert_compat_valid(compat) coerced = coerce_pandas_values(objs) - aligned = deep_align(coerced, join=join, copy=False, indexes=indexes) + aligned = deep_align(coerced, join=join, copy=False, indexes=indexes, + fill_value=fill_value) expanded = expand_variable_dicts(aligned) coord_names, noncoord_names = determine_coords(coerced) @@ -470,7 +476,7 @@ def merge_core(objs, return variables, coord_names, dict(dims) -def merge(objects, compat='no_conflicts', join='outer'): +def merge(objects, compat='no_conflicts', join='outer', fill_value=dtypes.NA): """Merge any number of xarray objects into a single Dataset as variables. Parameters @@ -492,6 +498,8 @@ def merge(objects, compat='no_conflicts', join='outer'): of all non-null values. join : {'outer', 'inner', 'left', 'right', 'exact'}, optional How to combine objects with different indexes. + fill_value : scalar, optional + Value to use for newly missing values Returns ------- @@ -529,7 +537,8 @@ def merge(objects, compat='no_conflicts', join='outer'): obj.to_dataset() if isinstance(obj, DataArray) else obj for obj in objects] - variables, coord_names, dims = merge_core(dict_like_objects, compat, join) + variables, coord_names, dims = merge_core(dict_like_objects, compat, join, + fill_value=fill_value) # TODO: don't always recompute indexes merged = Dataset._construct_direct( variables, coord_names, dims, indexes=None) @@ -537,7 +546,8 @@ def merge(objects, compat='no_conflicts', join='outer'): return merged -def dataset_merge_method(dataset, other, overwrite_vars, compat, join): +def dataset_merge_method(dataset, other, overwrite_vars, compat, join, + fill_value=dtypes.NA): """Guts of the Dataset.merge method.""" # we are locked into supporting overwrite_vars for the Dataset.merge @@ -565,7 +575,8 @@ def dataset_merge_method(dataset, other, overwrite_vars, compat, join): objs = [dataset, other_no_overwrite, other_overwrite] priority_arg = 2 - return merge_core(objs, compat, join, priority_arg=priority_arg) + return merge_core(objs, compat, join, priority_arg=priority_arg, + fill_value=fill_value) def dataset_update_method(dataset, other): diff --git a/xarray/core/missing.py b/xarray/core/missing.py index 50c420206cd..3931512325e 100644 --- a/xarray/core/missing.py +++ b/xarray/core/missing.py @@ -14,7 +14,7 @@ from .variable import Variable, broadcast_variables -class BaseInterpolator(object): +class BaseInterpolator: '''gerneric interpolator class for normalizing interpolation methods''' cons_kwargs = {} # type: Dict[str, Any] call_kwargs = {} # type: Dict[str, Any] diff --git a/xarray/core/nputils.py b/xarray/core/nputils.py index 14fbec72341..e96e7911d6d 100644 --- a/xarray/core/nputils.py +++ b/xarray/core/nputils.py @@ -120,7 +120,7 @@ def _advanced_indexer_subspaces(key): return mixed_positions, vindex_positions -class NumpyVIndexAdapter(object): +class NumpyVIndexAdapter: """Object that implements indexing like vindex on a np.ndarray. This is a pure Python implementation of (some of) the logic in this NumPy diff --git a/xarray/core/options.py b/xarray/core/options.py index c9d26c3e577..d441a81d325 100644 --- a/xarray/core/options.py +++ b/xarray/core/options.py @@ -69,7 +69,7 @@ def _get_keep_attrs(default): " True, False or 'default'.") -class set_options(object): +class set_options: """Set options for xarray in a controlled context. Currently supported options: diff --git a/xarray/core/pycompat.py b/xarray/core/pycompat.py index 0df0e727303..aa2cc5a0f03 100644 --- a/xarray/core/pycompat.py +++ b/xarray/core/pycompat.py @@ -1,6 +1,5 @@ -# flake8: noqa import sys -import typing +from collections import abc import numpy as np @@ -13,6 +12,38 @@ except ImportError: # pragma: no cover dask_array_type = () -# Ensure we have some more recent additions to the typing module. -# Note that TYPE_CHECKING itself is not available on Python 3.5.1. -TYPE_CHECKING = sys.version >= '3.5.3' and typing.TYPE_CHECKING + +if sys.version < '3.5.3': + TYPE_CHECKING = False + + class _ABCDummyBrackets(type(abc.Mapping)): # abc.ABCMeta + def __getitem__(cls, name): + return cls + + class Mapping(abc.Mapping, metaclass=_ABCDummyBrackets): + pass + + class MutableMapping(abc.MutableMapping, metaclass=_ABCDummyBrackets): + pass + + class MutableSet(abc.MutableSet, metaclass=_ABCDummyBrackets): + pass + +else: + from typing import TYPE_CHECKING # noqa: F401 + + # from typing import Mapping, MutableMapping, MutableSet + + # The above confuses mypy 0.700; + # see: https://github.com/python/mypy/issues/6652 + # As a workaround, use: + # + # from typing import Mapping, MutableMapping, MutableSet + # try: + # from .pycompat import Mapping, MutableMapping, MutableSet + # except ImportError: + # pass + # + # This is only necessary in modules that define subclasses of the + # abstract collections; when only type inference is needed, one can just + # use typing also in Python 3.5.0~3.5.2 (although mypy will misbehave). diff --git a/xarray/core/resample.py b/xarray/core/resample.py index 40298f14d08..9e9e3b79e13 100644 --- a/xarray/core/resample.py +++ b/xarray/core/resample.py @@ -4,7 +4,7 @@ RESAMPLE_DIM = '__resample_dim__' -class Resample(object): +class Resample: """An object that extends the `GroupBy` object with additional logic for handling specialized re-sampling operations. diff --git a/xarray/core/resample_cftime.py b/xarray/core/resample_cftime.py index 161945f118d..e7f41be8667 100644 --- a/xarray/core/resample_cftime.py +++ b/xarray/core/resample_cftime.py @@ -45,7 +45,7 @@ import pandas as pd -class CFTimeGrouper(object): +class CFTimeGrouper: """This is a simple container for the grouping parameters that implements a single method, the only one required for resampling in xarray. It cannot be used in a call to groupby like a pandas.Grouper object can.""" diff --git a/xarray/core/rolling.py b/xarray/core/rolling.py index 8a974e2da72..c113cfebe2a 100644 --- a/xarray/core/rolling.py +++ b/xarray/core/rolling.py @@ -12,7 +12,7 @@ from .pycompat import dask_array_type -class Rolling(object): +class Rolling: """A object that implements the moving window pattern. See Also @@ -170,15 +170,15 @@ def construct(self, window_dim, stride=1, fill_value=dtypes.NA): -------- >>> da = DataArray(np.arange(8).reshape(2, 4), dims=('a', 'b')) >>> - >>> rolling = da.rolling(a=3) - >>> rolling.to_datarray('window_dim') + >>> rolling = da.rolling(b=3) + >>> rolling.construct('window_dim') array([[[np.nan, np.nan, 0], [np.nan, 0, 1], [0, 1, 2], [1, 2, 3]], [[np.nan, np.nan, 4], [np.nan, 4, 5], [4, 5, 6], [5, 6, 7]]]) Dimensions without coordinates: a, b, window_dim >>> - >>> rolling = da.rolling(a=3, center=True) - >>> rolling.to_datarray('window_dim') + >>> rolling = da.rolling(b=3, center=True) + >>> rolling.construct('window_dim') array([[[np.nan, 0, 1], [0, 1, 2], [1, 2, 3], [2, 3, np.nan]], [[np.nan, 4, 5], [4, 5, 6], [5, 6, 7], [6, 7, np.nan]]]) @@ -435,7 +435,7 @@ def construct(self, window_dim, stride=1, fill_value=dtypes.NA): **{self.dim: slice(None, None, stride)}) -class Coarsen(object): +class Coarsen: """A object that implements the coarsen. See Also diff --git a/xarray/core/utils.py b/xarray/core/utils.py index 349c8f98dc5..94787dd35e2 100644 --- a/xarray/core/utils.py +++ b/xarray/core/utils.py @@ -7,15 +7,28 @@ import re import warnings from collections import OrderedDict -from collections.abc import Iterable, Mapping, MutableMapping, MutableSet +from typing import (AbstractSet, Any, Callable, Container, Dict, Hashable, + Iterable, Iterator, Optional, Sequence, + Tuple, TypeVar, cast) import numpy as np import pandas as pd from .pycompat import dask_array_type +from typing import Mapping, MutableMapping, MutableSet +try: # Fix typed collections in Python 3.5.0~3.5.2 + from .pycompat import Mapping, MutableMapping, MutableSet # noqa: F811 +except ImportError: + pass -def _check_inplace(inplace, default=False): + +K = TypeVar('K') +V = TypeVar('V') +T = TypeVar('T') + + +def _check_inplace(inplace: bool, default: bool = False) -> bool: if inplace is None: inplace = default else: @@ -26,16 +39,16 @@ def _check_inplace(inplace, default=False): return inplace -def alias_message(old_name, new_name): +def alias_message(old_name: str, new_name: str) -> str: return '%s has been deprecated. Use %s instead.' % (old_name, new_name) -def alias_warning(old_name, new_name, stacklevel=3): +def alias_warning(old_name: str, new_name: str, stacklevel: int = 3) -> None: warnings.warn(alias_message(old_name, new_name), FutureWarning, stacklevel=stacklevel) -def alias(obj, old_name): +def alias(obj: Callable[..., T], old_name: str) -> Callable[..., T]: assert isinstance(old_name, str) @functools.wraps(obj) @@ -46,7 +59,7 @@ def wrapper(*args, **kwargs): return wrapper -def _maybe_cast_to_cftimeindex(index): +def _maybe_cast_to_cftimeindex(index: pd.Index) -> pd.Index: from ..coding.cftimeindex import CFTimeIndex if index.dtype == 'O': @@ -58,7 +71,7 @@ def _maybe_cast_to_cftimeindex(index): return index -def safe_cast_to_index(array): +def safe_cast_to_index(array: Any) -> pd.Index: """Given an array, safely cast it to a pandas.Index. If it is already a pandas.Index, return it unchanged. @@ -79,7 +92,9 @@ def safe_cast_to_index(array): return _maybe_cast_to_cftimeindex(index) -def multiindex_from_product_levels(levels, names=None): +def multiindex_from_product_levels(levels: Sequence[pd.Index], + names: Optional[Sequence[str]] = None + ) -> pd.MultiIndex: """Creating a MultiIndex from a product without refactorizing levels. Keeping levels the same gives back the original labels when we unstack. @@ -117,7 +132,7 @@ def maybe_wrap_array(original, new_array): return new_array -def equivalent(first, second): +def equivalent(first: T, second: T) -> bool: """Compare two objects for equivalence (identity or equality), using array_equiv if either object is an ndarray """ @@ -131,8 +146,8 @@ def equivalent(first, second): (pd.isnull(first) and pd.isnull(second))) -def peek_at(iterable): - """Returns the first value from iterable, as well as a new iterable with +def peek_at(iterable: Iterable[T]) -> Tuple[T, Iterator[T]]: + """Returns the first value from iterable, as well as a new iterator with the same content as the original iterable """ gen = iter(iterable) @@ -140,7 +155,9 @@ def peek_at(iterable): return peek, itertools.chain([peek], gen) -def update_safety_check(first_dict, second_dict, compat=equivalent): +def update_safety_check(first_dict: MutableMapping[K, V], + second_dict: Mapping[K, V], + compat: Callable[[V, V], bool] = equivalent) -> None: """Check the safety of updating one dictionary with another. Raises ValueError if dictionaries have non-compatible values for any key, @@ -162,7 +179,10 @@ def update_safety_check(first_dict, second_dict, compat=equivalent): 'overriding values; conflicting key %r' % k) -def remove_incompatible_items(first_dict, second_dict, compat=equivalent): +def remove_incompatible_items(first_dict: MutableMapping[K, V], + second_dict: Mapping[K, V], + compat: Callable[[V, V], bool] = equivalent + ) -> None: """Remove incompatible items from the first dictionary in-place. Items are retained if their keys are found in both dictionaries and the @@ -177,21 +197,22 @@ def remove_incompatible_items(first_dict, second_dict, compat=equivalent): checks for equivalence. """ for k in list(first_dict): - if (k not in second_dict or - (k in second_dict and - not compat(first_dict[k], second_dict[k]))): + if k not in second_dict or not compat(first_dict[k], second_dict[k]): del first_dict[k] -def is_dict_like(value): +def is_dict_like(value: Any) -> bool: return hasattr(value, 'keys') and hasattr(value, '__getitem__') -def is_full_slice(value): +def is_full_slice(value: Any) -> bool: return isinstance(value, slice) and value == slice(None) -def either_dict_or_kwargs(pos_kwargs, kw_kwargs, func_name): +def either_dict_or_kwargs(pos_kwargs: Optional[Mapping[Hashable, T]], + kw_kwargs: Mapping[str, T], + func_name: str + ) -> Mapping[Hashable, T]: if pos_kwargs is not None: if not is_dict_like(pos_kwargs): raise ValueError('the first argument to .%s must be a dictionary' @@ -201,10 +222,12 @@ def either_dict_or_kwargs(pos_kwargs, kw_kwargs, func_name): 'arguments to .%s' % func_name) return pos_kwargs else: - return kw_kwargs + # Need an explicit cast to appease mypy due to invariance; see + # https://github.com/python/mypy/issues/6228 + return cast(Mapping[Hashable, T], kw_kwargs) -def is_scalar(value): +def is_scalar(value: Any) -> bool: """Whether to treat a value as a scalar. Any non-iterable, string, or 0-D array @@ -215,7 +238,7 @@ def is_scalar(value): isinstance(value, (Iterable, ) + dask_array_type)) -def is_valid_numpy_dtype(dtype): +def is_valid_numpy_dtype(dtype: Any) -> bool: try: np.dtype(dtype) except (TypeError, ValueError): @@ -224,15 +247,17 @@ def is_valid_numpy_dtype(dtype): return True -def to_0d_object_array(value): - """Given a value, wrap it in a 0-D numpy.ndarray with dtype=object.""" +def to_0d_object_array(value: Any) -> np.ndarray: + """Given a value, wrap it in a 0-D numpy.ndarray with dtype=object. + """ result = np.empty((), dtype=object) result[()] = value return result -def to_0d_array(value): - """Given a value, wrap it in a 0-D numpy.ndarray.""" +def to_0d_array(value: Any) -> np.ndarray: + """Given a value, wrap it in a 0-D numpy.ndarray. + """ if np.isscalar(value) or (isinstance(value, np.ndarray) and value.ndim == 0): return np.array(value) @@ -240,7 +265,8 @@ def to_0d_array(value): return to_0d_object_array(value) -def dict_equiv(first, second, compat=equivalent): +def dict_equiv(first: Mapping[K, V], second: Mapping[K, V], + compat: Callable[[V, V], bool] = equivalent) -> bool: """Test equivalence of two dict-like objects. If any of the values are numpy arrays, compare them correctly. @@ -266,7 +292,10 @@ def dict_equiv(first, second, compat=equivalent): return True -def ordered_dict_intersection(first_dict, second_dict, compat=equivalent): +def ordered_dict_intersection(first_dict: Mapping[K, V], + second_dict: Mapping[K, V], + compat: Callable[[V, V], bool] = equivalent + ) -> MutableMapping[K, V]: """Return the intersection of two dictionaries as a new OrderedDict. Items are retained if their keys are found in both dictionaries and the @@ -290,11 +319,10 @@ def ordered_dict_intersection(first_dict, second_dict, compat=equivalent): return new_dict -class SingleSlotPickleMixin(object): +class SingleSlotPickleMixin: """Mixin class to add the ability to pickle objects whose state is defined by a single __slots__ attribute. Only necessary under Python 2. """ - def __getstate__(self): return getattr(self, self.__slots__[0]) @@ -302,160 +330,123 @@ def __setstate__(self, state): setattr(self, self.__slots__[0], state) -class Frozen(Mapping, SingleSlotPickleMixin): +class Frozen(Mapping[K, V], SingleSlotPickleMixin): """Wrapper around an object implementing the mapping interface to make it immutable. If you really want to modify the mapping, the mutable version is saved under the `mapping` attribute. """ __slots__ = ['mapping'] - def __init__(self, mapping): + def __init__(self, mapping: Mapping[K, V]): self.mapping = mapping - def __getitem__(self, key): + def __getitem__(self, key: K) -> V: return self.mapping[key] - def __iter__(self): + def __iter__(self) -> Iterator[K]: return iter(self.mapping) - def __len__(self): + def __len__(self) -> int: return len(self.mapping) - def __contains__(self, key): + def __contains__(self, key: object) -> bool: return key in self.mapping - def __repr__(self): + def __repr__(self) -> str: return '%s(%r)' % (type(self).__name__, self.mapping) -def FrozenOrderedDict(*args, **kwargs): +def FrozenOrderedDict(*args, **kwargs) -> Frozen: return Frozen(OrderedDict(*args, **kwargs)) -class SortedKeysDict(MutableMapping, SingleSlotPickleMixin): +class SortedKeysDict(MutableMapping[K, V], SingleSlotPickleMixin): """An wrapper for dictionary-like objects that always iterates over its items in sorted order by key but is otherwise equivalent to the underlying mapping. """ __slots__ = ['mapping'] - def __init__(self, mapping=None): + def __init__(self, mapping: Optional[MutableMapping[K, V]] = None): self.mapping = {} if mapping is None else mapping - def __getitem__(self, key): + def __getitem__(self, key: K) -> V: return self.mapping[key] - def __setitem__(self, key, value): + def __setitem__(self, key: K, value: V) -> None: self.mapping[key] = value - def __delitem__(self, key): + def __delitem__(self, key: K) -> None: del self.mapping[key] - def __iter__(self): + def __iter__(self) -> Iterator[K]: return iter(sorted(self.mapping)) - def __len__(self): + def __len__(self) -> int: return len(self.mapping) - def __contains__(self, key): + def __contains__(self, key: object) -> bool: return key in self.mapping - def __repr__(self): + def __repr__(self) -> str: return '%s(%r)' % (type(self).__name__, self.mapping) - def copy(self): - return type(self)(self.mapping.copy()) - - -class ChainMap(MutableMapping, SingleSlotPickleMixin): - """Partial backport of collections.ChainMap from Python>=3.3 - - Don't return this from any public APIs, since some of the public methods - for a MutableMapping are missing (they will raise a NotImplementedError) - """ - __slots__ = ['maps'] - - def __init__(self, *maps): - self.maps = maps - - def __getitem__(self, key): - for mapping in self.maps: - try: - return mapping[key] - except KeyError: - pass - raise KeyError(key) - def __setitem__(self, key, value): - self.maps[0][key] = value - - def __delitem__(self, value): # pragma: no cover - raise NotImplementedError - - def __iter__(self): - seen = set() - for mapping in self.maps: - for item in mapping: - if item not in seen: - yield item - seen.add(item) - - def __len__(self): - raise len(iter(self)) - - -class OrderedSet(MutableSet): +class OrderedSet(MutableSet[T]): """A simple ordered set. The API matches the builtin set, but it preserves insertion order of elements, like an OrderedDict. """ - - def __init__(self, values=None): - self._ordered_dict = OrderedDict() + def __init__(self, values: Optional[AbstractSet[T]] = None): + self._ordered_dict = OrderedDict() # type: MutableMapping[T, None] if values is not None: - self |= values + # Disable type checking - both mypy and PyCharm believes that + # we're altering the type of self in place (see signature of + # MutableSet.__ior__) + self |= values # type: ignore # Required methods for MutableSet - def __contains__(self, value): + def __contains__(self, value: object) -> bool: return value in self._ordered_dict - def __iter__(self): + def __iter__(self) -> Iterator[T]: return iter(self._ordered_dict) - def __len__(self): + def __len__(self) -> int: return len(self._ordered_dict) - def add(self, value): + def add(self, value: T) -> None: self._ordered_dict[value] = None - def discard(self, value): + def discard(self, value: T) -> None: del self._ordered_dict[value] # Additional methods - def update(self, values): - self |= values + def update(self, values: AbstractSet[T]) -> None: + # See comment on __init__ re. type checking + self |= values # type: ignore - def __repr__(self): + def __repr__(self) -> str: return '%s(%r)' % (type(self).__name__, list(self)) -class NdimSizeLenMixin(object): +class NdimSizeLenMixin: """Mixin class that extends a class that defines a ``shape`` property to one that also defines ``ndim``, ``size`` and ``__len__``. """ @property - def ndim(self): + def ndim(self: Any) -> int: return len(self.shape) @property - def size(self): + def size(self: Any) -> int: # cast to int so that shape = () gives size = 1 return int(np.prod(self.shape)) - def __len__(self): + def __len__(self: Any) -> int: try: return self.shape[0] except IndexError: @@ -470,27 +461,27 @@ class NDArrayMixin(NdimSizeLenMixin): `dtype`, `shape` and `__getitem__`. """ @property - def dtype(self): + def dtype(self: Any) -> np.dtype: return self.array.dtype @property - def shape(self): + def shape(self: Any) -> Tuple[int]: return self.array.shape - def __getitem__(self, key): + def __getitem__(self: Any, key): return self.array[key] - def __repr__(self): + def __repr__(self: Any) -> str: return '%s(array=%r)' % (type(self).__name__, self.array) -class ReprObject(object): - """Object that prints as the given value, for use with sentinel values.""" - +class ReprObject: + """Object that prints as the given value, for use with sentinel values. + """ def __init__(self, value: str): self._value = value - def __repr__(self): + def __repr__(self) -> str: return self._value @@ -506,16 +497,16 @@ def close_on_error(f): raise -def is_remote_uri(path): +def is_remote_uri(path: str) -> bool: return bool(re.search(r'^https?\://', path)) -def is_grib_path(path): +def is_grib_path(path: str) -> bool: _, ext = os.path.splitext(path) return ext in ['.grib', '.grb', '.grib2', '.grb2'] -def is_uniform_spaced(arr, **kwargs): +def is_uniform_spaced(arr, **kwargs) -> bool: """Return True if values of an array are uniformly spaced and sorted. >>> is_uniform_spaced(range(5)) @@ -527,11 +518,12 @@ def is_uniform_spaced(arr, **kwargs): """ arr = np.array(arr, dtype=float) diffs = np.diff(arr) - return np.isclose(diffs.min(), diffs.max(), **kwargs) + return bool(np.isclose(diffs.min(), diffs.max(), **kwargs)) -def hashable(v): - """Determine whether `v` can be hashed.""" +def hashable(v: Any) -> bool: + """Determine whether `v` can be hashed. + """ try: hash(v) except TypeError: @@ -543,9 +535,10 @@ def not_implemented(*args, **kwargs): return NotImplemented -def decode_numpy_dict_values(attrs): +def decode_numpy_dict_values(attrs: Mapping[K, V]) -> Dict[K, V]: """Convert attribute values from numpy objects to native Python objects, - for use in to_dict""" + for use in to_dict + """ attrs = dict(attrs) for k, v in attrs.items(): if isinstance(v, np.ndarray): @@ -565,46 +558,43 @@ def ensure_us_time_resolution(val): return val -class HiddenKeyDict(MutableMapping): - ''' - Acts like a normal dictionary, but hides certain keys. - ''' +class HiddenKeyDict(MutableMapping[K, V]): + """Acts like a normal dictionary, but hides certain keys. + """ # ``__init__`` method required to create instance from class. - def __init__(self, data, hidden_keys): + def __init__(self, data: MutableMapping[K, V], hidden_keys: Iterable[K]): self._data = data - if type(hidden_keys) not in (list, tuple): - raise TypeError("hidden_keys must be a list or tuple") - self._hidden_keys = hidden_keys + self._hidden_keys = frozenset(hidden_keys) - def _raise_if_hidden(self, key): + def _raise_if_hidden(self, key: K) -> None: if key in self._hidden_keys: raise KeyError('Key `%r` is hidden.' % key) # The next five methods are requirements of the ABC. - def __setitem__(self, key, value): + def __setitem__(self, key: K, value: V) -> None: self._raise_if_hidden(key) self._data[key] = value - def __getitem__(self, key): + def __getitem__(self, key: K) -> V: self._raise_if_hidden(key) return self._data[key] - def __delitem__(self, key): + def __delitem__(self, key: K) -> None: self._raise_if_hidden(key) del self._data[key] - def __iter__(self): + def __iter__(self) -> Iterator[K]: for k in self._data: if k not in self._hidden_keys: yield k - def __len__(self): - num_hidden = sum([k in self._hidden_keys for k in self._data]) + def __len__(self) -> int: + num_hidden = len(self._hidden_keys & self._data.keys()) return len(self._data) - num_hidden -def get_temp_dimname(dims, new_dim): +def get_temp_dimname(dims: Container[Hashable], new_dim: Hashable) -> Hashable: """ Get an new dimension name based on new_dim, that is not used in dims. If the same name exists, we add an underscore(s) in the head. @@ -618,5 +608,5 @@ def get_temp_dimname(dims, new_dim): -> ['__rolling'] """ while new_dim in dims: - new_dim = '_' + new_dim + new_dim = '_' + str(new_dim) return new_dim diff --git a/xarray/core/variable.py b/xarray/core/variable.py index d6b64e7d458..41f8795b595 100644 --- a/xarray/core/variable.py +++ b/xarray/core/variable.py @@ -1121,7 +1121,7 @@ def roll(self, shifts=None, **shifts_kwargs): result = result._roll_one_dim(dim, count) return result - def transpose(self, *dims): + def transpose(self, *dims) -> 'Variable': """Return a new Variable object with transposed dimensions. Parameters @@ -1155,6 +1155,10 @@ def transpose(self, *dims): return type(self)(dims, data, self._attrs, self._encoding, fastpath=True) + @property + def T(self) -> 'Variable': + return self.transpose() + def expand_dims(self, *args): import warnings warnings.warn('Variable.expand_dims is deprecated: use ' @@ -1902,7 +1906,8 @@ def copy(self, deep=True, data=None): Parameters ---------- deep : bool, optional - Deep is always ignored. + Deep is ignored when data is given. Whether the data array is + loaded into memory and copied onto the new object. Default is True. data : array_like, optional Data to use in the new object. Must have same shape as original. @@ -1913,7 +1918,14 @@ def copy(self, deep=True, data=None): data copied from original. """ if data is None: - data = self._data + if deep: + # self._data should be a `PandasIndexAdapter` instance at this + # point, which doesn't have a copy method, so make a deep copy + # of the underlying `pandas.MultiIndex` and create a new + # `PandasIndexAdapter` instance with it. + data = PandasIndexAdapter(self._data.array.copy(deep=True)) + else: + data = self._data else: data = as_compatible_data(data) if self.shape != data.shape: diff --git a/xarray/plot/facetgrid.py b/xarray/plot/facetgrid.py index 4f0232236f8..4a8d77d7b86 100644 --- a/xarray/plot/facetgrid.py +++ b/xarray/plot/facetgrid.py @@ -30,7 +30,7 @@ def _nicetitle(coord, value, maxchar, template): return title -class FacetGrid(object): +class FacetGrid: """ Initialize the matplotlib figure and FacetGrid object. diff --git a/xarray/plot/plot.py b/xarray/plot/plot.py index 8e2457603d6..316d4fb4dd9 100644 --- a/xarray/plot/plot.py +++ b/xarray/plot/plot.py @@ -410,7 +410,7 @@ def hist(darray, figsize=None, size=None, aspect=None, ax=None, **kwargs): # MUST run before any 2d plotting functions are defined since # _plot2d decorator adds them as methods here. -class _PlotMethods(object): +class _PlotMethods: """ Enables use of xarray.plot functions as attributes on a DataArray. For example, DataArray.plot.imshow diff --git a/xarray/plot/utils.py b/xarray/plot/utils.py index 21523ede4cd..0a507993cd6 100644 --- a/xarray/plot/utils.py +++ b/xarray/plot/utils.py @@ -265,7 +265,8 @@ def _determine_cmap_params(plot_data, vmin=None, vmax=None, cmap=None, if extend is None: extend = _determine_extend(calc_data, vmin, vmax) - if levels is not None or isinstance(norm, mpl.colors.BoundaryNorm): + if ((levels is not None or isinstance(norm, mpl.colors.BoundaryNorm)) + and (not isinstance(cmap, mpl.colors.Colormap))): cmap, newnorm = _build_discrete_cmap(cmap, levels, extend, filled) norm = newnorm if norm is None else norm diff --git a/xarray/testing.py b/xarray/testing.py index 794c0614925..eb8a0e8603d 100644 --- a/xarray/testing.py +++ b/xarray/testing.py @@ -1,8 +1,12 @@ """Testing functions exposed to the user API""" +from collections import OrderedDict + import numpy as np +import pandas as pd from xarray.core import duck_array_ops from xarray.core import formatting +from xarray.core.indexes import default_indexes def _decode_string_data(data): @@ -143,8 +147,37 @@ def assert_allclose(a, b, rtol=1e-05, atol=1e-08, decode_bytes=True): .format(type(a))) -def assert_combined_tile_ids_equal(dict1, dict2): - assert len(dict1) == len(dict2) - for k, v in dict1.items(): - assert k in dict2.keys() - assert_equal(dict1[k], dict2[k]) +def _assert_indexes_invariants_checks(indexes, possible_coord_variables, dims): + import xarray as xr + + assert isinstance(indexes, OrderedDict), indexes + assert all(isinstance(v, pd.Index) for v in indexes.values()), \ + {k: type(v) for k, v in indexes.items()} + + index_vars = {k for k, v in possible_coord_variables.items() + if isinstance(v, xr.IndexVariable)} + assert indexes.keys() <= index_vars, (set(indexes), index_vars) + + # Note: when we support non-default indexes, these checks should be opt-in + # only! + defaults = default_indexes(possible_coord_variables, dims) + assert indexes.keys() == defaults.keys(), \ + (set(indexes), set(defaults)) + assert all(v.equals(defaults[k]) for k, v in indexes.items()), \ + (indexes, defaults) + + +def _assert_indexes_invariants(a): + """Separate helper function for checking indexes invariants only.""" + import xarray as xr + + if isinstance(a, xr.DataArray): + if a._indexes is not None: + _assert_indexes_invariants_checks(a._indexes, a._coords, a.dims) + elif isinstance(a, xr.Dataset): + if a._indexes is not None: + _assert_indexes_invariants_checks( + a._indexes, a._variables, a._dims) + elif isinstance(a, xr.Variable): + # no indexes + pass diff --git a/xarray/tests/__init__.py b/xarray/tests/__init__.py index 4ebcc29a61e..e9d670e4dd9 100644 --- a/xarray/tests/__init__.py +++ b/xarray/tests/__init__.py @@ -13,8 +13,7 @@ from xarray.core import utils from xarray.core.options import set_options from xarray.core.indexing import ExplicitlyIndexed -from xarray.testing import (assert_equal, assert_identical, # noqa: F401 - assert_allclose, assert_combined_tile_ids_equal) +import xarray.testing from xarray.plot.utils import import_seaborn try: @@ -152,13 +151,13 @@ def __getitem__(self, key): raise UnexpectedDataAccess("Tried accessing data") -class ReturnItem(object): +class ReturnItem: def __getitem__(self, key): return key -class IndexerMaker(object): +class IndexerMaker: def __init__(self, indexer_cls): self._indexer_cls = indexer_cls @@ -180,3 +179,25 @@ def source_ndarray(array): if base is None: base = array return base + + +# Internal versions of xarray's test functions that validate additional +# invariants +# TODO: add more invariant checks. + +def assert_equal(a, b): + xarray.testing.assert_equal(a, b) + xarray.testing._assert_indexes_invariants(a) + xarray.testing._assert_indexes_invariants(b) + + +def assert_identical(a, b): + xarray.testing.assert_identical(a, b) + xarray.testing._assert_indexes_invariants(a) + xarray.testing._assert_indexes_invariants(b) + + +def assert_allclose(a, b, **kwargs): + xarray.testing.assert_allclose(a, b, **kwargs) + xarray.testing._assert_indexes_invariants(a) + xarray.testing._assert_indexes_invariants(b) diff --git a/xarray/tests/test_accessors.py b/xarray/tests/test_accessors.py index ae95bae3a93..6bda5772143 100644 --- a/xarray/tests/test_accessors.py +++ b/xarray/tests/test_accessors.py @@ -9,7 +9,7 @@ has_dask, raises_regex, requires_dask) -class TestDatetimeAccessor(object): +class TestDatetimeAccessor: @pytest.fixture(autouse=True) def setup(self): nt = 100 diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index a20ba2df229..f31d3bf4f9b 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -19,7 +19,7 @@ import xarray as xr from xarray import ( DataArray, Dataset, backends, open_dataarray, open_dataset, open_mfdataset, - save_mfdataset) + save_mfdataset, load_dataset, load_dataarray) from xarray.backends.common import robust_getitem from xarray.backends.netCDF4_ import _extract_nc4_variable_encoding from xarray.backends.pydap_ import PydapDataStore @@ -141,13 +141,13 @@ def create_boolean_data(): return Dataset({'x': ('t', [True, False, False, True], attributes)}) -class TestCommon(object): +class TestCommon: def test_robust_getitem(self): class UnreliableArrayFailure(Exception): pass - class UnreliableArray(object): + class UnreliableArray: def __init__(self, array, failures=1): self.array = array self.failures = failures @@ -168,11 +168,11 @@ def __getitem__(self, key): assert actual == 0 -class NetCDF3Only(object): +class NetCDF3Only: pass -class DatasetIOBase(object): +class DatasetIOBase: engine = None # type: Optional[str] file_format = None # type: Optional[str] @@ -1391,7 +1391,7 @@ def test_auto_chunk(self): original = create_test_data().chunk() with self.roundtrip( - original, open_kwargs={'auto_chunk': False}) as actual: + original, open_kwargs={'chunks': None}) as actual: for k, v in actual.variables.items(): # only index variables should be in memory assert v._in_memory == (k in actual.dims) @@ -1399,19 +1399,101 @@ def test_auto_chunk(self): assert v.chunks is None with self.roundtrip( - original, open_kwargs={'auto_chunk': True}) as actual: + original, open_kwargs={'chunks': 'auto'}) as actual: for k, v in actual.variables.items(): # only index variables should be in memory assert v._in_memory == (k in actual.dims) # chunk size should be the same as original assert v.chunks == original[k].chunks + def test_manual_chunk(self): + original = create_test_data().chunk({'dim1': 3, 'dim2': 4, 'dim3': 3}) + + # All of these should return non-chunked arrays + NO_CHUNKS = (None, 0, {}) + for no_chunk in NO_CHUNKS: + open_kwargs = {'chunks': no_chunk} + with self.roundtrip(original, open_kwargs=open_kwargs) as actual: + for k, v in actual.variables.items(): + # only index variables should be in memory + assert v._in_memory == (k in actual.dims) + # there should be no chunks + assert v.chunks is None + + # uniform arrays + for i in range(2, 6): + rechunked = original.chunk(chunks=i) + open_kwargs = {'chunks': i} + with self.roundtrip(original, open_kwargs=open_kwargs) as actual: + for k, v in actual.variables.items(): + # only index variables should be in memory + assert v._in_memory == (k in actual.dims) + # chunk size should be the same as rechunked + assert v.chunks == rechunked[k].chunks + + chunks = {'dim1': 2, 'dim2': 3, 'dim3': 5} + rechunked = original.chunk(chunks=chunks) + + open_kwargs = {'chunks': chunks, 'overwrite_encoded_chunks': True} + with self.roundtrip(original, open_kwargs=open_kwargs) as actual: + for k, v in actual.variables.items(): + assert v.chunks == rechunked[k].chunks + + with self.roundtrip(actual) as auto: + # encoding should have changed + for k, v in actual.variables.items(): + assert v.chunks == rechunked[k].chunks + + assert_identical(actual, auto) + assert_identical(actual.load(), auto.load()) + + def test_warning_on_bad_chunks(self): + original = create_test_data().chunk({'dim1': 4, 'dim2': 3, 'dim3': 5}) + + bad_chunks = (2, {'dim2': (3, 3, 2, 1)}) + for chunks in bad_chunks: + kwargs = {'chunks': chunks} + with pytest.warns(UserWarning): + with self.roundtrip(original, open_kwargs=kwargs) as actual: + for k, v in actual.variables.items(): + # only index variables should be in memory + assert v._in_memory == (k in actual.dims) + + good_chunks = ({'dim2': 3}, {'dim3': 10}) + for chunks in good_chunks: + kwargs = {'chunks': chunks} + with pytest.warns(None) as record: + with self.roundtrip(original, open_kwargs=kwargs) as actual: + for k, v in actual.variables.items(): + # only index variables should be in memory + assert v._in_memory == (k in actual.dims) + assert len(record) == 0 + + def test_deprecate_auto_chunk(self): + original = create_test_data().chunk() + with pytest.warns(FutureWarning): + with self.roundtrip( + original, open_kwargs={'auto_chunk': True}) as actual: + for k, v in actual.variables.items(): + # only index variables should be in memory + assert v._in_memory == (k in actual.dims) + # chunk size should be the same as original + assert v.chunks == original[k].chunks + + with pytest.warns(FutureWarning): + with self.roundtrip( + original, open_kwargs={'auto_chunk': False}) as actual: + for k, v in actual.variables.items(): + # only index variables should be in memory + assert v._in_memory == (k in actual.dims) + # there should be no chunks + assert v.chunks is None + def test_write_uneven_dask_chunks(self): # regression for GH#2225 original = create_test_data().chunk({'dim1': 3, 'dim2': 4, 'dim3': 3}) - with self.roundtrip( - original, open_kwargs={'auto_chunk': True}) as actual: + original, open_kwargs={'chunks': 'auto'}) as actual: for k, v in actual.data_vars.items(): print(k) assert v.chunks == actual[k].chunks @@ -2114,7 +2196,7 @@ def test_open_mfdataset_manyfiles(readengine, nfiles, parallel, chunks, @requires_scipy_or_netCDF4 -class TestOpenMFDatasetWithDataVarsAndCoordsKw(object): +class TestOpenMFDatasetWithDataVarsAndCoordsKw: coord_name = 'lon' var_name = 'v1' @@ -2559,10 +2641,27 @@ def test_save_mfdataset_compute_false_roundtrip(self): with open_mfdataset([tmp1, tmp2]) as actual: assert_identical(actual, original) + def test_load_dataset(self): + with create_tmp_file() as tmp: + original = Dataset({'foo': ('x', np.random.randn(10))}) + original.to_netcdf(tmp) + ds = load_dataset(tmp) + # this would fail if we used open_dataset instead of load_dataset + ds.to_netcdf(tmp) + + def test_load_dataarray(self): + with create_tmp_file() as tmp: + original = Dataset({'foo': ('x', np.random.randn(10))}) + original.to_netcdf(tmp) + ds = load_dataarray(tmp) + # this would fail if we used open_dataarray instead of + # load_dataarray + ds.to_netcdf(tmp) + @requires_scipy_or_netCDF4 @requires_pydap -class TestPydap(object): +class TestPydap: def convert_to_pydap_dataset(self, original): from pydap.model import GridType, BaseType, DatasetType ds = DatasetType('bears', **original.attrs) @@ -2695,7 +2794,7 @@ def test_weakrefs(self): @requires_cfgrib -class TestCfGrib(object): +class TestCfGrib: def test_read(self): expected = {'number': 2, 'time': 3, 'isobaricInhPa': 2, 'latitude': 3, @@ -2718,7 +2817,7 @@ def test_read_filter_by_keys(self): @requires_pseudonetcdf @pytest.mark.filterwarnings('ignore:IOAPI_ISPH is assumed to be 6370000') -class TestPseudoNetCDFFormat(object): +class TestPseudoNetCDFFormat: def open(self, path, **kwargs): return open_dataset(path, engine='pseudonetcdf', **kwargs) @@ -2981,7 +3080,7 @@ def create_tmp_geotiff(nx=4, ny=3, nz=3, @requires_rasterio -class TestRasterio(object): +class TestRasterio: @requires_scipy_or_netCDF4 def test_serialization(self): @@ -3350,6 +3449,33 @@ def test_rasterio_vrt(self): assert actual_shape == expected_shape assert expected_val.all() == actual_val.all() + def test_rasterio_vrt_with_transform_and_size(self): + # Test open_rasterio() support of WarpedVRT with transform, width and + # height (issue #2864) + import rasterio + from rasterio.warp import calculate_default_transform + from affine import Affine + with create_tmp_geotiff() as (tmp_file, expected): + with rasterio.open(tmp_file) as src: + # Estimate the transform, width and height + # for a change of resolution + # tmp_file initial res is (1000,2000) (default values) + trans, w, h = calculate_default_transform( + src.crs, src.crs, src.width, src.height, + resolution=500, *src.bounds) + with rasterio.vrt.WarpedVRT(src, transform=trans, + width=w, height=h) as vrt: + expected_shape = (vrt.width, vrt.height) + expected_res = vrt.res + expected_transform = vrt.transform + with xr.open_rasterio(vrt) as da: + actual_shape = (da.sizes['x'], da.sizes['y']) + actual_res = da.res + actual_transform = Affine(*da.transform) + assert actual_res == expected_res + assert actual_shape == expected_shape + assert actual_transform == expected_transform + @network def test_rasterio_vrt_network(self): import rasterio @@ -3383,7 +3509,7 @@ def test_rasterio_vrt_network(self): assert_equal(expected_val, actual_val) -class TestEncodingInvalid(object): +class TestEncodingInvalid: def test_extract_nc4_variable_encoding(self): var = xr.Variable(('x',), [1, 2, 3], {}, {'foo': 'bar'}) @@ -3412,7 +3538,7 @@ class MiscObject: @requires_netCDF4 -class TestValidateAttrs(object): +class TestValidateAttrs: def test_validating_attrs(self): def new_dataset(): return Dataset({'data': ('y', np.arange(10.0))}, @@ -3502,7 +3628,7 @@ def new_dataset_and_coord_attrs(): @requires_scipy_or_netCDF4 -class TestDataArrayToNetCDF(object): +class TestDataArrayToNetCDF: def test_dataarray_to_netcdf_no_name(self): original_da = DataArray(np.arange(12).reshape((3, 4))) diff --git a/xarray/tests/test_cftimeindex_resample.py b/xarray/tests/test_cftimeindex_resample.py index 636f9ef7b0e..7aca4492680 100644 --- a/xarray/tests/test_cftimeindex_resample.py +++ b/xarray/tests/test_cftimeindex_resample.py @@ -6,31 +6,47 @@ import xarray as xr from xarray.core.resample_cftime import CFTimeGrouper + pytest.importorskip('cftime') pytest.importorskip('pandas', minversion='0.24') -@pytest.fixture( - params=[ - dict(start='2004-01-01T12:07:01', periods=91, freq='3D'), - dict(start='1892-01-03T12:07:01', periods=15, freq='41987T'), - dict(start='2004-01-01T12:07:01', periods=7, freq='3Q-AUG'), - dict(start='1892-01-03T12:07:01', periods=10, freq='3AS-JUN') - ], - ids=['3D', '41987T', '3Q_AUG', '3AS_JUN'] -) -def time_range_kwargs(request): - return request.param - - -@pytest.fixture() -def datetime_index(time_range_kwargs): - return pd.date_range(**time_range_kwargs) - - -@pytest.fixture() -def cftime_index(time_range_kwargs): - return xr.cftime_range(**time_range_kwargs) +# Create a list of pairs of similar-length initial and resample frequencies +# that cover: +# - Resampling from shorter to longer frequencies +# - Resampling from longer to shorter frequencies +# - Resampling from one initial frequency to another. +# These are used to test the cftime version of resample against pandas +# with a standard calendar. +FREQS = [ + ('8003D', '4001D'), + ('8003D', '16006D'), + ('8003D', '21AS'), + ('6H', '3H'), + ('6H', '12H'), + ('6H', '400T'), + ('3D', 'D'), + ('3D', '6D'), + ('11D', 'MS'), + ('3MS', 'MS'), + ('3MS', '6MS'), + ('3MS', '85D'), + ('7M', '3M'), + ('7M', '14M'), + ('7M', '2QS-APR'), + ('43QS-AUG', '21QS-AUG'), + ('43QS-AUG', '86QS-AUG'), + ('43QS-AUG', '11A-JUN'), + ('11Q-JUN', '5Q-JUN'), + ('11Q-JUN', '22Q-JUN'), + ('11Q-JUN', '51MS'), + ('3AS-MAR', 'AS-MAR'), + ('3AS-MAR', '6AS-MAR'), + ('3AS-MAR', '14Q-FEB'), + ('7A-MAY', '3A-MAY'), + ('7A-MAY', '14A-MAY'), + ('7A-MAY', '85M') +] def da(index): @@ -38,65 +54,31 @@ def da(index): coords=[index], dims=['time']) -@pytest.mark.parametrize('freq', [ - '700T', '8001T', - '12H', '8001H', - '8D', '8001D', - '2MS', '3MS', - '2QS-AUG', '3QS-SEP', - '3AS-MAR', '4A-MAY']) -@pytest.mark.parametrize('closed', [None, 'right']) -@pytest.mark.parametrize('label', [None, 'right']) +@pytest.mark.parametrize('freqs', FREQS, ids=lambda x: '{}->{}'.format(*x)) +@pytest.mark.parametrize('closed', [None, 'left', 'right']) +@pytest.mark.parametrize('label', [None, 'left', 'right']) @pytest.mark.parametrize('base', [24, 31]) -def test_resampler(freq, closed, label, base, - datetime_index, cftime_index): - # Fairly extensive testing for standard/proleptic Gregorian calendar - # For any frequencies which are not greater-than-day and anchored - # at the end, the default values for closed and label are 'left'. - loffset = '12H' - try: - da_datetime = da(datetime_index).resample( - time=freq, closed=closed, label=label, base=base, - loffset=loffset).mean() - except ValueError: - with pytest.raises(ValueError): - da(cftime_index).resample( - time=freq, closed=closed, label=label, base=base, - loffset=loffset).mean() - else: - da_cftime = da(cftime_index).resample(time=freq, closed=closed, - label=label, base=base, - loffset=loffset).mean() - da_cftime['time'] = da_cftime.indexes['time'].to_datetimeindex() - xr.testing.assert_identical(da_cftime, da_datetime) - +def test_resample(freqs, closed, label, base): + initial_freq, resample_freq = freqs + start = '2000-01-01T12:07:01' + index_kwargs = dict(start=start, periods=5, freq=initial_freq) + datetime_index = pd.date_range(**index_kwargs) + cftime_index = xr.cftime_range(**index_kwargs) -@pytest.mark.parametrize('freq', [ - '2M', '3M', - '2Q-JUN', '3Q-JUL', - '3A-FEB', '4A-APR']) -@pytest.mark.parametrize('closed', ['left', None]) -@pytest.mark.parametrize('label', ['left', None]) -@pytest.mark.parametrize('base', [17, 24]) -def test_resampler_end_super_day(freq, closed, label, base, - datetime_index, cftime_index): - # Fairly extensive testing for standard/proleptic Gregorian calendar. - # For greater-than-day frequencies anchored at the end, the default values - # for closed and label are 'right'. loffset = '12H' try: da_datetime = da(datetime_index).resample( - time=freq, closed=closed, label=label, base=base, + time=resample_freq, closed=closed, label=label, base=base, loffset=loffset).mean() except ValueError: with pytest.raises(ValueError): da(cftime_index).resample( - time=freq, closed=closed, label=label, base=base, + time=resample_freq, closed=closed, label=label, base=base, loffset=loffset).mean() else: - da_cftime = da(cftime_index).resample(time=freq, closed=closed, - label=label, base=base, - loffset=loffset).mean() + da_cftime = da(cftime_index).resample( + time=resample_freq, closed=closed, + label=label, base=base, loffset=loffset).mean() da_cftime['time'] = da_cftime.indexes['time'].to_datetimeindex() xr.testing.assert_identical(da_cftime, da_datetime) @@ -111,6 +93,7 @@ def test_closed_label_defaults(freq, expected): assert CFTimeGrouper(freq=freq).label == expected +@pytest.mark.filterwarnings('ignore:Converting a CFTimeIndex') @pytest.mark.parametrize('calendar', ['gregorian', 'noleap', 'all_leap', '360_day', 'julian']) def test_calendars(calendar): diff --git a/xarray/tests/test_coding_strings.py b/xarray/tests/test_coding_strings.py index c50376a5841..98824c9136c 100644 --- a/xarray/tests/test_coding_strings.py +++ b/xarray/tests/test_coding_strings.py @@ -107,6 +107,24 @@ def test_CharacterArrayCoder_encode(data): assert_identical(actual, expected) +@pytest.mark.parametrize( + ['original', 'expected_char_dim_name'], + [ + (Variable(('x',), [b'ab', b'cdef']), + 'string4'), + (Variable(('x',), [b'ab', b'cdef'], encoding={'char_dim_name': 'foo'}), + 'foo') + ] +) +def test_CharacterArrayCoder_char_dim_name(original, expected_char_dim_name): + coder = strings.CharacterArrayCoder() + encoded = coder.encode(original) + roundtripped = coder.decode(encoded) + assert encoded.dims[-1] == expected_char_dim_name + assert roundtripped.encoding['char_dim_name'] == expected_char_dim_name + assert roundtripped.dims[-1] == original.dims[-1] + + def test_StackedBytesArray(): array = np.array([[b'a', b'b', b'c'], [b'd', b'e', b'f']], dtype='S') actual = strings.StackedBytesArray(array) diff --git a/xarray/tests/test_combine.py b/xarray/tests/test_combine.py index 0d03b6e0cdf..1d8ed169d29 100644 --- a/xarray/tests/test_combine.py +++ b/xarray/tests/test_combine.py @@ -13,12 +13,12 @@ _infer_tile_ids_from_nested_list, _new_tile_id) from . import ( - InaccessibleArray, assert_array_equal, assert_combined_tile_ids_equal, + InaccessibleArray, assert_array_equal, assert_equal, assert_identical, raises_regex, requires_dask) from .test_dataset import create_test_data -class TestConcatDataset(object): +class TestConcatDataset: def test_concat(self): # TODO: simplify and split this test case @@ -238,7 +238,7 @@ def test_concat_multiindex(self): assert isinstance(actual.x.to_index(), pd.MultiIndex) -class TestConcatDataArray(object): +class TestConcatDataArray: def test_concat(self): ds = Dataset({'foo': (['x', 'y'], np.random.random((2, 3))), 'bar': (['x', 'y'], np.random.random((2, 3)))}, @@ -307,7 +307,7 @@ def test_concat_lazy(self): assert combined.dims == ('z', 'x', 'y') -class TestAutoCombine(object): +class TestAutoCombine: @pytest.mark.parametrize("combine", [_auto_combine_1d, auto_combine]) @requires_dask # only for toolz @@ -418,7 +418,14 @@ def test_auto_combine_no_concat(self): assert_identical(expected, actual) -class TestTileIDsFromNestedList(object): +def assert_combined_tile_ids_equal(dict1, dict2): + assert len(dict1) == len(dict2) + for k, v in dict1.items(): + assert k in dict2.keys() + assert_equal(dict1[k], dict2[k]) + + +class TestTileIDsFromNestedList: def test_1d(self): ds = create_test_data input = [ds(0), ds(1)] @@ -526,7 +533,7 @@ def _create_tile_ids(shape): @requires_dask # only for toolz -class TestCombineND(object): +class TestCombineND: @pytest.mark.parametrize("old_id, new_id", [((3, 0, 1), (0, 1)), ((0, 0), (0,)), ((1,), ()), @@ -591,7 +598,7 @@ def test_concat_twice(self, create_combined_ids, concat_dim): assert_equal(result, expected) -class TestCheckShapeTileIDs(object): +class TestCheckShapeTileIDs: def test_check_depths(self): ds = create_test_data(0) combined_tile_ids = {(0,): ds, (0, 1): ds} @@ -609,7 +616,7 @@ def test_check_lengths(self): @requires_dask # only for toolz -class TestAutoCombineND(object): +class TestAutoCombineND: def test_single_dataset(self): objs = [Dataset({'x': [0]}), Dataset({'x': [1]})] actual = auto_combine(objs) @@ -705,7 +712,7 @@ def test_combine_concat_over_redundant_nesting(self): assert_identical(expected, actual) -class TestAutoCombineUsingCoords(object): +class TestAutoCombineUsingCoords: def test_order_inferred_from_coords(self): data = create_test_data() objs = [data.isel(dim2=slice(4, 9)), data.isel(dim2=slice(4))] diff --git a/xarray/tests/test_computation.py b/xarray/tests/test_computation.py index 1003c531018..e3f49fd8d07 100644 --- a/xarray/tests/test_computation.py +++ b/xarray/tests/test_computation.py @@ -42,7 +42,7 @@ def test_signature_properties(): def test_result_name(): - class Named(object): + class Named: def __init__(self, name=None): self.name = name diff --git a/xarray/tests/test_conventions.py b/xarray/tests/test_conventions.py index 27f5e7ec079..b9690c211f4 100644 --- a/xarray/tests/test_conventions.py +++ b/xarray/tests/test_conventions.py @@ -19,7 +19,7 @@ from .test_backends import CFEncodedBase -class TestBoolTypeArray(object): +class TestBoolTypeArray: def test_booltype_array(self): x = np.array([1, 0, 1, 1, 0], dtype='i1') bx = conventions.BoolTypeArray(x) @@ -28,7 +28,7 @@ def test_booltype_array(self): dtype=np.bool)) -class TestNativeEndiannessArray(object): +class TestNativeEndiannessArray: def test(self): x = np.arange(5, dtype='>i8') expected = np.arange(5, dtype='int64') @@ -67,7 +67,7 @@ def test_decode_cf_with_conflicting_fill_missing_value(): @requires_cftime_or_netCDF4 -class TestEncodeCFVariable(object): +class TestEncodeCFVariable: def test_incompatible_attributes(self): invalid_vars = [ Variable(['t'], pd.date_range('2000-01-01', periods=3), @@ -132,7 +132,7 @@ def test_string_object_warning(self): @requires_cftime_or_netCDF4 -class TestDecodeCF(object): +class TestDecodeCF: def test_dataset(self): original = Dataset({ 't': ('t', [0, 1, 2], {'units': 'days since 2000-01-01'}), diff --git a/xarray/tests/test_dask.py b/xarray/tests/test_dask.py index b6a70794c23..0c55fe919d6 100644 --- a/xarray/tests/test_dask.py +++ b/xarray/tests/test_dask.py @@ -22,7 +22,7 @@ dd = pytest.importorskip('dask.dataframe') -class DaskTestCase(object): +class DaskTestCase: def assertLazyAnd(self, expected, actual, test): with (dask.config.set(scheduler='single-threaded') @@ -586,7 +586,7 @@ def test_from_dask_variable(self): self.assertLazyAndIdentical(self.lazy_array, a) -class TestToDaskDataFrame(object): +class TestToDaskDataFrame: def test_to_dask_dataframe(self): # Test conversion of Datasets to dask DataFrames diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index 4975071dad8..9471ec144c0 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -3,6 +3,7 @@ from collections import OrderedDict from copy import deepcopy from textwrap import dedent +import sys import numpy as np import pandas as pd @@ -22,7 +23,7 @@ requires_scipy, source_ndarray) -class TestDataArray(object): +class TestDataArray: @pytest.fixture(autouse=True) def setup(self): self.attrs = {'attr1': 'value1', 'attr2': 2929} @@ -500,6 +501,14 @@ def test_getitem_dataarray(self): assert_equal(da[ind], da[[0, 1]]) assert_equal(da[ind], da[ind.values]) + def test_getitem_empty_index(self): + da = DataArray(np.arange(12).reshape((3, 4)), dims=['x', 'y']) + assert_identical(da[{'x': []}], + DataArray(np.zeros((0, 4)), dims=['x', 'y'])) + assert_identical(da.loc[{'y': []}], + DataArray(np.zeros((3, 0)), dims=['x', 'y'])) + assert_identical(da[[]], DataArray(np.zeros((0, 4)), dims=['x', 'y'])) + def test_setitem(self): # basic indexing should work as numpy's indexing tuples = [(0, 0), (0, slice(None, None)), @@ -1250,6 +1259,18 @@ def test_reindex_like_no_index(self): ValueError, 'different size for unlabeled'): foo.reindex_like(bar) + @pytest.mark.parametrize('fill_value', [dtypes.NA, 2, 2.0]) + def test_reindex_fill_value(self, fill_value): + foo = DataArray([10, 20], dims='y', coords={'y': [0, 1]}) + bar = DataArray([10, 20, 30], dims='y', coords={'y': [0, 1, 2]}) + if fill_value == dtypes.NA: + # if we supply the default, we expect the missing value for a + # float array + fill_value = np.nan + actual = x.reindex_like(bar, fill_value=fill_value) + expected = DataArray([10, 20, fill_value], coords=[('y', [0, 1, 2])]) + assert_identical(expected, actual) + @pytest.mark.filterwarnings('ignore:Indexer has dimensions') def test_reindex_regressions(self): # regression test for #279 @@ -1277,6 +1298,18 @@ def test_reindex_method(self): expected = DataArray([10, 20, np.nan], coords=[('y', y)]) assert_identical(expected, actual) + @pytest.mark.parametrize('fill_value', [dtypes.NA, 2, 2.0]) + def test_reindex_fill_value(self, fill_value): + x = DataArray([10, 20], dims='y', coords={'y': [0, 1]}) + y = [0, 1, 2] + if fill_value == dtypes.NA: + # if we supply the default, we expect the missing value for a + # float array + fill_value = np.nan + actual = x.reindex(y=y, fill_value=fill_value) + expected = DataArray([10, 20, fill_value], coords=[('y', y)]) + assert_identical(expected, actual) + def test_rename(self): renamed = self.dv.rename('bar') assert_identical( @@ -1303,7 +1336,7 @@ def test_expand_dims_error(self): coords={'x': np.linspace(0.0, 1.0, 3)}, attrs={'key': 'entry'}) - with raises_regex(ValueError, 'dim should be str or'): + with raises_regex(TypeError, 'dim should be str or'): array.expand_dims(0) with raises_regex(ValueError, 'lengths of dim and axis'): # dims and axis argument should be the same length @@ -1328,6 +1361,16 @@ def test_expand_dims_error(self): array.expand_dims(dim=['y', 'z'], axis=[2, -4]) array.expand_dims(dim=['y', 'z'], axis=[2, 3]) + array = DataArray(np.random.randn(3, 4), dims=['x', 'dim_0'], + coords={'x': np.linspace(0.0, 1.0, 3)}, + attrs={'key': 'entry'}) + with pytest.raises(TypeError): + array.expand_dims(OrderedDict((("new_dim", 3.2),))) + + # Attempt to use both dim and kwargs + with pytest.raises(ValueError): + array.expand_dims(OrderedDict((("d", 4),)), e=4) + def test_expand_dims(self): array = DataArray(np.random.randn(3, 4), dims=['x', 'dim_0'], coords={'x': np.linspace(0.0, 1.0, 3)}, @@ -1392,6 +1435,46 @@ def test_expand_dims_with_scalar_coordinate(self): roundtripped = actual.squeeze(['z'], drop=False) assert_identical(array, roundtripped) + def test_expand_dims_with_greater_dim_size(self): + array = DataArray(np.random.randn(3, 4), dims=['x', 'dim_0'], + coords={'x': np.linspace(0.0, 1.0, 3), 'z': 1.0}, + attrs={'key': 'entry'}) + # For python 3.5 and earlier this has to be an ordered dict, to + # maintain insertion order. + actual = array.expand_dims( + OrderedDict((('y', 2), ('z', 1), ('dim_1', ['a', 'b', 'c'])))) + + expected_coords = OrderedDict(( + ('y', [0, 1]), ('z', [1.0]), ('dim_1', ['a', 'b', 'c']), + ('x', np.linspace(0, 1, 3)), ('dim_0', range(4)))) + expected = DataArray(array.values * np.ones([2, 1, 3, 3, 4]), + coords=expected_coords, + dims=list(expected_coords.keys()), + attrs={'key': 'entry'} + ).drop(['y', 'dim_0']) + assert_identical(expected, actual) + + # Test with kwargs instead of passing dict to dim arg. + + # TODO: only the code under the if-statement is needed when python 3.5 + # is no longer supported. + python36_plus = sys.version_info[0] == 3 and sys.version_info[1] > 5 + if python36_plus: + other_way = array.expand_dims(dim_1=['a', 'b', 'c']) + + other_way_expected = DataArray( + array.values * np.ones([3, 3, 4]), + coords={'dim_1': ['a', 'b', 'c'], + 'x': np.linspace(0, 1, 3), + 'dim_0': range(4), 'z': 1.0}, + dims=['dim_1', 'x', 'dim_0'], + attrs={'key': 'entry'}).drop('dim_0') + assert_identical(other_way_expected, other_way) + else: + # In python 3.5, using dim_kwargs should raise a ValueError. + with raises_regex(ValueError, "dim_kwargs isn't"): + array.expand_dims(e=["l", "m", "n"]) + def test_set_index(self): indexes = [self.mindex.get_level_values(n) for n in self.mindex.names] coords = {idx.name: ('x', idx) for idx in indexes} @@ -3238,6 +3321,32 @@ def test_copy_with_data(self): expected.data = new_data assert_identical(expected, actual) + @pytest.mark.xfail(raises=AssertionError) + @pytest.mark.parametrize('deep, expected_orig', [ + [True, + xr.DataArray(xr.IndexVariable('a', np.array([1, 2])), + coords={'a': [1, 2]}, dims=['a'])], + [False, + xr.DataArray(xr.IndexVariable('a', np.array([999, 2])), + coords={'a': [999, 2]}, dims=['a'])]]) + def test_copy_coords(self, deep, expected_orig): + """The test fails for the shallow copy, and apparently only on Windows + for some reason. In windows coords seem to be immutable unless it's one + dataarray deep copied from another.""" + da = xr.DataArray( + np.ones([2, 2, 2]), + coords={'a': [1, 2], 'b': ['x', 'y'], 'c': [0, 1]}, + dims=['a', 'b', 'c']) + da_cp = da.copy(deep) + da_cp['a'].data[0] = 999 + + expected_cp = xr.DataArray( + xr.IndexVariable('a', np.array([999, 2])), + coords={'a': [999, 2]}, dims=['a']) + assert_identical(da_cp['a'], expected_cp) + + assert_identical(da['a'], expected_orig) + def test_real_and_imag(self): array = DataArray(1 + 2j) assert_identical(array.real, DataArray(1)) @@ -3535,6 +3644,7 @@ def test_rolling_wrapped_bottleneck(da, name, center, min_periods): @pytest.mark.parametrize('center', (True, False, None)) @pytest.mark.parametrize('min_periods', (1, None)) @pytest.mark.parametrize('window', (7, 8)) +@pytest.mark.xfail(reason='https://github.com/pydata/xarray/issues/2940') def test_rolling_wrapped_dask(da_dask, name, center, min_periods, window): pytest.importorskip('dask.array') # dask version @@ -3703,7 +3813,7 @@ def test_name_in_masking(): assert da.where((da > 5).rename('YokoOno'), drop=True).name == name -class TestIrisConversion(object): +class TestIrisConversion: @requires_iris def test_to_and_from_iris(self): import iris diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index 777a8e84a3f..b47e26328ad 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -81,7 +81,7 @@ def lazy_inaccessible(k, v): k, v in self._variables.items()) -class TestDataset(object): +class TestDataset: def test_repr(self): data = create_test_data(seed=123) data.attrs['foo'] = 'bar' @@ -267,7 +267,7 @@ def test_constructor_0d(self): actual = Dataset({'x': arg}) assert_identical(expected, actual) - class Arbitrary(object): + class Arbitrary: pass d = pd.Timestamp('2000-01-01T12') @@ -1619,6 +1619,54 @@ def test_reindex_method(self): actual = ds.reindex_like(alt, method='pad') assert_identical(expected, actual) + @pytest.mark.parametrize('fill_value', [dtypes.NA, 2, 2.0]) + def test_reindex_fill_value(self, fill_value): + ds = Dataset({'x': ('y', [10, 20]), 'y': [0, 1]}) + y = [0, 1, 2] + actual = ds.reindex(y=y, fill_value=fill_value) + if fill_value == dtypes.NA: + # if we supply the default, we expect the missing value for a + # float array + fill_value = np.nan + expected = Dataset({'x': ('y', [10, 20, fill_value]), 'y': y}) + assert_identical(expected, actual) + + @pytest.mark.parametrize('fill_value', [dtypes.NA, 2, 2.0]) + def test_reindex_like_fill_value(self, fill_value): + ds = Dataset({'x': ('y', [10, 20]), 'y': [0, 1]}) + y = [0, 1, 2] + alt = Dataset({'y': y}) + actual = ds.reindex_like(alt, fill_value=fill_value) + if fill_value == dtypes.NA: + # if we supply the default, we expect the missing value for a + # float array + fill_value = np.nan + expected = Dataset({'x': ('y', [10, 20, fill_value]), 'y': y}) + assert_identical(expected, actual) + + @pytest.mark.parametrize('fill_value', [dtypes.NA, 2, 2.0]) + def test_align_fill_value(self, fill_value): + x = Dataset({'foo': DataArray([1, 2], dims=['x'], + coords={'x': [1, 2]})}) + y = Dataset({'bar': DataArray([1, 2], dims=['x'], + coords={'x': [1, 3]})}) + x2, y2 = align(x, y, join='outer', fill_value=fill_value) + if fill_value == dtypes.NA: + # if we supply the default, we expect the missing value for a + # float array + fill_value = np.nan + + expected_x2 = Dataset( + {'foo': DataArray([1, 2, fill_value], + dims=['x'], + coords={'x': [1, 2, 3]})}) + expected_y2 = Dataset( + {'bar': DataArray([1, fill_value, 2], + dims=['x'], + coords={'x': [1, 2, 3]})}) + assert_identical(expected_x2, x2) + assert_identical(expected_y2, y2) + def test_align(self): left = create_test_data() right = left.copy(deep=True) @@ -1885,6 +1933,7 @@ def test_drop_dims(self): def test_copy(self): data = create_test_data() + data.attrs['Test'] = [1, 2, 3] for copied in [data.copy(deep=False), copy(data)]: assert_identical(data, copied) @@ -1899,12 +1948,18 @@ def test_copy(self): copied['foo'] = ('z', np.arange(5)) assert 'foo' not in data + copied.attrs['foo'] = 'bar' + assert 'foo' not in data.attrs + assert data.attrs['Test'] is copied.attrs['Test'] + for copied in [data.copy(deep=True), deepcopy(data)]: assert_identical(data, copied) for k, v0 in data.variables.items(): v1 = copied.variables[k] assert v0 is not v1 + assert data.attrs['Test'] is not copied.attrs['Test'] + def test_copy_with_data(self): orig = create_test_data() new_data = {k: np.random.randn(*v.shape) @@ -1916,6 +1971,33 @@ def test_copy_with_data(self): expected[k].data = v assert_identical(expected, actual) + @pytest.mark.xfail(raises=AssertionError) + @pytest.mark.parametrize('deep, expected_orig', [ + [True, + xr.DataArray(xr.IndexVariable('a', np.array([1, 2])), + coords={'a': [1, 2]}, dims=['a'])], + [False, + xr.DataArray(xr.IndexVariable('a', np.array([999, 2])), + coords={'a': [999, 2]}, dims=['a'])]]) + def test_copy_coords(self, deep, expected_orig): + """The test fails for the shallow copy, and apparently only on Windows + for some reason. In windows coords seem to be immutable unless it's one + dataset deep copied from another.""" + ds = xr.DataArray( + np.ones([2, 2, 2]), + coords={'a': [1, 2], 'b': ['x', 'y'], 'c': [0, 1]}, + dims=['a', 'b', 'c'], + name='value').to_dataset() + ds_cp = ds.copy(deep=deep) + ds_cp.coords['a'].data[0] = 999 + + expected_cp = xr.DataArray( + xr.IndexVariable('a', np.array([999, 2])), + coords={'a': [999, 2]}, dims=['a']) + assert_identical(ds_cp.coords['a'], expected_cp) + + assert_identical(ds.coords['a'], expected_orig) + def test_copy_with_data_errors(self): orig = create_test_data() new_var1 = np.arange(orig['var1'].size).reshape(orig['var1'].shape) @@ -2030,7 +2112,24 @@ def test_expand_dims_error(self): with raises_regex(ValueError, 'already exists'): original.expand_dims(dim=['z']) - def test_expand_dims(self): + original = Dataset({'x': ('a', np.random.randn(3)), + 'y': (['b', 'a'], np.random.randn(4, 3)), + 'z': ('a', np.random.randn(3))}, + coords={'a': np.linspace(0, 1, 3), + 'b': np.linspace(0, 1, 4), + 'c': np.linspace(0, 1, 5)}, + attrs={'key': 'entry'}) + with raises_regex(TypeError, 'value of new dimension'): + original.expand_dims(OrderedDict((("d", 3.2),))) + + # TODO: only the code under the if-statement is needed when python 3.5 + # is no longer supported. + python36_plus = sys.version_info[0] == 3 and sys.version_info[1] > 5 + if python36_plus: + with raises_regex(ValueError, 'both keyword and positional'): + original.expand_dims(OrderedDict((("d", 4),)), e=4) + + def test_expand_dims_int(self): original = Dataset({'x': ('a', np.random.randn(3)), 'y': (['b', 'a'], np.random.randn(4, 3))}, coords={'a': np.linspace(0, 1, 3), @@ -2063,6 +2162,92 @@ def test_expand_dims(self): roundtripped = actual.squeeze('z') assert_identical(original, roundtripped) + def test_expand_dims_coords(self): + original = Dataset({'x': ('a', np.array([1, 2, 3]))}) + expected = Dataset( + {'x': (('b', 'a'), np.array([[1, 2, 3], [1, 2, 3]]))}, + coords={'b': [1, 2]}, + ) + actual = original.expand_dims(OrderedDict(b=[1, 2])) + assert_identical(expected, actual) + assert 'b' not in original._coord_names + + def test_expand_dims_existing_scalar_coord(self): + original = Dataset({'x': 1}, {'a': 2}) + expected = Dataset({'x': (('a',), [1])}, {'a': [2]}) + actual = original.expand_dims('a') + assert_identical(expected, actual) + + def test_isel_expand_dims_roundtrip(self): + original = Dataset({'x': (('a',), [1])}, {'a': [2]}) + actual = original.isel(a=0).expand_dims('a') + assert_identical(actual, original) + + def test_expand_dims_mixed_int_and_coords(self): + # Test expanding one dimension to have size > 1 that doesn't have + # coordinates, and also expanding another dimension to have size > 1 + # that DOES have coordinates. + original = Dataset({'x': ('a', np.random.randn(3)), + 'y': (['b', 'a'], np.random.randn(4, 3))}, + coords={'a': np.linspace(0, 1, 3), + 'b': np.linspace(0, 1, 4), + 'c': np.linspace(0, 1, 5)}) + + actual = original.expand_dims( + OrderedDict((("d", 4), ("e", ["l", "m", "n"])))) + + expected = Dataset( + {'x': xr.DataArray(original['x'].values * np.ones([4, 3, 3]), + coords=dict(d=range(4), + e=['l', 'm', 'n'], + a=np.linspace(0, 1, 3)), + dims=['d', 'e', 'a']).drop('d'), + 'y': xr.DataArray(original['y'].values * np.ones([4, 3, 4, 3]), + coords=dict(d=range(4), + e=['l', 'm', 'n'], + b=np.linspace(0, 1, 4), + a=np.linspace(0, 1, 3)), + dims=['d', 'e', 'b', 'a']).drop('d')}, + coords={'c': np.linspace(0, 1, 5)}) + assert_identical(actual, expected) + + @pytest.mark.skipif( + sys.version_info[:2] > (3, 5), + reason="we only raise these errors for Python 3.5", + ) + def test_expand_dims_kwargs_python35(self): + original = Dataset({'x': ('a', np.random.randn(3))}) + with raises_regex(ValueError, "dim_kwargs isn't"): + original.expand_dims(e=["l", "m", "n"]) + with raises_regex(TypeError, "must be an OrderedDict"): + original.expand_dims({'e': ["l", "m", "n"]}) + + @pytest.mark.skipif( + sys.version_info[:2] < (3, 6), + reason='keyword arguments are only ordered on Python 3.6+', + ) + def test_expand_dims_kwargs_python36plus(self): + original = Dataset({'x': ('a', np.random.randn(3)), + 'y': (['b', 'a'], np.random.randn(4, 3))}, + coords={'a': np.linspace(0, 1, 3), + 'b': np.linspace(0, 1, 4), + 'c': np.linspace(0, 1, 5)}, + attrs={'key': 'entry'}) + other_way = original.expand_dims(e=["l", "m", "n"]) + other_way_expected = Dataset( + {'x': xr.DataArray(original['x'].values * np.ones([3, 3]), + coords=dict(e=['l', 'm', 'n'], + a=np.linspace(0, 1, 3)), + dims=['e', 'a']), + 'y': xr.DataArray(original['y'].values * np.ones([3, 4, 3]), + coords=dict(e=['l', 'm', 'n'], + b=np.linspace(0, 1, 4), + a=np.linspace(0, 1, 3)), + dims=['e', 'b', 'a'])}, + coords={'c': np.linspace(0, 1, 5)}, + attrs={'key': 'entry'}) + assert_identical(other_way_expected, other_way) + def test_set_index(self): expected = create_test_multiindex() mindex = expected['x'].to_index() diff --git a/xarray/tests/test_duck_array_ops.py b/xarray/tests/test_duck_array_ops.py index 5d425f648bd..75ab5f52a1b 100644 --- a/xarray/tests/test_duck_array_ops.py +++ b/xarray/tests/test_duck_array_ops.py @@ -20,7 +20,7 @@ requires_dask) -class TestOps(object): +class TestOps: @pytest.fixture(autouse=True) def setUp(self): @@ -86,6 +86,8 @@ def test_count(self): expected = array([[1, 2, 3], [3, 2, 1]]) assert_array_equal(expected, count(self.x, axis=-1)) + assert 1 == count(np.datetime64('2000-01-01')) + def test_where_type_promotion(self): result = where([True, False], [1, 2], ['a', 'b']) assert_array_equal(result, np.array([1, 'b'], dtype=object)) diff --git a/xarray/tests/test_extensions.py b/xarray/tests/test_extensions.py index 1b6e665bdae..e67e7a0f6a0 100644 --- a/xarray/tests/test_extensions.py +++ b/xarray/tests/test_extensions.py @@ -9,19 +9,19 @@ @xr.register_dataset_accessor('example_accessor') @xr.register_dataarray_accessor('example_accessor') -class ExampleAccessor(object): +class ExampleAccessor: """For the pickling tests below.""" def __init__(self, xarray_obj): self.obj = xarray_obj -class TestAccessor(object): +class TestAccessor: def test_register(self): @xr.register_dataset_accessor('demo') @xr.register_dataarray_accessor('demo') - class DemoAccessor(object): + class DemoAccessor: """Demo accessor.""" def __init__(self, xarray_obj): @@ -52,7 +52,7 @@ def foo(self): with pytest.warns(Warning, match='overriding a preexisting attribute'): @xr.register_dataarray_accessor('demo') - class Foo(object): + class Foo: pass # it didn't get registered again @@ -80,7 +80,7 @@ def test_broken_accessor(self): # regression test for GH933 @xr.register_dataset_accessor('stupid_accessor') - class BrokenAccessor(object): + class BrokenAccessor: def __init__(self, xarray_obj): raise AttributeError('broken') diff --git a/xarray/tests/test_formatting.py b/xarray/tests/test_formatting.py index 82b7b86bb76..8a3c95a4962 100644 --- a/xarray/tests/test_formatting.py +++ b/xarray/tests/test_formatting.py @@ -10,7 +10,7 @@ from . import raises_regex -class TestFormatting(object): +class TestFormatting: def test_get_indexer_at_least_n_items(self): cases = [ @@ -303,6 +303,17 @@ def test_diff_dataset_repr(self): actual = formatting.diff_dataset_repr(ds_a, ds_b, 'identical') assert actual == expected + def test_array_repr(self): + ds = xr.Dataset(coords={'foo': [1, 2, 3], 'bar': [1, 2, 3]}) + ds[(1, 2)] = xr.DataArray([0], dims='test') + actual = formatting.array_repr(ds[(1, 2)]) + expected = dedent("""\ + + array([0]) + Dimensions without coordinates: test""") + + assert actual == expected + def test_set_numpy_options(): original_options = np.get_printoptions() diff --git a/xarray/tests/test_indexing.py b/xarray/tests/test_indexing.py index 14b79c71ca4..9301abb5e32 100644 --- a/xarray/tests/test_indexing.py +++ b/xarray/tests/test_indexing.py @@ -12,7 +12,7 @@ B = IndexerMaker(indexing.BasicIndexer) -class TestIndexers(object): +class TestIndexers: def set_to_zero(self, x, i): x = x.copy() x[i] = 0 @@ -129,7 +129,7 @@ def test_indexer(data, x, expected_pos, expected_idx=None): pd.MultiIndex.from_product([[1, 2], [-1, -2]])) -class TestLazyArray(object): +class TestLazyArray: def test_slice_slice(self): I = ReturnItem() # noqa: E741 # allow ambiguous name for size in [100, 99]: @@ -244,7 +244,7 @@ def check_indexing(v_eager, v_lazy, indexers): check_indexing(v_eager, v_lazy, indexers) -class TestCopyOnWriteArray(object): +class TestCopyOnWriteArray: def test_setitem(self): original = np.arange(10) wrapped = indexing.CopyOnWriteArray(original) @@ -268,7 +268,7 @@ def test_index_scalar(self): assert np.array(x[B[0]][B[()]]) == 'foo' -class TestMemoryCachedArray(object): +class TestMemoryCachedArray: def test_wrapper(self): original = indexing.LazilyOuterIndexedArray(np.arange(10)) wrapped = indexing.MemoryCachedArray(original) @@ -381,7 +381,7 @@ def test_vectorized_indexer(): np.arange(5, dtype=np.int64))) -class Test_vectorized_indexer(object): +class Test_vectorized_indexer: @pytest.fixture(autouse=True) def setup(self): self.data = indexing.NumpyIndexingAdapter(np.random.randn(10, 12, 13)) diff --git a/xarray/tests/test_interp.py b/xarray/tests/test_interp.py index 5596bfb3bfb..8347d54bd1e 100644 --- a/xarray/tests/test_interp.py +++ b/xarray/tests/test_interp.py @@ -291,7 +291,7 @@ def test_errors(use_dask): if use_dask: da = get_example_data(3) else: - da = get_example_data(1) + da = get_example_data(0) result = da.interp(x=[-1, 1, 3], kwargs={'fill_value': 0.0}) assert not np.isnan(result.values).any() diff --git a/xarray/tests/test_merge.py b/xarray/tests/test_merge.py index 4f26d616ce7..0d76db1d1ee 100644 --- a/xarray/tests/test_merge.py +++ b/xarray/tests/test_merge.py @@ -2,13 +2,13 @@ import pytest import xarray as xr -from xarray.core import merge +from xarray.core import merge, dtypes from . import raises_regex from .test_dataset import create_test_data -class TestMergeInternals(object): +class TestMergeInternals: def test_broadcast_dimension_size(self): actual = merge.broadcast_dimension_size( [xr.Variable('x', [1]), xr.Variable('y', [2, 1])]) @@ -19,11 +19,11 @@ def test_broadcast_dimension_size(self): assert actual == {'x': 1, 'y': 2} with pytest.raises(ValueError): - actual = merge.broadcast_dimension_size( + merge.broadcast_dimension_size( [xr.Variable(('x', 'y'), [[1, 2]]), xr.Variable('y', [2])]) -class TestMergeFunction(object): +class TestMergeFunction: def test_merge_arrays(self): data = create_test_data() actual = xr.merge([data.var1, data.var2]) @@ -128,7 +128,7 @@ def test_merge_no_conflicts_broadcast(self): assert expected.identical(actual) -class TestMergeMethod(object): +class TestMergeMethod: def test_merge(self): data = create_test_data() @@ -213,6 +213,21 @@ def test_merge_auto_align(self): assert expected.identical(ds1.merge(ds2, join='inner')) assert expected.identical(ds2.merge(ds1, join='inner')) + @pytest.mark.parametrize('fill_value', [dtypes.NA, 2, 2.0]) + def test_merge_fill_value(self, fill_value): + ds1 = xr.Dataset({'a': ('x', [1, 2]), 'x': [0, 1]}) + ds2 = xr.Dataset({'b': ('x', [3, 4]), 'x': [1, 2]}) + if fill_value == dtypes.NA: + # if we supply the default, we expect the missing value for a + # float array + fill_value = np.nan + expected = xr.Dataset({'a': ('x', [1, 2, fill_value]), + 'b': ('x', [fill_value, 3, 4])}, + {'x': [0, 1, 2]}) + assert expected.identical(ds1.merge(ds2, fill_value=fill_value)) + assert expected.identical(ds2.merge(ds1, fill_value=fill_value)) + assert expected.identical(xr.merge([ds1, ds2], fill_value=fill_value)) + def test_merge_no_conflicts(self): ds1 = xr.Dataset({'a': ('x', [1, 2]), 'x': [0, 1]}) ds2 = xr.Dataset({'a': ('x', [2, 3]), 'x': [1, 2]}) diff --git a/xarray/tests/test_options.py b/xarray/tests/test_options.py index 1508503f7eb..34bcba58020 100644 --- a/xarray/tests/test_options.py +++ b/xarray/tests/test_options.py @@ -81,7 +81,7 @@ def create_test_dataarray_attrs(seed=0, var='var1'): return da -class TestAttrRetention(object): +class TestAttrRetention: def test_dataset_attr_retention(self): # Use .mean() for all tests: a typical reduction operation ds = create_test_dataset_attrs() diff --git a/xarray/tests/test_plot.py b/xarray/tests/test_plot.py index c0e03b5791c..759a2974ca6 100644 --- a/xarray/tests/test_plot.py +++ b/xarray/tests/test_plot.py @@ -4,12 +4,10 @@ import numpy as np import pandas as pd import pytest -from numpy.testing import assert_array_equal import xarray as xr import xarray.plot as xplt from xarray import DataArray -from xarray.coding.times import _import_cftime from xarray.plot.plot import _infer_interval_breaks from xarray.plot.utils import ( _build_discrete_cmap, _color_palette, _determine_cmap_params, @@ -65,7 +63,7 @@ def easy_array(shape, start=0, stop=1): @requires_matplotlib -class PlotTestCase(object): +class PlotTestCase: @pytest.fixture(autouse=True) def setup(self): yield @@ -512,7 +510,7 @@ def test_hist_coord_with_interval(self): @requires_matplotlib -class TestDetermineCmapParams(object): +class TestDetermineCmapParams: @pytest.fixture(autouse=True) def setUp(self): self.data = np.linspace(0, 1, num=100) @@ -539,6 +537,25 @@ def test_cmap_sequential_option(self): cmap_params = _determine_cmap_params(self.data) assert cmap_params['cmap'] == 'magma' + def test_do_nothing_if_provided_cmap(self): + cmap_list = [ + mpl.colors.LinearSegmentedColormap.from_list('name', ['r', 'g']), + mpl.colors.ListedColormap(['r', 'g', 'b']) + ] + + # can't parametrize with mpl objects when mpl is absent + for cmap in cmap_list: + cmap_params = _determine_cmap_params(self.data, + cmap=cmap, + levels=7) + assert cmap_params['cmap'] is cmap + + def test_do_something_if_provided_str_cmap(self): + cmap = 'RdBu_r' + cmap_params = _determine_cmap_params(self.data, cmap=cmap, levels=7) + assert cmap_params['cmap'] is not cmap + assert isinstance(cmap_params['cmap'], mpl.colors.ListedColormap) + def test_cmap_sequential_explicit_option(self): with xr.set_options(cmap_sequential=mpl.cm.magma): cmap_params = _determine_cmap_params(self.data) @@ -706,7 +723,7 @@ def test_norm_sets_vmin_vmax(self): @requires_matplotlib -class TestDiscreteColorMap(object): +class TestDiscreteColorMap: @pytest.fixture(autouse=True) def setUp(self): x = np.arange(start=0, stop=10, step=2) @@ -793,7 +810,7 @@ def test_discrete_colormap_provided_boundary_norm(self): np.testing.assert_allclose(primitive.levels, norm.boundaries) -class Common2dMixin(object): +class Common2dMixin: """ Common tests for 2d plotting go here. @@ -1907,7 +1924,7 @@ def test_plot_seaborn_no_import_warning(): @requires_matplotlib -class TestAxesKwargs(object): +class TestAxesKwargs: @pytest.mark.parametrize('da', test_da_list) @pytest.mark.parametrize('xincrease', [True, False]) def test_xincrease_kwarg(self, da, xincrease): diff --git a/xarray/tests/test_tutorial.py b/xarray/tests/test_tutorial.py index 2bb2cfb0415..841f4f7c832 100644 --- a/xarray/tests/test_tutorial.py +++ b/xarray/tests/test_tutorial.py @@ -9,7 +9,7 @@ @network -class TestLoadDataset(object): +class TestLoadDataset: @pytest.fixture(autouse=True) def setUp(self): self.testfile = 'tiny' diff --git a/xarray/tests/test_ufuncs.py b/xarray/tests/test_ufuncs.py index 2f6ca37bd2a..2c60fb3861e 100644 --- a/xarray/tests/test_ufuncs.py +++ b/xarray/tests/test_ufuncs.py @@ -107,7 +107,7 @@ def test_kwargs(): @requires_np113 def test_xarray_defers_to_unrecognized_type(): - class Other(object): + class Other: def __array_ufunc__(self, *args, **kwargs): return 'other' diff --git a/xarray/tests/test_utils.py b/xarray/tests/test_utils.py index e98ab5cde4c..bcd960e4e29 100644 --- a/xarray/tests/test_utils.py +++ b/xarray/tests/test_utils.py @@ -5,19 +5,16 @@ import pandas as pd import pytest -import xarray as xr from xarray.coding.cftimeindex import CFTimeIndex from xarray.core import duck_array_ops, utils from xarray.core.utils import either_dict_or_kwargs -from xarray.testing import assert_identical from . import ( - assert_array_equal, has_cftime, has_cftime_or_netCDF4, requires_cftime, - requires_dask) + assert_array_equal, has_cftime, has_cftime_or_netCDF4, requires_dask) from .test_coding_times import _all_cftime_date_types -class TestAlias(object): +class TestAlias: def test(self): def new_method(): pass @@ -95,7 +92,7 @@ def test_multiindex_from_product_levels_non_unique(): np.testing.assert_array_equal(result.levels[1], [1, 2]) -class TestArrayEquiv(object): +class TestArrayEquiv: def test_0d(self): # verify our work around for pd.isnull not working for 0-dimensional # object arrays @@ -105,7 +102,7 @@ def test_0d(self): assert not duck_array_ops.array_equiv(0, np.array(1, dtype=object)) -class TestDictionaries(object): +class TestDictionaries: @pytest.fixture(autouse=True) def setup(self): self.x = {'a': 'A', 'b': 'B'} @@ -178,19 +175,6 @@ def test_sorted_keys_dict(self): assert repr(utils.SortedKeysDict()) == \ "SortedKeysDict({})" - def test_chain_map(self): - m = utils.ChainMap({'x': 0, 'y': 1}, {'x': -100, 'z': 2}) - assert 'x' in m - assert 'y' in m - assert 'z' in m - assert m['x'] == 0 - assert m['y'] == 1 - assert m['z'] == 2 - m['x'] = 100 - assert m['x'] == 100 - assert m.maps[0]['x'] == 100 - assert set(m) == {'x', 'y', 'z'} - def test_repr_object(): obj = utils.ReprObject('foo') @@ -213,7 +197,7 @@ def test_is_grib_path(): assert utils.is_grib_path('example.grb2') -class Test_is_uniform_and_sorted(object): +class Test_is_uniform_and_sorted: def test_sorted_uniform(self): assert utils.is_uniform_spaced(np.arange(5)) @@ -234,7 +218,7 @@ def test_relative_tolerance(self): assert utils.is_uniform_spaced([0, 0.97, 2], rtol=0.1) -class Test_hashable(object): +class Test_hashable: def test_hashable(self): for v in [False, 1, (2, ), (3, 4), 'four']: diff --git a/xarray/tests/test_variable.py b/xarray/tests/test_variable.py index eec8d268026..4ddd114d767 100644 --- a/xarray/tests/test_variable.py +++ b/xarray/tests/test_variable.py @@ -26,7 +26,7 @@ raises_regex, requires_dask, source_ndarray) -class VariableSubclassobjects(object): +class VariableSubclassobjects: def test_properties(self): data = 0.5 * np.arange(10) v = self.cls(['time'], data, {'foo': 'bar'}) @@ -195,7 +195,7 @@ def test_index_0d_not_a_time(self): def test_index_0d_object(self): - class HashableItemWrapper(object): + class HashableItemWrapper: def __init__(self, item): self.item = item @@ -1892,7 +1892,7 @@ def test_coarsen_2d(self): super(TestIndexVariable, self).test_coarsen_2d() -class TestAsCompatibleData(object): +class TestAsCompatibleData: def test_unchanged_types(self): types = (np.asarray, PandasIndexAdapter, LazilyOuterIndexedArray) for t in types: @@ -2033,7 +2033,7 @@ def test_raise_no_warning_for_nan_in_binary_ops(): assert len(record) == 0 -class TestBackendIndexing(object): +class TestBackendIndexing: """ Make sure all the array wrappers can be indexed. """ @pytest.fixture(autouse=True) diff --git a/xarray/tutorial.py b/xarray/tutorial.py index f54cf7b3889..1a977450ed6 100644 --- a/xarray/tutorial.py +++ b/xarray/tutorial.py @@ -27,7 +27,7 @@ def open_dataset(name, cache=True, cache_dir=_default_cache_dir, github_url='https://github.com/pydata/xarray-data', branch='master', **kws): """ - Load a dataset from the online repository (requires internet). + Open a dataset from the online repository (requires internet). If a local copy is found then always use that to avoid network traffic. @@ -91,17 +91,12 @@ def open_dataset(name, cache=True, cache_dir=_default_cache_dir, def load_dataset(*args, **kwargs): """ - `load_dataset` will be removed a future version of xarray. The current - behavior of this function can be achived by using - `tutorial.open_dataset(...).load()`. + Open, load into memory, and close a dataset from the online repository + (requires internet). See Also -------- open_dataset """ - warnings.warn( - "load_dataset` will be removed in a future version of xarray. The " - "current behavior of this function can be achived by using " - "`tutorial.open_dataset(...).load()`.", - DeprecationWarning, stacklevel=2) - return open_dataset(*args, **kwargs).load() + with open_dataset(*args, **kwargs) as ds: + return ds.load() diff --git a/xarray/ufuncs.py b/xarray/ufuncs.py index dcba208436e..c4261d465e9 100644 --- a/xarray/ufuncs.py +++ b/xarray/ufuncs.py @@ -35,7 +35,7 @@ def _dispatch_priority(obj): return -1 -class _UFuncDispatcher(object): +class _UFuncDispatcher: """Wrapper for dispatching ufuncs.""" def __init__(self, name):