From 717c216a936de9cde2885d9a34bddfa17e90de18 Mon Sep 17 00:00:00 2001
From: "Phillip J. Wolfram" <phillipwolfram@gmail.com>
Date: Tue, 28 Mar 2017 12:55:33 -0600
Subject: [PATCH 1/8] Adds marked tests as slow, flakey, and optionalci

Tests are marked using the following py.test options:

 * @slow which is run on default, not run if `--skip-slow`.
 * @optionalci is run on default, not run if `--skip-optional-ci`
 * @flakey which is not run on default, run if `--run-flakey`

Note, the `optionalci` category is needed in order to mark
tests that are environment dependent when run on continuous
integration (CI) hardware, e.g., tests that fail due to
variable loads on travis CI.
---
 .travis.yml                   | 32 +++++++++++------------
 appveyor.yml                  |  2 +-
 conftest.py                   |  9 +++++++
 doc/whats-new.rst             |  6 +++++
 xarray/tests/__init__.py      | 15 +++++++++++
 xarray/tests/test_backends.py | 34 ++++++++++++++++--------
 xarray/tests/test_plot.py     | 49 +++++++++++++++++++++++++++++++++--
 7 files changed, 117 insertions(+), 30 deletions(-)
 create mode 100644 conftest.py

diff --git a/.travis.yml b/.travis.yml
index 13c5362e530..47478c439f7 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -11,46 +11,46 @@ matrix:
   fast_finish: true
   include:
   - python: 2.7
-    env: CONDA_ENV=py27-min
+    env: CONDA_ENV=py27-min EXTRA_FLAGS=--skip-optional-ci
   - python: 2.7
-    env: CONDA_ENV=py27-cdat+pynio
+    env: CONDA_ENV=py27-cdat+pynio EXTRA_FLAGS=--skip-optional-ci
   - python: 3.4
-    env: CONDA_ENV=py34
+    env: CONDA_ENV=py34 EXTRA_FLAGS=--skip-optional-ci
   - python: 3.5
-    env: CONDA_ENV=py35
+    env: CONDA_ENV=py35 EXTRA_FLAGS=--skip-optional-ci
   - python: 3.6
-    env: CONDA_ENV=py36
+    env: CONDA_ENV=py36 EXTRA_FLAGS=--skip-optional-ci
   - python: 3.6
-    env: CONDA_ENV=py36-pydap
+    env: CONDA_ENV=py36-pydap EXTRA_FLAGS=--run-flakey
   - python: 3.6
-    env: CONDA_ENV=py36-netcdf4-dev
+    env: CONDA_ENV=py36-netcdf4-dev EXTRA_FLAGS=--run-flakey
     addons:
       apt_packages:
         - libhdf5-serial-dev
         - netcdf-bin
         - libnetcdf-dev
   - python: 3.6
-    env: CONDA_ENV=py36-dask-dev
+    env: CONDA_ENV=py36-dask-dev EXTRA_FLAGS=--run-flakey
   - python: 3.6
-    env: CONDA_ENV=py36-pandas-dev
+    env: CONDA_ENV=py36-pandas-dev EXTRA_FLAGS=--run-flakey
   - python: 3.6
-    env: CONDA_ENV=py36-condaforge-rc
+    env: CONDA_ENV=py36-condaforge-rc EXTRA_FLAGS=--run-flakey
   allow_failures:
   - python: 3.6
-    env: CONDA_ENV=py36-pydap
+    env: CONDA_ENV=py36-pydap EXTRA_FLAGS=--run-flakey
   - python: 3.6
-    env: CONDA_ENV=py36-netcdf4-dev
+    env: CONDA_ENV=py36-netcdf4-dev EXTRA_FLAGS=--run-flakey
     addons:
       apt_packages:
         - libhdf5-serial-dev
         - netcdf-bin
         - libnetcdf-dev
   - python: 3.6
-    env: CONDA_ENV=py36-dask-dev
+    env: CONDA_ENV=py36-dask-dev EXTRA_FLAGS=--run-flakey
   - python: 3.6
-    env: CONDA_ENV=py36-pandas-dev
+    env: CONDA_ENV=py36-pandas-dev EXTRA_FLAGS=--run-flakey
   - python: 3.6
-    env: CONDA_ENV=py36-condaforge-rc
+    env: CONDA_ENV=py36-condaforge-rc EXTRA_FLAGS=--run-flakey
 
 before_install:
   - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then
@@ -76,7 +76,7 @@ install:
   - python setup.py install
 
 script:
-  - py.test xarray --cov=xarray --cov-report term-missing --verbose
+  - py.test xarray --cov=xarray --cov-report term-missing --verbose $EXTRA_FLAGS
 
 after_success:
   - coveralls
diff --git a/appveyor.yml b/appveyor.yml
index 9f04ad8534c..f1c2bbdce5d 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -38,4 +38,4 @@ install:
 build: false
 
 test_script:
-  - "py.test xarray --verbose"
+  - "py.test xarray --verbose --skip-optional-ci"
diff --git a/conftest.py b/conftest.py
new file mode 100644
index 00000000000..396f324dcb5
--- /dev/null
+++ b/conftest.py
@@ -0,0 +1,9 @@
+import pytest
+
+def pytest_addoption(parser):
+    parser.addoption("--run-flakey", action="store_true",
+                     help="runs flakey tests")
+    parser.addoption("--skip-optional-ci", action="store_true",
+                     help="skips optional tests continuous integration (CI)")
+    parser.addoption("--skip-slow", action="store_true",
+                     help="skips slow tests")
diff --git a/doc/whats-new.rst b/doc/whats-new.rst
index bc7b0fd6c5c..04444f9cab7 100644
--- a/doc/whats-new.rst
+++ b/doc/whats-new.rst
@@ -41,6 +41,12 @@ By `Henry S. Harrison <https://hsharrison.github.io>`_.
   Note, the default is ``autoclose=False``, which is consistent with previous
   xarray behavior.  By `Phillip J. Wolfram <https://github.com/pwolfram>`_.
 
+- Enhances tests by use of ``@slow``, ``@flakey``, and ``@optionalci``
+  decorrators, which are controled via ``--run-flakey``,
+  ``--skip-optional-ci``, and ``--skip-slow``command line arguments 
+  to ``py.test``.  By `Stephan Hoyer <https://github.com/shoyer>`_ and
+  `Phillip J. Wolfram <https://github.com/pwolfram>`_.
+
 Bug fixes
 ~~~~~~~~~
 - ``rolling`` now keeps its original dimension order (:issue:`1125`).
diff --git a/xarray/tests/__init__.py b/xarray/tests/__init__.py
index 843b8bfdfbd..7ce4bf57bf2 100644
--- a/xarray/tests/__init__.py
+++ b/xarray/tests/__init__.py
@@ -114,6 +114,21 @@ def requires_bottleneck(test):
     return test if has_bottleneck else pytest.mark.skip('requires bottleneck')(test)
 
 
+flakey = pytest.mark.skipif(
+    not pytest.config.getoption("--run-flakey"),
+    reason="set --run-flakey option to run flakey tests")
+
+optionalci = pytest.mark.skipif(
+    pytest.config.getoption("--skip-optional-ci"),
+    reason=("set --skip-optional-ci option to skip tests in CI, "
+            "e.g., due to travis CI resource issues"))
+
+
+slow = pytest.mark.skipif(
+    pytest.config.getoption("--skip-slow"),
+    reason="set --skip-slow option to run slow tests")
+
+
 class TestCase(unittest.TestCase):
     if PY3:
         # Python 3 assertCountEqual is roughly equivalent to Python 2
diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py
index 7fe2d936506..f4f22623160 100644
--- a/xarray/tests/test_backends.py
+++ b/xarray/tests/test_backends.py
@@ -26,7 +26,8 @@
 
 from . import (TestCase, requires_scipy, requires_netCDF4, requires_pydap,
                requires_scipy_or_netCDF4, requires_dask, requires_h5netcdf,
-               requires_pynio, has_netCDF4, has_scipy, assert_allclose)
+               requires_pynio, has_netCDF4, has_scipy, assert_allclose,
+               flakey, optionalci, slow)
 from .test_dataset import create_test_data
 
 try:
@@ -1059,8 +1060,9 @@ def test_encoding_unlimited_dims(self):
                 ds.to_netcdf(tmp_file, engine='h5netcdf', unlimited_dims=['y'])
 
 # tests pending h5netcdf fix
-# class H5NetCDFDataTestAutocloseTrue(H5NetCDFDataTest):
-#     autoclose = True
+@flakey
+class H5NetCDFDataTestAutocloseTrue(H5NetCDFDataTest):
+    autoclose = True
 
 
 class OpenMFDatasetManyFilesTest(TestCase):
@@ -1105,32 +1107,42 @@ def test_3_autoclose_pynio(self):
 
     # use of autoclose=True with h5netcdf broken because of
     # probable h5netcdf error, uncomment when fixed to test
-    # @requires_dask
-    # @requires_h5netcdf
-    # def test_4_autoclose_h5netcdf(self):
-    #     self.validate_open_mfdataset_autoclose(engine=['h5netcdf'])
+    @requires_dask
+    @requires_h5netcdf
+    @flakey
+    def test_4_autoclose_h5netcdf(self):
+        self.validate_open_mfdataset_autoclose(engine=['h5netcdf'])
 
     @requires_dask
     @requires_netCDF4
+    @optionalci
+    @slow
     def test_1_open_large_num_files_netcdf4(self):
         self.validate_open_mfdataset_large_num_files(engine=['netcdf4'])
 
     @requires_dask
     @requires_scipy
+    @optionalci
+    @slow
     def test_2_open_large_num_files_scipy(self):
         self.validate_open_mfdataset_large_num_files(engine=['scipy'])
 
     @requires_dask
     @requires_pynio
+    @optionalci
+    @slow
     def test_3_open_large_num_files_pynio(self):
         self.validate_open_mfdataset_large_num_files(engine=['pynio'])
 
     # use of autoclose=True with h5netcdf broken because of
     # probable h5netcdf error, uncomment when fixed to test
-    # @requires_dask
-    # @requires_h5netcdf
-    # def test_4_open_large_num_files_h5netcdf(self):
-    #     self.validate_open_mfdataset_large_num_files(engine=['h5netcdf'])
+    @requires_dask
+    @requires_h5netcdf
+    @flakey
+    @optionalci
+    @slow
+    def test_4_open_large_num_files_h5netcdf(self):
+        self.validate_open_mfdataset_large_num_files(engine=['h5netcdf'])
 
 
 @requires_dask
diff --git a/xarray/tests/test_plot.py b/xarray/tests/test_plot.py
index 55ca5b0d929..621c1680492 100644
--- a/xarray/tests/test_plot.py
+++ b/xarray/tests/test_plot.py
@@ -26,7 +26,7 @@
                                _build_discrete_cmap,
                                _color_palette)
 
-from . import TestCase, requires_matplotlib
+from . import TestCase, requires_matplotlib, slow
 
 
 def text_in_fig():
@@ -74,6 +74,7 @@ def pass_in_axis(self, plotmethod):
         plotmethod(ax=axes[0])
         self.assertTrue(axes[0].has_data())
 
+    @slow
     def imshow_called(self, plotmethod):
         plotmethod()
         images = plt.gca().findobj(mpl.image.AxesImage)
@@ -100,6 +101,7 @@ def test_2d_before_squeeze(self):
     def test2d_uniform_calls_imshow(self):
         self.assertTrue(self.imshow_called(self.darray[:, :, 0].plot.imshow))
 
+    @slow
     def test2d_nonuniform_calls_contourf(self):
         a = self.darray[:, :, 0]
         a.coords['dim_1'] = [2, 1, 89]
@@ -139,6 +141,7 @@ def test_datetime_dimension(self):
         ax = plt.gca()
         self.assertTrue(ax.has_data())
 
+    @slow
     def test_convenient_facetgrid(self):
         a = easy_array((10, 15, 4))
         d = DataArray(a, dims=['y', 'x', 'z'])
@@ -155,6 +158,7 @@ def test_convenient_facetgrid(self):
         with self.assertRaisesRegexp(ValueError, '[Ff]acet'):
             d[0].plot(x='x', y='y', col='z', ax=plt.gca())
 
+    @slow
     def test_subplot_kws(self):
         a = easy_array((10, 15, 4))
         d = DataArray(a, dims=['y', 'x', 'z'])
@@ -169,6 +173,7 @@ def test_subplot_kws(self):
             except AttributeError:
                 self.assertEqual(ax.get_axis_bgcolor(), 'r')
 
+    @slow
     def test_plot_size(self):
         self.darray[:, 0, 0].plot(figsize=(13, 5))
         assert tuple(plt.gcf().get_size_inches()) == (13, 5)
@@ -194,6 +199,7 @@ def test_plot_size(self):
         with self.assertRaisesRegexp(ValueError, 'cannot provide `aspect`'):
             self.darray.plot(aspect=1)
 
+    @slow
     def test_convenient_facetgrid_4d(self):
         a = easy_array((10, 15, 2, 3))
         d = DataArray(a, dims=['y', 'x', 'columns', 'rows'])
@@ -248,6 +254,7 @@ def test_primitive_returned(self):
         p = self.darray.plot.line()
         self.assertTrue(isinstance(p[0], mpl.lines.Line2D))
 
+    @slow
     def test_plot_nans(self):
         self.darray[1] = np.nan
         self.darray.plot.line()
@@ -299,6 +306,7 @@ def test_primitive_returned(self):
         h = self.darray.plot.hist()
         self.assertTrue(isinstance(h[-1][0], mpl.patches.Rectangle))
 
+    @slow
     def test_plot_nans(self):
         self.darray[0, 0, 0] = np.nan
         self.darray.plot.hist()
@@ -327,6 +335,7 @@ def test_center(self):
         self.assertIsNone(cmap_params['levels'])
         self.assertIsNone(cmap_params['norm'])
 
+    @slow
     def test_integer_levels(self):
         data = self.data + 1
 
@@ -466,11 +475,13 @@ def setUp(self):
         self.data_min = distance.min()
         self.data_max = distance.max()
 
+    @slow
     def test_recover_from_seaborn_jet_exception(self):
         pal = _color_palette('jet', 4)
         self.assertTrue(type(pal) == np.ndarray)
         self.assertEqual(len(pal), 4)
 
+    @slow
     def test_build_discrete_cmap(self):
         for (cmap, levels, extend, filled) in [('jet', [0, 1], 'both', False),
                                                ('hot', [-4, 4], 'max', True)]:
@@ -486,6 +497,7 @@ def test_build_discrete_cmap(self):
             else:
                 self.assertEqual(ncmap.colorbar_extend, 'max')
 
+    @slow
     def test_discrete_colormap_list_of_levels(self):
         for extend, levels in [('max', [-1, 2, 4, 8, 10]),
                                ('both', [2, 5, 10, 11]),
@@ -502,6 +514,7 @@ def test_discrete_colormap_list_of_levels(self):
                     self.assertEqual('max', primitive.cmap.colorbar_extend)
                 self.assertEqual(len(levels) - 1, len(primitive.cmap.colors))
 
+    @slow
     def test_discrete_colormap_int_levels(self):
         for extend, levels, vmin, vmax in [('neither', 7, None, None),
                                            ('neither', 7, None, 20),
@@ -813,10 +826,12 @@ def test_facetgrid_cmap(self):
         self.assertTrue(len(set(m.get_cmap().name for m in fg._mappables)) == 1)
 
 
+@slow
 class TestContourf(Common2dMixin, PlotTestCase):
 
     plotfunc = staticmethod(xplt.contourf)
 
+    @slow
     def test_contourf_called(self):
         # Having both statements ensures the test works properly
         self.assertFalse(self.contourf_called(self.darray.plot.imshow))
@@ -826,6 +841,7 @@ def test_primitive_artist_returned(self):
         artist = self.plotmethod()
         self.assertTrue(isinstance(artist, mpl.contour.QuadContourSet))
 
+    @slow
     def test_extend(self):
         artist = self.plotmethod()
         self.assertEqual(artist.extend, 'neither')
@@ -843,6 +859,7 @@ def test_extend(self):
         artist = self.plotmethod(vmin=-10, vmax=0)
         self.assertEqual(artist.extend, 'max')
 
+    @slow
     def test_2d_coord_names(self):
         self.plotmethod(x='x2d', y='y2d')
         # make sure labels came out ok
@@ -850,6 +867,7 @@ def test_2d_coord_names(self):
         self.assertEqual('x2d', ax.get_xlabel())
         self.assertEqual('y2d', ax.get_ylabel())
 
+    @slow
     def test_levels(self):
         artist = self.plotmethod(levels=[-0.5, -0.4, 0.1])
         self.assertEqual(artist.extend, 'both')
@@ -858,6 +876,7 @@ def test_levels(self):
         self.assertEqual(artist.extend, 'neither')
 
 
+@slow
 class TestContour(Common2dMixin, PlotTestCase):
 
     plotfunc = staticmethod(xplt.contour)
@@ -898,6 +917,7 @@ def list_of_colors_in_cmap_deprecated(self):
         with self.assertRaises(Exception):
             self.plotmethod(cmap=['k', 'b'])
 
+    @slow
     def test_2d_coord_names(self):
         self.plotmethod(x='x2d', y='y2d')
         # make sure labels came out ok
@@ -924,6 +944,7 @@ def test_everything_plotted(self):
         artist = self.plotmethod()
         self.assertEqual(artist.get_array().size, self.darray.size)
 
+    @slow
     def test_2d_coord_names(self):
         self.plotmethod(x='x2d', y='y2d')
         # make sure labels came out ok
@@ -941,11 +962,12 @@ def test_dont_infer_interval_breaks_for_cartopy(self):
         # Let cartopy handle the axis limits and artist size
         self.assertTrue(artist.get_array().size <= self.darray.size)
 
-
+@slow
 class TestImshow(Common2dMixin, PlotTestCase):
 
     plotfunc = staticmethod(xplt.imshow)
 
+    @slow
     def test_imshow_called(self):
         # Having both statements ensures the test works properly
         self.assertFalse(self.imshow_called(self.darray.plot.contourf))
@@ -960,6 +982,7 @@ def test_default_aspect_is_auto(self):
         self.darray.plot.imshow()
         self.assertEqual('auto', plt.gca().get_aspect())
 
+    @slow
     def test_cannot_change_mpl_aspect(self):
 
         with self.assertRaisesRegexp(ValueError, 'not available in xarray'):
@@ -970,10 +993,12 @@ def test_cannot_change_mpl_aspect(self):
         self.assertEqual('auto', plt.gca().get_aspect())
         assert tuple(plt.gcf().get_size_inches()) == (10, 5)
 
+    @slow
     def test_primitive_artist_returned(self):
         artist = self.plotmethod()
         self.assertTrue(isinstance(artist, mpl.image.AxesImage))
 
+    @slow
     def test_seaborn_palette_needs_levels(self):
         try:
             import seaborn
@@ -994,6 +1019,7 @@ def setUp(self):
                                 coords={'z': ['a', 'b', 'c']})
         self.g = xplt.FacetGrid(self.darray, col='z')
 
+    @slow
     def test_no_args(self):
         self.g.map_dataarray(xplt.contourf, 'x', 'y')
 
@@ -1008,6 +1034,7 @@ def test_no_args(self):
             fontsize = ax.title.get_size()
             self.assertLessEqual(fontsize, 12)
 
+    @slow
     def test_names_appear_somewhere(self):
         self.darray.name = 'testvar'
         self.g.map_dataarray(xplt.contourf, 'x', 'y')
@@ -1019,6 +1046,7 @@ def test_names_appear_somewhere(self):
         for label in ['x', 'y']:
             self.assertIn(label, alltxt)
 
+    @slow
     def test_text_not_super_long(self):
         self.darray.coords['z'] = [100 * letter for letter in 'abc']
         g = xplt.FacetGrid(self.darray, col='z')
@@ -1030,6 +1058,7 @@ def test_text_not_super_long(self):
         t0 = g.axes[0, 0].get_title()
         self.assertTrue(t0.endswith('...'))
 
+    @slow
     def test_colorbar(self):
         vmin = self.darray.values.min()
         vmax = self.darray.values.max()
@@ -1043,6 +1072,7 @@ def test_colorbar(self):
 
         self.assertEqual(1, len(find_possible_colorbars()))
 
+    @slow
     def test_empty_cell(self):
         g = xplt.FacetGrid(self.darray, col='z', col_wrap=2)
         g.map_dataarray(xplt.imshow, 'x', 'y')
@@ -1051,10 +1081,12 @@ def test_empty_cell(self):
         self.assertFalse(bottomright.has_data())
         self.assertFalse(bottomright.get_visible())
 
+    @slow
     def test_norow_nocol_error(self):
         with self.assertRaisesRegexp(ValueError, r'[Rr]ow'):
             xplt.FacetGrid(self.darray)
 
+    @slow
     def test_groups(self):
         self.g.map_dataarray(xplt.imshow, 'x', 'y')
         upperleft_dict = self.g.name_dicts[0, 0]
@@ -1063,16 +1095,19 @@ def test_groups(self):
 
         self.assertDataArrayEqual(upperleft_array, z0)
 
+    @slow
     def test_float_index(self):
         self.darray.coords['z'] = [0.1, 0.2, 0.4]
         g = xplt.FacetGrid(self.darray, col='z')
         g.map_dataarray(xplt.imshow, 'x', 'y')
 
+    @slow
     def test_nonunique_index_error(self):
         self.darray.coords['z'] = [0.1, 0.2, 0.2]
         with self.assertRaisesRegexp(ValueError, r'[Uu]nique'):
             xplt.FacetGrid(self.darray, col='z')
 
+    @slow
     def test_robust(self):
         z = np.zeros((20, 20, 2))
         darray = DataArray(z, dims=['y', 'x', 'z'])
@@ -1094,6 +1129,7 @@ def test_robust(self):
         largest = max(abs(x) for x in numbers)
         self.assertLess(largest, 21)
 
+    @slow
     def test_can_set_vmin_vmax(self):
         vmin, vmax = 50.0, 1000.0
         expected = np.array((vmin, vmax))
@@ -1103,12 +1139,14 @@ def test_can_set_vmin_vmax(self):
             clim = np.array(image.get_clim())
             self.assertTrue(np.allclose(expected, clim))
 
+    @slow
     def test_can_set_norm(self):
         norm = mpl.colors.SymLogNorm(0.1)
         self.g.map_dataarray(xplt.imshow, 'x', 'y', norm=norm)
         for image in plt.gcf().findobj(mpl.image.AxesImage):
             self.assertIs(image.norm, norm)
 
+    @slow
     def test_figure_size(self):
 
         self.assertArrayEqual(self.g.fig.get_size_inches(), (10, 3))
@@ -1131,6 +1169,7 @@ def test_figure_size(self):
         with self.assertRaisesRegexp(ValueError, "Can't use"):
             g = xplt.plot(self.darray, row=2, col='z', ax=plt.gca(), size=6)
 
+    @slow
     def test_num_ticks(self):
         nticks = 99
         maxticks = nticks + 1
@@ -1145,10 +1184,12 @@ def test_num_ticks(self):
             self.assertGreaterEqual(xticks, nticks / 2.0)
             self.assertGreaterEqual(yticks, nticks / 2.0)
 
+    @slow
     def test_map(self):
         self.g.map(plt.contourf, 'x', 'y', Ellipsis)
         self.g.map(lambda: None)
 
+    @slow
     def test_map_dataset(self):
         g = xplt.FacetGrid(self.darray.to_dataset(name='foo'), col='z')
         g.map(plt.contourf, 'x', 'y', 'foo')
@@ -1167,6 +1208,7 @@ def test_map_dataset(self):
         self.assertIn('colors!', text_in_fig())
         self.assertEqual(1, len(find_possible_colorbars()))
 
+    @slow
     def test_set_axis_labels(self):
         g = self.g.map_dataarray(xplt.contourf, 'x', 'y')
         g.set_axis_labels('longitude', 'latitude')
@@ -1174,6 +1216,7 @@ def test_set_axis_labels(self):
         for label in ['longitude', 'latitude']:
             self.assertIn(label, alltxt)
 
+    @slow
     def test_facetgrid_colorbar(self):
         a = easy_array((10, 15, 4))
         d = DataArray(a, dims=['y', 'x', 'z'], name='foo')
@@ -1187,6 +1230,7 @@ def test_facetgrid_colorbar(self):
         d.plot.imshow(x='x', y='y', col='z', add_colorbar=False)
         self.assertEqual(0, len(find_possible_colorbars()))
 
+    @slow
     def test_facetgrid_polar(self):
         # test if polar projection in FacetGrid does not raise an exception
         self.darray.plot.pcolormesh(col='z',
@@ -1206,6 +1250,7 @@ def setUp(self):
 
         self.darray = darray
 
+    @slow
     def test_default_labels(self):
         g = xplt.FacetGrid(self.darray, col='col', row='row')
         self.assertEqual((2, 3), g.axes.shape)

From c18a9d5161a6b3405d93da8ad2fa15f97beb07c9 Mon Sep 17 00:00:00 2001
From: Stephan Hoyer <shoyer@gmail.com>
Date: Sun, 2 Apr 2017 18:05:43 -0700
Subject: [PATCH 2/8] Spelling: flakey -> flaky

---
 .travis.yml                   | 20 ++++++++++----------
 conftest.py                   |  4 ++--
 doc/whats-new.rst             |  6 +++---
 xarray/tests/__init__.py      |  6 +++---
 xarray/tests/test_backends.py |  8 ++++----
 5 files changed, 22 insertions(+), 22 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 47478c439f7..d1af0abd413 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -21,36 +21,36 @@ matrix:
   - python: 3.6
     env: CONDA_ENV=py36 EXTRA_FLAGS=--skip-optional-ci
   - python: 3.6
-    env: CONDA_ENV=py36-pydap EXTRA_FLAGS=--run-flakey
+    env: CONDA_ENV=py36-pydap EXTRA_FLAGS=--run-flaky
   - python: 3.6
-    env: CONDA_ENV=py36-netcdf4-dev EXTRA_FLAGS=--run-flakey
+    env: CONDA_ENV=py36-netcdf4-dev EXTRA_FLAGS=--run-flaky
     addons:
       apt_packages:
         - libhdf5-serial-dev
         - netcdf-bin
         - libnetcdf-dev
   - python: 3.6
-    env: CONDA_ENV=py36-dask-dev EXTRA_FLAGS=--run-flakey
+    env: CONDA_ENV=py36-dask-dev EXTRA_FLAGS=--run-flaky
   - python: 3.6
-    env: CONDA_ENV=py36-pandas-dev EXTRA_FLAGS=--run-flakey
+    env: CONDA_ENV=py36-pandas-dev EXTRA_FLAGS=--run-flaky
   - python: 3.6
-    env: CONDA_ENV=py36-condaforge-rc EXTRA_FLAGS=--run-flakey
+    env: CONDA_ENV=py36-condaforge-rc EXTRA_FLAGS=--run-flaky
   allow_failures:
   - python: 3.6
-    env: CONDA_ENV=py36-pydap EXTRA_FLAGS=--run-flakey
+    env: CONDA_ENV=py36-pydap EXTRA_FLAGS=--run-flaky
   - python: 3.6
-    env: CONDA_ENV=py36-netcdf4-dev EXTRA_FLAGS=--run-flakey
+    env: CONDA_ENV=py36-netcdf4-dev EXTRA_FLAGS=--run-flaky
     addons:
       apt_packages:
         - libhdf5-serial-dev
         - netcdf-bin
         - libnetcdf-dev
   - python: 3.6
-    env: CONDA_ENV=py36-dask-dev EXTRA_FLAGS=--run-flakey
+    env: CONDA_ENV=py36-dask-dev EXTRA_FLAGS=--run-flaky
   - python: 3.6
-    env: CONDA_ENV=py36-pandas-dev EXTRA_FLAGS=--run-flakey
+    env: CONDA_ENV=py36-pandas-dev EXTRA_FLAGS=--run-flaky
   - python: 3.6
-    env: CONDA_ENV=py36-condaforge-rc EXTRA_FLAGS=--run-flakey
+    env: CONDA_ENV=py36-condaforge-rc EXTRA_FLAGS=--run-flaky
 
 before_install:
   - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then
diff --git a/conftest.py b/conftest.py
index 396f324dcb5..ffa91a4817d 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,8 +1,8 @@
 import pytest
 
 def pytest_addoption(parser):
-    parser.addoption("--run-flakey", action="store_true",
-                     help="runs flakey tests")
+    parser.addoption("--run-flaky", action="store_true",
+                     help="runs flaky tests")
     parser.addoption("--skip-optional-ci", action="store_true",
                      help="skips optional tests continuous integration (CI)")
     parser.addoption("--skip-slow", action="store_true",
diff --git a/doc/whats-new.rst b/doc/whats-new.rst
index 7fdc1cb51e5..3460fbf3f6c 100644
--- a/doc/whats-new.rst
+++ b/doc/whats-new.rst
@@ -50,9 +50,9 @@ By `Henry S. Harrison <https://hsharrison.github.io>`_.
   cleaner output.  The full metadata is still available as ``data.attrs``.
   By `Zac Hatfield-Dodds <https://github.com/Zac-HD>`_.
 
-- Enhances tests by use of ``@slow``, ``@flakey``, and ``@optionalci``
-  decorrators, which are controled via ``--run-flakey``,
-  ``--skip-optional-ci``, and ``--skip-slow``command line arguments 
+- Enhances tests by use of ``@slow``, ``@flaky``, and ``@optionalci``
+  decorrators, which are controled via ``--run-flaky``,
+  ``--skip-optional-ci``, and ``--skip-slow``command line arguments
   to ``py.test``.  By `Stephan Hoyer <https://github.com/shoyer>`_ and
   `Phillip J. Wolfram <https://github.com/pwolfram>`_.
 
diff --git a/xarray/tests/__init__.py b/xarray/tests/__init__.py
index 7ce4bf57bf2..19df29d9416 100644
--- a/xarray/tests/__init__.py
+++ b/xarray/tests/__init__.py
@@ -114,9 +114,9 @@ def requires_bottleneck(test):
     return test if has_bottleneck else pytest.mark.skip('requires bottleneck')(test)
 
 
-flakey = pytest.mark.skipif(
-    not pytest.config.getoption("--run-flakey"),
-    reason="set --run-flakey option to run flakey tests")
+flaky = pytest.mark.skipif(
+    not pytest.config.getoption("--run-flaky"),
+    reason="set --run-flaky option to run flaky tests")
 
 optionalci = pytest.mark.skipif(
     pytest.config.getoption("--skip-optional-ci"),
diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py
index d726a5dc19d..de01364b2c8 100644
--- a/xarray/tests/test_backends.py
+++ b/xarray/tests/test_backends.py
@@ -27,7 +27,7 @@
 from . import (TestCase, requires_scipy, requires_netCDF4, requires_pydap,
                requires_scipy_or_netCDF4, requires_dask, requires_h5netcdf,
                requires_pynio, has_netCDF4, has_scipy, assert_allclose,
-               flakey, optionalci, slow)
+               flaky, optionalci, slow)
 from .test_dataset import create_test_data
 
 try:
@@ -1060,7 +1060,7 @@ def test_encoding_unlimited_dims(self):
                 ds.to_netcdf(tmp_file, engine='h5netcdf', unlimited_dims=['y'])
 
 # tests pending h5netcdf fix
-@flakey
+@flaky
 class H5NetCDFDataTestAutocloseTrue(H5NetCDFDataTest):
     autoclose = True
 
@@ -1109,7 +1109,7 @@ def test_3_autoclose_pynio(self):
     # probable h5netcdf error, uncomment when fixed to test
     @requires_dask
     @requires_h5netcdf
-    @flakey
+    @flaky
     def test_4_autoclose_h5netcdf(self):
         self.validate_open_mfdataset_autoclose(engine=['h5netcdf'])
 
@@ -1138,7 +1138,7 @@ def test_3_open_large_num_files_pynio(self):
     # probable h5netcdf error, uncomment when fixed to test
     @requires_dask
     @requires_h5netcdf
-    @flakey
+    @flaky
     @optionalci
     @slow
     def test_4_open_large_num_files_h5netcdf(self):

From 5a4cba5d1523093c46b9c6b5ec7d597d8c53ce62 Mon Sep 17 00:00:00 2001
From: Stephan Hoyer <shoyer@gmail.com>
Date: Sun, 2 Apr 2017 18:31:19 -0700
Subject: [PATCH 3/8] Test suite adjustments

---
 .travis.yml                            | 38 ++++++++++++++++----------
 appveyor.yml                           |  2 +-
 ci/requirements-py27-cdat+pynio.yml    | 18 ++++++------
 ci/requirements-py27-windows.yml       | 10 +++++--
 ci/requirements-py35.yml               |  7 +++--
 ci/requirements-py36-condaforge-rc.yml |  5 ++--
 ci/requirements-py36-dask-dev.yml      | 10 +++++--
 ci/requirements-py36-netcdf4-dev.yml   |  5 +++-
 ci/requirements-py36-pandas-dev.yml    | 15 ++++++----
 ci/requirements-py36-pydap.yml         |  5 +++-
 ci/requirements-py36-windows.yml       | 11 ++++++--
 ci/requirements-py36.yml               |  7 +++--
 conftest.py                            |  6 ++--
 doc/whats-new.rst                      |  5 ++--
 xarray/tests/__init__.py               |  6 ----
 xarray/tests/test_backends.py          | 20 ++++++++------
 16 files changed, 103 insertions(+), 67 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index d1af0abd413..b7c35231867 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -11,46 +11,54 @@ matrix:
   fast_finish: true
   include:
   - python: 2.7
-    env: CONDA_ENV=py27-min EXTRA_FLAGS=--skip-optional-ci
+    env: CONDA_ENV=py27-min
   - python: 2.7
-    env: CONDA_ENV=py27-cdat+pynio EXTRA_FLAGS=--skip-optional-ci
+    env: CONDA_ENV=py27-cdat+pynio
   - python: 3.4
-    env: CONDA_ENV=py34 EXTRA_FLAGS=--skip-optional-ci
+    env: CONDA_ENV=py34
   - python: 3.5
-    env: CONDA_ENV=py35 EXTRA_FLAGS=--skip-optional-ci
+    env: CONDA_ENV=py35
   - python: 3.6
-    env: CONDA_ENV=py36 EXTRA_FLAGS=--skip-optional-ci
+    env: CONDA_ENV=py36
   - python: 3.6
-    env: CONDA_ENV=py36-pydap EXTRA_FLAGS=--run-flaky
+    env:
+    - CONDA_ENV=py36
+    - EXTRA_FLAGS="--run-flaky"
   - python: 3.6
-    env: CONDA_ENV=py36-netcdf4-dev EXTRA_FLAGS=--run-flaky
+    env: CONDA_ENV=py36-pydap
+  - python: 3.6
+    env: CONDA_ENV=py36-netcdf4-dev
     addons:
       apt_packages:
         - libhdf5-serial-dev
         - netcdf-bin
         - libnetcdf-dev
   - python: 3.6
-    env: CONDA_ENV=py36-dask-dev EXTRA_FLAGS=--run-flaky
+    env: CONDA_ENV=py36-dask-dev
   - python: 3.6
-    env: CONDA_ENV=py36-pandas-dev EXTRA_FLAGS=--run-flaky
+    env: CONDA_ENV=py36-pandas-dev
   - python: 3.6
-    env: CONDA_ENV=py36-condaforge-rc EXTRA_FLAGS=--run-flaky
+    env: CONDA_ENV=py36-condaforge-rc
   allow_failures:
   - python: 3.6
-    env: CONDA_ENV=py36-pydap EXTRA_FLAGS=--run-flaky
+    env:
+    - CONDA_ENV=py36
+    - EXTRA_FLAGS="--run-flaky"
+  - python: 3.6
+    env: CONDA_ENV=py36-pydap
   - python: 3.6
-    env: CONDA_ENV=py36-netcdf4-dev EXTRA_FLAGS=--run-flaky
+    env: CONDA_ENV=py36-netcdf4-dev
     addons:
       apt_packages:
         - libhdf5-serial-dev
         - netcdf-bin
         - libnetcdf-dev
   - python: 3.6
-    env: CONDA_ENV=py36-dask-dev EXTRA_FLAGS=--run-flaky
+    env: CONDA_ENV=py36-dask-dev
   - python: 3.6
-    env: CONDA_ENV=py36-pandas-dev EXTRA_FLAGS=--run-flaky
+    env: CONDA_ENV=py36-pandas-dev
   - python: 3.6
-    env: CONDA_ENV=py36-condaforge-rc EXTRA_FLAGS=--run-flaky
+    env: CONDA_ENV=py36-condaforge-rc
 
 before_install:
   - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then
diff --git a/appveyor.yml b/appveyor.yml
index 97af6e34556..28d6918ca4d 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -43,4 +43,4 @@ install:
 build: false
 
 test_script:
-  - "py.test xarray --verbose --skip-optional-ci"
+  - "py.test xarray --verbose"
diff --git a/ci/requirements-py27-cdat+pynio.yml b/ci/requirements-py27-cdat+pynio.yml
index 3d2df5cffd1..62ba59ce4ca 100644
--- a/ci/requirements-py27-cdat+pynio.yml
+++ b/ci/requirements-py27-cdat+pynio.yml
@@ -3,20 +3,22 @@ channels:
   - conda-forge
 dependencies:
   - python=2.7
+  - bottleneck
   - cdat-lite
+  - cyordereddict
   - dask
   - distributed
-  - pytest
+  - h5py
+  - h5netcdf
+  - matplotlib
+  - netcdf4
   - numpy
-  - pandas>=0.15.0
+  - pandas
   - pynio
-  - netcdf4
+  - pytest
   - scipy
-  - pytest-cov
-  - cyordereddict
-  - h5py
-  - netcdf4
-  - matplotlib
   - seaborn
+  - toolz
   - pip:
     - coveralls
+    - pytest-cov
diff --git a/ci/requirements-py27-windows.yml b/ci/requirements-py27-windows.yml
index 7f2a51d4c97..caa77627acc 100644
--- a/ci/requirements-py27-windows.yml
+++ b/ci/requirements-py27-windows.yml
@@ -4,10 +4,14 @@ channels:
 dependencies:
   - python=2.7
   - dask
+  - distributed
+  - h5py
+  - h5netcdf
   - matplotlib
-  - netCDF4
+  - netcdf4
+  - pytest
   - numpy
   - pandas
-  - pip
-  - pytest
   - scipy
+  - seaborn
+  - toolz
diff --git a/ci/requirements-py35.yml b/ci/requirements-py35.yml
index df69d89c520..f23562d2247 100644
--- a/ci/requirements-py35.yml
+++ b/ci/requirements-py35.yml
@@ -3,18 +3,19 @@ channels:
   - conda-forge
 dependencies:
   - python=3.5
-  - cython
+  - bottleneck
   - dask
   - distributed
   - h5py
+  - h5netcdf
   - matplotlib
   - netcdf4
   - pytest
   - numpy
   - pandas
-  - seaborn
   - scipy
+  - seaborn
+  - toolz
   - pip:
     - coveralls
     - pytest-cov
-    - h5netcdf
diff --git a/ci/requirements-py36-condaforge-rc.yml b/ci/requirements-py36-condaforge-rc.yml
index 436192907ff..ef8a29462c9 100644
--- a/ci/requirements-py36-condaforge-rc.yml
+++ b/ci/requirements-py36-condaforge-rc.yml
@@ -4,10 +4,11 @@ channels:
   - conda-forge
 dependencies:
   - python=3.6
-  - cython
+  - bottleneck
   - dask
   - distributed
   - h5py
+  - h5netcdf
   - matplotlib
   - netcdf4
   - pytest
@@ -15,7 +16,7 @@ dependencies:
   - pandas
   - seaborn
   - scipy
+  - toolz
   - pip:
     - coveralls
     - pytest-cov
-    - h5netcdf
diff --git a/ci/requirements-py36-dask-dev.yml b/ci/requirements-py36-dask-dev.yml
index 463e41d3600..ab60ceb0ab9 100644
--- a/ci/requirements-py36-dask-dev.yml
+++ b/ci/requirements-py36-dask-dev.yml
@@ -3,13 +3,19 @@ channels:
   - conda-forge
 dependencies:
   - python=3.6
-  - cython
+  - bottleneck
+  - h5py
+  - h5netcdf
+  - matplotlib
+  - netcdf4
   - pytest
   - numpy
   - pandas
+  - seaborn
   - scipy
   - toolz
   - pip:
     - coveralls
     - pytest-cov
-    - git+https://github.com/blaze/dask.git
+    - git+https://github.com/dask/dask.git
+    - git+https://github.com/dask/distributed.git
diff --git a/ci/requirements-py36-netcdf4-dev.yml b/ci/requirements-py36-netcdf4-dev.yml
index 84f9fb3ffca..d6dff09825d 100644
--- a/ci/requirements-py36-netcdf4-dev.yml
+++ b/ci/requirements-py36-netcdf4-dev.yml
@@ -3,16 +3,19 @@ channels:
   - conda-forge
 dependencies:
   - python=3.6
+  - bottleneck
   - cython
   - dask
   - distributed
   - h5py
+  - h5netcdf
+  - matplotlib
   - pytest
   - numpy
   - pandas
   - scipy
+  - toolz
   - pip:
     - coveralls
     - pytest-cov
-    - h5netcdf
     - git+https://github.com/Unidata/netcdf4-python.git
diff --git a/ci/requirements-py36-pandas-dev.yml b/ci/requirements-py36-pandas-dev.yml
index 9c814a07b77..c525fe7c115 100644
--- a/ci/requirements-py36-pandas-dev.yml
+++ b/ci/requirements-py36-pandas-dev.yml
@@ -3,14 +3,19 @@ channels:
   - conda-forge
 dependencies:
   - python=3.6
-  - cython=0.23.4
+  - bottleneck
+  - cython
+  - dask
+  - distributed
+  - h5py
+  - h5netcdf
+  - matplotlib
+  - netcdf4
   - pytest
-  - numpy=1.10.1
-  - netcdf4=1.1.9
-  - scipy=0.16.0
+  - numpy
+  - scipy
   - toolz
   - pip:
     - coveralls
     - pytest-cov
-    - dask
     - git+https://github.com/pydata/pandas.git
diff --git a/ci/requirements-py36-pydap.yml b/ci/requirements-py36-pydap.yml
index 2197ea3864d..e541b598051 100644
--- a/ci/requirements-py36-pydap.yml
+++ b/ci/requirements-py36-pydap.yml
@@ -3,14 +3,17 @@ channels:
   - conda-forge
 dependencies:
   - python=3.6
+  - bottleneck
   - dask
   - distributed
   - h5py
+  - h5netcdf
+  - matplotlib
   - netcdf4
   - pytest
   - numpy
-  - pandas
   - scipy
+  - toolz
   - pip:
     - coveralls
     - pytest-cov
diff --git a/ci/requirements-py36-windows.yml b/ci/requirements-py36-windows.yml
index 48818479a39..25af7d31005 100644
--- a/ci/requirements-py36-windows.yml
+++ b/ci/requirements-py36-windows.yml
@@ -3,11 +3,16 @@ channels:
   - conda-forge
 dependencies:
   - python=3.6
+  - bottleneck
   - dask
+  - distributed
+  - h5py
+  - h5netcdf
   - matplotlib
-  - netCDF4
+  - netcdf4
+  - pytest
   - numpy
   - pandas
-  - pip
-  - pytest
   - scipy
+  - seaborn
+  - toolz
diff --git a/ci/requirements-py36.yml b/ci/requirements-py36.yml
index a7ba4166eef..4f587079e27 100644
--- a/ci/requirements-py36.yml
+++ b/ci/requirements-py36.yml
@@ -3,18 +3,19 @@ channels:
   - conda-forge
 dependencies:
   - python=3.6
-  - cython
+  - bottleneck
   - dask
   - distributed
   - h5py
+  - h5netcdf
   - matplotlib
   - netcdf4
   - pytest
   - numpy
   - pandas
-  - seaborn
   - scipy
+  - seaborn
+  - toolz
   - pip:
     - coveralls
     - pytest-cov
-    - h5netcdf
diff --git a/conftest.py b/conftest.py
index ffa91a4817d..acbdb0f0765 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,9 +1,9 @@
-import pytest
+"""Configuration for pytest."""
+
 
 def pytest_addoption(parser):
+    """Add command-line flags for pytest."""
     parser.addoption("--run-flaky", action="store_true",
                      help="runs flaky tests")
-    parser.addoption("--skip-optional-ci", action="store_true",
-                     help="skips optional tests continuous integration (CI)")
     parser.addoption("--skip-slow", action="store_true",
                      help="skips slow tests")
diff --git a/doc/whats-new.rst b/doc/whats-new.rst
index 3460fbf3f6c..aad440298a8 100644
--- a/doc/whats-new.rst
+++ b/doc/whats-new.rst
@@ -50,9 +50,8 @@ By `Henry S. Harrison <https://hsharrison.github.io>`_.
   cleaner output.  The full metadata is still available as ``data.attrs``.
   By `Zac Hatfield-Dodds <https://github.com/Zac-HD>`_.
 
-- Enhances tests by use of ``@slow``, ``@flaky``, and ``@optionalci``
-  decorrators, which are controled via ``--run-flaky``,
-  ``--skip-optional-ci``, and ``--skip-slow``command line arguments
+- Enhanced tests suite by use of ``@slow`` and ``@flaky`` decorators, which are
+  controlled via ``--run-flaky`` and ``--skip-slow`` command line arguments
   to ``py.test``.  By `Stephan Hoyer <https://github.com/shoyer>`_ and
   `Phillip J. Wolfram <https://github.com/pwolfram>`_.
 
diff --git a/xarray/tests/__init__.py b/xarray/tests/__init__.py
index 19df29d9416..2454c6bf787 100644
--- a/xarray/tests/__init__.py
+++ b/xarray/tests/__init__.py
@@ -118,12 +118,6 @@ def requires_bottleneck(test):
     not pytest.config.getoption("--run-flaky"),
     reason="set --run-flaky option to run flaky tests")
 
-optionalci = pytest.mark.skipif(
-    pytest.config.getoption("--skip-optional-ci"),
-    reason=("set --skip-optional-ci option to skip tests in CI, "
-            "e.g., due to travis CI resource issues"))
-
-
 slow = pytest.mark.skipif(
     pytest.config.getoption("--skip-slow"),
     reason="set --skip-slow option to run slow tests")
diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py
index de01364b2c8..ff82c158779 100644
--- a/xarray/tests/test_backends.py
+++ b/xarray/tests/test_backends.py
@@ -27,7 +27,7 @@
 from . import (TestCase, requires_scipy, requires_netCDF4, requires_pydap,
                requires_scipy_or_netCDF4, requires_dask, requires_h5netcdf,
                requires_pynio, has_netCDF4, has_scipy, assert_allclose,
-               flaky, optionalci, slow)
+               flaky, slow)
 from .test_dataset import create_test_data
 
 try:
@@ -1059,6 +1059,7 @@ def test_encoding_unlimited_dims(self):
             with pytest.warns(UserWarning):
                 ds.to_netcdf(tmp_file, engine='h5netcdf', unlimited_dims=['y'])
 
+
 # tests pending h5netcdf fix
 @flaky
 class H5NetCDFDataTestAutocloseTrue(H5NetCDFDataTest):
@@ -1106,40 +1107,43 @@ def test_3_autoclose_pynio(self):
         self.validate_open_mfdataset_autoclose(engine=['pynio'])
 
     # use of autoclose=True with h5netcdf broken because of
-    # probable h5netcdf error, uncomment when fixed to test
+    # probable h5netcdf error
     @requires_dask
     @requires_h5netcdf
-    @flaky
+    @pytest.pytest.mark.xfail
     def test_4_autoclose_h5netcdf(self):
         self.validate_open_mfdataset_autoclose(engine=['h5netcdf'])
 
+    # These tests below are marked as flaky (and skipped by default) because
+    # they fail sometimes on Travis-CI, for no clear reason.
+
     @requires_dask
     @requires_netCDF4
-    @optionalci
+    @flaky
     @slow
     def test_1_open_large_num_files_netcdf4(self):
         self.validate_open_mfdataset_large_num_files(engine=['netcdf4'])
 
     @requires_dask
     @requires_scipy
-    @optionalci
+    @flaky
     @slow
     def test_2_open_large_num_files_scipy(self):
         self.validate_open_mfdataset_large_num_files(engine=['scipy'])
 
     @requires_dask
     @requires_pynio
-    @optionalci
+    @flaky
     @slow
     def test_3_open_large_num_files_pynio(self):
         self.validate_open_mfdataset_large_num_files(engine=['pynio'])
 
     # use of autoclose=True with h5netcdf broken because of
-    # probable h5netcdf error, uncomment when fixed to test
+    # probable h5netcdf error
     @requires_dask
     @requires_h5netcdf
     @flaky
-    @optionalci
+    @pytest.mark.xfail
     @slow
     def test_4_open_large_num_files_h5netcdf(self):
         self.validate_open_mfdataset_large_num_files(engine=['h5netcdf'])

From 26aff95d557b8fcdaf161cf672c4e3887456d4cd Mon Sep 17 00:00:00 2001
From: Stephan Hoyer <shoyer@gmail.com>
Date: Sun, 2 Apr 2017 18:35:52 -0700
Subject: [PATCH 4/8] Typos

---
 xarray/tests/test_backends.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py
index ff82c158779..0a3a0ba7821 100644
--- a/xarray/tests/test_backends.py
+++ b/xarray/tests/test_backends.py
@@ -1061,7 +1061,7 @@ def test_encoding_unlimited_dims(self):
 
 
 # tests pending h5netcdf fix
-@flaky
+@pytest.mark.xfail
 class H5NetCDFDataTestAutocloseTrue(H5NetCDFDataTest):
     autoclose = True
 
@@ -1073,8 +1073,8 @@ def validate_open_mfdataset_autoclose(self, engine, nfiles=10):
         # test standard open_mfdataset approach with too many files
         with create_tmp_files(nfiles) as tmpfiles:
             for readengine in engine:
-                writeengine = \
-                        readengine if readengine != 'pynio' else 'netcdf4'
+                writeengine = (readengine if readengine != 'pynio'
+                               else 'netcdf4')
                 # split into multiple sets of temp files
                 for ii in original.x.values:
                     subds = original.isel(x=slice(ii, ii+1))
@@ -1110,7 +1110,7 @@ def test_3_autoclose_pynio(self):
     # probable h5netcdf error
     @requires_dask
     @requires_h5netcdf
-    @pytest.pytest.mark.xfail
+    @pytest.mark.xfail
     def test_4_autoclose_h5netcdf(self):
         self.validate_open_mfdataset_autoclose(engine=['h5netcdf'])
 

From 278a5667d2f6553e265e71569f472b787bfa3264 Mon Sep 17 00:00:00 2001
From: Stephan Hoyer <shoyer@gmail.com>
Date: Sun, 2 Apr 2017 19:05:38 -0700
Subject: [PATCH 5/8] Remove bottleneck from most CI builds

---
 ci/requirements-py27-cdat+pynio.yml    | 1 -
 ci/requirements-py35.yml               | 1 -
 ci/requirements-py36-condaforge-rc.yml | 1 -
 ci/requirements-py36-dask-dev.yml      | 1 -
 ci/requirements-py36-netcdf4-dev.yml   | 1 -
 ci/requirements-py36-pandas-dev.yml    | 1 -
 ci/requirements-py36-pydap.yml         | 1 -
 ci/requirements-py36-windows.yml       | 1 -
 ci/requirements-py36.yml               | 1 -
 9 files changed, 9 deletions(-)

diff --git a/ci/requirements-py27-cdat+pynio.yml b/ci/requirements-py27-cdat+pynio.yml
index 62ba59ce4ca..c88263fcfba 100644
--- a/ci/requirements-py27-cdat+pynio.yml
+++ b/ci/requirements-py27-cdat+pynio.yml
@@ -3,7 +3,6 @@ channels:
   - conda-forge
 dependencies:
   - python=2.7
-  - bottleneck
   - cdat-lite
   - cyordereddict
   - dask
diff --git a/ci/requirements-py35.yml b/ci/requirements-py35.yml
index f23562d2247..f6a62ac72a6 100644
--- a/ci/requirements-py35.yml
+++ b/ci/requirements-py35.yml
@@ -3,7 +3,6 @@ channels:
   - conda-forge
 dependencies:
   - python=3.5
-  - bottleneck
   - dask
   - distributed
   - h5py
diff --git a/ci/requirements-py36-condaforge-rc.yml b/ci/requirements-py36-condaforge-rc.yml
index ef8a29462c9..8426ca2df42 100644
--- a/ci/requirements-py36-condaforge-rc.yml
+++ b/ci/requirements-py36-condaforge-rc.yml
@@ -4,7 +4,6 @@ channels:
   - conda-forge
 dependencies:
   - python=3.6
-  - bottleneck
   - dask
   - distributed
   - h5py
diff --git a/ci/requirements-py36-dask-dev.yml b/ci/requirements-py36-dask-dev.yml
index ab60ceb0ab9..8606e15d614 100644
--- a/ci/requirements-py36-dask-dev.yml
+++ b/ci/requirements-py36-dask-dev.yml
@@ -3,7 +3,6 @@ channels:
   - conda-forge
 dependencies:
   - python=3.6
-  - bottleneck
   - h5py
   - h5netcdf
   - matplotlib
diff --git a/ci/requirements-py36-netcdf4-dev.yml b/ci/requirements-py36-netcdf4-dev.yml
index d6dff09825d..033d1f41b4d 100644
--- a/ci/requirements-py36-netcdf4-dev.yml
+++ b/ci/requirements-py36-netcdf4-dev.yml
@@ -3,7 +3,6 @@ channels:
   - conda-forge
 dependencies:
   - python=3.6
-  - bottleneck
   - cython
   - dask
   - distributed
diff --git a/ci/requirements-py36-pandas-dev.yml b/ci/requirements-py36-pandas-dev.yml
index c525fe7c115..ebcec868f76 100644
--- a/ci/requirements-py36-pandas-dev.yml
+++ b/ci/requirements-py36-pandas-dev.yml
@@ -3,7 +3,6 @@ channels:
   - conda-forge
 dependencies:
   - python=3.6
-  - bottleneck
   - cython
   - dask
   - distributed
diff --git a/ci/requirements-py36-pydap.yml b/ci/requirements-py36-pydap.yml
index e541b598051..c10bd93f928 100644
--- a/ci/requirements-py36-pydap.yml
+++ b/ci/requirements-py36-pydap.yml
@@ -3,7 +3,6 @@ channels:
   - conda-forge
 dependencies:
   - python=3.6
-  - bottleneck
   - dask
   - distributed
   - h5py
diff --git a/ci/requirements-py36-windows.yml b/ci/requirements-py36-windows.yml
index 25af7d31005..e84a9346b59 100644
--- a/ci/requirements-py36-windows.yml
+++ b/ci/requirements-py36-windows.yml
@@ -3,7 +3,6 @@ channels:
   - conda-forge
 dependencies:
   - python=3.6
-  - bottleneck
   - dask
   - distributed
   - h5py
diff --git a/ci/requirements-py36.yml b/ci/requirements-py36.yml
index 4f587079e27..be78d32ddb1 100644
--- a/ci/requirements-py36.yml
+++ b/ci/requirements-py36.yml
@@ -3,7 +3,6 @@ channels:
   - conda-forge
 dependencies:
   - python=3.6
-  - bottleneck
   - dask
   - distributed
   - h5py

From 673e6dc520c3d422c727a03d0920d9662800af97 Mon Sep 17 00:00:00 2001
From: Stephan Hoyer <shoyer@gmail.com>
Date: Sun, 2 Apr 2017 20:22:38 -0700
Subject: [PATCH 6/8] xfail for cross engine read write netcdf4

---
 xarray/tests/test_backends.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py
index 0a3a0ba7821..1950f9802c4 100644
--- a/xarray/tests/test_backends.py
+++ b/xarray/tests/test_backends.py
@@ -1025,13 +1025,11 @@ def test_complex(self):
         with self.roundtrip(expected) as actual:
             self.assertDatasetEqual(expected, actual)
 
+    @pytest.mark.xfail(reason='https://github.com/pydata/xarray/issues/535')
     def test_cross_engine_read_write_netcdf4(self):
         # Drop dim3, because its labels include strings. These appear to be
         # not properly read with python-netCDF4, which converts them into
         # unicode instead of leaving them as bytes.
-        if PY3:
-            raise unittest.SkipTest('see https://github.com/pydata/xarray/issues/535')
-
         data = create_test_data().drop('dim3')
         data.attrs['foo'] = 'bar'
         valid_engines = ['netcdf4', 'h5netcdf']

From 283c80f2a51fa319594841f5582eadf6edda36fb Mon Sep 17 00:00:00 2001
From: "Phillip J. Wolfram" <phillipwolfram@gmail.com>
Date: Sun, 2 Apr 2017 22:26:23 -0600
Subject: [PATCH 7/8] Try fix: Closes open file (for Windows error)

```
WindowsError: [Error 32] The process cannot access the file because it
is being used by another process
```
---
 xarray/tests/test_backends.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py
index 1950f9802c4..091d66794e5 100644
--- a/xarray/tests/test_backends.py
+++ b/xarray/tests/test_backends.py
@@ -1047,6 +1047,7 @@ def test_read_byte_attrs_as_unicode(self):
             actual = open_dataset(tmp_file)
             expected = Dataset(attrs={'foo': 'bar'})
             self.assertDatasetIdentical(expected, actual)
+            actual.close()
 
     def test_encoding_unlimited_dims(self):
         ds = Dataset({'x': ('y', np.arange(10.0))})

From 57d33249c233afc6cc0d71f88e16398eb0406f70 Mon Sep 17 00:00:00 2001
From: "Phillip J. Wolfram" <phillipwolfram@gmail.com>
Date: Sun, 2 Apr 2017 22:26:23 -0600
Subject: [PATCH 8/8] Try fix: Closes open file (for Windows error)

```
WindowsError: [Error 32] The process cannot access the file because it
is being used by another process
```
---
 xarray/tests/test_backends.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py
index 091d66794e5..0e73b710663 100644
--- a/xarray/tests/test_backends.py
+++ b/xarray/tests/test_backends.py
@@ -120,8 +120,8 @@ def test_write_store(self):
             expected.dump_to_store(store)
             # we need to cf decode the store because it has time and
             # non-dimension coordinates
-            actual = xr.decode_cf(store)
-            self.assertDatasetAllClose(expected, actual)
+            with xr.decode_cf(store) as actual:
+                self.assertDatasetAllClose(expected, actual)
 
     def check_dtypes_roundtripped(self, expected, actual):
         for k in expected:
@@ -1044,10 +1044,9 @@ def test_read_byte_attrs_as_unicode(self):
         with create_tmp_file() as tmp_file:
             with nc4.Dataset(tmp_file, 'w') as nc:
                 nc.foo = b'bar'
-            actual = open_dataset(tmp_file)
-            expected = Dataset(attrs={'foo': 'bar'})
-            self.assertDatasetIdentical(expected, actual)
-            actual.close()
+            with open_dataset(tmp_file) as actual:
+                expected = Dataset(attrs={'foo': 'bar'})
+                self.assertDatasetIdentical(expected, actual)
 
     def test_encoding_unlimited_dims(self):
         ds = Dataset({'x': ('y', np.arange(10.0))})