diff --git a/.travis.yml b/.travis.yml index 13c5362e530..47478c439f7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,46 +11,46 @@ matrix: fast_finish: true include: - python: 2.7 - env: CONDA_ENV=py27-min + env: CONDA_ENV=py27-min EXTRA_FLAGS=--skip-optional-ci - python: 2.7 - env: CONDA_ENV=py27-cdat+pynio + env: CONDA_ENV=py27-cdat+pynio EXTRA_FLAGS=--skip-optional-ci - python: 3.4 - env: CONDA_ENV=py34 + env: CONDA_ENV=py34 EXTRA_FLAGS=--skip-optional-ci - python: 3.5 - env: CONDA_ENV=py35 + env: CONDA_ENV=py35 EXTRA_FLAGS=--skip-optional-ci - python: 3.6 - env: CONDA_ENV=py36 + env: CONDA_ENV=py36 EXTRA_FLAGS=--skip-optional-ci - python: 3.6 - env: CONDA_ENV=py36-pydap + env: CONDA_ENV=py36-pydap EXTRA_FLAGS=--run-flakey - python: 3.6 - env: CONDA_ENV=py36-netcdf4-dev + env: CONDA_ENV=py36-netcdf4-dev EXTRA_FLAGS=--run-flakey addons: apt_packages: - libhdf5-serial-dev - netcdf-bin - libnetcdf-dev - python: 3.6 - env: CONDA_ENV=py36-dask-dev + env: CONDA_ENV=py36-dask-dev EXTRA_FLAGS=--run-flakey - python: 3.6 - env: CONDA_ENV=py36-pandas-dev + env: CONDA_ENV=py36-pandas-dev EXTRA_FLAGS=--run-flakey - python: 3.6 - env: CONDA_ENV=py36-condaforge-rc + env: CONDA_ENV=py36-condaforge-rc EXTRA_FLAGS=--run-flakey allow_failures: - python: 3.6 - env: CONDA_ENV=py36-pydap + env: CONDA_ENV=py36-pydap EXTRA_FLAGS=--run-flakey - python: 3.6 - env: CONDA_ENV=py36-netcdf4-dev + env: CONDA_ENV=py36-netcdf4-dev EXTRA_FLAGS=--run-flakey addons: apt_packages: - libhdf5-serial-dev - netcdf-bin - libnetcdf-dev - python: 3.6 - env: CONDA_ENV=py36-dask-dev + env: CONDA_ENV=py36-dask-dev EXTRA_FLAGS=--run-flakey - python: 3.6 - env: CONDA_ENV=py36-pandas-dev + env: CONDA_ENV=py36-pandas-dev EXTRA_FLAGS=--run-flakey - python: 3.6 - env: CONDA_ENV=py36-condaforge-rc + env: CONDA_ENV=py36-condaforge-rc EXTRA_FLAGS=--run-flakey before_install: - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then @@ -76,7 +76,7 @@ install: - python setup.py install script: - - py.test xarray --cov=xarray --cov-report term-missing --verbose + - py.test xarray --cov=xarray --cov-report term-missing --verbose $EXTRA_FLAGS after_success: - coveralls diff --git a/appveyor.yml b/appveyor.yml index 9f04ad8534c..f1c2bbdce5d 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -38,4 +38,4 @@ install: build: false test_script: - - "py.test xarray --verbose" + - "py.test xarray --verbose --skip-optional-ci" diff --git a/conftest.py b/conftest.py new file mode 100644 index 00000000000..396f324dcb5 --- /dev/null +++ b/conftest.py @@ -0,0 +1,9 @@ +import pytest + +def pytest_addoption(parser): + parser.addoption("--run-flakey", action="store_true", + help="runs flakey tests") + parser.addoption("--skip-optional-ci", action="store_true", + help="skips optional tests continuous integration (CI)") + parser.addoption("--skip-slow", action="store_true", + help="skips slow tests") diff --git a/doc/whats-new.rst b/doc/whats-new.rst index bc7b0fd6c5c..1fb15927814 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -41,6 +41,12 @@ By `Henry S. Harrison `_. Note, the default is ``autoclose=False``, which is consistent with previous xarray behavior. By `Phillip J. Wolfram `_. +- Enhances tests by use of slow, flakey, and optionalci decorrators, which + are controled via ``--run-flakey``, ``--skip-optional-ci``, and + ``--skip-slow`` command line arguments to ``py.test``. + By `Stephan Hoyer `_ and + `Phillip J. Wolfram `_. + Bug fixes ~~~~~~~~~ - ``rolling`` now keeps its original dimension order (:issue:`1125`). diff --git a/xarray/tests/__init__.py b/xarray/tests/__init__.py index 843b8bfdfbd..7ce4bf57bf2 100644 --- a/xarray/tests/__init__.py +++ b/xarray/tests/__init__.py @@ -114,6 +114,21 @@ def requires_bottleneck(test): return test if has_bottleneck else pytest.mark.skip('requires bottleneck')(test) +flakey = pytest.mark.skipif( + not pytest.config.getoption("--run-flakey"), + reason="set --run-flakey option to run flakey tests") + +optionalci = pytest.mark.skipif( + pytest.config.getoption("--skip-optional-ci"), + reason=("set --skip-optional-ci option to skip tests in CI, " + "e.g., due to travis CI resource issues")) + + +slow = pytest.mark.skipif( + pytest.config.getoption("--skip-slow"), + reason="set --skip-slow option to run slow tests") + + class TestCase(unittest.TestCase): if PY3: # Python 3 assertCountEqual is roughly equivalent to Python 2 diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 7fe2d936506..f4f22623160 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -26,7 +26,8 @@ from . import (TestCase, requires_scipy, requires_netCDF4, requires_pydap, requires_scipy_or_netCDF4, requires_dask, requires_h5netcdf, - requires_pynio, has_netCDF4, has_scipy, assert_allclose) + requires_pynio, has_netCDF4, has_scipy, assert_allclose, + flakey, optionalci, slow) from .test_dataset import create_test_data try: @@ -1059,8 +1060,9 @@ def test_encoding_unlimited_dims(self): ds.to_netcdf(tmp_file, engine='h5netcdf', unlimited_dims=['y']) # tests pending h5netcdf fix -# class H5NetCDFDataTestAutocloseTrue(H5NetCDFDataTest): -# autoclose = True +@flakey +class H5NetCDFDataTestAutocloseTrue(H5NetCDFDataTest): + autoclose = True class OpenMFDatasetManyFilesTest(TestCase): @@ -1105,32 +1107,42 @@ def test_3_autoclose_pynio(self): # use of autoclose=True with h5netcdf broken because of # probable h5netcdf error, uncomment when fixed to test - # @requires_dask - # @requires_h5netcdf - # def test_4_autoclose_h5netcdf(self): - # self.validate_open_mfdataset_autoclose(engine=['h5netcdf']) + @requires_dask + @requires_h5netcdf + @flakey + def test_4_autoclose_h5netcdf(self): + self.validate_open_mfdataset_autoclose(engine=['h5netcdf']) @requires_dask @requires_netCDF4 + @optionalci + @slow def test_1_open_large_num_files_netcdf4(self): self.validate_open_mfdataset_large_num_files(engine=['netcdf4']) @requires_dask @requires_scipy + @optionalci + @slow def test_2_open_large_num_files_scipy(self): self.validate_open_mfdataset_large_num_files(engine=['scipy']) @requires_dask @requires_pynio + @optionalci + @slow def test_3_open_large_num_files_pynio(self): self.validate_open_mfdataset_large_num_files(engine=['pynio']) # use of autoclose=True with h5netcdf broken because of # probable h5netcdf error, uncomment when fixed to test - # @requires_dask - # @requires_h5netcdf - # def test_4_open_large_num_files_h5netcdf(self): - # self.validate_open_mfdataset_large_num_files(engine=['h5netcdf']) + @requires_dask + @requires_h5netcdf + @flakey + @optionalci + @slow + def test_4_open_large_num_files_h5netcdf(self): + self.validate_open_mfdataset_large_num_files(engine=['h5netcdf']) @requires_dask diff --git a/xarray/tests/test_plot.py b/xarray/tests/test_plot.py index 55ca5b0d929..621c1680492 100644 --- a/xarray/tests/test_plot.py +++ b/xarray/tests/test_plot.py @@ -26,7 +26,7 @@ _build_discrete_cmap, _color_palette) -from . import TestCase, requires_matplotlib +from . import TestCase, requires_matplotlib, slow def text_in_fig(): @@ -74,6 +74,7 @@ def pass_in_axis(self, plotmethod): plotmethod(ax=axes[0]) self.assertTrue(axes[0].has_data()) + @slow def imshow_called(self, plotmethod): plotmethod() images = plt.gca().findobj(mpl.image.AxesImage) @@ -100,6 +101,7 @@ def test_2d_before_squeeze(self): def test2d_uniform_calls_imshow(self): self.assertTrue(self.imshow_called(self.darray[:, :, 0].plot.imshow)) + @slow def test2d_nonuniform_calls_contourf(self): a = self.darray[:, :, 0] a.coords['dim_1'] = [2, 1, 89] @@ -139,6 +141,7 @@ def test_datetime_dimension(self): ax = plt.gca() self.assertTrue(ax.has_data()) + @slow def test_convenient_facetgrid(self): a = easy_array((10, 15, 4)) d = DataArray(a, dims=['y', 'x', 'z']) @@ -155,6 +158,7 @@ def test_convenient_facetgrid(self): with self.assertRaisesRegexp(ValueError, '[Ff]acet'): d[0].plot(x='x', y='y', col='z', ax=plt.gca()) + @slow def test_subplot_kws(self): a = easy_array((10, 15, 4)) d = DataArray(a, dims=['y', 'x', 'z']) @@ -169,6 +173,7 @@ def test_subplot_kws(self): except AttributeError: self.assertEqual(ax.get_axis_bgcolor(), 'r') + @slow def test_plot_size(self): self.darray[:, 0, 0].plot(figsize=(13, 5)) assert tuple(plt.gcf().get_size_inches()) == (13, 5) @@ -194,6 +199,7 @@ def test_plot_size(self): with self.assertRaisesRegexp(ValueError, 'cannot provide `aspect`'): self.darray.plot(aspect=1) + @slow def test_convenient_facetgrid_4d(self): a = easy_array((10, 15, 2, 3)) d = DataArray(a, dims=['y', 'x', 'columns', 'rows']) @@ -248,6 +254,7 @@ def test_primitive_returned(self): p = self.darray.plot.line() self.assertTrue(isinstance(p[0], mpl.lines.Line2D)) + @slow def test_plot_nans(self): self.darray[1] = np.nan self.darray.plot.line() @@ -299,6 +306,7 @@ def test_primitive_returned(self): h = self.darray.plot.hist() self.assertTrue(isinstance(h[-1][0], mpl.patches.Rectangle)) + @slow def test_plot_nans(self): self.darray[0, 0, 0] = np.nan self.darray.plot.hist() @@ -327,6 +335,7 @@ def test_center(self): self.assertIsNone(cmap_params['levels']) self.assertIsNone(cmap_params['norm']) + @slow def test_integer_levels(self): data = self.data + 1 @@ -466,11 +475,13 @@ def setUp(self): self.data_min = distance.min() self.data_max = distance.max() + @slow def test_recover_from_seaborn_jet_exception(self): pal = _color_palette('jet', 4) self.assertTrue(type(pal) == np.ndarray) self.assertEqual(len(pal), 4) + @slow def test_build_discrete_cmap(self): for (cmap, levels, extend, filled) in [('jet', [0, 1], 'both', False), ('hot', [-4, 4], 'max', True)]: @@ -486,6 +497,7 @@ def test_build_discrete_cmap(self): else: self.assertEqual(ncmap.colorbar_extend, 'max') + @slow def test_discrete_colormap_list_of_levels(self): for extend, levels in [('max', [-1, 2, 4, 8, 10]), ('both', [2, 5, 10, 11]), @@ -502,6 +514,7 @@ def test_discrete_colormap_list_of_levels(self): self.assertEqual('max', primitive.cmap.colorbar_extend) self.assertEqual(len(levels) - 1, len(primitive.cmap.colors)) + @slow def test_discrete_colormap_int_levels(self): for extend, levels, vmin, vmax in [('neither', 7, None, None), ('neither', 7, None, 20), @@ -813,10 +826,12 @@ def test_facetgrid_cmap(self): self.assertTrue(len(set(m.get_cmap().name for m in fg._mappables)) == 1) +@slow class TestContourf(Common2dMixin, PlotTestCase): plotfunc = staticmethod(xplt.contourf) + @slow def test_contourf_called(self): # Having both statements ensures the test works properly self.assertFalse(self.contourf_called(self.darray.plot.imshow)) @@ -826,6 +841,7 @@ def test_primitive_artist_returned(self): artist = self.plotmethod() self.assertTrue(isinstance(artist, mpl.contour.QuadContourSet)) + @slow def test_extend(self): artist = self.plotmethod() self.assertEqual(artist.extend, 'neither') @@ -843,6 +859,7 @@ def test_extend(self): artist = self.plotmethod(vmin=-10, vmax=0) self.assertEqual(artist.extend, 'max') + @slow def test_2d_coord_names(self): self.plotmethod(x='x2d', y='y2d') # make sure labels came out ok @@ -850,6 +867,7 @@ def test_2d_coord_names(self): self.assertEqual('x2d', ax.get_xlabel()) self.assertEqual('y2d', ax.get_ylabel()) + @slow def test_levels(self): artist = self.plotmethod(levels=[-0.5, -0.4, 0.1]) self.assertEqual(artist.extend, 'both') @@ -858,6 +876,7 @@ def test_levels(self): self.assertEqual(artist.extend, 'neither') +@slow class TestContour(Common2dMixin, PlotTestCase): plotfunc = staticmethod(xplt.contour) @@ -898,6 +917,7 @@ def list_of_colors_in_cmap_deprecated(self): with self.assertRaises(Exception): self.plotmethod(cmap=['k', 'b']) + @slow def test_2d_coord_names(self): self.plotmethod(x='x2d', y='y2d') # make sure labels came out ok @@ -924,6 +944,7 @@ def test_everything_plotted(self): artist = self.plotmethod() self.assertEqual(artist.get_array().size, self.darray.size) + @slow def test_2d_coord_names(self): self.plotmethod(x='x2d', y='y2d') # make sure labels came out ok @@ -941,11 +962,12 @@ def test_dont_infer_interval_breaks_for_cartopy(self): # Let cartopy handle the axis limits and artist size self.assertTrue(artist.get_array().size <= self.darray.size) - +@slow class TestImshow(Common2dMixin, PlotTestCase): plotfunc = staticmethod(xplt.imshow) + @slow def test_imshow_called(self): # Having both statements ensures the test works properly self.assertFalse(self.imshow_called(self.darray.plot.contourf)) @@ -960,6 +982,7 @@ def test_default_aspect_is_auto(self): self.darray.plot.imshow() self.assertEqual('auto', plt.gca().get_aspect()) + @slow def test_cannot_change_mpl_aspect(self): with self.assertRaisesRegexp(ValueError, 'not available in xarray'): @@ -970,10 +993,12 @@ def test_cannot_change_mpl_aspect(self): self.assertEqual('auto', plt.gca().get_aspect()) assert tuple(plt.gcf().get_size_inches()) == (10, 5) + @slow def test_primitive_artist_returned(self): artist = self.plotmethod() self.assertTrue(isinstance(artist, mpl.image.AxesImage)) + @slow def test_seaborn_palette_needs_levels(self): try: import seaborn @@ -994,6 +1019,7 @@ def setUp(self): coords={'z': ['a', 'b', 'c']}) self.g = xplt.FacetGrid(self.darray, col='z') + @slow def test_no_args(self): self.g.map_dataarray(xplt.contourf, 'x', 'y') @@ -1008,6 +1034,7 @@ def test_no_args(self): fontsize = ax.title.get_size() self.assertLessEqual(fontsize, 12) + @slow def test_names_appear_somewhere(self): self.darray.name = 'testvar' self.g.map_dataarray(xplt.contourf, 'x', 'y') @@ -1019,6 +1046,7 @@ def test_names_appear_somewhere(self): for label in ['x', 'y']: self.assertIn(label, alltxt) + @slow def test_text_not_super_long(self): self.darray.coords['z'] = [100 * letter for letter in 'abc'] g = xplt.FacetGrid(self.darray, col='z') @@ -1030,6 +1058,7 @@ def test_text_not_super_long(self): t0 = g.axes[0, 0].get_title() self.assertTrue(t0.endswith('...')) + @slow def test_colorbar(self): vmin = self.darray.values.min() vmax = self.darray.values.max() @@ -1043,6 +1072,7 @@ def test_colorbar(self): self.assertEqual(1, len(find_possible_colorbars())) + @slow def test_empty_cell(self): g = xplt.FacetGrid(self.darray, col='z', col_wrap=2) g.map_dataarray(xplt.imshow, 'x', 'y') @@ -1051,10 +1081,12 @@ def test_empty_cell(self): self.assertFalse(bottomright.has_data()) self.assertFalse(bottomright.get_visible()) + @slow def test_norow_nocol_error(self): with self.assertRaisesRegexp(ValueError, r'[Rr]ow'): xplt.FacetGrid(self.darray) + @slow def test_groups(self): self.g.map_dataarray(xplt.imshow, 'x', 'y') upperleft_dict = self.g.name_dicts[0, 0] @@ -1063,16 +1095,19 @@ def test_groups(self): self.assertDataArrayEqual(upperleft_array, z0) + @slow def test_float_index(self): self.darray.coords['z'] = [0.1, 0.2, 0.4] g = xplt.FacetGrid(self.darray, col='z') g.map_dataarray(xplt.imshow, 'x', 'y') + @slow def test_nonunique_index_error(self): self.darray.coords['z'] = [0.1, 0.2, 0.2] with self.assertRaisesRegexp(ValueError, r'[Uu]nique'): xplt.FacetGrid(self.darray, col='z') + @slow def test_robust(self): z = np.zeros((20, 20, 2)) darray = DataArray(z, dims=['y', 'x', 'z']) @@ -1094,6 +1129,7 @@ def test_robust(self): largest = max(abs(x) for x in numbers) self.assertLess(largest, 21) + @slow def test_can_set_vmin_vmax(self): vmin, vmax = 50.0, 1000.0 expected = np.array((vmin, vmax)) @@ -1103,12 +1139,14 @@ def test_can_set_vmin_vmax(self): clim = np.array(image.get_clim()) self.assertTrue(np.allclose(expected, clim)) + @slow def test_can_set_norm(self): norm = mpl.colors.SymLogNorm(0.1) self.g.map_dataarray(xplt.imshow, 'x', 'y', norm=norm) for image in plt.gcf().findobj(mpl.image.AxesImage): self.assertIs(image.norm, norm) + @slow def test_figure_size(self): self.assertArrayEqual(self.g.fig.get_size_inches(), (10, 3)) @@ -1131,6 +1169,7 @@ def test_figure_size(self): with self.assertRaisesRegexp(ValueError, "Can't use"): g = xplt.plot(self.darray, row=2, col='z', ax=plt.gca(), size=6) + @slow def test_num_ticks(self): nticks = 99 maxticks = nticks + 1 @@ -1145,10 +1184,12 @@ def test_num_ticks(self): self.assertGreaterEqual(xticks, nticks / 2.0) self.assertGreaterEqual(yticks, nticks / 2.0) + @slow def test_map(self): self.g.map(plt.contourf, 'x', 'y', Ellipsis) self.g.map(lambda: None) + @slow def test_map_dataset(self): g = xplt.FacetGrid(self.darray.to_dataset(name='foo'), col='z') g.map(plt.contourf, 'x', 'y', 'foo') @@ -1167,6 +1208,7 @@ def test_map_dataset(self): self.assertIn('colors!', text_in_fig()) self.assertEqual(1, len(find_possible_colorbars())) + @slow def test_set_axis_labels(self): g = self.g.map_dataarray(xplt.contourf, 'x', 'y') g.set_axis_labels('longitude', 'latitude') @@ -1174,6 +1216,7 @@ def test_set_axis_labels(self): for label in ['longitude', 'latitude']: self.assertIn(label, alltxt) + @slow def test_facetgrid_colorbar(self): a = easy_array((10, 15, 4)) d = DataArray(a, dims=['y', 'x', 'z'], name='foo') @@ -1187,6 +1230,7 @@ def test_facetgrid_colorbar(self): d.plot.imshow(x='x', y='y', col='z', add_colorbar=False) self.assertEqual(0, len(find_possible_colorbars())) + @slow def test_facetgrid_polar(self): # test if polar projection in FacetGrid does not raise an exception self.darray.plot.pcolormesh(col='z', @@ -1206,6 +1250,7 @@ def setUp(self): self.darray = darray + @slow def test_default_labels(self): g = xplt.FacetGrid(self.darray, col='col', row='row') self.assertEqual((2, 3), g.axes.shape)