Adds marked tests as slow, flakey, and optionalci

Tests are marked using the following py.test options:

 * @slow which is run on default, not run if `--skip-slow`.
 * @optionalci is run on default, not run if `--skip-optional-ci`
 * @flakey which is not run on default, run if `--run-flakey`

Note, the `optionalci` category is needed in order to mark
tests that are environment dependent when run on continuous
integration (CI) hardware, e.g., tests that fail due to
variable loads on travis CI.

.travis.yml

@@ -11,46 +11,46 @@ matrix:
   fast_finish: true
   include:
   - python: 2.7
-    env: CONDA_ENV=py27-min
+    env: CONDA_ENV=py27-min EXTRA_FLAGS=--skip-optional-ci
   - python: 2.7
-    env: CONDA_ENV=py27-cdat+pynio
+    env: CONDA_ENV=py27-cdat+pynio EXTRA_FLAGS=--skip-optional-ci
   - python: 3.4
-    env: CONDA_ENV=py34
+    env: CONDA_ENV=py34 EXTRA_FLAGS=--skip-optional-ci
   - python: 3.5
-    env: CONDA_ENV=py35
+    env: CONDA_ENV=py35 EXTRA_FLAGS=--skip-optional-ci
   - python: 3.6
-    env: CONDA_ENV=py36
+    env: CONDA_ENV=py36 EXTRA_FLAGS=--skip-optional-ci
   - python: 3.6
-    env: CONDA_ENV=py36-pydap
+    env: CONDA_ENV=py36-pydap EXTRA_FLAGS=--run-flakey
   - python: 3.6
-    env: CONDA_ENV=py36-netcdf4-dev
+    env: CONDA_ENV=py36-netcdf4-dev EXTRA_FLAGS=--run-flakey
     addons:
       apt_packages:
         - libhdf5-serial-dev
         - netcdf-bin
         - libnetcdf-dev
   - python: 3.6
-    env: CONDA_ENV=py36-dask-dev
+    env: CONDA_ENV=py36-dask-dev EXTRA_FLAGS=--run-flakey
   - python: 3.6
-    env: CONDA_ENV=py36-pandas-dev
+    env: CONDA_ENV=py36-pandas-dev EXTRA_FLAGS=--run-flakey
   - python: 3.6
-    env: CONDA_ENV=py36-condaforge-rc
+    env: CONDA_ENV=py36-condaforge-rc EXTRA_FLAGS=--run-flakey
   allow_failures:
   - python: 3.6
-    env: CONDA_ENV=py36-pydap
+    env: CONDA_ENV=py36-pydap EXTRA_FLAGS=--run-flakey
   - python: 3.6
-    env: CONDA_ENV=py36-netcdf4-dev
+    env: CONDA_ENV=py36-netcdf4-dev EXTRA_FLAGS=--run-flakey
     addons:
       apt_packages:
         - libhdf5-serial-dev
         - netcdf-bin
         - libnetcdf-dev
   - python: 3.6
-    env: CONDA_ENV=py36-dask-dev
+    env: CONDA_ENV=py36-dask-dev EXTRA_FLAGS=--run-flakey
   - python: 3.6
-    env: CONDA_ENV=py36-pandas-dev
+    env: CONDA_ENV=py36-pandas-dev EXTRA_FLAGS=--run-flakey
   - python: 3.6
-    env: CONDA_ENV=py36-condaforge-rc
+    env: CONDA_ENV=py36-condaforge-rc EXTRA_FLAGS=--run-flakey
 
 before_install:
   - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then
@@ -76,7 +76,7 @@ install:
   - python setup.py install
 
 script:
-  - py.test xarray --cov=xarray --cov-report term-missing --verbose
+  - py.test xarray --cov=xarray --cov-report term-missing --verbose $EXTRA_FLAGS
 
 after_success:
   - coveralls

appveyor.yml

@@ -38,4 +38,4 @@ install:
 build: false
 
 test_script:
-  - "py.test xarray --verbose"
+  - "py.test xarray --verbose --skip-optional-ci"

conftest.py

@@ -0,0 +1,9 @@
+import pytest
+
+def pytest_addoption(parser):
+    parser.addoption("--run-flakey", action="store_true",
+                     help="runs flakey tests")
+    parser.addoption("--skip-optional-ci", action="store_true",
+                     help="skips optional tests continuous integration (CI)")
+    parser.addoption("--skip-slow", action="store_true",
+                     help="skips slow tests")

doc/whats-new.rst

@@ -41,6 +41,12 @@ By `Henry S. Harrison <https://hsharrison.github.io>`_.
   Note, the default is ``autoclose=False``, which is consistent with previous
   xarray behavior.  By `Phillip J. Wolfram <https://github.com/pwolfram>`_.
 
+- Enhances tests by use of slow, flakey, and optionalci decorrators, which
+  are controled via ``--run-flakey``, ``--skip-optional-ci``, and
+  ``--skip-slow`` command line arguments to ``py.test``.
+  By `Stephan Hoyer <https://github.com/shoyer>`_ and
+  `Phillip J. Wolfram <https://github.com/pwolfram>`_.
+
 Bug fixes
 ~~~~~~~~~
 - ``rolling`` now keeps its original dimension order (:issue:`1125`).

xarray/tests/__init__.py

@@ -114,6 +114,21 @@ def requires_bottleneck(test):
     return test if has_bottleneck else pytest.mark.skip('requires bottleneck')(test)
 
 
+flakey = pytest.mark.skipif(
+    not pytest.config.getoption("--run-flakey"),
+    reason="set --run-flakey option to run flakey tests")
+
+optionalci = pytest.mark.skipif(
+    pytest.config.getoption("--skip-optional-ci"),
+    reason=("set --skip-optional-ci option to skip tests in CI, "
+            "e.g., due to travis CI resource issues"))
+
+
+slow = pytest.mark.skipif(
+    pytest.config.getoption("--skip-slow"),
+    reason="set --skip-slow option to run slow tests")
+
+
 class TestCase(unittest.TestCase):
     if PY3:
         # Python 3 assertCountEqual is roughly equivalent to Python 2

xarray/tests/test_backends.py

@@ -26,7 +26,8 @@
 
 from . import (TestCase, requires_scipy, requires_netCDF4, requires_pydap,
                requires_scipy_or_netCDF4, requires_dask, requires_h5netcdf,
-               requires_pynio, has_netCDF4, has_scipy, assert_allclose)
+               requires_pynio, has_netCDF4, has_scipy, assert_allclose,
+               flakey, optionalci, slow)
 from .test_dataset import create_test_data
 
 try:
@@ -1059,8 +1060,9 @@ def test_encoding_unlimited_dims(self):
                 ds.to_netcdf(tmp_file, engine='h5netcdf', unlimited_dims=['y'])
 
 # tests pending h5netcdf fix
-# class H5NetCDFDataTestAutocloseTrue(H5NetCDFDataTest):
-#     autoclose = True
+@flakey
+class H5NetCDFDataTestAutocloseTrue(H5NetCDFDataTest):
+    autoclose = True
 
 
 class OpenMFDatasetManyFilesTest(TestCase):
@@ -1105,32 +1107,42 @@ def test_3_autoclose_pynio(self):
 
     # use of autoclose=True with h5netcdf broken because of
     # probable h5netcdf error, uncomment when fixed to test
-    # @requires_dask
-    # @requires_h5netcdf
-    # def test_4_autoclose_h5netcdf(self):
-    #     self.validate_open_mfdataset_autoclose(engine=['h5netcdf'])
+    @requires_dask
+    @requires_h5netcdf
+    @flakey
+    def test_4_autoclose_h5netcdf(self):
+        self.validate_open_mfdataset_autoclose(engine=['h5netcdf'])
 
     @requires_dask
     @requires_netCDF4
+    @optionalci
+    @slow
     def test_1_open_large_num_files_netcdf4(self):
         self.validate_open_mfdataset_large_num_files(engine=['netcdf4'])
 
     @requires_dask
     @requires_scipy
+    @optionalci
+    @slow
     def test_2_open_large_num_files_scipy(self):
         self.validate_open_mfdataset_large_num_files(engine=['scipy'])
 
     @requires_dask
     @requires_pynio
+    @optionalci
+    @slow
     def test_3_open_large_num_files_pynio(self):
         self.validate_open_mfdataset_large_num_files(engine=['pynio'])
 
     # use of autoclose=True with h5netcdf broken because of
     # probable h5netcdf error, uncomment when fixed to test
-    # @requires_dask
-    # @requires_h5netcdf
-    # def test_4_open_large_num_files_h5netcdf(self):
-    #     self.validate_open_mfdataset_large_num_files(engine=['h5netcdf'])
+    @requires_dask
+    @requires_h5netcdf
+    @flakey
+    @optionalci
+    @slow
+    def test_4_open_large_num_files_h5netcdf(self):
+        self.validate_open_mfdataset_large_num_files(engine=['h5netcdf'])
 
 
 @requires_dask