Skip to content

Commit

Permalink
Adds marked tests as slow, flakey, and optionalci
Browse files Browse the repository at this point in the history 
Tests are marked using the following py.test options:

 * @slow which is run on default, not run if `--skip-slow`.
 * @optionalci is run on default, not run if `--skip-optional-ci`
 * @flakey which is not run on default, run if `--run-flakey`

Note, the `optionalci` category is needed in order to mark
tests that are environment dependent when run on continuous
integration (CI) hardware, e.g., tests that fail due to
variable loads on travis CI.
  • Loading branch information
@pwolfram
pwolfram committed Mar 31, 2017
1 parent 371d034 commit 28d825f
Show file tree
Hide file tree
Showing 6 changed files with 178 additions and 15 deletions.
84 changes: 83 additions & 1 deletion .travis.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,29 +12,64 @@ matrix:
include:
- python: 2.7
env: CONDA_ENV=py27-min
- python: 2.7
env: CONDA_ENV=py27-min SKIP_CI=--skip-optional-ci
- python: 2.7
env: CONDA_ENV=py27-min RUN_FLAKEY=--run-flakey
- python: 2.7
env: CONDA_ENV=py27-cdat+pynio
- python: 2.7
env: CONDA_ENV=py27-cdat+pynio SKIP_CI=--skip-optional-ci
- python: 2.7
env: CONDA_ENV=py27-cdat+pynio RUN_FLAKEY=--run-flakey
- python: 3.4
env: CONDA_ENV=py34
- python: 3.4
env: CONDA_ENV=py34 SKIP_CI=--skip-optional-ci
- python: 3.4
env: CONDA_ENV=py34 RUN_FLAKEY=--run-flakey
- python: 3.5
env: CONDA_ENV=py35
- python: 3.5
env: CONDA_ENV=py35 SKIP_CI=--skip-optional-ci
- python: 3.5
env: CONDA_ENV=py35 RUN_FLAKEY=--run-flakey
- python: 3.6
env: CONDA_ENV=py36
- python: 3.6
env: CONDA_ENV=py36 SKIP_CI=--skip-optional-ci
- python: 3.6
env: CONDA_ENV=py36 RUN_FLAKEY=--run-flakey
- python: 3.6
env: CONDA_ENV=py36-pydap
- python: 3.6
env: CONDA_ENV=py36-pydap RUN_FLAKEY=--run-flakey
- python: 3.6
env: CONDA_ENV=py36-netcdf4-dev
addons:
apt_packages:
- libhdf5-serial-dev
- netcdf-bin
- libnetcdf-dev
- python: 3.6
env: CONDA_ENV=py36-netcdf4-dev RUN_FLAKEY=--run-flakey
addons:
apt_packages:
- libhdf5-serial-dev
- netcdf-bin
- libnetcdf-dev
- python: 3.6
env: CONDA_ENV=py36-dask-dev
- python: 3.6
env: CONDA_ENV=py36-dask-dev RUN_FLAKEY=--run-flakey
- python: 3.6
env: CONDA_ENV=py36-pandas-dev
- python: 3.6
env: CONDA_ENV=py36-pandas-dev RUN_FLAKEY=--run-flakey
- python: 3.6
env: CONDA_ENV=py36-condaforge-rc
- python: 3.6
env: CONDA_ENV=py36-condaforge-rc RUN_FLAKEY=--run-flakey
allow_failures:
- python: 3.6
env: CONDA_ENV=py36-pydap
Expand All @@ -51,6 +86,53 @@ matrix:
env: CONDA_ENV=py36-pandas-dev
- python: 3.6
env: CONDA_ENV=py36-condaforge-rc
- python: 2.7
env: CONDA_ENV=py27-min
- python: 2.7
env: CONDA_ENV=py27-cdat+pynio
- python: 3.4
env: CONDA_ENV=py34
- python: 3.5
env: CONDA_ENV=py35
- python: 3.6
env: CONDA_ENV=py36
- python: 3.6
env: CONDA_ENV=py36-pydap
- python: 2.7
env: CONDA_ENV=py27-min
- python: 2.7
env: CONDA_ENV=py27-cdat+pynio
- python: 3.4
env: CONDA_ENV=py34
- python: 3.5
env: CONDA_ENV=py35
- python: 3.6
env: CONDA_ENV=py36
- python: 2.7
env: CONDA_ENV=py27-min RUN_FLAKEY=--run-flakey
- python: 2.7
env: CONDA_ENV=py27-cdat+pynio RUN_FLAKEY=--run-flakey
- python: 3.4
env: CONDA_ENV=py34 RUN_FLAKEY=--run-flakey
- python: 3.5
env: CONDA_ENV=py35 RUN_FLAKEY=--run-flakey
- python: 3.6
env: CONDA_ENV=py36 RUN_FLAKEY=--run-flakey
- python: 3.6
env: CONDA_ENV=py36-pydap RUN_FLAKEY=--run-flakey
- python: 3.6
env: CONDA_ENV=py36-netcdf4-dev RUN_FLAKEY=--run-flakey
addons:
apt_packages:
- libhdf5-serial-dev
- netcdf-bin
- libnetcdf-dev
- python: 3.6
env: CONDA_ENV=py36-dask-dev RUN_FLAKEY=--run-flakey
- python: 3.6
env: CONDA_ENV=py36-pandas-dev RUN_FLAKEY=--run-flakey
- python: 3.6
env: CONDA_ENV=py36-condaforge-rc RUN_FLAKEY=--run-flakey

before_install:
- if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then
Expand All @@ -76,7 +158,7 @@ install:
- python setup.py install

script:
- py.test xarray --cov=xarray --cov-report term-missing --verbose
- py.test xarray --cov=xarray --cov-report term-missing --verbose $SKIP_CI $RUN_FLAKEY

after_success:
- coveralls
2 changes: 1 addition & 1 deletion appveyor.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,4 +38,4 @@ install:
build: false

test_script:
- "py.test xarray --verbose"
- "py.test xarray --verbose --skip-optional-ci"
9 changes: 9 additions & 0 deletions conftest.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
import pytest

def pytest_addoption(parser):
parser.addoption("--run-flakey", action="store_true",
help="runs flakey tests")
parser.addoption("--skip-optional-ci", action="store_true",
help="skips optional tests continuous integration (CI)")
parser.addoption("--skip-slow", action="store_true",
help="skips slow tests")
15 changes: 15 additions & 0 deletions xarray/tests/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,21 @@ def requires_bottleneck(test):
return test if has_bottleneck else pytest.mark.skip('requires bottleneck')(test)


flakey = pytest.mark.skipif(
not pytest.config.getoption("--run-flakey"),
reason="set --run-flakey option to run flakey tests")

optionalci = pytest.mark.skipif(
pytest.config.getoption("--skip-optional-ci"),
reason=("set --skip-optional-ci option to skip tests in CI, "
"e.g., due to travis CI resource issues"))


slow = pytest.mark.skipif(
pytest.config.getoption("--skip-slow"),
reason="set --skip-slow option to run slow tests")


class TestCase(unittest.TestCase):
if PY3:
# Python 3 assertCountEqual is roughly equivalent to Python 2
Expand Down
34 changes: 23 additions & 11 deletions xarray/tests/test_backends.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@

from . import (TestCase, requires_scipy, requires_netCDF4, requires_pydap,
requires_scipy_or_netCDF4, requires_dask, requires_h5netcdf,
requires_pynio, has_netCDF4, has_scipy, assert_allclose)
requires_pynio, has_netCDF4, has_scipy, assert_allclose,
flakey, optionalci, slow)
from .test_dataset import create_test_data

try:
Expand Down Expand Up @@ -1059,8 +1060,9 @@ def test_encoding_unlimited_dims(self):
ds.to_netcdf(tmp_file, engine='h5netcdf', unlimited_dims=['y'])

# tests pending h5netcdf fix
# class H5NetCDFDataTestAutocloseTrue(H5NetCDFDataTest):
# autoclose = True
@flakey
class H5NetCDFDataTestAutocloseTrue(H5NetCDFDataTest):
autoclose = True


class OpenMFDatasetManyFilesTest(TestCase):
Expand Down Expand Up @@ -1105,32 +1107,42 @@ def test_3_autoclose_pynio(self):

# use of autoclose=True with h5netcdf broken because of
# probable h5netcdf error, uncomment when fixed to test
# @requires_dask
# @requires_h5netcdf
# def test_4_autoclose_h5netcdf(self):
# self.validate_open_mfdataset_autoclose(engine=['h5netcdf'])
@requires_dask
@requires_h5netcdf
@flakey
def test_4_autoclose_h5netcdf(self):
self.validate_open_mfdataset_autoclose(engine=['h5netcdf'])

@requires_dask
@requires_netCDF4
@optionalci
@slow
def test_1_open_large_num_files_netcdf4(self):
self.validate_open_mfdataset_large_num_files(engine=['netcdf4'])

@requires_dask
@requires_scipy
@optionalci
@slow
def test_2_open_large_num_files_scipy(self):
self.validate_open_mfdataset_large_num_files(engine=['scipy'])

@requires_dask
@requires_pynio
@optionalci
@slow
def test_3_open_large_num_files_pynio(self):
self.validate_open_mfdataset_large_num_files(engine=['pynio'])

# use of autoclose=True with h5netcdf broken because of
# probable h5netcdf error, uncomment when fixed to test
# @requires_dask
# @requires_h5netcdf
# def test_4_open_large_num_files_h5netcdf(self):
# self.validate_open_mfdataset_large_num_files(engine=['h5netcdf'])
@requires_dask
@requires_h5netcdf
@flakey
@optionalci
@slow
def test_4_open_large_num_files_h5netcdf(self):
self.validate_open_mfdataset_large_num_files(engine=['h5netcdf'])


@requires_dask
Expand Down
Loading

0 comments on commit 28d825f

Please sign in to comment.