From 4680597338190b5430ef5625066d1254d2221ae2 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Mon, 1 Jul 2019 22:01:30 -0700 Subject: [PATCH 01/16] Fix the ability to run network and flaky tests The old setup didn't seem to work on CI, even when we explicitly passed the relevant flags. --- conftest.py | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/conftest.py b/conftest.py index ffceb27e753..9ab4d543b9f 100644 --- a/conftest.py +++ b/conftest.py @@ -10,20 +10,11 @@ def pytest_addoption(parser): parser.addoption("--run-network-tests", action="store_true", help="runs tests requiring a network connection") - -def pytest_collection_modifyitems(config, items): - - if not config.getoption("--run-flaky"): - skip_flaky = pytest.mark.skip( - reason="set --run-flaky option to run flaky tests") - for item in items: - if "flaky" in item.keywords: - item.add_marker(skip_flaky) - - if not config.getoption("--run-network-tests"): - skip_network = pytest.mark.skip( - reason="set --run-network-tests option to run tests requiring an" - "internet connection") - for item in items: - if "network" in item.keywords: - item.add_marker(skip_network) +def pytest_runtest_setup(item): + # based on https://stackoverflow.com/questions/47559524 + if 'flaky' in item.keywords and not item.config.getoption("--run-flaky"): + pytest.skip("set --run-flaky option to run flaky tests") + if ('network' in item.keywords + and not item.config.getoption("--run-network-tests")): + pytest.skip("set --run-network-tests to run test requiring an " + "internet connection") From c55a03b6454df2f8e3b64b902b379caa8fe1632a Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Mon, 1 Jul 2019 22:03:05 -0700 Subject: [PATCH 02/16] fix flake8 --- conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/conftest.py b/conftest.py index 9ab4d543b9f..177e689591f 100644 --- a/conftest.py +++ b/conftest.py @@ -10,6 +10,7 @@ def pytest_addoption(parser): parser.addoption("--run-network-tests", action="store_true", help="runs tests requiring a network connection") + def pytest_runtest_setup(item): # based on https://stackoverflow.com/questions/47559524 if 'flaky' in item.keywords and not item.config.getoption("--run-flaky"): From 207c200822093dd1aceda29e5ade7ae61578420b Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Mon, 1 Jul 2019 22:29:11 -0700 Subject: [PATCH 03/16] verbose tests --- ci/azure/unit-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/azure/unit-tests.yml b/ci/azure/unit-tests.yml index 9a92a1b3bf5..0d965ab179d 100644 --- a/ci/azure/unit-tests.yml +++ b/ci/azure/unit-tests.yml @@ -9,7 +9,7 @@ steps: - bash: | source activate test_env - pytest xarray --junitxml=junit/test-results.xml \ + pytest xarray -v --junitxml=junit/test-results.xml \ --cov=xarray --cov-config=ci/.coveragerc --cov-report=xml $EXTRA_FLAGS displayName: Run tests From 4b3fe246fd49f600f32cb1603804295c28b7ceda Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Wed, 3 Jul 2019 11:17:04 -0700 Subject: [PATCH 04/16] Fix use of pytest_extra_flags --- ci/azure/unit-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/azure/unit-tests.yml b/ci/azure/unit-tests.yml index 84d78c6219a..6d0beb61735 100644 --- a/ci/azure/unit-tests.yml +++ b/ci/azure/unit-tests.yml @@ -11,7 +11,7 @@ steps: source activate test_env pytest xarray --junitxml=junit/test-results.xml \ --cov=xarray --cov-config=ci/.coveragerc --cov-report=xml \ - --cov-report=html $EXTRA_FLAGS + --cov-report=html $(pytest_extra_flags) displayName: Run tests - bash: | From 528cb63e2d65cbe8a2061f3892e13994ebe32eed Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Wed, 3 Jul 2019 20:44:02 -0700 Subject: [PATCH 05/16] Update conda --- azure-pipelines.yml | 1 - ci/azure/install.yml | 4 ++++ ci/azure/unit-tests.yml | 17 ++++++----------- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9f516926a5e..13d866dcf15 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -24,7 +24,6 @@ jobs: pytest_extra_flags: --run-flaky --run-network-tests allow_failure: true # TODO: add numpy-dev to the test-matrix - continueOnError: ${{ variables['allow_failure'] }} pool: vmImage: 'ubuntu-16.04' steps: diff --git a/ci/azure/install.yml b/ci/azure/install.yml index 8d9b3daec0e..9f2a6f2e293 100644 --- a/ci/azure/install.yml +++ b/ci/azure/install.yml @@ -5,6 +5,10 @@ steps: - template: add-conda-to-path.yml +- bash: | + conda update conda + displayName: Update conda + - bash: | conda env create -n test_env --file ${{ parameters.env_file }} displayName: Install dependencies diff --git a/ci/azure/unit-tests.yml b/ci/azure/unit-tests.yml index 6d0beb61735..8af27354cc5 100644 --- a/ci/azure/unit-tests.yml +++ b/ci/azure/unit-tests.yml @@ -9,21 +9,16 @@ steps: - bash: | source activate test_env - pytest xarray --junitxml=junit/test-results.xml \ + pytest xarray \ + --test-run-title="$(conda_env) $(pytest_extra_flags)" + --junitxml=junit/test-results.xml \ --cov=xarray --cov-config=ci/.coveragerc --cov-report=xml \ - --cov-report=html $(pytest_extra_flags) + $(pytest_extra_flags) displayName: Run tests - bash: | bash <(curl https://codecov.io/bash) -t 688f4d53-31bb-49b5-8370-4ce6f792cf3d displayName: Upload coverage to codecov.io - # TODO: publish coverage results to Azure, once we can merge them across - # multiple jobs: https://stackoverflow.com/questions/56776185 - -- task: PublishTestResults@2 - condition: succeededOrFailed() - inputs: - testResultsFiles: '**/test-*.xml' - failTaskOnFailedTests: true - testRunTitle: 'Test results for Python $(conda_env) $(pytest_extra_flags)' +# TODO: publish coverage results to Azure, once we can merge them across +# multiple jobs: https://stackoverflow.com/questions/56776185 From e4d443c8ea6120f7d8f3537635c15f8eb1cdad3c Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Wed, 3 Jul 2019 21:31:33 -0700 Subject: [PATCH 06/16] add --yes --- ci/azure/install.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/azure/install.yml b/ci/azure/install.yml index 9f2a6f2e293..2c5a6d21fa5 100644 --- a/ci/azure/install.yml +++ b/ci/azure/install.yml @@ -6,7 +6,7 @@ steps: - template: add-conda-to-path.yml - bash: | - conda update conda + conda update conda --yes displayName: Update conda - bash: | From 227348e694ff37eb847f0e6713841caab99a373b Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Wed, 3 Jul 2019 21:36:00 -0700 Subject: [PATCH 07/16] fix allowed failure --- ci/azure/unit-tests.yml | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/ci/azure/unit-tests.yml b/ci/azure/unit-tests.yml index 8af27354cc5..a3d2f886dc0 100644 --- a/ci/azure/unit-tests.yml +++ b/ci/azure/unit-tests.yml @@ -7,14 +7,27 @@ steps: python -OO -c "import xarray" displayName: Import xarray -- bash: | - source activate test_env - pytest xarray \ - --test-run-title="$(conda_env) $(pytest_extra_flags)" - --junitxml=junit/test-results.xml \ - --cov=xarray --cov-config=ci/.coveragerc --cov-report=xml \ - $(pytest_extra_flags) - displayName: Run tests +# Work around for allowed test failures: +# https://github.com/microsoft/azure-pipelines-tasks/issues/9302 +- ${{ if not(eq(variables.allow_failure, 'true')) }}: + - bash: | + source activate test_env + pytest xarray \ + --test-run-title="$(conda_env) $(pytest_extra_flags)" + --junitxml=junit/test-results.xml \ + --cov=xarray --cov-config=ci/.coveragerc --cov-report=xml \ + $(pytest_extra_flags) + displayName: Run tests +- ${{ if eq(variables.allow_failure, 'true') }}: + - script: | + source activate test_env + pytest xarray \ + --test-run-title="$(conda_env) $(pytest_extra_flags)" + --junitxml=junit/test-results.xml \ + --cov=xarray --cov-config=ci/.coveragerc --cov-report=xml \ + $(pytest_extra_flags) + exit 0 + displayName: Run tests - bash: | bash <(curl https://codecov.io/bash) -t 688f4d53-31bb-49b5-8370-4ce6f792cf3d From 367dd9dbd65c69fe8dfda83aaed7e7a16840be85 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Wed, 3 Jul 2019 21:52:47 -0700 Subject: [PATCH 08/16] fixup allow failures --- ci/azure/unit-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/azure/unit-tests.yml b/ci/azure/unit-tests.yml index a3d2f886dc0..e02a7d9346a 100644 --- a/ci/azure/unit-tests.yml +++ b/ci/azure/unit-tests.yml @@ -13,7 +13,7 @@ steps: - bash: | source activate test_env pytest xarray \ - --test-run-title="$(conda_env) $(pytest_extra_flags)" + --test-run-title="$(conda_env) $(pytest_extra_flags)" \ --junitxml=junit/test-results.xml \ --cov=xarray --cov-config=ci/.coveragerc --cov-report=xml \ $(pytest_extra_flags) @@ -22,7 +22,7 @@ steps: - script: | source activate test_env pytest xarray \ - --test-run-title="$(conda_env) $(pytest_extra_flags)" + --test-run-title="$(conda_env) $(pytest_extra_flags)" \ --junitxml=junit/test-results.xml \ --cov=xarray --cov-config=ci/.coveragerc --cov-report=xml \ $(pytest_extra_flags) From 192dfee24e42e64f9d1a9213a53245c4c3597593 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Wed, 3 Jul 2019 21:59:51 -0700 Subject: [PATCH 09/16] Update conda in base --- ci/azure/install.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/azure/install.yml b/ci/azure/install.yml index 2c5a6d21fa5..6974328de48 100644 --- a/ci/azure/install.yml +++ b/ci/azure/install.yml @@ -6,6 +6,7 @@ steps: - template: add-conda-to-path.yml - bash: | + source activate base conda update conda --yes displayName: Update conda From 236c20cd445ac0948f66c96723c6c23ae85bac4e Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Wed, 3 Jul 2019 22:02:17 -0700 Subject: [PATCH 10/16] Don't update conda --- ci/azure/install.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/ci/azure/install.yml b/ci/azure/install.yml index 6974328de48..8d9b3daec0e 100644 --- a/ci/azure/install.yml +++ b/ci/azure/install.yml @@ -5,11 +5,6 @@ steps: - template: add-conda-to-path.yml -- bash: | - source activate base - conda update conda --yes - displayName: Update conda - - bash: | conda env create -n test_env --file ${{ parameters.env_file }} displayName: Install dependencies From 16263379420282d896d810b15423a8bb4d7fa6b4 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Wed, 3 Jul 2019 22:38:51 -0700 Subject: [PATCH 11/16] Tweak conditional --- ci/azure/unit-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/azure/unit-tests.yml b/ci/azure/unit-tests.yml index e02a7d9346a..b91d045599c 100644 --- a/ci/azure/unit-tests.yml +++ b/ci/azure/unit-tests.yml @@ -9,7 +9,7 @@ steps: # Work around for allowed test failures: # https://github.com/microsoft/azure-pipelines-tasks/issues/9302 -- ${{ if not(eq(variables.allow_failure, 'true')) }}: +- ${{ if not(variables.allow_failure) }}: - bash: | source activate test_env pytest xarray \ @@ -18,7 +18,7 @@ steps: --cov=xarray --cov-config=ci/.coveragerc --cov-report=xml \ $(pytest_extra_flags) displayName: Run tests -- ${{ if eq(variables.allow_failure, 'true') }}: +- ${{ if variables.allow_failure }}: - script: | source activate test_env pytest xarray \ From 7206713f3d3030f453d0b56397d5d5dc10329f4f Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Wed, 3 Jul 2019 22:42:04 -0700 Subject: [PATCH 12/16] Fix pandas.Panel fix --- xarray/tests/test_dataarray.py | 4 ++-- xarray/tests/test_dataset.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py index 5c81c843d00..58b69e7a30c 100644 --- a/xarray/tests/test_dataarray.py +++ b/xarray/tests/test_dataarray.py @@ -313,7 +313,7 @@ def test_constructor_from_self_described(self): actual = DataArray(series) assert_equal(expected[0].reset_coords('x', drop=True), actual) - if hasattr(pd, 'Panel'): + if LooseVersion(pd.__version__) < '0.25.0': with warnings.catch_warnings(): warnings.filterwarnings('ignore', r'\W*Panel is deprecated') panel = pd.Panel({0: frame}) @@ -2971,7 +2971,7 @@ def test_to_pandas(self): # roundtrips for shape in [(3,), (3, 4), (3, 4, 5)]: - if len(shape) > 2 and not hasattr(pd, 'Panel'): + if len(shape) > 2 and not LooseVersion(pd.__version__) < '0.25.0': continue dims = list('abc')[:len(shape)] da = DataArray(np.random.randn(*shape), dims=dims) diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py index ac759ac9c24..93ff87ab47f 100644 --- a/xarray/tests/test_dataset.py +++ b/xarray/tests/test_dataset.py @@ -20,7 +20,7 @@ from xarray.core.pycompat import integer_types from . import ( - InaccessibleArray, UnexpectedDataAccess, assert_allclose, + LooseVersion, InaccessibleArray, UnexpectedDataAccess, assert_allclose, assert_array_equal, assert_equal, assert_identical, has_cftime, has_dask, raises_regex, requires_bottleneck, requires_cftime, requires_dask, requires_numbagg, requires_scipy, source_ndarray) @@ -389,7 +389,7 @@ def test_constructor_pandas_single(self): DataArray(np.random.rand(4, 3), dims=['a', 'b']), # df ] - if hasattr(pd, 'Panel'): + if LooseVersion(pd.__version__) < '0.25.0': das.append( DataArray(np.random.rand(4, 3, 2), dims=['a', 'b', 'c'])) From 89cea7f06dbbf3f4ba36ec6da09a5d26f8e27d46 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Wed, 3 Jul 2019 23:20:24 -0700 Subject: [PATCH 13/16] Try an alternative for allowed failures --- ci/azure/unit-tests.yml | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/ci/azure/unit-tests.yml b/ci/azure/unit-tests.yml index b91d045599c..fbf67a39cbb 100644 --- a/ci/azure/unit-tests.yml +++ b/ci/azure/unit-tests.yml @@ -9,25 +9,14 @@ steps: # Work around for allowed test failures: # https://github.com/microsoft/azure-pipelines-tasks/issues/9302 -- ${{ if not(variables.allow_failure) }}: - - bash: | - source activate test_env - pytest xarray \ - --test-run-title="$(conda_env) $(pytest_extra_flags)" \ - --junitxml=junit/test-results.xml \ - --cov=xarray --cov-config=ci/.coveragerc --cov-report=xml \ - $(pytest_extra_flags) - displayName: Run tests -- ${{ if variables.allow_failure }}: - - script: | - source activate test_env - pytest xarray \ - --test-run-title="$(conda_env) $(pytest_extra_flags)" \ - --junitxml=junit/test-results.xml \ - --cov=xarray --cov-config=ci/.coveragerc --cov-report=xml \ - $(pytest_extra_flags) - exit 0 - displayName: Run tests +- bash: | + source activate test_env + pytest xarray \ + --test-run-title="$(conda_env) $(pytest_extra_flags)" \ + --junitxml=junit/test-results.xml \ + --cov=xarray --cov-config=ci/.coveragerc --cov-report=xml \ + $(pytest_extra_flags) || $ALLOW_FAILURE == "true" + displayName: Run tests - bash: | bash <(curl https://codecov.io/bash) -t 688f4d53-31bb-49b5-8370-4ce6f792cf3d From a8ba10cea2bb8f0282418924d79783c86e00f545 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Wed, 3 Jul 2019 23:21:10 -0700 Subject: [PATCH 14/16] exit --- ci/azure/unit-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/azure/unit-tests.yml b/ci/azure/unit-tests.yml index fbf67a39cbb..c3c9528c951 100644 --- a/ci/azure/unit-tests.yml +++ b/ci/azure/unit-tests.yml @@ -15,7 +15,7 @@ steps: --test-run-title="$(conda_env) $(pytest_extra_flags)" \ --junitxml=junit/test-results.xml \ --cov=xarray --cov-config=ci/.coveragerc --cov-report=xml \ - $(pytest_extra_flags) || $ALLOW_FAILURE == "true" + $(pytest_extra_flags) || exit $ALLOW_FAILURE == "true" displayName: Run tests - bash: | From b99ac49d94d5c637c334b0ca40a949c7747080e6 Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Thu, 4 Jul 2019 12:48:59 -0700 Subject: [PATCH 15/16] tweak exit --- ci/azure/unit-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/azure/unit-tests.yml b/ci/azure/unit-tests.yml index c3c9528c951..d3d02ba09fe 100644 --- a/ci/azure/unit-tests.yml +++ b/ci/azure/unit-tests.yml @@ -15,7 +15,7 @@ steps: --test-run-title="$(conda_env) $(pytest_extra_flags)" \ --junitxml=junit/test-results.xml \ --cov=xarray --cov-config=ci/.coveragerc --cov-report=xml \ - $(pytest_extra_flags) || exit $ALLOW_FAILURE == "true" + $(pytest_extra_flags) || exit [[ "$ALLOW_FAILURE" == "true" ]] displayName: Run tests - bash: | From 89dac822e3174bae54b6e1bd652005911a2e7d5b Mon Sep 17 00:00:00 2001 From: Stephan Hoyer Date: Thu, 4 Jul 2019 13:02:14 -0700 Subject: [PATCH 16/16] fix bash --- ci/azure/unit-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/azure/unit-tests.yml b/ci/azure/unit-tests.yml index d3d02ba09fe..9c43a068191 100644 --- a/ci/azure/unit-tests.yml +++ b/ci/azure/unit-tests.yml @@ -15,7 +15,7 @@ steps: --test-run-title="$(conda_env) $(pytest_extra_flags)" \ --junitxml=junit/test-results.xml \ --cov=xarray --cov-config=ci/.coveragerc --cov-report=xml \ - $(pytest_extra_flags) || exit [[ "$ALLOW_FAILURE" == "true" ]] + $(pytest_extra_flags) || [ "$ALLOW_FAILURE" = "true" ] displayName: Run tests - bash: |