diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 9f516926a5e..13d866dcf15 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -24,7 +24,6 @@ jobs:
         pytest_extra_flags: --run-flaky --run-network-tests
         allow_failure: true
       # TODO: add numpy-dev to the test-matrix
-  continueOnError: ${{ variables['allow_failure'] }}
   pool:
     vmImage: 'ubuntu-16.04'
   steps:
diff --git a/ci/azure/unit-tests.yml b/ci/azure/unit-tests.yml
index 84d78c6219a..9c43a068191 100644
--- a/ci/azure/unit-tests.yml
+++ b/ci/azure/unit-tests.yml
@@ -7,23 +7,20 @@ steps:
     python -OO -c "import xarray"
   displayName: Import xarray
 
+# Work around for allowed test failures:
+# https://github.com/microsoft/azure-pipelines-tasks/issues/9302
 - bash: |
     source activate test_env
-    pytest xarray --junitxml=junit/test-results.xml \
+    pytest xarray \
+    --test-run-title="$(conda_env) $(pytest_extra_flags)" \
+    --junitxml=junit/test-results.xml \
     --cov=xarray --cov-config=ci/.coveragerc --cov-report=xml \
-    --cov-report=html $EXTRA_FLAGS
+    $(pytest_extra_flags) || [ "$ALLOW_FAILURE" = "true" ]
   displayName: Run tests
 
 - bash: |
     bash <(curl https://codecov.io/bash) -t 688f4d53-31bb-49b5-8370-4ce6f792cf3d
   displayName: Upload coverage to codecov.io
 
-  # TODO: publish coverage results to Azure, once we can merge them across
-  # multiple jobs: https://stackoverflow.com/questions/56776185
-
-- task: PublishTestResults@2
-  condition: succeededOrFailed()
-  inputs:
-    testResultsFiles: '**/test-*.xml'
-    failTaskOnFailedTests: true
-    testRunTitle: 'Test results for Python $(conda_env) $(pytest_extra_flags)'
+# TODO: publish coverage results to Azure, once we can merge them across
+# multiple jobs: https://stackoverflow.com/questions/56776185
diff --git a/conftest.py b/conftest.py
index ffceb27e753..177e689591f 100644
--- a/conftest.py
+++ b/conftest.py
@@ -11,19 +11,11 @@ def pytest_addoption(parser):
                      help="runs tests requiring a network connection")
 
 
-def pytest_collection_modifyitems(config, items):
-
-    if not config.getoption("--run-flaky"):
-        skip_flaky = pytest.mark.skip(
-            reason="set --run-flaky option to run flaky tests")
-        for item in items:
-            if "flaky" in item.keywords:
-                item.add_marker(skip_flaky)
-
-    if not config.getoption("--run-network-tests"):
-        skip_network = pytest.mark.skip(
-            reason="set --run-network-tests option to run tests requiring an"
-            "internet connection")
-        for item in items:
-            if "network" in item.keywords:
-                item.add_marker(skip_network)
+def pytest_runtest_setup(item):
+    # based on https://stackoverflow.com/questions/47559524
+    if 'flaky' in item.keywords and not item.config.getoption("--run-flaky"):
+        pytest.skip("set --run-flaky option to run flaky tests")
+    if ('network' in item.keywords
+            and not item.config.getoption("--run-network-tests")):
+        pytest.skip("set --run-network-tests to run test requiring an "
+                    "internet connection")
diff --git a/xarray/tests/test_dataarray.py b/xarray/tests/test_dataarray.py
index 5c81c843d00..58b69e7a30c 100644
--- a/xarray/tests/test_dataarray.py
+++ b/xarray/tests/test_dataarray.py
@@ -313,7 +313,7 @@ def test_constructor_from_self_described(self):
         actual = DataArray(series)
         assert_equal(expected[0].reset_coords('x', drop=True), actual)
 
-        if hasattr(pd, 'Panel'):
+        if LooseVersion(pd.__version__) < '0.25.0':
             with warnings.catch_warnings():
                 warnings.filterwarnings('ignore', r'\W*Panel is deprecated')
                 panel = pd.Panel({0: frame})
@@ -2971,7 +2971,7 @@ def test_to_pandas(self):
 
         # roundtrips
         for shape in [(3,), (3, 4), (3, 4, 5)]:
-            if len(shape) > 2 and not hasattr(pd, 'Panel'):
+            if len(shape) > 2 and not LooseVersion(pd.__version__) < '0.25.0':
                 continue
             dims = list('abc')[:len(shape)]
             da = DataArray(np.random.randn(*shape), dims=dims)
diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py
index ac759ac9c24..93ff87ab47f 100644
--- a/xarray/tests/test_dataset.py
+++ b/xarray/tests/test_dataset.py
@@ -20,7 +20,7 @@
 from xarray.core.pycompat import integer_types
 
 from . import (
-    InaccessibleArray, UnexpectedDataAccess, assert_allclose,
+    LooseVersion, InaccessibleArray, UnexpectedDataAccess, assert_allclose,
     assert_array_equal, assert_equal, assert_identical, has_cftime, has_dask,
     raises_regex, requires_bottleneck, requires_cftime, requires_dask,
     requires_numbagg, requires_scipy, source_ndarray)
@@ -389,7 +389,7 @@ def test_constructor_pandas_single(self):
             DataArray(np.random.rand(4, 3), dims=['a', 'b']),  # df
         ]
 
-        if hasattr(pd, 'Panel'):
+        if LooseVersion(pd.__version__) < '0.25.0':
             das.append(
                 DataArray(np.random.rand(4, 3, 2), dims=['a', 'b', 'c']))