diff --git a/.azure_pipeline.yml b/.azure_pipeline.yml index ff01864b..5f1418a9 100644 --- a/.azure_pipeline.yml +++ b/.azure_pipeline.yml @@ -3,10 +3,10 @@ jobs: strategy: matrix: - # windows-py38: - # imageName: "vs2017-win2016" - # python.version: "3.8" - # tox.env: py38 + windows-py38: + imageName: "vs2017-win2016" + python.version: "3.8" + tox.env: py38 windows-py35: imageName: "vs2017-win2016" python.version: "3.5" @@ -16,10 +16,10 @@ jobs: python.version: "2.7" tox.env: py27 - # macos-py38: - # imageName: "macos-10.13" - # python.version: "3.8" - # tox.env: py38 + macos-py38: + imageName: "macos-10.13" + python.version: "3.8" + tox.env: py38 macos-py35: imageName: "macos-10.13" python.version: "3.5" @@ -41,11 +41,11 @@ jobs: python.version: "3.8" tox.env: "py38" joblib.tests: "true" - # linux-py38-high-memory: - # imageName: "ubuntu-16.04" - # python.version: "3.8" - # tox.env: py38 - # RUN_MEMORY: "true" + linux-py38-high-memory: + imageName: "ubuntu-16.04" + python.version: "3.8" + tox.env: py38 + RUN_MEMORY: "true" linux-py37: imageName: "ubuntu-16.04" python.version: "3.7" diff --git a/tests/_test_process_executor.py b/tests/_test_process_executor.py index 16b1d5cc..55e1fe15 100644 --- a/tests/_test_process_executor.py +++ b/tests/_test_process_executor.py @@ -22,6 +22,7 @@ import sys import time import shutil +import platform import pytest import weakref import tempfile @@ -535,9 +536,20 @@ def test_shutdown_race_issue12456(self): self.executor.map(str, [2] * (self.worker_count + 1)) self.executor.shutdown() + + @pytest.mark.skipif( + platform.python_implementation() != "CPython" or + (sys.version_info >= (3, 8, 0) and sys.version_info < (3, 8, 2)), + reason="Underlying bug fixed upstream starting Python 3.8.2") def test_no_stale_references(self): # Issue #16284: check that the executors don't unnecessarily hang onto # references. + + # This test has to be skipped on early Python 3.8 versions because of a + # low-level reference cycle inside the pickle module for early versions + # of Python 3.8 preventing stale references from being collected. See + # cloudpipe/cloudpickle#327 as well as + # https://bugs.python.org/issue39492 my_object = MyObject() collect = threading.Event() _ = weakref.ref(my_object, lambda obj: collect.set()) # noqa @@ -793,11 +805,19 @@ def test_max_depth(self, kill_workers): ctx=self.context) @pytest.mark.high_memory - @pytest.mark.skipif(sys.version_info < (3, 4), - reason="Limitation only present for python3") - def test_failure_on_large_data_send(self): + @pytest.mark.skipif( + sys.version_info[:2] < (3, 8), + reason="These Pythons cannot pickle objects of size > 2 ** 31GB") + def test_no_failure_on_large_data_send(self): data = b'\x00' * int(2.2e9) + self.executor.submit(id, data).result() + @pytest.mark.high_memory + @pytest.mark.skipif( + sys.version_info[:2] >= (3, 8), + reason="These Pythons can pickle objects of size > 2 ** 31GB") + def test_expected_failure_on_large_data_send(self): + data = b'\x00' * int(2.2e9) with pytest.raises(RuntimeError): self.executor.submit(id, data).result()