From b773269ddeb83632b04e3626fca370e182e7c17e Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Wed, 29 Jan 2020 14:03:28 -0800 Subject: [PATCH 1/4] Try to use pickle5 when available As there are some improvements for buffer serialization in pickle protocol 5 and these can be opted in using a backports package on older Python versions. Try to leverage the backport package when available and fallback to the builtin pickle otherwise. --- distributed/protocol/pickle.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/distributed/protocol/pickle.py b/distributed/protocol/pickle.py index 629fb962fb..8ad9a3f947 100644 --- a/distributed/protocol/pickle.py +++ b/distributed/protocol/pickle.py @@ -5,7 +5,12 @@ if sys.version_info.major == 2: import cPickle as pickle -else: +elif sys.version_info.major == 3 and sys.version_info.minor < 8: + try: + import pickle5 as pickle + except ImportError: + import pickle +elif sys.version_info.minor: import pickle logger = logging.getLogger(__name__) From db74076056881f43b64cc1eb7bff5814ae1f5079 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Wed, 29 Jan 2020 14:51:14 -0800 Subject: [PATCH 2/4] Install `pickle5` for Python pre-3.8 This is a backport of Python's pickle 5 protocol for Python 3.6 and 3.7. So install it for those versions. In Python 3.8+ this is included in Python natively. So skip installing it for more recent Python versions. --- dev-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/dev-requirements.txt b/dev-requirements.txt index a367f706e7..a3f1507fc6 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -11,3 +11,4 @@ pytest >= 3.2 prometheus_client >= 0.6.0 jupyter-server-proxy >= 1.1.0 pytest-asyncio +pickle5; python_version >= '3.6' and python_version < '3.8' From 5bbc8737cf48d8932d64505e2be0ca7eb8ae84fe Mon Sep 17 00:00:00 2001 From: jakirkham Date: Sun, 16 Feb 2020 11:42:58 -0800 Subject: [PATCH 3/4] Update distributed/protocol/pickle.py --- distributed/protocol/pickle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distributed/protocol/pickle.py b/distributed/protocol/pickle.py index 8ad9a3f947..9dc740e61e 100644 --- a/distributed/protocol/pickle.py +++ b/distributed/protocol/pickle.py @@ -10,7 +10,7 @@ import pickle5 as pickle except ImportError: import pickle -elif sys.version_info.minor: +else: import pickle logger = logging.getLogger(__name__) From f4ad9ae9d946861be535c09774afd23a0d612b75 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Sun, 16 Feb 2020 11:44:24 -0800 Subject: [PATCH 4/4] Drop Python 2 case --- distributed/protocol/pickle.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/distributed/protocol/pickle.py b/distributed/protocol/pickle.py index 9dc740e61e..50612782d4 100644 --- a/distributed/protocol/pickle.py +++ b/distributed/protocol/pickle.py @@ -3,9 +3,7 @@ import cloudpickle -if sys.version_info.major == 2: - import cPickle as pickle -elif sys.version_info.major == 3 and sys.version_info.minor < 8: +if sys.version_info.major == 3 and sys.version_info.minor < 8: try: import pickle5 as pickle except ImportError: