Fix cloudpickle incompatibilities on early Python 3.5 versions

.github/workflows/testing.yml

@@ -29,7 +29,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python_version: [3.5, 3.6, 3.7, 3.8, "pypy3"]
+        python_version: ["3.5.0", 3.6, 3.7, 3.8, "pypy3"]
         exclude:
           # Do not test all minor versions on all platforms, especially if they
           # are not the oldest/newest supported versions
@@ -50,31 +50,34 @@ jobs:
     steps:
     - uses: actions/checkout@v1
     - name: Set up Python ${{ matrix.python_version }}
-      uses: actions/setup-python@v1
+      uses: goanpeca/setup-miniconda@v1
       with:
-        python-version: ${{ matrix.python_version }}
+        auto-update-conda: true
+        python-version: ${{ matrix.python-version }}
+        auto-activate-base: false
+        activate-environment: testenv
     - name: Install project and dependencies
-      shell: bash
+      shell: bash -l {0}
       run: |
         python -m pip install --upgrade pip
         python -m pip install -e .
         python -m pip install -r dev-requirements.txt
         python ci/install_coverage_subprocess_pth.py
         export
     - name: Install optional typing_extensions in Python 3.6
-      shell: bash
+      shell: bash -l {0}
       run: python -m pip install typing-extensions
       if: matrix.python_version == '3.6'
     - name: Display Python version
-      shell: bash
+      shell: bash -l {0}
       run: python -c "import sys; print(sys.version)"
     - name: Look for syntax errors/undefined names
-      shell: bash
+      shell: bash -l {0}
       run: |
         python -m flake8 . --count --verbose --select=E901,E999,F821,F822,F823 \
           --show-source --statistics
     - name: Test with pytest
-      shell: bash
+      shell: bash -l {0}
       run: |
         COVERAGE_PROCESS_START=$GITHUB_WORKSPACE/.coveragerc \
           PYTHONPATH='.:tests' python -m pytest -r s

CHANGES.md

@@ -1,6 +1,10 @@
 1.4.1 (in development)
 ======================
 
+- Fix incompatibilities between cloudpickle 1.4.0 and Python 3.5.0/1/2
+  introduced by the new support of cloudpickle for pickling typing constructs.
+  ([issue #360](https://github.com/cloudpipe/cloudpickle/issues/360))
+
 - Restore compat with loading dynamic classes pickled with cloudpickle
   version 1.2.1 that would reference the `types.ClassType` attribute.
   ([PR #359](https://github.com/cloudpipe/cloudpickle/pull/359))

cloudpickle/cloudpickle.py

@@ -61,7 +61,7 @@
 import typing
 from enum import Enum
 
-from typing import Generic, Union, Tuple, Callable, ClassVar
+from typing import Generic, Union, Tuple, Callable
 from pickle import _Pickler as Pickler
 from pickle import _getattribute
 from io import BytesIO
@@ -73,6 +73,11 @@
 except ImportError:
     _typing_extensions = Literal = Final = None
 
+if sys.version_info > (3, 5, 2):
+    from typing import ClassVar
+else:  # pragma: no cover
+    ClassVar = None
+
 
 # cloudpickle is meant for inter process communication: we expect all
 # communicating processes to run the same Python version hence we favor
@@ -432,10 +437,24 @@ def _extract_class_dict(cls):
 if sys.version_info[:2] < (3, 7):  # pragma: no branch
     def _is_parametrized_type_hint(obj):
         # This is very cheap but might generate false positives.
-        origin = getattr(obj, '__origin__', None)  # typing Constructs
-        values = getattr(obj, '__values__', None)  # typing_extensions.Literal
-        type_ = getattr(obj, '__type__', None)     # typing_extensions.Final
-        return origin is not None or values is not None or type_ is not None
+        # genral typing Constructs
+        is_typing = getattr(obj, '__origin__', None) is not None
+
+        # typing_extensions.Literal
+        is_litteral = getattr(obj, '__values__', None) is not None
+
+        # typing_extensions.Final
+        is_final = getattr(obj, '__type__', None) is not None
+
+        # typing.Union/Tuple for old Python 3.5
+        is_union = getattr(obj, '__union_params__', None) is not None
+        is_tuple = getattr(obj, '__tuple_params__', None) is not None
+        is_callable = (
+            getattr(obj, '__result__', None) is not None and
+            getattr(obj, '__args__', None) is not None
+        )
+        return any((is_typing, is_litteral, is_final, is_union, is_tuple,
+                    is_callable))
 
     def _create_parametrized_type_hint(origin, args):
         return origin[args]
@@ -971,14 +990,37 @@ def _save_parametrized_type_hint(self, obj):
                 initargs = (Final, obj.__type__)
             elif type(obj) is type(ClassVar):
                 initargs = (ClassVar, obj.__type__)
-            elif type(obj) in [type(Union), type(Tuple), type(Generic)]:
-                initargs = (obj.__origin__, obj.__args__)
+            elif type(obj) is type(Generic):
+                if sys.version_info < (3, 5, 2):  # pragma: no cover
+                    initargs = (obj.__origin__, obj.__parameters__)
+                else:
+                    initargs = (obj.__origin__, obj.__args__)
+            elif type(obj) is type(Union):
+                if sys.version_info < (3, 5, 2):  # pragma: no cover
+                    initargs = (Union, obj.__union_params__)
+                else:
+                    initargs = (Union, obj.__args__)
+            elif type(obj) is type(Tuple):
+                if sys.version_info < (3, 5, 2):  # pragma: no cover
+                    initargs = (Tuple, obj.__tuple_params__)
+                else:
+                    initargs = (Tuple, obj.__args__)
             elif type(obj) is type(Callable):
-                args = obj.__args__
-                if args[0] is Ellipsis:
-                    initargs = (obj.__origin__, args)
+                if sys.version_info < (3, 5, 2):  # pragma: no cover
+                    args = obj.__args__
+                    result = obj.__result__
+                    if args != Ellipsis:
+                        if isinstance(args, tuple):
+                            args = list(args)
+                        else:
+                            args = [args]
                 else:
-                    initargs = (obj.__origin__, (list(args[:-1]), args[-1]))
+                    (*args, result) = obj.__args__
+                    if len(args) == 1 and args[0] is Ellipsis:
+                        args = Ellipsis
+                    else:
+                        args = list(args)
+                initargs = (Callable, (args, result))
             else:  # pragma: no cover
                 raise pickle.PicklingError(
                     "Cloudpickle Error: Unknown type {}".format(type(obj))

tests/cloudpickle_test.py

@@ -2083,12 +2083,18 @@ class C(typing.Generic[T]):
 
             def check_generic(generic, origin, type_value):
                 assert generic.__origin__ is origin
-                assert len(generic.__args__) == 1
-                assert generic.__args__[0] is type_value
 
-                assert len(origin.__orig_bases__) == 1
-                ob = origin.__orig_bases__[0]
-                assert ob.__origin__ is typing.Generic
+                if sys.version_info > (3, 5, 2):
+                    assert len(generic.__args__) == 1
+                    assert generic.__args__[0] is type_value
+                    assert len(origin.__orig_bases__) == 1
+                    ob = origin.__orig_bases__[0]
+                    assert ob.__origin__ is typing.Generic
+                else:  # Python 3.5.[0-1-2]
+                    assert len(generic.__parameters__) == 1
+                    assert generic.__parameters__[0] is type_value
+                    assert len(origin.__bases__) == 1
+                    ob = origin.__bases__[0]
                 assert len(ob.__parameters__) == 1
 
                 return "ok"
@@ -2182,21 +2188,26 @@ def _all_types_to_test():
     class C(typing.Generic[T]):
         pass
 
-    return [
+    types_to_test = [
         C, C[int],
-        T, typing.Any, typing.NoReturn, typing.Optional,
-        typing.Generic, typing.Union, typing.ClassVar,
+        T, typing.Any, typing.Optional,
+        typing.Generic, typing.Union,
         typing.Optional[int],
         typing.Generic[T],
         typing.Callable[[int], typing.Any],
         typing.Callable[..., typing.Any],
         typing.Callable[[], typing.Any],
         typing.Tuple[int, ...],
         typing.Tuple[int, C[int]],
-        typing.ClassVar[C[int]],
         typing.List[int],
         typing.Dict[int, str],
     ]
+    if sys.version_info > (3, 5, 2):
+        types_to_test.append(typing.ClassVar)
+        types_to_test.append(typing.ClassVar[C[int]])
+    if sys.version_info > (3, 5, 3):
+        types_to_test.append(typing.NoReturn)
+    return types_to_test
 
 
 if __name__ == '__main__':