From a86050976e391529dd374122abb2026052ff11fc Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Thu, 14 May 2020 10:57:34 +0200 Subject: [PATCH 01/89] backport python3.8 save_reduce into old CloudPickler --- cloudpickle/cloudpickle.py | 651 ++++++++++++++++---------------- cloudpickle/cloudpickle_fast.py | 248 +----------- 2 files changed, 324 insertions(+), 575 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index c639daab1..27e0e51b9 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -78,6 +78,17 @@ else: # pragma: no cover ClassVar = None +if sys.version_info >= (3, 8): + from types import CellType +else: + def f(): + a = 1 + + def g(): + return a + return g + CellType = type(f().__closure__[0]) + # cloudpickle is meant for inter process communication: we expect all # communicating processes to run the same Python version hence we favor @@ -101,6 +112,265 @@ _extract_code_globals_cache = weakref.WeakKeyDictionary() +def _function_setstate(obj, state): + """Update the state of a dynaamic function. + + As __closure__ and __globals__ are readonly attributes of a function, we + cannot rely on the native setstate routine of pickle.load_build, that calls + setattr on items of the slotstate. Instead, we have to modify them inplace. + """ + state, slotstate = state + obj.__dict__.update(state) + + obj_globals = slotstate.pop("__globals__") + obj_closure = slotstate.pop("__closure__") + # _cloudpickle_subimports is a set of submodules that must be loaded for + # the pickled function to work correctly at unpickling time. Now that these + # submodules are depickled (hence imported), they can be removed from the + # object's state (the object state only served as a reference holder to + # these submodules) + slotstate.pop("_cloudpickle_submodules") + + obj.__globals__.update(obj_globals) + obj.__globals__["__builtins__"] = __builtins__ + + if obj_closure is not None: + for i, cell in enumerate(obj_closure): + try: + value = cell.cell_contents + except ValueError: # cell is empty + continue + cell_set(obj.__closure__[i], value) + + for k, v in slotstate.items(): + setattr(obj, k, v) + + +def _function_getstate(func): + # - Put func's dynamic attributes (stored in func.__dict__) in state. These + # attributes will be restored at unpickling time using + # f.__dict__.update(state) + # - Put func's members into slotstate. Such attributes will be restored at + # unpickling time by iterating over slotstate and calling setattr(func, + # slotname, slotvalue) + slotstate = { + "__name__": func.__name__, + "__qualname__": func.__qualname__, + "__annotations__": func.__annotations__, + "__kwdefaults__": func.__kwdefaults__, + "__defaults__": func.__defaults__, + "__module__": func.__module__, + "__doc__": func.__doc__, + "__closure__": func.__closure__, + } + + f_globals_ref = _extract_code_globals(func.__code__) + f_globals = {k: func.__globals__[k] for k in f_globals_ref if k in + func.__globals__} + + closure_values = ( + list(map(_get_cell_contents, func.__closure__)) + if func.__closure__ is not None else () + ) + + # Extract currently-imported submodules used by func. Storing these modules + # in a smoke _cloudpickle_subimports attribute of the object's state will + # trigger the side effect of importing these modules at unpickling time + # (which is necessary for func to work correctly once depickled) + slotstate["_cloudpickle_submodules"] = _find_imported_submodules( + func.__code__, itertools.chain(f_globals.values(), closure_values)) + slotstate["__globals__"] = f_globals + + state = func.__dict__ + return state, slotstate + + +class FunctionSaverMixin: + # function reducers are defined as instance methods of CloudPickler + # objects, as they rely on a CloudPickler attribute (globals_ref) + def _dynamic_function_reduce(self, func): + """Reduce a function that is not pickleable via attribute lookup.""" + newargs = self._function_getnewargs(func) + state = _function_getstate(func) + return (types.FunctionType, newargs, state, None, None, + _function_setstate) + + def _function_reduce(self, obj): + """Reducer for function objects. + + If obj is a top-level attribute of a file-backed module, this + reducer returns NotImplemented, making the CloudPickler fallback to + traditional _pickle.Pickler routines to save obj. Otherwise, it reduces + obj using a custom cloudpickle reducer designed specifically to handle + dynamic functions. + + As opposed to cloudpickle.py, There no special handling for builtin + pypy functions because cloudpickle_fast is CPython-specific. + """ + if _is_importable_by_name(obj): + return NotImplemented + else: + return self._dynamic_function_reduce(obj) + + def _function_getnewargs(self, func): + code = func.__code__ + + # base_globals represents the future global namespace of func at + # unpickling time. Looking it up and storing it in + # CloudpiPickler.globals_ref allow functions sharing the same globals + # at pickling time to also share them once unpickled, at one condition: + # since globals_ref is an attribute of a CloudPickler instance, and + # that a new CloudPickler is created each time pickle.dump or + # pickle.dumps is called, functions also need to be saved within the + # same invocation of cloudpickle.dump/cloudpickle.dumps (for example: + # cloudpickle.dumps([f1, f2])). There is no such limitation when using + # CloudPickler.dump, as long as the multiple invocations are bound to + # the same CloudPickler. + base_globals = self.globals_ref.setdefault(id(func.__globals__), {}) + + if base_globals == {}: + # Add module attributes used to resolve relative imports + # instructions inside func. + for k in ["__package__", "__name__", "__path__", "__file__"]: + if k in func.__globals__: + base_globals[k] = func.__globals__[k] + + # Do not bind the free variables before the function is created to + # avoid infinite recursion. + if func.__closure__ is None: + closure = None + else: + closure = tuple( + _make_empty_cell() for _ in range(len(code.co_freevars))) + + return code, base_globals, None, None, closure + + +def _class_getnewargs(obj): + type_kwargs = {} + if "__slots__" in obj.__dict__: + type_kwargs["__slots__"] = obj.__slots__ + + __dict__ = obj.__dict__.get('__dict__', None) + if isinstance(__dict__, property): + type_kwargs['__dict__'] = __dict__ + + return (type(obj), obj.__name__, _get_bases(obj), type_kwargs, + _get_or_create_tracker_id(obj), None) + + +def _enum_getnewargs(obj): + members = dict((e.name, e.value) for e in obj) + return (obj.__bases__, obj.__name__, obj.__qualname__, members, + obj.__module__, _get_or_create_tracker_id(obj), None) + + +def _class_reduce(obj): + """Select the reducer depending on the dynamic nature of the class obj""" + if obj is type(None): # noqa + return type, (None,) + elif obj is type(Ellipsis): + return type, (Ellipsis,) + elif obj is type(NotImplemented): + return type, (NotImplemented,) + elif obj in _BUILTIN_TYPE_NAMES: + return _builtin_type, (_BUILTIN_TYPE_NAMES[obj],) + elif not _is_importable_by_name(obj): + return _dynamic_class_reduce(obj) + return NotImplemented + + +def _dynamic_class_reduce(obj): + """ + Save a class that can't be stored as module global. + + This method is used to serialize classes that are defined inside + functions, or that otherwise can't be serialized as attribute lookups + from global modules. + """ + if Enum is not None and issubclass(obj, Enum): + return ( + _make_skeleton_enum, _enum_getnewargs(obj), _enum_getstate(obj), + None, None, _class_setstate + ) + else: + return ( + _make_skeleton_class, _class_getnewargs(obj), _class_getstate(obj), + None, None, _class_setstate + ) + + +def _class_getstate(obj): + clsdict = _extract_class_dict(obj) + clsdict.pop('__weakref__', None) + + if issubclass(type(obj), abc.ABCMeta): + # If obj is an instance of an ABCMeta subclass, dont pickle the + # cache/negative caches populated during isinstance/issubclass + # checks, but pickle the list of registered subclasses of obj. + clsdict.pop('_abc_cache', None) + clsdict.pop('_abc_negative_cache', None) + clsdict.pop('_abc_negative_cache_version', None) + registry = clsdict.pop('_abc_registry', None) + if registry is None: + # in Python3.7+, the abc caches and registered subclasses of a + # class are bundled into the single _abc_impl attribute + clsdict.pop('_abc_impl', None) + (registry, _, _, _) = abc._get_dump(obj) + + clsdict["_abc_impl"] = [subclass_weakref() + for subclass_weakref in registry] + else: + # In the above if clause, registry is a set of weakrefs -- in + # this case, registry is a WeakSet + clsdict["_abc_impl"] = [type_ for type_ in registry] + + if "__slots__" in clsdict: + # pickle string length optimization: member descriptors of obj are + # created automatically from obj's __slots__ attribute, no need to + # save them in obj's state + if isinstance(obj.__slots__, str): + clsdict.pop(obj.__slots__) + else: + for k in obj.__slots__: + clsdict.pop(k, None) + + clsdict.pop('__dict__', None) # unpicklable property object + + return (clsdict, {}) + + +def _enum_getstate(obj): + clsdict, slotstate = _class_getstate(obj) + + members = dict((e.name, e.value) for e in obj) + # Cleanup the clsdict that will be passed to _rehydrate_skeleton_class: + # Those attributes are already handled by the metaclass. + for attrname in ["_generate_next_value_", "_member_names_", + "_member_map_", "_member_type_", + "_value2member_map_"]: + clsdict.pop(attrname, None) + for member in members: + clsdict.pop(member) + # Special handling of Enum subclasses + return clsdict, slotstate + + +def _class_setstate(obj, state): + state, slotstate = state + registry = None + for attrname, attr in state.items(): + if attrname == "_abc_impl": + registry = attr + else: + setattr(obj, attrname, attr) + if registry is not None: + for subclass in registry: + obj.register(subclass) + + return obj + + def _get_or_create_tracker_id(class_def): with _DYNAMIC_CLASS_TRACKER_LOCK: class_tracker_id = _DYNAMIC_CLASS_TRACKER_BY_CLASS.get(class_def) @@ -460,9 +730,10 @@ def _create_parametrized_type_hint(origin, args): return origin[args] -class CloudPickler(Pickler): +class CloudPickler(FunctionSaverMixin, Pickler): dispatch = Pickler.dispatch.copy() + dispatch_table = dict() def __init__(self, file, protocol=None): if protocol is None: @@ -482,6 +753,17 @@ def dump(self, obj): else: raise + def _cell_reduce(obj): + """Cell (containing values of a function's free variables) reducer""" + try: + obj.cell_contents + except ValueError: # cell is empty + return _make_empty_cell, () + else: + return _make_cell, (obj.cell_contents, ) + + dispatch_table[CellType] = _cell_reduce + def save_typevar(self, obj): self.save_reduce(*_typevar_reduce(obj), obj=obj) @@ -541,7 +823,30 @@ def save_function(self, obj, name=None): elif PYPY and isinstance(obj.__code__, builtin_code_type): return self.save_pypy_builtin_func(obj) else: - return self.save_function_tuple(obj) + return self._save_reduce_pickle5( + *self._dynamic_function_reduce(obj), obj=obj + ) + + def _save_reduce_pickle5(self, func, args, state=None, listitems=None, + dictitems=None, state_setter=None, obj=None): + save = self.save + write = self.write + self.save_reduce( + func, args, state=None, listitems=listitems, dictitems=dictitems, + obj=obj + ) + # backport of the Python 3.8 state_setter pickle operations + save(state_setter) + save(obj) # simple BINGET opcode as obj is already memoized. + save(state) + write(pickle.TUPLE2) + # Trigger a state_setter(obj, state) function call. + write(pickle.REDUCE) + # The purpose of state_setter is to carry-out an + # inplace modification of obj. We do not care about what the + # method might return, so its output is eventually removed from + # the stack. + write(pickle.POP) dispatch[types.FunctionType] = save_function @@ -569,246 +874,6 @@ def save_pypy_builtin_func(self, obj): obj.__dict__) self.save_reduce(*rv, obj=obj) - def _save_dynamic_enum(self, obj, clsdict): - """Special handling for dynamic Enum subclasses - - Use a dedicated Enum constructor (inspired by EnumMeta.__call__) as the - EnumMeta metaclass has complex initialization that makes the Enum - subclasses hold references to their own instances. - """ - members = dict((e.name, e.value) for e in obj) - - self.save_reduce( - _make_skeleton_enum, - (obj.__bases__, obj.__name__, obj.__qualname__, - members, obj.__module__, _get_or_create_tracker_id(obj), None), - obj=obj - ) - - # Cleanup the clsdict that will be passed to _rehydrate_skeleton_class: - # Those attributes are already handled by the metaclass. - for attrname in ["_generate_next_value_", "_member_names_", - "_member_map_", "_member_type_", - "_value2member_map_"]: - clsdict.pop(attrname, None) - for member in members: - clsdict.pop(member) - - def save_dynamic_class(self, obj): - """Save a class that can't be stored as module global. - - This method is used to serialize classes that are defined inside - functions, or that otherwise can't be serialized as attribute lookups - from global modules. - """ - clsdict = _extract_class_dict(obj) - clsdict.pop('__weakref__', None) - - if issubclass(type(obj), abc.ABCMeta): - # If obj is an instance of an ABCMeta subclass, dont pickle the - # cache/negative caches populated during isinstance/issubclass - # checks, but pickle the list of registered subclasses of obj. - clsdict.pop('_abc_cache', None) - clsdict.pop('_abc_negative_cache', None) - clsdict.pop('_abc_negative_cache_version', None) - registry = clsdict.pop('_abc_registry', None) - if registry is None: - # in Python3.7+, the abc caches and registered subclasses of a - # class are bundled into the single _abc_impl attribute - clsdict.pop('_abc_impl', None) - (registry, _, _, _) = abc._get_dump(obj) - - clsdict["_abc_impl"] = [subclass_weakref() - for subclass_weakref in registry] - else: - # In the above if clause, registry is a set of weakrefs -- in - # this case, registry is a WeakSet - clsdict["_abc_impl"] = [type_ for type_ in registry] - - # On PyPy, __doc__ is a readonly attribute, so we need to include it in - # the initial skeleton class. This is safe because we know that the - # doc can't participate in a cycle with the original class. - type_kwargs = {'__doc__': clsdict.pop('__doc__', None)} - - if "__slots__" in clsdict: - type_kwargs['__slots__'] = obj.__slots__ - # pickle string length optimization: member descriptors of obj are - # created automatically from obj's __slots__ attribute, no need to - # save them in obj's state - if isinstance(obj.__slots__, str): - clsdict.pop(obj.__slots__) - else: - for k in obj.__slots__: - clsdict.pop(k, None) - - # If type overrides __dict__ as a property, include it in the type - # kwargs. In Python 2, we can't set this attribute after construction. - # XXX: can this ever happen in Python 3? If so add a test. - __dict__ = clsdict.pop('__dict__', None) - if isinstance(__dict__, property): - type_kwargs['__dict__'] = __dict__ - - save = self.save - write = self.write - - # We write pickle instructions explicitly here to handle the - # possibility that the type object participates in a cycle with its own - # __dict__. We first write an empty "skeleton" version of the class and - # memoize it before writing the class' __dict__ itself. We then write - # instructions to "rehydrate" the skeleton class by restoring the - # attributes from the __dict__. - # - # A type can appear in a cycle with its __dict__ if an instance of the - # type appears in the type's __dict__ (which happens for the stdlib - # Enum class), or if the type defines methods that close over the name - # of the type, (which is common for Python 2-style super() calls). - - # Push the rehydration function. - save(_rehydrate_skeleton_class) - - # Mark the start of the args tuple for the rehydration function. - write(pickle.MARK) - - # Create and memoize an skeleton class with obj's name and bases. - if Enum is not None and issubclass(obj, Enum): - # Special handling of Enum subclasses - self._save_dynamic_enum(obj, clsdict) - else: - # "Regular" class definition: - tp = type(obj) - self.save_reduce(_make_skeleton_class, - (tp, obj.__name__, _get_bases(obj), type_kwargs, - _get_or_create_tracker_id(obj), None), - obj=obj) - - # Now save the rest of obj's __dict__. Any references to obj - # encountered while saving will point to the skeleton class. - save(clsdict) - - # Write a tuple of (skeleton_class, clsdict). - write(pickle.TUPLE) - - # Call _rehydrate_skeleton_class(skeleton_class, clsdict) - write(pickle.REDUCE) - - def save_function_tuple(self, func): - """ Pickles an actual func object. - - A func comprises: code, globals, defaults, closure, and dict. We - extract and save these, injecting reducing functions at certain points - to recreate the func object. Keep in mind that some of these pieces - can contain a ref to the func itself. Thus, a naive save on these - pieces could trigger an infinite loop of save's. To get around that, - we first create a skeleton func object using just the code (this is - safe, since this won't contain a ref to the func), and memoize it as - soon as it's created. The other stuff can then be filled in later. - """ - if is_tornado_coroutine(func): - self.save_reduce(_rebuild_tornado_coroutine, (func.__wrapped__,), - obj=func) - return - - save = self.save - write = self.write - - code, f_globals, defaults, closure_values, dct, base_globals = self.extract_func_data(func) - - save(_fill_function) # skeleton function updater - write(pickle.MARK) # beginning of tuple that _fill_function expects - - # Extract currently-imported submodules used by func. Storing these - # modules in a smoke _cloudpickle_subimports attribute of the object's - # state will trigger the side effect of importing these modules at - # unpickling time (which is necessary for func to work correctly once - # depickled) - submodules = _find_imported_submodules( - code, - itertools.chain(f_globals.values(), closure_values or ()), - ) - - # create a skeleton function object and memoize it - save(_make_skel_func) - save(( - code, - len(closure_values) if closure_values is not None else -1, - base_globals, - )) - write(pickle.REDUCE) - self.memoize(func) - - # save the rest of the func data needed by _fill_function - state = { - 'globals': f_globals, - 'defaults': defaults, - 'dict': dct, - 'closure_values': closure_values, - 'module': func.__module__, - 'name': func.__name__, - 'doc': func.__doc__, - '_cloudpickle_submodules': submodules - } - if hasattr(func, '__annotations__'): - state['annotations'] = func.__annotations__ - if hasattr(func, '__qualname__'): - state['qualname'] = func.__qualname__ - if hasattr(func, '__kwdefaults__'): - state['kwdefaults'] = func.__kwdefaults__ - save(state) - write(pickle.TUPLE) - write(pickle.REDUCE) # applies _fill_function on the tuple - - def extract_func_data(self, func): - """ - Turn the function into a tuple of data necessary to recreate it: - code, globals, defaults, closure_values, dict - """ - code = func.__code__ - - # extract all global ref's - func_global_refs = _extract_code_globals(code) - - # process all variables referenced by global environment - f_globals = {} - for var in func_global_refs: - if var in func.__globals__: - f_globals[var] = func.__globals__[var] - - # defaults requires no processing - defaults = func.__defaults__ - - # process closure - closure = ( - list(map(_get_cell_contents, func.__closure__)) - if func.__closure__ is not None - else None - ) - - # save the dict - dct = func.__dict__ - - # base_globals represents the future global namespace of func at - # unpickling time. Looking it up and storing it in globals_ref allow - # functions sharing the same globals at pickling time to also - # share them once unpickled, at one condition: since globals_ref is - # an attribute of a Cloudpickler instance, and that a new CloudPickler is - # created each time pickle.dump or pickle.dumps is called, functions - # also need to be saved within the same invokation of - # cloudpickle.dump/cloudpickle.dumps (for example: cloudpickle.dumps([f1, f2])). There - # is no such limitation when using Cloudpickler.dump, as long as the - # multiple invokations are bound to the same Cloudpickler. - base_globals = self.globals_ref.setdefault(id(func.__globals__), {}) - - if base_globals == {}: - # Add module attributes used to resolve relative imports - # instructions inside func. - for k in ["__package__", "__name__", "__path__", "__file__"]: - # Some built-in functions/methods such as object.__new__ have - # their __globals__ set to None in PyPy - if func.__globals__ is not None and k in func.__globals__: - base_globals[k] = func.__globals__[k] - - return (code, f_globals, defaults, closure, dct, base_globals) - def save_getset_descriptor(self, obj): return self.save_reduce(getattr, (obj.__objclass__, obj.__name__)) @@ -841,7 +906,7 @@ def save_global(self, obj, name=None, pack=struct.pack): elif name is not None: Pickler.save_global(self, obj, name=name) elif not _is_importable_by_name(obj, name=name): - self.save_dynamic_class(obj) + self._save_reduce_pickle5(*_dynamic_class_reduce(obj), obj=obj) else: Pickler.save_global(self, obj, name=name) @@ -1145,78 +1210,24 @@ def __reduce__(cls): return cls.__name__ -def _fill_function(*args): - """Fills in the rest of function data into the skeleton function object - - The skeleton itself is create by _make_skel_func(). - """ - if len(args) == 2: - func = args[0] - state = args[1] - elif len(args) == 5: - # Backwards compat for cloudpickle v0.4.0, after which the `module` - # argument was introduced - func = args[0] - keys = ['globals', 'defaults', 'dict', 'closure_values'] - state = dict(zip(keys, args[1:])) - elif len(args) == 6: - # Backwards compat for cloudpickle v0.4.1, after which the function - # state was passed as a dict to the _fill_function it-self. - func = args[0] - keys = ['globals', 'defaults', 'dict', 'module', 'closure_values'] - state = dict(zip(keys, args[1:])) +def _make_empty_cell(): + if sys.version_info >= (3, 8): + from types import CellType + return CellType else: - raise ValueError('Unexpected _fill_value arguments: %r' % (args,)) - - # - At pickling time, any dynamic global variable used by func is - # serialized by value (in state['globals']). - # - At unpickling time, func's __globals__ attribute is initialized by - # first retrieving an empty isolated namespace that will be shared - # with other functions pickled from the same original module - # by the same CloudPickler instance and then updated with the - # content of state['globals'] to populate the shared isolated - # namespace with all the global variables that are specifically - # referenced for this function. - func.__globals__.update(state['globals']) - - func.__defaults__ = state['defaults'] - func.__dict__ = state['dict'] - if 'annotations' in state: - func.__annotations__ = state['annotations'] - if 'doc' in state: - func.__doc__ = state['doc'] - if 'name' in state: - func.__name__ = state['name'] - if 'module' in state: - func.__module__ = state['module'] - if 'qualname' in state: - func.__qualname__ = state['qualname'] - if 'kwdefaults' in state: - func.__kwdefaults__ = state['kwdefaults'] - # _cloudpickle_subimports is a set of submodules that must be loaded for - # the pickled function to work correctly at unpickling time. Now that these - # submodules are depickled (hence imported), they can be removed from the - # object's state (the object state only served as a reference holder to - # these submodules) - if '_cloudpickle_submodules' in state: - state.pop('_cloudpickle_submodules') + if False: + # trick the compiler into creating an empty cell in our lambda + cell = None + raise AssertionError('this route should not be executed') - cells = func.__closure__ - if cells is not None: - for cell, value in zip(cells, state['closure_values']): - if value is not _empty_cell_value: - cell_set(cell, value) + return (lambda: cell).__closure__[0] - return func - -def _make_empty_cell(): - if False: - # trick the compiler into creating an empty cell in our lambda - cell = None - raise AssertionError('this route should not be executed') - - return (lambda: cell).__closure__[0] +def _make_cell(value=_empty_cell_value): + cell = _make_empty_cell() + if value != _empty_cell_value: + cell_set(cell, value) + return cell def _make_skel_func(code, cell_count, base_globals=None): @@ -1259,24 +1270,6 @@ class id will also reuse this class definition. return _lookup_class_or_track(class_tracker_id, skeleton_class) -def _rehydrate_skeleton_class(skeleton_class, class_dict): - """Put attributes from `class_dict` back on `skeleton_class`. - - See CloudPickler.save_dynamic_class for more info. - """ - registry = None - for attrname, attr in class_dict.items(): - if attrname == "_abc_impl": - registry = attr - else: - setattr(skeleton_class, attrname, attr) - if registry is not None: - for subclass in registry: - skeleton_class.register(subclass) - - return skeleton_class - - def _make_skeleton_enum(bases, name, qualname, members, module, class_tracker_id, extra): """Build dynamic enum with an empty __dict__ to be filled once memoized diff --git a/cloudpickle/cloudpickle_fast.py b/cloudpickle/cloudpickle_fast.py index 7df95f8dc..eb2ec648a 100644 --- a/cloudpickle/cloudpickle_fast.py +++ b/cloudpickle/cloudpickle_fast.py @@ -29,6 +29,7 @@ _find_imported_submodules, _get_cell_contents, _is_importable_by_name, _builtin_type, Enum, _get_or_create_tracker_id, _make_skeleton_class, _make_skeleton_enum, _extract_class_dict, dynamic_subimport, subimport, _typevar_reduce, _get_bases, + FunctionSaverMixin ) load, loads = _pickle.load, _pickle.loads @@ -67,25 +68,6 @@ def dumps(obj, protocol=None, buffer_callback=None): # COLLECTION OF OBJECTS __getnewargs__-LIKE METHODS # ------------------------------------------------- -def _class_getnewargs(obj): - type_kwargs = {} - if "__slots__" in obj.__dict__: - type_kwargs["__slots__"] = obj.__slots__ - - __dict__ = obj.__dict__.get('__dict__', None) - if isinstance(__dict__, property): - type_kwargs['__dict__'] = __dict__ - - return (type(obj), obj.__name__, _get_bases(obj), type_kwargs, - _get_or_create_tracker_id(obj), None) - - -def _enum_getnewargs(obj): - members = dict((e.name, e.value) for e in obj) - return (obj.__bases__, obj.__name__, obj.__qualname__, members, - obj.__module__, _get_or_create_tracker_id(obj), None) - - # COLLECTION OF OBJECTS RECONSTRUCTORS # ------------------------------------ def _file_reconstructor(retval): @@ -94,89 +76,6 @@ def _file_reconstructor(retval): # COLLECTION OF OBJECTS STATE GETTERS # ----------------------------------- -def _function_getstate(func): - # - Put func's dynamic attributes (stored in func.__dict__) in state. These - # attributes will be restored at unpickling time using - # f.__dict__.update(state) - # - Put func's members into slotstate. Such attributes will be restored at - # unpickling time by iterating over slotstate and calling setattr(func, - # slotname, slotvalue) - slotstate = { - "__name__": func.__name__, - "__qualname__": func.__qualname__, - "__annotations__": func.__annotations__, - "__kwdefaults__": func.__kwdefaults__, - "__defaults__": func.__defaults__, - "__module__": func.__module__, - "__doc__": func.__doc__, - "__closure__": func.__closure__, - } - - f_globals_ref = _extract_code_globals(func.__code__) - f_globals = {k: func.__globals__[k] for k in f_globals_ref if k in - func.__globals__} - - closure_values = ( - list(map(_get_cell_contents, func.__closure__)) - if func.__closure__ is not None else () - ) - - # Extract currently-imported submodules used by func. Storing these modules - # in a smoke _cloudpickle_subimports attribute of the object's state will - # trigger the side effect of importing these modules at unpickling time - # (which is necessary for func to work correctly once depickled) - slotstate["_cloudpickle_submodules"] = _find_imported_submodules( - func.__code__, itertools.chain(f_globals.values(), closure_values)) - slotstate["__globals__"] = f_globals - - state = func.__dict__ - return state, slotstate - - -def _class_getstate(obj): - clsdict = _extract_class_dict(obj) - clsdict.pop('__weakref__', None) - - if issubclass(type(obj), abc.ABCMeta): - # If obj is an instance of an ABCMeta subclass, dont pickle the - # cache/negative caches populated during isinstance/issubclass - # checks, but pickle the list of registered subclasses of obj. - clsdict.pop('_abc_impl', None) - (registry, _, _, _) = abc._get_dump(obj) - clsdict["_abc_impl"] = [subclass_weakref() - for subclass_weakref in registry] - - if "__slots__" in clsdict: - # pickle string length optimization: member descriptors of obj are - # created automatically from obj's __slots__ attribute, no need to - # save them in obj's state - if isinstance(obj.__slots__, str): - clsdict.pop(obj.__slots__) - else: - for k in obj.__slots__: - clsdict.pop(k, None) - - clsdict.pop('__dict__', None) # unpicklable property object - - return (clsdict, {}) - - -def _enum_getstate(obj): - clsdict, slotstate = _class_getstate(obj) - - members = dict((e.name, e.value) for e in obj) - # Cleanup the clsdict that will be passed to _rehydrate_skeleton_class: - # Those attributes are already handled by the metaclass. - for attrname in ["_generate_next_value_", "_member_names_", - "_member_map_", "_member_type_", - "_value2member_map_"]: - clsdict.pop(attrname, None) - for member in members: - clsdict.pop(member) - # Special handling of Enum subclasses - return clsdict, slotstate - - # COLLECTIONS OF OBJECTS REDUCERS # ------------------------------- # A reducer is a function taking a single argument (obj), and that returns a @@ -303,97 +202,13 @@ def _weakset_reduce(obj): return weakref.WeakSet, (list(obj),) -def _dynamic_class_reduce(obj): - """ - Save a class that can't be stored as module global. - - This method is used to serialize classes that are defined inside - functions, or that otherwise can't be serialized as attribute lookups - from global modules. - """ - if Enum is not None and issubclass(obj, Enum): - return ( - _make_skeleton_enum, _enum_getnewargs(obj), _enum_getstate(obj), - None, None, _class_setstate - ) - else: - return ( - _make_skeleton_class, _class_getnewargs(obj), _class_getstate(obj), - None, None, _class_setstate - ) - - -def _class_reduce(obj): - """Select the reducer depending on the dynamic nature of the class obj""" - if obj is type(None): # noqa - return type, (None,) - elif obj is type(Ellipsis): - return type, (Ellipsis,) - elif obj is type(NotImplemented): - return type, (NotImplemented,) - elif obj in _BUILTIN_TYPE_NAMES: - return _builtin_type, (_BUILTIN_TYPE_NAMES[obj],) - elif not _is_importable_by_name(obj): - return _dynamic_class_reduce(obj) - return NotImplemented - - # COLLECTIONS OF OBJECTS STATE SETTERS # ------------------------------------ # state setters are called at unpickling time, once the object is created and # it has to be updated to how it was at unpickling time. -def _function_setstate(obj, state): - """Update the state of a dynaamic function. - - As __closure__ and __globals__ are readonly attributes of a function, we - cannot rely on the native setstate routine of pickle.load_build, that calls - setattr on items of the slotstate. Instead, we have to modify them inplace. - """ - state, slotstate = state - obj.__dict__.update(state) - - obj_globals = slotstate.pop("__globals__") - obj_closure = slotstate.pop("__closure__") - # _cloudpickle_subimports is a set of submodules that must be loaded for - # the pickled function to work correctly at unpickling time. Now that these - # submodules are depickled (hence imported), they can be removed from the - # object's state (the object state only served as a reference holder to - # these submodules) - slotstate.pop("_cloudpickle_submodules") - - obj.__globals__.update(obj_globals) - obj.__globals__["__builtins__"] = __builtins__ - - if obj_closure is not None: - for i, cell in enumerate(obj_closure): - try: - value = cell.cell_contents - except ValueError: # cell is empty - continue - obj.__closure__[i].cell_contents = value - - for k, v in slotstate.items(): - setattr(obj, k, v) - - -def _class_setstate(obj, state): - state, slotstate = state - registry = None - for attrname, attr in state.items(): - if attrname == "_abc_impl": - registry = attr - else: - setattr(obj, attrname, attr) - if registry is not None: - for subclass in registry: - obj.register(subclass) - - return obj - - -class CloudPickler(Pickler): +class CloudPickler(FunctionSaverMixin, Pickler): """Fast C Pickler extension with additional reducing routines. CloudPickler's extensions exist into into: @@ -484,65 +299,6 @@ def reducer_override(self, obj): # fallback to save_global, including the Pickler's distpatch_table return NotImplemented - # function reducers are defined as instance methods of CloudPickler - # objects, as they rely on a CloudPickler attribute (globals_ref) - def _dynamic_function_reduce(self, func): - """Reduce a function that is not pickleable via attribute lookup.""" - newargs = self._function_getnewargs(func) - state = _function_getstate(func) - return (types.FunctionType, newargs, state, None, None, - _function_setstate) - - def _function_reduce(self, obj): - """Reducer for function objects. - - If obj is a top-level attribute of a file-backed module, this - reducer returns NotImplemented, making the CloudPickler fallback to - traditional _pickle.Pickler routines to save obj. Otherwise, it reduces - obj using a custom cloudpickle reducer designed specifically to handle - dynamic functions. - - As opposed to cloudpickle.py, There no special handling for builtin - pypy functions because cloudpickle_fast is CPython-specific. - """ - if _is_importable_by_name(obj): - return NotImplemented - else: - return self._dynamic_function_reduce(obj) - - def _function_getnewargs(self, func): - code = func.__code__ - - # base_globals represents the future global namespace of func at - # unpickling time. Looking it up and storing it in - # CloudpiPickler.globals_ref allow functions sharing the same globals - # at pickling time to also share them once unpickled, at one condition: - # since globals_ref is an attribute of a CloudPickler instance, and - # that a new CloudPickler is created each time pickle.dump or - # pickle.dumps is called, functions also need to be saved within the - # same invocation of cloudpickle.dump/cloudpickle.dumps (for example: - # cloudpickle.dumps([f1, f2])). There is no such limitation when using - # CloudPickler.dump, as long as the multiple invocations are bound to - # the same CloudPickler. - base_globals = self.globals_ref.setdefault(id(func.__globals__), {}) - - if base_globals == {}: - # Add module attributes used to resolve relative imports - # instructions inside func. - for k in ["__package__", "__name__", "__path__", "__file__"]: - if k in func.__globals__: - base_globals[k] = func.__globals__[k] - - # Do not bind the free variables before the function is created to - # avoid infinite recursion. - if func.__closure__ is None: - closure = None - else: - closure = tuple( - types.CellType() for _ in range(len(code.co_freevars))) - - return code, base_globals, None, None, closure - def dump(self, obj): try: return Pickler.dump(self, obj) From ee722f1889b47f87bd2100965c05ec3891b6bd4c Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Thu, 14 May 2020 11:01:56 +0200 Subject: [PATCH 02/89] python 3.8 compat --- cloudpickle/cloudpickle.py | 14 +++++--------- cloudpickle/cloudpickle_fast.py | 2 +- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 27e0e51b9..a4a35430e 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -1211,16 +1211,12 @@ def __reduce__(cls): def _make_empty_cell(): - if sys.version_info >= (3, 8): - from types import CellType - return CellType - else: - if False: - # trick the compiler into creating an empty cell in our lambda - cell = None - raise AssertionError('this route should not be executed') + if False: + # trick the compiler into creating an empty cell in our lambda + cell = None + raise AssertionError('this route should not be executed') - return (lambda: cell).__closure__[0] + return (lambda: cell).__closure__[0] def _make_cell(value=_empty_cell_value): diff --git a/cloudpickle/cloudpickle_fast.py b/cloudpickle/cloudpickle_fast.py index eb2ec648a..541a0a061 100644 --- a/cloudpickle/cloudpickle_fast.py +++ b/cloudpickle/cloudpickle_fast.py @@ -29,7 +29,7 @@ _find_imported_submodules, _get_cell_contents, _is_importable_by_name, _builtin_type, Enum, _get_or_create_tracker_id, _make_skeleton_class, _make_skeleton_enum, _extract_class_dict, dynamic_subimport, subimport, _typevar_reduce, _get_bases, - FunctionSaverMixin + FunctionSaverMixin, _class_reduce ) load, loads = _pickle.load, _pickle.loads From d69ca868f7ccbd535802932ef875b310eb89c091 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Thu, 14 May 2020 11:40:12 +0200 Subject: [PATCH 03/89] use the same Cloudpickler class for all Pythons --- cloudpickle/__init__.py | 8 +- cloudpickle/cloudpickle.py | 670 +++----------------------------- cloudpickle/cloudpickle_fast.py | 437 +++++++++++++++++++-- tests/cloudpickle_test.py | 8 +- 4 files changed, 455 insertions(+), 668 deletions(-) diff --git a/cloudpickle/__init__.py b/cloudpickle/__init__.py index c3ecfdbc7..ae5f85e53 100644 --- a/cloudpickle/__init__.py +++ b/cloudpickle/__init__.py @@ -1,11 +1,7 @@ from __future__ import absolute_import -import sys -import pickle - -from cloudpickle.cloudpickle import * -if sys.version_info[:2] >= (3, 8): - from cloudpickle.cloudpickle_fast import CloudPickler, dumps, dump +from cloudpickle.cloudpickle import * # noqa +from cloudpickle.cloudpickle_fast import CloudPickler, dumps, dump # noqa __version__ = '1.5.0dev0' diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index a4a35430e..a0da593a6 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -112,265 +112,6 @@ def g(): _extract_code_globals_cache = weakref.WeakKeyDictionary() -def _function_setstate(obj, state): - """Update the state of a dynaamic function. - - As __closure__ and __globals__ are readonly attributes of a function, we - cannot rely on the native setstate routine of pickle.load_build, that calls - setattr on items of the slotstate. Instead, we have to modify them inplace. - """ - state, slotstate = state - obj.__dict__.update(state) - - obj_globals = slotstate.pop("__globals__") - obj_closure = slotstate.pop("__closure__") - # _cloudpickle_subimports is a set of submodules that must be loaded for - # the pickled function to work correctly at unpickling time. Now that these - # submodules are depickled (hence imported), they can be removed from the - # object's state (the object state only served as a reference holder to - # these submodules) - slotstate.pop("_cloudpickle_submodules") - - obj.__globals__.update(obj_globals) - obj.__globals__["__builtins__"] = __builtins__ - - if obj_closure is not None: - for i, cell in enumerate(obj_closure): - try: - value = cell.cell_contents - except ValueError: # cell is empty - continue - cell_set(obj.__closure__[i], value) - - for k, v in slotstate.items(): - setattr(obj, k, v) - - -def _function_getstate(func): - # - Put func's dynamic attributes (stored in func.__dict__) in state. These - # attributes will be restored at unpickling time using - # f.__dict__.update(state) - # - Put func's members into slotstate. Such attributes will be restored at - # unpickling time by iterating over slotstate and calling setattr(func, - # slotname, slotvalue) - slotstate = { - "__name__": func.__name__, - "__qualname__": func.__qualname__, - "__annotations__": func.__annotations__, - "__kwdefaults__": func.__kwdefaults__, - "__defaults__": func.__defaults__, - "__module__": func.__module__, - "__doc__": func.__doc__, - "__closure__": func.__closure__, - } - - f_globals_ref = _extract_code_globals(func.__code__) - f_globals = {k: func.__globals__[k] for k in f_globals_ref if k in - func.__globals__} - - closure_values = ( - list(map(_get_cell_contents, func.__closure__)) - if func.__closure__ is not None else () - ) - - # Extract currently-imported submodules used by func. Storing these modules - # in a smoke _cloudpickle_subimports attribute of the object's state will - # trigger the side effect of importing these modules at unpickling time - # (which is necessary for func to work correctly once depickled) - slotstate["_cloudpickle_submodules"] = _find_imported_submodules( - func.__code__, itertools.chain(f_globals.values(), closure_values)) - slotstate["__globals__"] = f_globals - - state = func.__dict__ - return state, slotstate - - -class FunctionSaverMixin: - # function reducers are defined as instance methods of CloudPickler - # objects, as they rely on a CloudPickler attribute (globals_ref) - def _dynamic_function_reduce(self, func): - """Reduce a function that is not pickleable via attribute lookup.""" - newargs = self._function_getnewargs(func) - state = _function_getstate(func) - return (types.FunctionType, newargs, state, None, None, - _function_setstate) - - def _function_reduce(self, obj): - """Reducer for function objects. - - If obj is a top-level attribute of a file-backed module, this - reducer returns NotImplemented, making the CloudPickler fallback to - traditional _pickle.Pickler routines to save obj. Otherwise, it reduces - obj using a custom cloudpickle reducer designed specifically to handle - dynamic functions. - - As opposed to cloudpickle.py, There no special handling for builtin - pypy functions because cloudpickle_fast is CPython-specific. - """ - if _is_importable_by_name(obj): - return NotImplemented - else: - return self._dynamic_function_reduce(obj) - - def _function_getnewargs(self, func): - code = func.__code__ - - # base_globals represents the future global namespace of func at - # unpickling time. Looking it up and storing it in - # CloudpiPickler.globals_ref allow functions sharing the same globals - # at pickling time to also share them once unpickled, at one condition: - # since globals_ref is an attribute of a CloudPickler instance, and - # that a new CloudPickler is created each time pickle.dump or - # pickle.dumps is called, functions also need to be saved within the - # same invocation of cloudpickle.dump/cloudpickle.dumps (for example: - # cloudpickle.dumps([f1, f2])). There is no such limitation when using - # CloudPickler.dump, as long as the multiple invocations are bound to - # the same CloudPickler. - base_globals = self.globals_ref.setdefault(id(func.__globals__), {}) - - if base_globals == {}: - # Add module attributes used to resolve relative imports - # instructions inside func. - for k in ["__package__", "__name__", "__path__", "__file__"]: - if k in func.__globals__: - base_globals[k] = func.__globals__[k] - - # Do not bind the free variables before the function is created to - # avoid infinite recursion. - if func.__closure__ is None: - closure = None - else: - closure = tuple( - _make_empty_cell() for _ in range(len(code.co_freevars))) - - return code, base_globals, None, None, closure - - -def _class_getnewargs(obj): - type_kwargs = {} - if "__slots__" in obj.__dict__: - type_kwargs["__slots__"] = obj.__slots__ - - __dict__ = obj.__dict__.get('__dict__', None) - if isinstance(__dict__, property): - type_kwargs['__dict__'] = __dict__ - - return (type(obj), obj.__name__, _get_bases(obj), type_kwargs, - _get_or_create_tracker_id(obj), None) - - -def _enum_getnewargs(obj): - members = dict((e.name, e.value) for e in obj) - return (obj.__bases__, obj.__name__, obj.__qualname__, members, - obj.__module__, _get_or_create_tracker_id(obj), None) - - -def _class_reduce(obj): - """Select the reducer depending on the dynamic nature of the class obj""" - if obj is type(None): # noqa - return type, (None,) - elif obj is type(Ellipsis): - return type, (Ellipsis,) - elif obj is type(NotImplemented): - return type, (NotImplemented,) - elif obj in _BUILTIN_TYPE_NAMES: - return _builtin_type, (_BUILTIN_TYPE_NAMES[obj],) - elif not _is_importable_by_name(obj): - return _dynamic_class_reduce(obj) - return NotImplemented - - -def _dynamic_class_reduce(obj): - """ - Save a class that can't be stored as module global. - - This method is used to serialize classes that are defined inside - functions, or that otherwise can't be serialized as attribute lookups - from global modules. - """ - if Enum is not None and issubclass(obj, Enum): - return ( - _make_skeleton_enum, _enum_getnewargs(obj), _enum_getstate(obj), - None, None, _class_setstate - ) - else: - return ( - _make_skeleton_class, _class_getnewargs(obj), _class_getstate(obj), - None, None, _class_setstate - ) - - -def _class_getstate(obj): - clsdict = _extract_class_dict(obj) - clsdict.pop('__weakref__', None) - - if issubclass(type(obj), abc.ABCMeta): - # If obj is an instance of an ABCMeta subclass, dont pickle the - # cache/negative caches populated during isinstance/issubclass - # checks, but pickle the list of registered subclasses of obj. - clsdict.pop('_abc_cache', None) - clsdict.pop('_abc_negative_cache', None) - clsdict.pop('_abc_negative_cache_version', None) - registry = clsdict.pop('_abc_registry', None) - if registry is None: - # in Python3.7+, the abc caches and registered subclasses of a - # class are bundled into the single _abc_impl attribute - clsdict.pop('_abc_impl', None) - (registry, _, _, _) = abc._get_dump(obj) - - clsdict["_abc_impl"] = [subclass_weakref() - for subclass_weakref in registry] - else: - # In the above if clause, registry is a set of weakrefs -- in - # this case, registry is a WeakSet - clsdict["_abc_impl"] = [type_ for type_ in registry] - - if "__slots__" in clsdict: - # pickle string length optimization: member descriptors of obj are - # created automatically from obj's __slots__ attribute, no need to - # save them in obj's state - if isinstance(obj.__slots__, str): - clsdict.pop(obj.__slots__) - else: - for k in obj.__slots__: - clsdict.pop(k, None) - - clsdict.pop('__dict__', None) # unpicklable property object - - return (clsdict, {}) - - -def _enum_getstate(obj): - clsdict, slotstate = _class_getstate(obj) - - members = dict((e.name, e.value) for e in obj) - # Cleanup the clsdict that will be passed to _rehydrate_skeleton_class: - # Those attributes are already handled by the metaclass. - for attrname in ["_generate_next_value_", "_member_names_", - "_member_map_", "_member_type_", - "_value2member_map_"]: - clsdict.pop(attrname, None) - for member in members: - clsdict.pop(member) - # Special handling of Enum subclasses - return clsdict, slotstate - - -def _class_setstate(obj, state): - state, slotstate = state - registry = None - for attrname, attr in state.items(): - if attrname == "_abc_impl": - registry = attr - else: - setattr(obj, attrname, attr) - if registry is not None: - for subclass in registry: - obj.register(subclass) - - return obj - - def _get_or_create_tracker_id(class_def): with _DYNAMIC_CLASS_TRACKER_LOCK: class_tracker_id = _DYNAMIC_CLASS_TRACKER_BY_CLASS.get(class_def) @@ -728,373 +469,60 @@ def _is_parametrized_type_hint(obj): def _create_parametrized_type_hint(origin, args): return origin[args] - - -class CloudPickler(FunctionSaverMixin, Pickler): - - dispatch = Pickler.dispatch.copy() - dispatch_table = dict() - - def __init__(self, file, protocol=None): - if protocol is None: - protocol = DEFAULT_PROTOCOL - Pickler.__init__(self, file, protocol=protocol) - # map ids to dictionary. used to ensure that functions can share global env - self.globals_ref = {} - - def dump(self, obj): - self.inject_addons() - try: - return Pickler.dump(self, obj) - except RuntimeError as e: - if 'recursion' in e.args[0]: - msg = """Could not pickle object as excessively deep recursion required.""" - raise pickle.PicklingError(msg) - else: - raise - - def _cell_reduce(obj): - """Cell (containing values of a function's free variables) reducer""" - try: - obj.cell_contents - except ValueError: # cell is empty - return _make_empty_cell, () +else: + _is_parametrized_type_hint = None + _create_parametrized_type_hint = None + + +def parametrized_type_hint_getinitargs(obj): + # The distorted type check sematic for typing construct becomes: + # ``type(obj) is type(TypeHint)``, which means "obj is a + # parametrized TypeHint" + if type(obj) is type(Literal): # pragma: no branch + initargs = (Literal, obj.__values__) + elif type(obj) is type(Final): # pragma: no branch + initargs = (Final, obj.__type__) + elif type(obj) is type(ClassVar): + initargs = (ClassVar, obj.__type__) + elif type(obj) is type(Generic): + parameters = obj.__parameters__ + if len(obj.__parameters__) > 0: + # in early Python 3.5, __parameters__ was sometimes + # preferred to __args__ + initargs = (obj.__origin__, parameters) else: - return _make_cell, (obj.cell_contents, ) - - dispatch_table[CellType] = _cell_reduce - - def save_typevar(self, obj): - self.save_reduce(*_typevar_reduce(obj), obj=obj) - - dispatch[typing.TypeVar] = save_typevar - - def save_memoryview(self, obj): - self.save(obj.tobytes()) - - dispatch[memoryview] = save_memoryview - - def save_module(self, obj): - """ - Save a module as an import - """ - if _is_dynamic(obj): - obj.__dict__.pop('__builtins__', None) - self.save_reduce(dynamic_subimport, (obj.__name__, vars(obj)), - obj=obj) + initargs = (obj.__origin__, obj.__args__) + elif type(obj) is type(Union): + if sys.version_info < (3, 5, 3): # pragma: no cover + initargs = (Union, obj.__union_params__) else: - self.save_reduce(subimport, (obj.__name__,), obj=obj) - - dispatch[types.ModuleType] = save_module - - def save_codeobject(self, obj): - """ - Save a code object - """ - if hasattr(obj, "co_posonlyargcount"): # pragma: no branch - args = ( - obj.co_argcount, obj.co_posonlyargcount, - obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, - obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, - obj.co_varnames, obj.co_filename, obj.co_name, - obj.co_firstlineno, obj.co_lnotab, obj.co_freevars, - obj.co_cellvars - ) + initargs = (Union, obj.__args__) + elif type(obj) is type(Tuple): + if sys.version_info < (3, 5, 3): # pragma: no cover + initargs = (Tuple, obj.__tuple_params__) else: - args = ( - obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals, - obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts, - obj.co_names, obj.co_varnames, obj.co_filename, - obj.co_name, obj.co_firstlineno, obj.co_lnotab, - obj.co_freevars, obj.co_cellvars - ) - self.save_reduce(types.CodeType, args, obj=obj) - - dispatch[types.CodeType] = save_codeobject - - def save_function(self, obj, name=None): - """ Registered with the dispatch to handle all function types. - - Determines what kind of function obj is (e.g. lambda, defined at - interactive prompt, etc) and handles the pickling appropriately. - """ - if _is_importable_by_name(obj, name=name): - return Pickler.save_global(self, obj, name=name) - elif PYPY and isinstance(obj.__code__, builtin_code_type): - return self.save_pypy_builtin_func(obj) + initargs = (Tuple, obj.__args__) + elif type(obj) is type(Callable): + if sys.version_info < (3, 5, 3): # pragma: no cover + args = obj.__args__ + result = obj.__result__ + if args != Ellipsis: + if isinstance(args, tuple): + args = list(args) + else: + args = [args] else: - return self._save_reduce_pickle5( - *self._dynamic_function_reduce(obj), obj=obj - ) - - def _save_reduce_pickle5(self, func, args, state=None, listitems=None, - dictitems=None, state_setter=None, obj=None): - save = self.save - write = self.write - self.save_reduce( - func, args, state=None, listitems=listitems, dictitems=dictitems, - obj=obj + (*args, result) = obj.__args__ + if len(args) == 1 and args[0] is Ellipsis: + args = Ellipsis + else: + args = list(args) + initargs = (Callable, (args, result)) + else: # pragma: no cover + raise pickle.PicklingError( + "Cloudpickle Error: Unknown type {}".format(type(obj)) ) - # backport of the Python 3.8 state_setter pickle operations - save(state_setter) - save(obj) # simple BINGET opcode as obj is already memoized. - save(state) - write(pickle.TUPLE2) - # Trigger a state_setter(obj, state) function call. - write(pickle.REDUCE) - # The purpose of state_setter is to carry-out an - # inplace modification of obj. We do not care about what the - # method might return, so its output is eventually removed from - # the stack. - write(pickle.POP) - - dispatch[types.FunctionType] = save_function - - def save_pypy_builtin_func(self, obj): - """Save pypy equivalent of builtin functions. - - PyPy does not have the concept of builtin-functions. Instead, - builtin-functions are simple function instances, but with a - builtin-code attribute. - Most of the time, builtin functions should be pickled by attribute. But - PyPy has flaky support for __qualname__, so some builtin functions such - as float.__new__ will be classified as dynamic. For this reason only, - we created this special routine. Because builtin-functions are not - expected to have closure or globals, there is no additional hack - (compared the one already implemented in pickle) to protect ourselves - from reference cycles. A simple (reconstructor, newargs, obj.__dict__) - tuple is save_reduced. - - Note also that PyPy improved their support for __qualname__ in v3.6, so - this routing should be removed when cloudpickle supports only PyPy 3.6 - and later. - """ - rv = (types.FunctionType, (obj.__code__, {}, obj.__name__, - obj.__defaults__, obj.__closure__), - obj.__dict__) - self.save_reduce(*rv, obj=obj) - - def save_getset_descriptor(self, obj): - return self.save_reduce(getattr, (obj.__objclass__, obj.__name__)) - - dispatch[types.GetSetDescriptorType] = save_getset_descriptor - - def save_global(self, obj, name=None, pack=struct.pack): - """ - Save a "global". - - The name of this method is somewhat misleading: all types get - dispatched here. - """ - if obj is type(None): - return self.save_reduce(type, (None,), obj=obj) - elif obj is type(Ellipsis): - return self.save_reduce(type, (Ellipsis,), obj=obj) - elif obj is type(NotImplemented): - return self.save_reduce(type, (NotImplemented,), obj=obj) - elif obj in _BUILTIN_TYPE_NAMES: - return self.save_reduce( - _builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj) - - if sys.version_info[:2] < (3, 7) and _is_parametrized_type_hint(obj): # noqa # pragma: no branch - # Parametrized typing constructs in Python < 3.7 are not compatible - # with type checks and ``isinstance`` semantics. For this reason, - # it is easier to detect them using a duck-typing-based check - # (``_is_parametrized_type_hint``) than to populate the Pickler's - # dispatch with type-specific savers. - self._save_parametrized_type_hint(obj) - elif name is not None: - Pickler.save_global(self, obj, name=name) - elif not _is_importable_by_name(obj, name=name): - self._save_reduce_pickle5(*_dynamic_class_reduce(obj), obj=obj) - else: - Pickler.save_global(self, obj, name=name) - - dispatch[type] = save_global - - def save_instancemethod(self, obj): - # Memoization rarely is ever useful due to python bounding - if obj.__self__ is None: - self.save_reduce(getattr, (obj.im_class, obj.__name__)) - else: - self.save_reduce(types.MethodType, (obj.__func__, obj.__self__), obj=obj) - - dispatch[types.MethodType] = save_instancemethod - - def save_property(self, obj): - # properties not correctly saved in python - self.save_reduce(property, (obj.fget, obj.fset, obj.fdel, obj.__doc__), - obj=obj) - - dispatch[property] = save_property - - def save_classmethod(self, obj): - orig_func = obj.__func__ - self.save_reduce(type(obj), (orig_func,), obj=obj) - - dispatch[classmethod] = save_classmethod - dispatch[staticmethod] = save_classmethod - - def save_itemgetter(self, obj): - """itemgetter serializer (needed for namedtuple support)""" - class Dummy: - def __getitem__(self, item): - return item - items = obj(Dummy()) - if not isinstance(items, tuple): - items = (items,) - return self.save_reduce(operator.itemgetter, items) - - if type(operator.itemgetter) is type: - dispatch[operator.itemgetter] = save_itemgetter - - def save_attrgetter(self, obj): - """attrgetter serializer""" - class Dummy(object): - def __init__(self, attrs, index=None): - self.attrs = attrs - self.index = index - def __getattribute__(self, item): - attrs = object.__getattribute__(self, "attrs") - index = object.__getattribute__(self, "index") - if index is None: - index = len(attrs) - attrs.append(item) - else: - attrs[index] = ".".join([attrs[index], item]) - return type(self)(attrs, index) - attrs = [] - obj(Dummy(attrs)) - return self.save_reduce(operator.attrgetter, tuple(attrs)) - - if type(operator.attrgetter) is type: - dispatch[operator.attrgetter] = save_attrgetter - - def save_file(self, obj): - """Save a file""" - - if not hasattr(obj, 'name') or not hasattr(obj, 'mode'): - raise pickle.PicklingError("Cannot pickle files that do not map to an actual file") - if obj is sys.stdout: - return self.save_reduce(getattr, (sys, 'stdout'), obj=obj) - if obj is sys.stderr: - return self.save_reduce(getattr, (sys, 'stderr'), obj=obj) - if obj is sys.stdin: - raise pickle.PicklingError("Cannot pickle standard input") - if obj.closed: - raise pickle.PicklingError("Cannot pickle closed files") - if hasattr(obj, 'isatty') and obj.isatty(): - raise pickle.PicklingError("Cannot pickle files that map to tty objects") - if 'r' not in obj.mode and '+' not in obj.mode: - raise pickle.PicklingError("Cannot pickle files that are not opened for reading: %s" % obj.mode) - - name = obj.name - - # TODO: also support binary mode files with io.BytesIO - retval = io.StringIO() - - try: - # Read the whole file - curloc = obj.tell() - obj.seek(0) - contents = obj.read() - obj.seek(curloc) - except IOError: - raise pickle.PicklingError("Cannot pickle file %s as it cannot be read" % name) - retval.write(contents) - retval.seek(curloc) - - retval.name = name - self.save(retval) - self.memoize(obj) - - def save_ellipsis(self, obj): - self.save_reduce(_gen_ellipsis, ()) - - def save_not_implemented(self, obj): - self.save_reduce(_gen_not_implemented, ()) - - dispatch[io.TextIOWrapper] = save_file - dispatch[type(Ellipsis)] = save_ellipsis - dispatch[type(NotImplemented)] = save_not_implemented - - def save_weakset(self, obj): - self.save_reduce(weakref.WeakSet, (list(obj),)) - - dispatch[weakref.WeakSet] = save_weakset - - def save_logger(self, obj): - self.save_reduce(logging.getLogger, (obj.name,), obj=obj) - - dispatch[logging.Logger] = save_logger - - def save_root_logger(self, obj): - self.save_reduce(logging.getLogger, (), obj=obj) - - dispatch[logging.RootLogger] = save_root_logger - - if hasattr(types, "MappingProxyType"): # pragma: no branch - def save_mappingproxy(self, obj): - self.save_reduce(types.MappingProxyType, (dict(obj),), obj=obj) - - dispatch[types.MappingProxyType] = save_mappingproxy - - """Special functions for Add-on libraries""" - def inject_addons(self): - """Plug in system. Register additional pickling functions if modules already loaded""" - pass - - if sys.version_info < (3, 7): # pragma: no branch - def _save_parametrized_type_hint(self, obj): - # The distorted type check sematic for typing construct becomes: - # ``type(obj) is type(TypeHint)``, which means "obj is a - # parametrized TypeHint" - if type(obj) is type(Literal): # pragma: no branch - initargs = (Literal, obj.__values__) - elif type(obj) is type(Final): # pragma: no branch - initargs = (Final, obj.__type__) - elif type(obj) is type(ClassVar): - initargs = (ClassVar, obj.__type__) - elif type(obj) is type(Generic): - parameters = obj.__parameters__ - if len(obj.__parameters__) > 0: - # in early Python 3.5, __parameters__ was sometimes - # preferred to __args__ - initargs = (obj.__origin__, parameters) - else: - initargs = (obj.__origin__, obj.__args__) - elif type(obj) is type(Union): - if sys.version_info < (3, 5, 3): # pragma: no cover - initargs = (Union, obj.__union_params__) - else: - initargs = (Union, obj.__args__) - elif type(obj) is type(Tuple): - if sys.version_info < (3, 5, 3): # pragma: no cover - initargs = (Tuple, obj.__tuple_params__) - else: - initargs = (Tuple, obj.__args__) - elif type(obj) is type(Callable): - if sys.version_info < (3, 5, 3): # pragma: no cover - args = obj.__args__ - result = obj.__result__ - if args != Ellipsis: - if isinstance(args, tuple): - args = list(args) - else: - args = [args] - else: - (*args, result) = obj.__args__ - if len(args) == 1 and args[0] is Ellipsis: - args = Ellipsis - else: - args = list(args) - initargs = (Callable, (args, result)) - else: # pragma: no cover - raise pickle.PicklingError( - "Cloudpickle Error: Unknown type {}".format(type(obj)) - ) - self.save_reduce(_create_parametrized_type_hint, initargs, obj=obj) - + return initargs # Tornado support diff --git a/cloudpickle/cloudpickle_fast.py b/cloudpickle/cloudpickle_fast.py index 541a0a061..39bbc6ee9 100644 --- a/cloudpickle/cloudpickle_fast.py +++ b/cloudpickle/cloudpickle_fast.py @@ -18,18 +18,26 @@ import _pickle import pickle import sys +import struct import types import weakref import typing -from _pickle import Pickler +if sys.version_info >= (3, 8): + from _pickle import Pickler +else: + from pickle import _Pickler as Pickler + from .cloudpickle import ( _is_dynamic, _extract_code_globals, _BUILTIN_TYPE_NAMES, DEFAULT_PROTOCOL, - _find_imported_submodules, _get_cell_contents, _is_importable_by_name, _builtin_type, - Enum, _get_or_create_tracker_id, _make_skeleton_class, _make_skeleton_enum, - _extract_class_dict, dynamic_subimport, subimport, _typevar_reduce, _get_bases, - FunctionSaverMixin, _class_reduce + _find_imported_submodules, _get_cell_contents, _is_importable_by_name, + _builtin_type, Enum, _get_or_create_tracker_id, _make_skeleton_class, + _make_skeleton_enum, _extract_class_dict, dynamic_subimport, subimport, + _typevar_reduce, _get_bases, _make_cell, _make_empty_cell, CellType, + _is_parametrized_type_hint, PYPY, cell_set, + parametrized_type_hint_getinitargs, _create_parametrized_type_hint, + builtin_code_type ) load, loads = _pickle.load, _pickle.loads @@ -46,7 +54,11 @@ def dump(obj, file, protocol=None, buffer_callback=None): Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure compatibility with older versions of Python. """ - CloudPickler(file, protocol=protocol, buffer_callback=buffer_callback).dump(obj) + if sys.version_info >= (3, 8): + CloudPickler( + file, protocol=protocol, buffer_callback=buffer_callback).dump(obj) + else: + CloudPickler(file, protocol=protocol).dump(obj) def dumps(obj, protocol=None, buffer_callback=None): @@ -60,7 +72,12 @@ def dumps(obj, protocol=None, buffer_callback=None): compatibility with older versions of Python. """ with io.BytesIO() as file: - cp = CloudPickler(file, protocol=protocol, buffer_callback=buffer_callback) + if sys.version_info >= (3, 8): + cp = CloudPickler( + file, protocol=protocol, buffer_callback=buffer_callback + ) + else: + cp = CloudPickler(file, protocol=protocol) cp.dump(obj) return file.getvalue() @@ -68,6 +85,25 @@ def dumps(obj, protocol=None, buffer_callback=None): # COLLECTION OF OBJECTS __getnewargs__-LIKE METHODS # ------------------------------------------------- +def _class_getnewargs(obj): + type_kwargs = {} + if "__slots__" in obj.__dict__: + type_kwargs["__slots__"] = obj.__slots__ + + __dict__ = obj.__dict__.get('__dict__', None) + if isinstance(__dict__, property): + type_kwargs['__dict__'] = __dict__ + + return (type(obj), obj.__name__, _get_bases(obj), type_kwargs, + _get_or_create_tracker_id(obj), None) + + +def _enum_getnewargs(obj): + members = dict((e.name, e.value) for e in obj) + return (obj.__bases__, obj.__name__, obj.__qualname__, members, + obj.__module__, _get_or_create_tracker_id(obj), None) + + # COLLECTION OF OBJECTS RECONSTRUCTORS # ------------------------------------ def _file_reconstructor(retval): @@ -76,6 +112,101 @@ def _file_reconstructor(retval): # COLLECTION OF OBJECTS STATE GETTERS # ----------------------------------- +def _function_getstate(func): + # - Put func's dynamic attributes (stored in func.__dict__) in state. These + # attributes will be restored at unpickling time using + # f.__dict__.update(state) + # - Put func's members into slotstate. Such attributes will be restored at + # unpickling time by iterating over slotstate and calling setattr(func, + # slotname, slotvalue) + slotstate = { + "__name__": func.__name__, + "__qualname__": func.__qualname__, + "__annotations__": func.__annotations__, + "__kwdefaults__": func.__kwdefaults__, + "__defaults__": func.__defaults__, + "__module__": func.__module__, + "__doc__": func.__doc__, + "__closure__": func.__closure__, + } + + f_globals_ref = _extract_code_globals(func.__code__) + f_globals = {k: func.__globals__[k] for k in f_globals_ref if k in + func.__globals__} + + closure_values = ( + list(map(_get_cell_contents, func.__closure__)) + if func.__closure__ is not None else () + ) + + # Extract currently-imported submodules used by func. Storing these modules + # in a smoke _cloudpickle_subimports attribute of the object's state will + # trigger the side effect of importing these modules at unpickling time + # (which is necessary for func to work correctly once depickled) + slotstate["_cloudpickle_submodules"] = _find_imported_submodules( + func.__code__, itertools.chain(f_globals.values(), closure_values)) + slotstate["__globals__"] = f_globals + + state = func.__dict__ + return state, slotstate + + +def _class_getstate(obj): + clsdict = _extract_class_dict(obj) + clsdict.pop('__weakref__', None) + + if issubclass(type(obj), abc.ABCMeta): + # If obj is an instance of an ABCMeta subclass, dont pickle the + # cache/negative caches populated during isinstance/issubclass + # checks, but pickle the list of registered subclasses of obj. + clsdict.pop('_abc_cache', None) + clsdict.pop('_abc_negative_cache', None) + clsdict.pop('_abc_negative_cache_version', None) + registry = clsdict.pop('_abc_registry', None) + if registry is None: + # in Python3.7+, the abc caches and registered subclasses of a + # class are bundled into the single _abc_impl attribute + clsdict.pop('_abc_impl', None) + (registry, _, _, _) = abc._get_dump(obj) + + clsdict["_abc_impl"] = [subclass_weakref() + for subclass_weakref in registry] + else: + # In the above if clause, registry is a set of weakrefs -- in + # this case, registry is a WeakSet + clsdict["_abc_impl"] = [type_ for type_ in registry] + + if "__slots__" in clsdict: + # pickle string length optimization: member descriptors of obj are + # created automatically from obj's __slots__ attribute, no need to + # save them in obj's state + if isinstance(obj.__slots__, str): + clsdict.pop(obj.__slots__) + else: + for k in obj.__slots__: + clsdict.pop(k, None) + + clsdict.pop('__dict__', None) # unpicklable property object + + return (clsdict, {}) + + +def _enum_getstate(obj): + clsdict, slotstate = _class_getstate(obj) + + members = dict((e.name, e.value) for e in obj) + # Cleanup the clsdict that will be passed to _rehydrate_skeleton_class: + # Those attributes are already handled by the metaclass. + for attrname in ["_generate_next_value_", "_member_names_", + "_member_map_", "_member_type_", + "_value2member_map_"]: + clsdict.pop(attrname, None) + for member in members: + clsdict.pop(member) + # Special handling of Enum subclasses + return clsdict, slotstate + + # COLLECTIONS OF OBJECTS REDUCERS # ------------------------------- # A reducer is a function taking a single argument (obj), and that returns a @@ -89,14 +220,23 @@ def _file_reconstructor(retval): def _code_reduce(obj): """codeobject reducer""" - args = ( - obj.co_argcount, obj.co_posonlyargcount, - obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, - obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, - obj.co_varnames, obj.co_filename, obj.co_name, - obj.co_firstlineno, obj.co_lnotab, obj.co_freevars, - obj.co_cellvars - ) + if hasattr(obj, "co_posonlyargcount"): # pragma: no branch + args = ( + obj.co_argcount, obj.co_posonlyargcount, + obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, + obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, + obj.co_varnames, obj.co_filename, obj.co_name, + obj.co_firstlineno, obj.co_lnotab, obj.co_freevars, + obj.co_cellvars + ) + else: + args = ( + obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals, + obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts, + obj.co_names, obj.co_varnames, obj.co_filename, + obj.co_name, obj.co_firstlineno, obj.co_lnotab, + obj.co_freevars, obj.co_cellvars + ) return types.CodeType, args @@ -105,9 +245,9 @@ def _cell_reduce(obj): try: obj.cell_contents except ValueError: # cell is empty - return types.CellType, () + return _make_empty_cell, () else: - return types.CellType, (obj.cell_contents,) + return _make_cell, (obj.cell_contents, ) def _classmethod_reduce(obj): @@ -202,12 +342,157 @@ def _weakset_reduce(obj): return weakref.WeakSet, (list(obj),) +def _dynamic_class_reduce(obj): + """ + Save a class that can't be stored as module global. + + This method is used to serialize classes that are defined inside + functions, or that otherwise can't be serialized as attribute lookups + from global modules. + """ + if Enum is not None and issubclass(obj, Enum): + return ( + _make_skeleton_enum, _enum_getnewargs(obj), _enum_getstate(obj), + None, None, _class_setstate + ) + else: + return ( + _make_skeleton_class, _class_getnewargs(obj), _class_getstate(obj), + None, None, _class_setstate + ) + + +def _class_reduce(obj): + """Select the reducer depending on the dynamic nature of the class obj""" + if obj is type(None): # noqa + return type, (None,) + elif obj is type(Ellipsis): + return type, (Ellipsis,) + elif obj is type(NotImplemented): + return type, (NotImplemented,) + elif obj in _BUILTIN_TYPE_NAMES: + return _builtin_type, (_BUILTIN_TYPE_NAMES[obj],) + elif not _is_importable_by_name(obj): + return _dynamic_class_reduce(obj) + return NotImplemented + + # COLLECTIONS OF OBJECTS STATE SETTERS # ------------------------------------ # state setters are called at unpickling time, once the object is created and # it has to be updated to how it was at unpickling time. +def _function_setstate(obj, state): + """Update the state of a dynaamic function. + + As __closure__ and __globals__ are readonly attributes of a function, we + cannot rely on the native setstate routine of pickle.load_build, that calls + setattr on items of the slotstate. Instead, we have to modify them inplace. + """ + state, slotstate = state + obj.__dict__.update(state) + + obj_globals = slotstate.pop("__globals__") + obj_closure = slotstate.pop("__closure__") + # _cloudpickle_subimports is a set of submodules that must be loaded for + # the pickled function to work correctly at unpickling time. Now that these + # submodules are depickled (hence imported), they can be removed from the + # object's state (the object state only served as a reference holder to + # these submodules) + slotstate.pop("_cloudpickle_submodules") + + obj.__globals__.update(obj_globals) + obj.__globals__["__builtins__"] = __builtins__ + + if obj_closure is not None: + for i, cell in enumerate(obj_closure): + try: + value = cell.cell_contents + except ValueError: # cell is empty + continue + cell_set(obj.__closure__[i], value) + + for k, v in slotstate.items(): + setattr(obj, k, v) + + +def _class_setstate(obj, state): + state, slotstate = state + registry = None + for attrname, attr in state.items(): + if attrname == "_abc_impl": + registry = attr + else: + setattr(obj, attrname, attr) + if registry is not None: + for subclass in registry: + obj.register(subclass) + + return obj + + +class FunctionSaverMixin: + # function reducers are defined as instance methods of CloudPickler + # objects, as they rely on a CloudPickler attribute (globals_ref) + def _dynamic_function_reduce(self, func): + """Reduce a function that is not pickleable via attribute lookup.""" + newargs = self._function_getnewargs(func) + state = _function_getstate(func) + return (types.FunctionType, newargs, state, None, None, + _function_setstate) + + def _function_reduce(self, obj): + """Reducer for function objects. + + If obj is a top-level attribute of a file-backed module, this + reducer returns NotImplemented, making the CloudPickler fallback to + traditional _pickle.Pickler routines to save obj. Otherwise, it reduces + obj using a custom cloudpickle reducer designed specifically to handle + dynamic functions. + + As opposed to cloudpickle.py, There no special handling for builtin + pypy functions because cloudpickle_fast is CPython-specific. + """ + if _is_importable_by_name(obj): + return NotImplemented + else: + return self._dynamic_function_reduce(obj) + + def _function_getnewargs(self, func): + code = func.__code__ + + # base_globals represents the future global namespace of func at + # unpickling time. Looking it up and storing it in + # CloudpiPickler.globals_ref allow functions sharing the same globals + # at pickling time to also share them once unpickled, at one condition: + # since globals_ref is an attribute of a CloudPickler instance, and + # that a new CloudPickler is created each time pickle.dump or + # pickle.dumps is called, functions also need to be saved within the + # same invocation of cloudpickle.dump/cloudpickle.dumps (for example: + # cloudpickle.dumps([f1, f2])). There is no such limitation when using + # CloudPickler.dump, as long as the multiple invocations are bound to + # the same CloudPickler. + base_globals = self.globals_ref.setdefault(id(func.__globals__), {}) + + if base_globals == {}: + # Add module attributes used to resolve relative imports + # instructions inside func. + for k in ["__package__", "__name__", "__path__", "__file__"]: + if k in func.__globals__: + base_globals[k] = func.__globals__[k] + + # Do not bind the free variables before the function is created to + # avoid infinite recursion. + if func.__closure__ is None: + closure = None + else: + closure = tuple( + _make_empty_cell() for _ in range(len(code.co_freevars))) + + return code, base_globals, None, None, closure + + class CloudPickler(FunctionSaverMixin, Pickler): """Fast C Pickler extension with additional reducing routines. @@ -223,27 +508,32 @@ class CloudPickler(FunctionSaverMixin, Pickler): # cloudpickle's own dispatch_table, containing the additional set of # objects (compared to the standard library pickle) that cloupickle can # serialize. - dispatch = {} - dispatch[classmethod] = _classmethod_reduce - dispatch[io.TextIOWrapper] = _file_reduce - dispatch[logging.Logger] = _logger_reduce - dispatch[logging.RootLogger] = _root_logger_reduce - dispatch[memoryview] = _memoryview_reduce - dispatch[property] = _property_reduce - dispatch[staticmethod] = _classmethod_reduce - dispatch[types.CellType] = _cell_reduce - dispatch[types.CodeType] = _code_reduce - dispatch[types.GetSetDescriptorType] = _getset_descriptor_reduce - dispatch[types.ModuleType] = _module_reduce - dispatch[types.MethodType] = _method_reduce - dispatch[types.MappingProxyType] = _mappingproxy_reduce - dispatch[weakref.WeakSet] = _weakset_reduce - dispatch[typing.TypeVar] = _typevar_reduce + _dispatch = {} + _dispatch[classmethod] = _classmethod_reduce + _dispatch[io.TextIOWrapper] = _file_reduce + _dispatch[logging.Logger] = _logger_reduce + _dispatch[logging.RootLogger] = _root_logger_reduce + _dispatch[memoryview] = _memoryview_reduce + _dispatch[property] = _property_reduce + _dispatch[staticmethod] = _classmethod_reduce + _dispatch[CellType] = _cell_reduce + _dispatch[types.CodeType] = _code_reduce + _dispatch[types.GetSetDescriptorType] = _getset_descriptor_reduce + _dispatch[types.ModuleType] = _module_reduce + _dispatch[types.MethodType] = _method_reduce + _dispatch[types.MappingProxyType] = _mappingproxy_reduce + _dispatch[weakref.WeakSet] = _weakset_reduce + _dispatch[typing.TypeVar] = _typevar_reduce def __init__(self, file, protocol=None, buffer_callback=None): if protocol is None: protocol = DEFAULT_PROTOCOL - Pickler.__init__(self, file, protocol=protocol, buffer_callback=buffer_callback) + if sys.version_info >= (3, 8): + Pickler.__init__( + self, file, protocol=protocol, buffer_callback=buffer_callback + ) + else: + Pickler.__init__(self, file, protocol=protocol) # map functions __globals__ attribute ids, to ensure that functions # sharing the same global namespace at pickling time also share their # global namespace at unpickling time. @@ -252,7 +542,7 @@ def __init__(self, file, protocol=None, buffer_callback=None): # Take into account potential custom reducers registered by external # modules self.dispatch_table = copyreg.dispatch_table.copy() - self.dispatch_table.update(self.dispatch) + self.dispatch_table.update(self._dispatch) self.proto = int(protocol) def reducer_override(self, obj): @@ -311,3 +601,80 @@ def dump(self, obj): raise pickle.PicklingError(msg) else: raise + if sys.version_info < (3, 8): + dispatch = Pickler.dispatch.copy() + + def save_function(self, obj, name=None): + """ Registered with the dispatch to handle all function types. + + Determines what kind of function obj is (e.g. lambda, defined at + interactive prompt, etc) and handles the pickling appropriately. + """ + if _is_importable_by_name(obj, name=name): + return Pickler.save_global(self, obj, name=name) + elif PYPY and isinstance(obj.__code__, builtin_code_type): + return self.save_pypy_builtin_func(obj) + else: + return self._save_reduce_pickle5( + *self._dynamic_function_reduce(obj), obj=obj + ) + + def _save_reduce_pickle5(self, func, args, state=None, listitems=None, + dictitems=None, state_setter=None, obj=None): + save = self.save + write = self.write + self.save_reduce( + func, args, state=None, listitems=listitems, + dictitems=dictitems, obj=obj + ) + # backport of the Python 3.8 state_setter pickle operations + save(state_setter) + save(obj) # simple BINGET opcode as obj is already memoized. + save(state) + write(pickle.TUPLE2) + # Trigger a state_setter(obj, state) function call. + write(pickle.REDUCE) + # The purpose of state_setter is to carry-out an + # inplace modification of obj. We do not care about what the + # method might return, so its output is eventually removed from + # the stack. + write(pickle.POP) + + dispatch[types.FunctionType] = save_function + + def save_global(self, obj, name=None, pack=struct.pack): + """ + Save a "global". + + The name of this method is somewhat misleading: all types get + dispatched here. + """ + if obj is type(None): # noqa + return self.save_reduce(type, (None,), obj=obj) + elif obj is type(Ellipsis): + return self.save_reduce(type, (Ellipsis,), obj=obj) + elif obj is type(NotImplemented): + return self.save_reduce(type, (NotImplemented,), obj=obj) + elif obj in _BUILTIN_TYPE_NAMES: + return self.save_reduce( + _builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj) + + if sys.version_info[:2] < (3, 7) and _is_parametrized_type_hint(obj): # noqa # pragma: no branch + # Parametrized typing constructs in Python < 3.7 are not + # compatible with type checks and ``isinstance`` semantics. For + # this reason, it is easier to detect them using a + # duck-typing-based check (``_is_parametrized_type_hint``) than + # to populate the Pickler's dispatch with type-specific savers. + self.save_reduce( + _create_parametrized_type_hint, + parametrized_type_hint_getinitargs(obj), + obj=obj + ) + elif name is not None: + Pickler.save_global(self, obj, name=name) + elif not _is_importable_by_name(obj, name=name): + self._save_reduce_pickle5(*_dynamic_class_reduce(obj), obj=obj) + else: + Pickler.save_global(self, obj, name=name) + + dispatch[type] = save_global diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index 890d4165d..76b170bd8 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -1228,12 +1228,8 @@ def f(): # some setup is required to allow pytest apimodules to be correctly # serializable. from cloudpickle import CloudPickler - if sys.version_info[:2] >= (3, 8): - from cloudpickle import cloudpickle_fast as cp_fast - CloudPickler.dispatch[ - type(py.builtin)] = cp_fast._module_reduce - else: - CloudPickler.dispatch[type(py.builtin)] = CloudPickler.save_module + from cloudpickle import cloudpickle_fast as cp_fast + CloudPickler._dispatch[type(py.builtin)] = cp_fast._module_reduce g = cloudpickle.loads(cloudpickle.dumps(f, protocol=self.protocol)) From 62646414c7de6d1ac31e45a4f1b1168d0eb334af Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Thu, 14 May 2020 11:43:04 +0200 Subject: [PATCH 04/89] increase flake8 max complexity --- .github/scripts/flake8_diff.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/flake8_diff.sh b/.github/scripts/flake8_diff.sh index 935d02392..0d4a1b748 100644 --- a/.github/scripts/flake8_diff.sh +++ b/.github/scripts/flake8_diff.sh @@ -83,7 +83,7 @@ check_files() { # that was not changed does not create failures # The github terminal is 127 characters wide git diff --unified=0 $COMMIT_RANGE -- $files | flake8 --diff --show-source \ - --max-complexity=10 --max-line-length=127 $options + --max-complexity=20 --max-line-length=127 $options fi } From dc9ba8404926278565f6425d0e1c050bde85d11c Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Thu, 14 May 2020 11:46:13 +0200 Subject: [PATCH 05/89] unused functions and imports --- cloudpickle/cloudpickle.py | 43 --------------------------------- cloudpickle/cloudpickle_fast.py | 4 ++- 2 files changed, 3 insertions(+), 44 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index a0da593a6..b6e971c76 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -42,29 +42,20 @@ """ from __future__ import print_function -import abc import builtins import dis -import io -import itertools -import logging import opcode -import operator import pickle import platform -import struct import sys import types import weakref import uuid import threading import typing -from enum import Enum from typing import Generic, Union, Tuple, Callable -from pickle import _Pickler as Pickler from pickle import _getattribute -from io import BytesIO from importlib._bootstrap import _find_spec try: # pragma: no branch @@ -545,40 +536,6 @@ def _rebuild_tornado_coroutine(func): return gen.coroutine(func) -# Shorthands for legacy support - -def dump(obj, file, protocol=None): - """Serialize obj as bytes streamed into file - - protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to - pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed - between processes running the same Python version. - - Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure - compatibility with older versions of Python. - """ - CloudPickler(file, protocol=protocol).dump(obj) - - -def dumps(obj, protocol=None): - """Serialize obj as a string of bytes allocated in memory - - protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to - pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed - between processes running the same Python version. - - Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure - compatibility with older versions of Python. - """ - file = BytesIO() - try: - cp = CloudPickler(file, protocol=protocol) - cp.dump(obj) - return file.getvalue() - finally: - file.close() - - # including pickles unloading functions in this namespace load = pickle.load loads = pickle.loads diff --git a/cloudpickle/cloudpickle_fast.py b/cloudpickle/cloudpickle_fast.py index 39bbc6ee9..be9ea6cd5 100644 --- a/cloudpickle/cloudpickle_fast.py +++ b/cloudpickle/cloudpickle_fast.py @@ -23,6 +23,8 @@ import weakref import typing +from enum import Enum + if sys.version_info >= (3, 8): from _pickle import Pickler else: @@ -32,7 +34,7 @@ from .cloudpickle import ( _is_dynamic, _extract_code_globals, _BUILTIN_TYPE_NAMES, DEFAULT_PROTOCOL, _find_imported_submodules, _get_cell_contents, _is_importable_by_name, - _builtin_type, Enum, _get_or_create_tracker_id, _make_skeleton_class, + _builtin_type, _get_or_create_tracker_id, _make_skeleton_class, _make_skeleton_enum, _extract_class_dict, dynamic_subimport, subimport, _typevar_reduce, _get_bases, _make_cell, _make_empty_cell, CellType, _is_parametrized_type_hint, PYPY, cell_set, From d5ed4d925315c86f03611798ded6456469cce228 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Thu, 14 May 2020 12:02:56 +0200 Subject: [PATCH 06/89] pypy compat --- cloudpickle/cloudpickle_fast.py | 36 +++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/cloudpickle/cloudpickle_fast.py b/cloudpickle/cloudpickle_fast.py index be9ea6cd5..d81a55512 100644 --- a/cloudpickle/cloudpickle_fast.py +++ b/cloudpickle/cloudpickle_fast.py @@ -15,7 +15,6 @@ import io import itertools import logging -import _pickle import pickle import sys import struct @@ -25,12 +24,6 @@ from enum import Enum -if sys.version_info >= (3, 8): - from _pickle import Pickler -else: - from pickle import _Pickler as Pickler - - from .cloudpickle import ( _is_dynamic, _extract_code_globals, _BUILTIN_TYPE_NAMES, DEFAULT_PROTOCOL, _find_imported_submodules, _get_cell_contents, _is_importable_by_name, @@ -42,7 +35,12 @@ builtin_code_type ) -load, loads = _pickle.load, _pickle.loads +if sys.version_info < (3, 8) or PYPY: + from pickle import _Pickler as Pickler +else: + from _pickle import Pickler + +load, loads = pickle.load, pickle.loads # Shorthands similar to pickle.dump/pickle.dumps @@ -606,6 +604,28 @@ def dump(self, obj): if sys.version_info < (3, 8): dispatch = Pickler.dispatch.copy() + def save_pypy_builtin_func(self, obj): + """Save pypy equivalent of builtin functions. + PyPy does not have the concept of builtin-functions. Instead, + builtin-functions are simple function instances, but with a + builtin-code attribute. + Most of the time, builtin functions should be pickled by attribute. + But PyPy has flaky support for __qualname__, so some builtin + functions such as float.__new__ will be classified as dynamic. For + this reason only, we created this special routine. Because + builtin-functions are not expected to have closure or globals, + there is no additional hack (compared the one already implemented + in pickle) to protect ourselves from reference cycles. A simple + (reconstructor, newargs, obj.__dict__) tuple is save_reduced. Note + also that PyPy improved their support for __qualname__ in v3.6, so + this routing should be removed when cloudpickle supports only PyPy + 3.6 and later. + """ + rv = (types.FunctionType, (obj.__code__, {}, obj.__name__, + obj.__defaults__, obj.__closure__), + obj.__dict__) + self.save_reduce(*rv, obj=obj) + def save_function(self, obj, name=None): """ Registered with the dispatch to handle all function types. From c44ad88fa1a1383f7cdbbfd51f99db749f150fa9 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Thu, 14 May 2020 12:15:38 +0200 Subject: [PATCH 07/89] be pickle5 friendly --- cloudpickle/cloudpickle_fast.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/cloudpickle/cloudpickle_fast.py b/cloudpickle/cloudpickle_fast.py index d81a55512..f02f3b535 100644 --- a/cloudpickle/cloudpickle_fast.py +++ b/cloudpickle/cloudpickle_fast.py @@ -54,7 +54,7 @@ def dump(obj, file, protocol=None, buffer_callback=None): Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure compatibility with older versions of Python. """ - if sys.version_info >= (3, 8): + if pickle.HIGHEST_PROTOCOL >= 5: CloudPickler( file, protocol=protocol, buffer_callback=buffer_callback).dump(obj) else: @@ -72,7 +72,7 @@ def dumps(obj, protocol=None, buffer_callback=None): compatibility with older versions of Python. """ with io.BytesIO() as file: - if sys.version_info >= (3, 8): + if pickle.HIGHEST_PROTOCOL >= 5: cp = CloudPickler( file, protocol=protocol, buffer_callback=buffer_callback ) @@ -528,7 +528,7 @@ class CloudPickler(FunctionSaverMixin, Pickler): def __init__(self, file, protocol=None, buffer_callback=None): if protocol is None: protocol = DEFAULT_PROTOCOL - if sys.version_info >= (3, 8): + if pickle.HIGHEST_PROTOCOL >= 5: Pickler.__init__( self, file, protocol=protocol, buffer_callback=buffer_callback ) @@ -601,7 +601,12 @@ def dump(self, obj): raise pickle.PicklingError(msg) else: raise - if sys.version_info < (3, 8): + + if pickle.HIGHEST_PROTOCOL < 5: + # backport of Python 3.8+ new save_reduce function. although the + # Python 3.8's save_reduce is not explicitly part of pickle 5 its + # availablity coincidates with the availablity of pickle protocol 5 in + # both the stdlib pickle and pickle5. dispatch = Pickler.dispatch.copy() def save_pypy_builtin_func(self, obj): From fd67f4b96066d14c319ab68077280181ab3be250 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Thu, 14 May 2020 13:02:11 +0200 Subject: [PATCH 08/89] duplicate top-level functions --- cloudpickle/cloudpickle_fast.py | 373 +++++++++++++++++--------------- tests/cloudpickle_test.py | 2 +- 2 files changed, 200 insertions(+), 175 deletions(-) diff --git a/cloudpickle/cloudpickle_fast.py b/cloudpickle/cloudpickle_fast.py index f02f3b535..c04b35325 100644 --- a/cloudpickle/cloudpickle_fast.py +++ b/cloudpickle/cloudpickle_fast.py @@ -35,51 +35,74 @@ builtin_code_type ) -if sys.version_info < (3, 8) or PYPY: - from pickle import _Pickler as Pickler -else: +if sys.version_info >= (3, 8) and not PYPY: from _pickle import Pickler + # Shorthands similar to pickle.dump/pickle.dumps -load, loads = pickle.load, pickle.loads - - -# Shorthands similar to pickle.dump/pickle.dumps -def dump(obj, file, protocol=None, buffer_callback=None): - """Serialize obj as bytes streamed into file + def dump(obj, file, protocol=None, buffer_callback=None): + """Serialize obj as bytes streamed into file - protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to - pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed - between processes running the same Python version. + protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to + pickle.HIGHEST_PROTOCOL. This setting favors maximum communication + speed between processes running the same Python version. - Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure - compatibility with older versions of Python. - """ - if pickle.HIGHEST_PROTOCOL >= 5: + Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure + compatibility with older versions of Python. + """ CloudPickler( - file, protocol=protocol, buffer_callback=buffer_callback).dump(obj) - else: - CloudPickler(file, protocol=protocol).dump(obj) - + file, protocol=protocol, buffer_callback=buffer_callback + ).dump(obj) -def dumps(obj, protocol=None, buffer_callback=None): - """Serialize obj as a string of bytes allocated in memory + def dumps(obj, protocol=None, buffer_callback=None): + """Serialize obj as a string of bytes allocated in memory - protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to - pickle.HIGHEST_PROTOCOL. This setting favors maximum communication speed - between processes running the same Python version. + protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to + pickle.HIGHEST_PROTOCOL. This setting favors maximum communication + speed between processes running the same Python version. - Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure - compatibility with older versions of Python. - """ - with io.BytesIO() as file: - if pickle.HIGHEST_PROTOCOL >= 5: + Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure + compatibility with older versions of Python. + """ + with io.BytesIO() as file: cp = CloudPickler( file, protocol=protocol, buffer_callback=buffer_callback ) - else: + cp.dump(obj) + return file.getvalue() + +else: + from pickle import _Pickler as Pickler + + # Shorthands similar to pickle.dump/pickle.dumps + def dump(obj, file, protocol=None): + """Serialize obj as bytes streamed into file + + protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to + pickle.HIGHEST_PROTOCOL. This setting favors maximum communication + speed between processes running the same Python version. + + Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure + compatibility with older versions of Python. + """ + CloudPickler(file, protocol=protocol).dump(obj) + + def dumps(obj, protocol=None): + """Serialize obj as a string of bytes allocated in memory + + protocol defaults to cloudpickle.DEFAULT_PROTOCOL which is an alias to + pickle.HIGHEST_PROTOCOL. This setting favors maximum communication + speed between processes running the same Python version. + + Set protocol=pickle.DEFAULT_PROTOCOL instead if you need to ensure + compatibility with older versions of Python. + """ + with io.BytesIO() as file: cp = CloudPickler(file, protocol=protocol) - cp.dump(obj) - return file.getvalue() + cp.dump(obj) + return file.getvalue() + + +load, loads = pickle.load, pickle.loads # COLLECTION OF OBJECTS __getnewargs__-LIKE METHODS @@ -432,7 +455,28 @@ def _class_setstate(obj, state): return obj -class FunctionSaverMixin: +class CloudPickler(Pickler): + # Take into account potential custom reducers registered by external + # modules + dispatch_table = copyreg.dispatch_table.copy() + # register the additional set of objects (compared to the standard library + # pickle) that cloupickle can serialize. + dispatch_table[classmethod] = _classmethod_reduce + dispatch_table[io.TextIOWrapper] = _file_reduce + dispatch_table[logging.Logger] = _logger_reduce + dispatch_table[logging.RootLogger] = _root_logger_reduce + dispatch_table[memoryview] = _memoryview_reduce + dispatch_table[property] = _property_reduce + dispatch_table[staticmethod] = _classmethod_reduce + dispatch_table[CellType] = _cell_reduce + dispatch_table[types.CodeType] = _code_reduce + dispatch_table[types.GetSetDescriptorType] = _getset_descriptor_reduce + dispatch_table[types.ModuleType] = _module_reduce + dispatch_table[types.MethodType] = _method_reduce + dispatch_table[types.MappingProxyType] = _mappingproxy_reduce + dispatch_table[weakref.WeakSet] = _weakset_reduce + dispatch_table[typing.TypeVar] = _typevar_reduce + # function reducers are defined as instance methods of CloudPickler # objects, as they rely on a CloudPickler attribute (globals_ref) def _dynamic_function_reduce(self, func): @@ -492,103 +536,6 @@ def _function_getnewargs(self, func): return code, base_globals, None, None, closure - -class CloudPickler(FunctionSaverMixin, Pickler): - """Fast C Pickler extension with additional reducing routines. - - CloudPickler's extensions exist into into: - - * its dispatch_table containing reducers that are called only if ALL - built-in saving functions were previously discarded. - * a special callback named "reducer_override", invoked before standard - function/class builtin-saving method (save_global), to serialize dynamic - functions - """ - - # cloudpickle's own dispatch_table, containing the additional set of - # objects (compared to the standard library pickle) that cloupickle can - # serialize. - _dispatch = {} - _dispatch[classmethod] = _classmethod_reduce - _dispatch[io.TextIOWrapper] = _file_reduce - _dispatch[logging.Logger] = _logger_reduce - _dispatch[logging.RootLogger] = _root_logger_reduce - _dispatch[memoryview] = _memoryview_reduce - _dispatch[property] = _property_reduce - _dispatch[staticmethod] = _classmethod_reduce - _dispatch[CellType] = _cell_reduce - _dispatch[types.CodeType] = _code_reduce - _dispatch[types.GetSetDescriptorType] = _getset_descriptor_reduce - _dispatch[types.ModuleType] = _module_reduce - _dispatch[types.MethodType] = _method_reduce - _dispatch[types.MappingProxyType] = _mappingproxy_reduce - _dispatch[weakref.WeakSet] = _weakset_reduce - _dispatch[typing.TypeVar] = _typevar_reduce - - def __init__(self, file, protocol=None, buffer_callback=None): - if protocol is None: - protocol = DEFAULT_PROTOCOL - if pickle.HIGHEST_PROTOCOL >= 5: - Pickler.__init__( - self, file, protocol=protocol, buffer_callback=buffer_callback - ) - else: - Pickler.__init__(self, file, protocol=protocol) - # map functions __globals__ attribute ids, to ensure that functions - # sharing the same global namespace at pickling time also share their - # global namespace at unpickling time. - self.globals_ref = {} - - # Take into account potential custom reducers registered by external - # modules - self.dispatch_table = copyreg.dispatch_table.copy() - self.dispatch_table.update(self._dispatch) - self.proto = int(protocol) - - def reducer_override(self, obj): - """Type-agnostic reducing callback for function and classes. - - For performance reasons, subclasses of the C _pickle.Pickler class - cannot register custom reducers for functions and classes in the - dispatch_table. Reducer for such types must instead implemented in the - special reducer_override method. - - Note that method will be called for any object except a few - builtin-types (int, lists, dicts etc.), which differs from reducers in - the Pickler's dispatch_table, each of them being invoked for objects of - a specific type only. - - This property comes in handy for classes: although most classes are - instances of the ``type`` metaclass, some of them can be instances of - other custom metaclasses (such as enum.EnumMeta for example). In - particular, the metaclass will likely not be known in advance, and thus - cannot be special-cased using an entry in the dispatch_table. - reducer_override, among other things, allows us to register a reducer - that will be called for any class, independently of its type. - - - Notes: - - * reducer_override has the priority over dispatch_table-registered - reducers. - * reducer_override can be used to fix other limitations of cloudpickle - for other types that suffered from type-specific reducers, such as - Exceptions. See https://github.com/cloudpipe/cloudpickle/issues/248 - """ - t = type(obj) - try: - is_anyclass = issubclass(t, type) - except TypeError: # t is not a class (old Boost; see SF #502085) - is_anyclass = False - - if is_anyclass: - return _class_reduce(obj) - elif isinstance(obj, types.FunctionType): - return self._function_reduce(obj) - else: - # fallback to save_global, including the Pickler's distpatch_table - return NotImplemented - def dump(self, obj): try: return Pickler.dump(self, obj) @@ -602,49 +549,91 @@ def dump(self, obj): else: raise - if pickle.HIGHEST_PROTOCOL < 5: - # backport of Python 3.8+ new save_reduce function. although the - # Python 3.8's save_reduce is not explicitly part of pickle 5 its - # availablity coincidates with the availablity of pickle protocol 5 in - # both the stdlib pickle and pickle5. - dispatch = Pickler.dispatch.copy() - - def save_pypy_builtin_func(self, obj): - """Save pypy equivalent of builtin functions. - PyPy does not have the concept of builtin-functions. Instead, - builtin-functions are simple function instances, but with a - builtin-code attribute. - Most of the time, builtin functions should be pickled by attribute. - But PyPy has flaky support for __qualname__, so some builtin - functions such as float.__new__ will be classified as dynamic. For - this reason only, we created this special routine. Because - builtin-functions are not expected to have closure or globals, - there is no additional hack (compared the one already implemented - in pickle) to protect ourselves from reference cycles. A simple - (reconstructor, newargs, obj.__dict__) tuple is save_reduced. Note - also that PyPy improved their support for __qualname__ in v3.6, so - this routing should be removed when cloudpickle supports only PyPy - 3.6 and later. + if pickle.HIGHEST_PROTOCOL >= 5: + # Implementation of the reducer_override callback, in order to + # efficiently serialize dynamic functions and classes by subclassing + # the C-implemented Pickler. + # TODO: decorrelate reducer_override (which is tied to CPython's + # implementation - would it make sense to backport it to pypy? - and + # pickle's protocol 5 which is implementation agnostic. Currently, the + # availability of both notions coincide on CPython's pickle and the + # pickle5 backport, but it may not be the case anymore when pypy + # implements protocol 5 + def __init__(self, file, protocol=None, buffer_callback=None): + if protocol is None: + protocol = DEFAULT_PROTOCOL + Pickler.__init__( + self, file, protocol=protocol, buffer_callback=buffer_callback + ) + # map functions __globals__ attribute ids, to ensure that functions + # sharing the same global namespace at pickling time also share + # their global namespace at unpickling time. + self.globals_ref = {} + self.proto = int(protocol) + + def reducer_override(self, obj): + """Type-agnostic reducing callback for function and classes. + + For performance reasons, subclasses of the C _pickle.Pickler class + cannot register custom reducers for functions and classes in the + dispatch_table. Reducer for such types must instead implemented in + the special reducer_override method. + + Note that method will be called for any object except a few + builtin-types (int, lists, dicts etc.), which differs from reducers + in the Pickler's dispatch_table, each of them being invoked for + objects of a specific type only. + + This property comes in handy for classes: although most classes are + instances of the ``type`` metaclass, some of them can be instances + of other custom metaclasses (such as enum.EnumMeta for example). In + particular, the metaclass will likely not be known in advance, and + thus cannot be special-cased using an entry in the dispatch_table. + reducer_override, among other things, allows us to register a + reducer that will be called for any class, independently of its + type. + + + Notes: + + * reducer_override has the priority over dispatch_table-registered + reducers. + * reducer_override can be used to fix other limitations of + cloudpickle for other types that suffered from type-specific + reducers, such as Exceptions. See + https://github.com/cloudpipe/cloudpickle/issues/248 """ - rv = (types.FunctionType, (obj.__code__, {}, obj.__name__, - obj.__defaults__, obj.__closure__), - obj.__dict__) - self.save_reduce(*rv, obj=obj) + t = type(obj) + try: + is_anyclass = issubclass(t, type) + except TypeError: # t is not a class (old Boost; see SF #502085) + is_anyclass = False + + if is_anyclass: + return _class_reduce(obj) + elif isinstance(obj, types.FunctionType): + return self._function_reduce(obj) + else: + # fallback to save_global, including the Pickler's + # distpatch_table + return NotImplemented - def save_function(self, obj, name=None): - """ Registered with the dispatch to handle all function types. + else: + # When reducer_override is not available, hack the pure-Python + # Pickler's types.FunctionType and type savers. Note: the type saver + # must override Pickler.save_global, because pickle.py contains a + # hard-coded call to save_global when pickling meta-classes. + dispatch = Pickler.dispatch.copy() - Determines what kind of function obj is (e.g. lambda, defined at - interactive prompt, etc) and handles the pickling appropriately. - """ - if _is_importable_by_name(obj, name=name): - return Pickler.save_global(self, obj, name=name) - elif PYPY and isinstance(obj.__code__, builtin_code_type): - return self.save_pypy_builtin_func(obj) - else: - return self._save_reduce_pickle5( - *self._dynamic_function_reduce(obj), obj=obj - ) + def __init__(self, file, protocol=None): + if protocol is None: + protocol = DEFAULT_PROTOCOL + Pickler.__init__(self, file, protocol=protocol) + # map functions __globals__ attribute ids, to ensure that functions + # sharing the same global namespace at pickling time also share + # their global namespace at unpickling time. + self.globals_ref = {} + assert hasattr(self, 'proto') def _save_reduce_pickle5(self, func, args, state=None, listitems=None, dictitems=None, state_setter=None, obj=None): @@ -667,8 +656,6 @@ def _save_reduce_pickle5(self, func, args, state=None, listitems=None, # the stack. write(pickle.POP) - dispatch[types.FunctionType] = save_function - def save_global(self, obj, name=None, pack=struct.pack): """ Save a "global". @@ -703,5 +690,43 @@ def save_global(self, obj, name=None, pack=struct.pack): self._save_reduce_pickle5(*_dynamic_class_reduce(obj), obj=obj) else: Pickler.save_global(self, obj, name=name) - dispatch[type] = save_global + + def save_function(self, obj, name=None): + """ Registered with the dispatch to handle all function types. + + Determines what kind of function obj is (e.g. lambda, defined at + interactive prompt, etc) and handles the pickling appropriately. + """ + if _is_importable_by_name(obj, name=name): + return Pickler.save_global(self, obj, name=name) + elif PYPY and isinstance(obj.__code__, builtin_code_type): + return self.save_pypy_builtin_func(obj) + else: + return self._save_reduce_pickle5( + *self._dynamic_function_reduce(obj), obj=obj + ) + + def save_pypy_builtin_func(self, obj): + """Save pypy equivalent of builtin functions. + PyPy does not have the concept of builtin-functions. Instead, + builtin-functions are simple function instances, but with a + builtin-code attribute. + Most of the time, builtin functions should be pickled by attribute. + But PyPy has flaky support for __qualname__, so some builtin + functions such as float.__new__ will be classified as dynamic. For + this reason only, we created this special routine. Because + builtin-functions are not expected to have closure or globals, + there is no additional hack (compared the one already implemented + in pickle) to protect ourselves from reference cycles. A simple + (reconstructor, newargs, obj.__dict__) tuple is save_reduced. Note + also that PyPy improved their support for __qualname__ in v3.6, so + this routing should be removed when cloudpickle supports only PyPy + 3.6 and later. + """ + rv = (types.FunctionType, (obj.__code__, {}, obj.__name__, + obj.__defaults__, obj.__closure__), + obj.__dict__) + self.save_reduce(*rv, obj=obj) + + dispatch[types.FunctionType] = save_function diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index 76b170bd8..afff36665 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -1229,7 +1229,7 @@ def f(): # serializable. from cloudpickle import CloudPickler from cloudpickle import cloudpickle_fast as cp_fast - CloudPickler._dispatch[type(py.builtin)] = cp_fast._module_reduce + CloudPickler.dispatch_table[type(py.builtin)] = cp_fast._module_reduce g = cloudpickle.loads(cloudpickle.dumps(f, protocol=self.protocol)) From 9740b22d584efb85175aa5cf5617cf87db177887 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Thu, 14 May 2020 13:11:40 +0200 Subject: [PATCH 09/89] increase flake8 max-complexity --- .github/scripts/flake8_diff.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/flake8_diff.sh b/.github/scripts/flake8_diff.sh index 0d4a1b748..ced33f722 100644 --- a/.github/scripts/flake8_diff.sh +++ b/.github/scripts/flake8_diff.sh @@ -83,7 +83,7 @@ check_files() { # that was not changed does not create failures # The github terminal is 127 characters wide git diff --unified=0 $COMMIT_RANGE -- $files | flake8 --diff --show-source \ - --max-complexity=20 --max-line-length=127 $options + --max-complexity=40 --max-line-length=127 $options fi } From c657fe12ddd41d412940ae66090705a190bbe84d Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Thu, 14 May 2020 14:16:26 +0200 Subject: [PATCH 10/89] install tornado in the CI --- .github/workflows/testing.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index d7cc99f6e..b0fbfd285 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -61,9 +61,9 @@ jobs: python -m pip install -r dev-requirements.txt python ci/install_coverage_subprocess_pth.py export - - name: Install optional typing_extensions in Python 3.6 + - name: Install supported dependencies (only test in Python 3.6) shell: bash - run: python -m pip install typing-extensions + run: python -m pip install typing-extensions tornado if: matrix.python_version == '3.6' - name: Display Python version shell: bash From 1f2410fc7edbd4250832fa31eb30a1fc57dd1752 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 21 May 2020 14:40:09 -0700 Subject: [PATCH 11/89] Add `compat` Tries to use `pickle5` for `pickle` if available on older Python versions. --- cloudpickle/compat.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 cloudpickle/compat.py diff --git a/cloudpickle/compat.py b/cloudpickle/compat.py new file mode 100644 index 000000000..453386d05 --- /dev/null +++ b/cloudpickle/compat.py @@ -0,0 +1,14 @@ +import sys + + +if sys.version_info.major == 3 and sys.version_info.minor < 8: + try: + import pickle5 as pickle # noqa: F401 + import pickle5._pickle as _pickle # noqa: F401 + from pickle5._pickle import Pickler # noqa: F401 + except ImportError: + import pickle # noqa: F401 + from pickle import _Pickler as Pickler # noqa: F401 +else: + import pickle # noqa: F401 + from _pickle import Pickler # noqa: F401 From ebaf7f6559b80dd5c57ef1de5890a0f7f4a38e6f Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 21 May 2020 14:40:10 -0700 Subject: [PATCH 12/89] Use `compat` to provide `pickle` or `pickle5` --- cloudpickle/cloudpickle.py | 2 +- cloudpickle/cloudpickle_fast.py | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index b6e971c76..f698c04cd 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -45,7 +45,6 @@ import builtins import dis import opcode -import pickle import platform import sys import types @@ -54,6 +53,7 @@ import threading import typing +from .compat import pickle from typing import Generic, Union, Tuple, Callable from pickle import _getattribute from importlib._bootstrap import _find_spec diff --git a/cloudpickle/cloudpickle_fast.py b/cloudpickle/cloudpickle_fast.py index c04b35325..57321d0d2 100644 --- a/cloudpickle/cloudpickle_fast.py +++ b/cloudpickle/cloudpickle_fast.py @@ -15,7 +15,6 @@ import io import itertools import logging -import pickle import sys import struct import types @@ -24,6 +23,7 @@ from enum import Enum +from .compat import pickle, Pickler from .cloudpickle import ( _is_dynamic, _extract_code_globals, _BUILTIN_TYPE_NAMES, DEFAULT_PROTOCOL, _find_imported_submodules, _get_cell_contents, _is_importable_by_name, @@ -35,8 +35,8 @@ builtin_code_type ) -if sys.version_info >= (3, 8) and not PYPY: - from _pickle import Pickler + +if pickle.HIGHEST_PROTOCOL >= 5 and not PYPY: # Shorthands similar to pickle.dump/pickle.dumps def dump(obj, file, protocol=None, buffer_callback=None): @@ -71,8 +71,6 @@ def dumps(obj, protocol=None, buffer_callback=None): return file.getvalue() else: - from pickle import _Pickler as Pickler - # Shorthands similar to pickle.dump/pickle.dumps def dump(obj, file, protocol=None): """Serialize obj as bytes streamed into file From 1d6abbd51e817764c60f0efb85d61f33bda21cc8 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 21 May 2020 14:40:11 -0700 Subject: [PATCH 13/89] Import `pickle` from `compat` in tests --- tests/cloudpickle_file_test.py | 2 +- tests/cloudpickle_test.py | 16 +++++++++------- tests/testutils.py | 3 ++- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/tests/cloudpickle_file_test.py b/tests/cloudpickle_file_test.py index 4f05186e3..6f1099a19 100644 --- a/tests/cloudpickle_file_test.py +++ b/tests/cloudpickle_file_test.py @@ -1,7 +1,6 @@ from __future__ import unicode_literals import os -import pickle import shutil import sys import tempfile @@ -10,6 +9,7 @@ import pytest import cloudpickle +from cloudpickle.compat import pickle class CloudPickleFileTests(unittest.TestCase): diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index afff36665..1a3f6c17f 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -9,7 +9,6 @@ import logging import math from operator import itemgetter, attrgetter -import pickle import platform import random import shutil @@ -43,6 +42,7 @@ tornado = None import cloudpickle +from cloudpickle.compat import pickle from cloudpickle.cloudpickle import _is_dynamic from cloudpickle.cloudpickle import _make_empty_cell, cell_set from cloudpickle.cloudpickle import _extract_class_dict, _whichmodule @@ -519,7 +519,7 @@ def test_module_locals_behavior(self): pickled_func_path = os.path.join(self.tmpdir, 'local_func_g.pkl') child_process_script = ''' - import pickle + from cloudpickle.compat import pickle import gc with open("{pickled_func_path}", 'rb') as f: func = pickle.load(f) @@ -604,7 +604,7 @@ def test_load_dynamic_module_in_grandchild_process(self): child_process_module_file = os.path.join( self.tmpdir, 'dynamic_module_from_child_process.pkl') child_process_script = ''' - import pickle + from cloudpickle.compat import pickle import textwrap import cloudpickle @@ -624,7 +624,7 @@ def test_load_dynamic_module_in_grandchild_process(self): # The script ran by the process created by the child process child_of_child_process_script = """ ''' - import pickle + from cloudpickle.compat import pickle with open('{child_process_module_file}','rb') as fid: mod = pickle.load(fid) ''' """ @@ -679,7 +679,7 @@ def my_small_function(x, y): assert b'math' not in b def test_is_dynamic_module(self): - import pickle # decouple this test from global imports + from cloudpickle.compat import pickle import os.path import distutils import distutils.ccompiler @@ -969,7 +969,8 @@ def example(): # choose "subprocess" rather than "multiprocessing" because the latter # library uses fork to preserve the parent environment. - command = ("import pickle, base64; " + command = ("import base64; " + "from cloudpickle.compat import pickle; " "pickle.loads(base64.b32decode('" + base64.b32encode(s).decode('ascii') + "'))()") @@ -990,7 +991,8 @@ def example(): s = cloudpickle.dumps(example, protocol=self.protocol) - command = ("import pickle, base64; " + command = ("import base64; " + "from cloudpickle.compat import pickle; " "pickle.loads(base64.b32decode('" + base64.b32encode(s).decode('ascii') + "'))()") diff --git a/tests/testutils.py b/tests/testutils.py index 303d0a996..e0276f58c 100644 --- a/tests/testutils.py +++ b/tests/testutils.py @@ -4,7 +4,7 @@ import tempfile import base64 from subprocess import Popen, check_output, PIPE, STDOUT, CalledProcessError -from pickle import loads +from cloudpickle.compat import pickle from contextlib import contextmanager from concurrent.futures import ProcessPoolExecutor @@ -12,6 +12,7 @@ from cloudpickle import dumps from subprocess import TimeoutExpired +loads = pickle.loads TIMEOUT = 60 TEST_GLOBALS = "a test value" From fb34d27049eda2fbf2765dab5dfaf35e6ae11acf Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 21 May 2020 17:27:41 -0700 Subject: [PATCH 14/89] Add pickle5 to the dev requirements --- dev-requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev-requirements.txt b/dev-requirements.txt index 8dc4512d4..5d3bb10a9 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -3,6 +3,8 @@ flake8 pytest pytest-cov psutil +# To test on older Python versions +pickle5 >=0.0.10 ; python_version <= '3.7' # To be able to test tornado coroutines tornado # To be able to test numpy specific things From a15782f8fda1258c5c39f13ec6fa53bc011c8cd8 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 21 May 2020 19:41:46 -0700 Subject: [PATCH 15/89] Always remove `_abc_impl` from `clsdict` --- cloudpickle/cloudpickle_fast.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudpickle/cloudpickle_fast.py b/cloudpickle/cloudpickle_fast.py index 57321d0d2..072fc88f3 100644 --- a/cloudpickle/cloudpickle_fast.py +++ b/cloudpickle/cloudpickle_fast.py @@ -181,13 +181,13 @@ def _class_getstate(obj): # cache/negative caches populated during isinstance/issubclass # checks, but pickle the list of registered subclasses of obj. clsdict.pop('_abc_cache', None) + clsdict.pop('_abc_impl', None) clsdict.pop('_abc_negative_cache', None) clsdict.pop('_abc_negative_cache_version', None) registry = clsdict.pop('_abc_registry', None) if registry is None: # in Python3.7+, the abc caches and registered subclasses of a # class are bundled into the single _abc_impl attribute - clsdict.pop('_abc_impl', None) (registry, _, _, _) = abc._get_dump(obj) clsdict["_abc_impl"] = [subclass_weakref() From 35e39d03e75c346410a646f9f45dd3e4cf09c026 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 21 May 2020 19:43:00 -0700 Subject: [PATCH 16/89] Keep all abstract base class elements --- cloudpickle/cloudpickle.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index f698c04cd..54b5d0681 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -425,6 +425,8 @@ def _extract_class_dict(cls): inherited_dict.update(base.__dict__) to_remove = [] for name, value in clsdict.items(): + if name.startswith("_abc"): + continue try: base_value = inherited_dict[name] if value is base_value: From a03a1ab9bcb44ba566da7fdf3047e51a8deb0a2b Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Fri, 29 May 2020 20:33:20 +0200 Subject: [PATCH 17/89] call typing reducers inside reducer_override reducer_override is no longer Python 3.8+-only code --- cloudpickle/cloudpickle_fast.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cloudpickle/cloudpickle_fast.py b/cloudpickle/cloudpickle_fast.py index 072fc88f3..b39c07730 100644 --- a/cloudpickle/cloudpickle_fast.py +++ b/cloudpickle/cloudpickle_fast.py @@ -601,6 +601,11 @@ def reducer_override(self, obj): reducers, such as Exceptions. See https://github.com/cloudpipe/cloudpickle/issues/248 """ + if sys.version_info[:2] < (3, 7) and _is_parametrized_type_hint(obj): # noqa # pragma: no branch + return ( + _create_parametrized_type_hint, + parametrized_type_hint_getinitargs(obj) + ) t = type(obj) try: is_anyclass = issubclass(t, type) From cdaeec594f6d7b5e61bb2ba1542ab7788b35f179 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Fri, 29 May 2020 20:39:25 +0200 Subject: [PATCH 18/89] revert older temptative fixes --- cloudpickle/cloudpickle.py | 2 -- cloudpickle/cloudpickle_fast.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 54b5d0681..f698c04cd 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -425,8 +425,6 @@ def _extract_class_dict(cls): inherited_dict.update(base.__dict__) to_remove = [] for name, value in clsdict.items(): - if name.startswith("_abc"): - continue try: base_value = inherited_dict[name] if value is base_value: diff --git a/cloudpickle/cloudpickle_fast.py b/cloudpickle/cloudpickle_fast.py index b39c07730..4014ec35b 100644 --- a/cloudpickle/cloudpickle_fast.py +++ b/cloudpickle/cloudpickle_fast.py @@ -181,13 +181,13 @@ def _class_getstate(obj): # cache/negative caches populated during isinstance/issubclass # checks, but pickle the list of registered subclasses of obj. clsdict.pop('_abc_cache', None) - clsdict.pop('_abc_impl', None) clsdict.pop('_abc_negative_cache', None) clsdict.pop('_abc_negative_cache_version', None) registry = clsdict.pop('_abc_registry', None) if registry is None: # in Python3.7+, the abc caches and registered subclasses of a # class are bundled into the single _abc_impl attribute + clsdict.pop('_abc_impl', None) (registry, _, _, _) = abc._get_dump(obj) clsdict["_abc_impl"] = [subclass_weakref() From 95fdce0388851c4a765a689276896ae456673f62 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Fri, 29 May 2020 20:44:48 +0200 Subject: [PATCH 19/89] skip numpy + pickle5 test on Python 3.5 --- tests/cloudpickle_test.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index 1a3f6c17f..88d228121 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -2019,6 +2019,10 @@ def __getattr__(self, name): with pytest.raises(pickle.PicklingError, match='recursion'): cloudpickle.dumps(a) + @pytest.mark.skipif( + sys.version_info < (3, 6), + reason='numpy does not support pickle protocol 5 on Python 3.5' + ) def test_out_of_band_buffers(self): if self.protocol < 5: pytest.skip("Need Pickle Protocol 5 or later") From 38ed5f502ed1cd0ddecf39eec6fa8604ac706e1b Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Fri, 29 May 2020 21:01:03 +0200 Subject: [PATCH 20/89] don't try to install pickle5 on PYPY --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 5d3bb10a9..2176605bd 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -4,7 +4,7 @@ pytest pytest-cov psutil # To test on older Python versions -pickle5 >=0.0.10 ; python_version <= '3.7' +pickle5 >=0.0.10 ; python_version <= '3.7' and python_implementation == 'CPython' # To be able to test tornado coroutines tornado # To be able to test numpy specific things From 5334e797165082888a613857269ceed7a50029f0 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Wed, 10 Jun 2020 11:23:28 -0700 Subject: [PATCH 21/89] Simplify `version_info` check --- cloudpickle/compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudpickle/compat.py b/cloudpickle/compat.py index 453386d05..b362db55c 100644 --- a/cloudpickle/compat.py +++ b/cloudpickle/compat.py @@ -1,7 +1,7 @@ import sys -if sys.version_info.major == 3 and sys.version_info.minor < 8: +if sys.version_info < (3, 8): try: import pickle5 as pickle # noqa: F401 import pickle5._pickle as _pickle # noqa: F401 From abbb9780168db5631234a9591cca77214fcb042b Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 10 Jun 2020 20:27:34 +0200 Subject: [PATCH 22/89] Update cloudpickle/cloudpickle.py Co-authored-by: jakirkham --- cloudpickle/cloudpickle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index b6e971c76..ec34f0420 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -606,7 +606,7 @@ def _make_empty_cell(): def _make_cell(value=_empty_cell_value): cell = _make_empty_cell() - if value != _empty_cell_value: + if value is not _empty_cell_value: cell_set(cell, value) return cell From 53111d3a396610a101179dfb5ab8384deb50e466 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Wed, 10 Jun 2020 17:48:35 -0700 Subject: [PATCH 23/89] Tweak pickle5 imports --- cloudpickle/compat.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cloudpickle/compat.py b/cloudpickle/compat.py index b362db55c..aab0015d5 100644 --- a/cloudpickle/compat.py +++ b/cloudpickle/compat.py @@ -4,8 +4,7 @@ if sys.version_info < (3, 8): try: import pickle5 as pickle # noqa: F401 - import pickle5._pickle as _pickle # noqa: F401 - from pickle5._pickle import Pickler # noqa: F401 + from pickle5 import _Pickler as Pickler # noqa: F401 except ImportError: import pickle # noqa: F401 from pickle import _Pickler as Pickler # noqa: F401 From 9d566328a888a0a5e4dbd1ed22af1a12ea5c836f Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Sun, 14 Jun 2020 18:39:54 +0200 Subject: [PATCH 24/89] test best-effort cloudpickle backward compat --- .github/workflows/testing.yml | 7 +++ tests/generate_old_pickles.py | 82 +++++++++++++++++++++++++++++++++++ tests/test_backward_compat.py | 77 ++++++++++++++++++++++++++++++++ 3 files changed, 166 insertions(+) create mode 100644 tests/generate_old_pickles.py create mode 100644 tests/test_backward_compat.py diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index b0fbfd285..eccf08210 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -68,6 +68,13 @@ jobs: - name: Display Python version shell: bash run: python -c "import sys; print(sys.version)" + - name: Generate old pickles (backward compat) + shell: bash + run: | + git_head=$(git rev-parse HEAD) + git checkout v1.4.1 + python tests/generate_old_pickles.py + git checkout $(git_head) - name: Look for syntax errors/undefined names shell: bash run: | diff --git a/tests/generate_old_pickles.py b/tests/generate_old_pickles.py new file mode 100644 index 000000000..1ae28c70d --- /dev/null +++ b/tests/generate_old_pickles.py @@ -0,0 +1,82 @@ +"""scripts reproducing pickles used to test cloudpickle backward compat support + + +This file contains a few python scripts that generate pickles of canonical +objects whose pickling is supported by cloudpickle (dynamic functions, enums, +classes, modules etc). These scripts must be run with an "old" version of +cloudpickle. When testing, the generated pickle files are depickled using the +active cloudpickle branch to make sure that cloudpickle is able to depickle old +cloudpickle files. +""" +from pathlib import Path + +from enum import IntEnum +from types import ModuleType +from typing import TypeVar, Generic + +import cloudpickle + +PICKLE_DIRECTORY = Path(__file__).parent / "old_pickles" + + +def dump_obj(obj, filename): + with open(PICKLE_DIRECTORY / filename, "wb") as f: + cloudpickle.dump(obj, f) + + +def nested_function_generator(): + a = 1 + + def nested_func(b): + return a + b + + return nested_func + + +if __name__ == "__main__": + PICKLE_DIRECTORY.mkdir() + + # simple dynamic function + def simple_func(x: int, y=1): + return x + y + + dump_obj(simple_func, "simple_func.pkl") + + # simple dynamic class + class SimpleClass: + def __init__(self, attribute): + self.attribute = attribute + + dump_obj(SimpleClass, "simple_class.pkl") + + # simple dynamic module + dynamic_module = ModuleType("dynamic_module") + s = """if 1: + def f(x, y=1): + return x + y + """ + exec(s, vars(dynamic_module)) + assert dynamic_module.f(2, 1) == 3 + dump_obj(dynamic_module, "simple_module.pkl") + + # simple dynamic Enum + class DynamicEnum(IntEnum): + RED = 1 + BLUE = 2 + + dump_obj(DynamicEnum, "simple_enum.pkl") + + # complex dynanic function/classes involing various typing annotations + # supported since cloudpickle 1.4 + T = TypeVar("T") + + class MyClass(Generic[T]): + def __init__(self, attribute: T): + self.attribute = attribute + + dump_obj(MyClass, "class_with_type_hints.pkl") + + def add(x: MyClass[int], y: MyClass[int]): + return MyClass(x.attribute + y.attribute) + + dump_obj([MyClass, add], "function_with_type_hints.pkl") diff --git a/tests/test_backward_compat.py b/tests/test_backward_compat.py new file mode 100644 index 000000000..0d373957c --- /dev/null +++ b/tests/test_backward_compat.py @@ -0,0 +1,77 @@ +"""Limited, best-effort test suite regarding cloudpickle backward-compat. + +Cloudpickle does not officially support reading pickles files +generated with an older version of cloudpickle than the one used to read the +said pickles. However, this policy is not widely known among users that use +libraries that rely on cloudpickle such as mlflow, and is subject to confusion. + +As a compromise, this script make sure cloudpickle is backward compatible for a +few canonical use cases. Cloudpicke backward-compatitibility support remains a +best-effort initiative. +""" +import pickle + +from .generate_old_pickles import PICKLE_DIRECTORY + + +def load_obj(filename): + with open(PICKLE_DIRECTORY / filename, "rb") as f: + obj = pickle.load(f) + return obj + + +def test_simple_func(): + f = load_obj("simple_func.pkl") + assert f(1) == 2 + assert f(1, 1) == 2 + assert f(2, 2) == 4 + + +def test_simple_class(): + SimpleClass = load_obj("simple_class.pkl") + c = SimpleClass(1) + assert hasattr(c, "attribute") + assert c.attribute == 1 + + # test class tracking feature + assert SimpleClass is load_obj("simple_class.pkl") + + +def test_dynamic_module(): + mod = load_obj("simple_module.pkl") + assert hasattr(mod, "f") + assert mod.f(1) == 2 + assert mod.f(1, 1) == 2 + assert mod.f(2, 2) == 4 + + +def test_simple_enum(): + enum = load_obj("simple_enum.pkl") + assert hasattr(enum, "RED") + assert enum.RED == 1 + assert enum.BLUE == 2 + + # test enum tracking feature + new_enum = load_obj("simple_enum.pkl") + assert new_enum is enum + + +def test_complex_class(): + SimpleClass = load_obj("class_with_type_hints.pkl") + c = SimpleClass(1) + assert hasattr(c, "attribute") + assert c.attribute == 1 + + # test class tracking feature + assert SimpleClass is load_obj("class_with_type_hints.pkl") + + +def test_complex_function(): + MyClass, f = load_obj("function_with_type_hints.pkl") + assert len(f.__annotations__) > 0 + + a = MyClass(1) + b = MyClass(2) + + c = f(a, b) + assert c.attribute == 3 From 9e9241377068e6a6b7c554357ff875dcac89af97 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Sun, 14 Jun 2020 18:47:09 +0200 Subject: [PATCH 25/89] debug ci not launching --- .github/workflows/testing.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index eccf08210..b0fbfd285 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -68,13 +68,6 @@ jobs: - name: Display Python version shell: bash run: python -c "import sys; print(sys.version)" - - name: Generate old pickles (backward compat) - shell: bash - run: | - git_head=$(git rev-parse HEAD) - git checkout v1.4.1 - python tests/generate_old_pickles.py - git checkout $(git_head) - name: Look for syntax errors/undefined names shell: bash run: | From 09d9e33861705e44b4fd0f56a28991f77fffd494 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Sun, 14 Jun 2020 18:55:09 +0200 Subject: [PATCH 26/89] solve merge conflics mistakes --- cloudpickle/cloudpickle_fast.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cloudpickle/cloudpickle_fast.py b/cloudpickle/cloudpickle_fast.py index ad8ad416d..75920dd24 100644 --- a/cloudpickle/cloudpickle_fast.py +++ b/cloudpickle/cloudpickle_fast.py @@ -25,8 +25,8 @@ from enum import Enum from .cloudpickle import ( - _is_dynamic, _extract_code_globals, _BUILTIN_TYPE_NAMES, DEFAULT_PROTOCOL, - _find_imported_submodules, _get_cell_contents, _is_importable_by_name, + _extract_code_globals, _BUILTIN_TYPE_NAMES, DEFAULT_PROTOCOL, + _find_imported_submodules, _get_cell_contents, _is_importable, _builtin_type, _get_or_create_tracker_id, _make_skeleton_class, _make_skeleton_enum, _extract_class_dict, dynamic_subimport, subimport, _typevar_reduce, _get_bases, _make_cell, _make_empty_cell, CellType, @@ -687,7 +687,7 @@ def save_global(self, obj, name=None, pack=struct.pack): ) elif name is not None: Pickler.save_global(self, obj, name=name) - elif not _is_importable_by_name(obj, name=name): + elif not _is_importable(obj, name=name): self._save_reduce_pickle5(*_dynamic_class_reduce(obj), obj=obj) else: Pickler.save_global(self, obj, name=name) @@ -699,7 +699,7 @@ def save_function(self, obj, name=None): Determines what kind of function obj is (e.g. lambda, defined at interactive prompt, etc) and handles the pickling appropriately. """ - if _is_importable_by_name(obj, name=name): + if _is_importable(obj, name=name): return Pickler.save_global(self, obj, name=name) elif PYPY and isinstance(obj.__code__, builtin_code_type): return self.save_pypy_builtin_func(obj) From f74372a6cc61af85212b44b88dd1bd4bd53e869c Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Sun, 14 Jun 2020 18:57:15 +0200 Subject: [PATCH 27/89] generate old pickle files during CI --- .github/workflows/testing.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index b0fbfd285..eccf08210 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -68,6 +68,13 @@ jobs: - name: Display Python version shell: bash run: python -c "import sys; print(sys.version)" + - name: Generate old pickles (backward compat) + shell: bash + run: | + git_head=$(git rev-parse HEAD) + git checkout v1.4.1 + python tests/generate_old_pickles.py + git checkout $(git_head) - name: Look for syntax errors/undefined names shell: bash run: | From 829b9192b952a28bb5d1124f55b5e13efe259aee Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Sun, 14 Jun 2020 19:01:56 +0200 Subject: [PATCH 28/89] fixup! generate old pickle files during CI --- .github/workflows/testing.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index eccf08210..83d55aaa1 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -72,8 +72,9 @@ jobs: shell: bash run: | git_head=$(git rev-parse HEAD) + cp tests/generate_old_pickles.py tests/_generate_old_pickles.py git checkout v1.4.1 - python tests/generate_old_pickles.py + python tests/_generate_old_pickles.py git checkout $(git_head) - name: Look for syntax errors/undefined names shell: bash From f693401d584d724e2f6d6d92f20d43ca7b2c027c Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Sun, 14 Jun 2020 19:04:37 +0200 Subject: [PATCH 29/89] fix pathlib/open compat --- tests/generate_old_pickles.py | 2 +- tests/test_backward_compat.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/generate_old_pickles.py b/tests/generate_old_pickles.py index 1ae28c70d..cb27c6f25 100644 --- a/tests/generate_old_pickles.py +++ b/tests/generate_old_pickles.py @@ -20,7 +20,7 @@ def dump_obj(obj, filename): - with open(PICKLE_DIRECTORY / filename, "wb") as f: + with open(str(PICKLE_DIRECTORY / filename), "wb") as f: cloudpickle.dump(obj, f) diff --git a/tests/test_backward_compat.py b/tests/test_backward_compat.py index 0d373957c..12b60b8cc 100644 --- a/tests/test_backward_compat.py +++ b/tests/test_backward_compat.py @@ -15,7 +15,7 @@ def load_obj(filename): - with open(PICKLE_DIRECTORY / filename, "rb") as f: + with open(str(PICKLE_DIRECTORY / filename), "rb") as f: obj = pickle.load(f) return obj From 46405e5ccb25446e1ef708aa7d05a37f484387f6 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Sun, 14 Jun 2020 19:07:51 +0200 Subject: [PATCH 30/89] update python-nightly entry --- .github/workflows/testing.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 83d55aaa1..c99405833 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -114,6 +114,14 @@ jobs: python -m pip install -r dev-requirements.txt python -m pip install -e . python ci/install_coverage_subprocess_pth.py + - name: Generate old pickles (backward compat) + shell: bash + run: | + git_head=$(git rev-parse HEAD) + cp tests/generate_old_pickles.py tests/_generate_old_pickles.py + git checkout v1.4.1 + python tests/_generate_old_pickles.py + git checkout $(git_head) - name: Test with pytest run: | COVERAGE_PROCESS_START=$GITHUB_WORKSPACE/.coveragerc \ From df6fa9bf55349f247e832caa2c283c62551c54c1 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Mon, 15 Jun 2020 10:11:15 +0200 Subject: [PATCH 31/89] fix ci script --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index c99405833..3fb75a437 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -121,7 +121,7 @@ jobs: cp tests/generate_old_pickles.py tests/_generate_old_pickles.py git checkout v1.4.1 python tests/_generate_old_pickles.py - git checkout $(git_head) + git checkout ${git_head} - name: Test with pytest run: | COVERAGE_PROCESS_START=$GITHUB_WORKSPACE/.coveragerc \ From 20a522bd4c19359b926db51732d862dc1713205a Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 17 Jun 2020 20:23:20 +0200 Subject: [PATCH 32/89] fix variable reference in ci script --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 3fb75a437..a4fcc82ee 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -75,7 +75,7 @@ jobs: cp tests/generate_old_pickles.py tests/_generate_old_pickles.py git checkout v1.4.1 python tests/_generate_old_pickles.py - git checkout $(git_head) + git checkout ${git_head} - name: Look for syntax errors/undefined names shell: bash run: | From e5deaf6846c01e8c6e4e6e16250e548c2500947a Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 17 Jun 2020 20:34:59 +0200 Subject: [PATCH 33/89] restore backward compat with cloudpickle master --- cloudpickle/cloudpickle.py | 83 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 5da2204ea..27decf841 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -609,6 +609,71 @@ def __reduce__(cls): return cls.__name__ +def _fill_function(*args): + """Fills in the rest of function data into the skeleton function object + + The skeleton itself is create by _make_skel_func(). + """ + if len(args) == 2: + func = args[0] + state = args[1] + elif len(args) == 5: + # Backwards compat for cloudpickle v0.4.0, after which the `module` + # argument was introduced + func = args[0] + keys = ['globals', 'defaults', 'dict', 'closure_values'] + state = dict(zip(keys, args[1:])) + elif len(args) == 6: + # Backwards compat for cloudpickle v0.4.1, after which the function + # state was passed as a dict to the _fill_function it-self. + func = args[0] + keys = ['globals', 'defaults', 'dict', 'module', 'closure_values'] + state = dict(zip(keys, args[1:])) + else: + raise ValueError('Unexpected _fill_value arguments: %r' % (args,)) + + # - At pickling time, any dynamic global variable used by func is + # serialized by value (in state['globals']). + # - At unpickling time, func's __globals__ attribute is initialized by + # first retrieving an empty isolated namespace that will be shared + # with other functions pickled from the same original module + # by the same CloudPickler instance and then updated with the + # content of state['globals'] to populate the shared isolated + # namespace with all the global variables that are specifically + # referenced for this function. + func.__globals__.update(state['globals']) + + func.__defaults__ = state['defaults'] + func.__dict__ = state['dict'] + if 'annotations' in state: + func.__annotations__ = state['annotations'] + if 'doc' in state: + func.__doc__ = state['doc'] + if 'name' in state: + func.__name__ = state['name'] + if 'module' in state: + func.__module__ = state['module'] + if 'qualname' in state: + func.__qualname__ = state['qualname'] + if 'kwdefaults' in state: + func.__kwdefaults__ = state['kwdefaults'] + # _cloudpickle_subimports is a set of submodules that must be loaded for + # the pickled function to work correctly at unpickling time. Now that these + # submodules are depickled (hence imported), they can be removed from the + # object's state (the object state only served as a reference holder to + # these submodules) + if '_cloudpickle_submodules' in state: + state.pop('_cloudpickle_submodules') + + cells = func.__closure__ + if cells is not None: + for cell, value in zip(cells, state['closure_values']): + if value is not _empty_cell_value: + cell_set(cell, value) + + return func + + def _make_empty_cell(): if False: # trick the compiler into creating an empty cell in our lambda @@ -665,6 +730,24 @@ class id will also reuse this class definition. return _lookup_class_or_track(class_tracker_id, skeleton_class) +def _rehydrate_skeleton_class(skeleton_class, class_dict): + """Put attributes from `class_dict` back on `skeleton_class`. + + See CloudPickler.save_dynamic_class for more info. + """ + registry = None + for attrname, attr in class_dict.items(): + if attrname == "_abc_impl": + registry = attr + else: + setattr(skeleton_class, attrname, attr) + if registry is not None: + for subclass in registry: + skeleton_class.register(subclass) + + return skeleton_class + + def _make_skeleton_enum(bases, name, qualname, members, module, class_tracker_id, extra): """Build dynamic enum with an empty __dict__ to be filled once memoized From ff2baa93a313e0d679c72f6ad17875e52e9f03cc Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 17 Jun 2020 20:51:05 +0200 Subject: [PATCH 34/89] add deprecation warnings inside old reconstructors --- cloudpickle/cloudpickle.py | 7 +++++++ tests/test_backward_compat.py | 20 +++++++++++++++----- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 27decf841..3cb476798 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -53,6 +53,7 @@ import uuid import threading import typing +import warnings from typing import Generic, Union, Tuple, Callable from pickle import _getattribute @@ -695,6 +696,12 @@ def _make_skel_func(code, cell_count, base_globals=None): code and the correct number of cells in func_closure. All other func attributes (e.g. func_globals) are empty. """ + # This function is deprecated and should be removed in cloudpickle 1.7 + warnings.warn( + "A pickle file created using an old (<=1.4.1) version of cloudpicke " + "is currently being loaded. This is not supported by cloudpickle and " + "will break in cloudpickle 1.7", category=UserWarning + ) # This is backward-compatibility code: for cloudpickle versions between # 0.5.4 and 0.7, base_globals could be a string or None. base_globals # should now always be a dictionary. diff --git a/tests/test_backward_compat.py b/tests/test_backward_compat.py index 12b60b8cc..cdbc9b5ce 100644 --- a/tests/test_backward_compat.py +++ b/tests/test_backward_compat.py @@ -11,12 +11,22 @@ """ import pickle +import pytest + from .generate_old_pickles import PICKLE_DIRECTORY -def load_obj(filename): - with open(str(PICKLE_DIRECTORY / filename), "rb") as f: - obj = pickle.load(f) +def load_obj(filename, check_deprecation_warning=True): + try: + f = open(str(PICKLE_DIRECTORY / filename), "rb") + if check_deprecation_warning: + msg = "A pickle file created using an old" + with pytest.warns(UserWarning, match=msg): + obj = pickle.load(f) + else: + obj = pickle.load(f) + finally: + f.close() return obj @@ -46,13 +56,13 @@ def test_dynamic_module(): def test_simple_enum(): - enum = load_obj("simple_enum.pkl") + enum = load_obj("simple_enum.pkl", check_deprecation_warning=False) assert hasattr(enum, "RED") assert enum.RED == 1 assert enum.BLUE == 2 # test enum tracking feature - new_enum = load_obj("simple_enum.pkl") + new_enum = load_obj("simple_enum.pkl", check_deprecation_warning=False) assert new_enum is enum From 13a76f9c59fd049da0308bfac90a3ff212f7b392 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 17 Jun 2020 20:51:33 +0200 Subject: [PATCH 35/89] always use up-to-date copyreg --- cloudpickle/cloudpickle_fast.py | 40 ++++++++++++++++----------------- tests/cloudpickle_test.py | 18 +++++++++++++++ 2 files changed, 38 insertions(+), 20 deletions(-) diff --git a/cloudpickle/cloudpickle_fast.py b/cloudpickle/cloudpickle_fast.py index 75920dd24..4b52a1e8e 100644 --- a/cloudpickle/cloudpickle_fast.py +++ b/cloudpickle/cloudpickle_fast.py @@ -23,6 +23,7 @@ import typing from enum import Enum +from collections import ChainMap from .cloudpickle import ( _extract_code_globals, _BUILTIN_TYPE_NAMES, DEFAULT_PROTOCOL, @@ -457,26 +458,25 @@ def _class_setstate(obj, state): class CloudPickler(Pickler): - # Take into account potential custom reducers registered by external - # modules - dispatch_table = copyreg.dispatch_table.copy() - # register the additional set of objects (compared to the standard library - # pickle) that cloupickle can serialize. - dispatch_table[classmethod] = _classmethod_reduce - dispatch_table[io.TextIOWrapper] = _file_reduce - dispatch_table[logging.Logger] = _logger_reduce - dispatch_table[logging.RootLogger] = _root_logger_reduce - dispatch_table[memoryview] = _memoryview_reduce - dispatch_table[property] = _property_reduce - dispatch_table[staticmethod] = _classmethod_reduce - dispatch_table[CellType] = _cell_reduce - dispatch_table[types.CodeType] = _code_reduce - dispatch_table[types.GetSetDescriptorType] = _getset_descriptor_reduce - dispatch_table[types.ModuleType] = _module_reduce - dispatch_table[types.MethodType] = _method_reduce - dispatch_table[types.MappingProxyType] = _mappingproxy_reduce - dispatch_table[weakref.WeakSet] = _weakset_reduce - dispatch_table[typing.TypeVar] = _typevar_reduce + # set of reducers defined and used by cloudpickle (private) + _dispatch_table = {} + _dispatch_table[classmethod] = _classmethod_reduce + _dispatch_table[io.TextIOWrapper] = _file_reduce + _dispatch_table[logging.Logger] = _logger_reduce + _dispatch_table[logging.RootLogger] = _root_logger_reduce + _dispatch_table[memoryview] = _memoryview_reduce + _dispatch_table[property] = _property_reduce + _dispatch_table[staticmethod] = _classmethod_reduce + _dispatch_table[CellType] = _cell_reduce + _dispatch_table[types.CodeType] = _code_reduce + _dispatch_table[types.GetSetDescriptorType] = _getset_descriptor_reduce + _dispatch_table[types.ModuleType] = _module_reduce + _dispatch_table[types.MethodType] = _method_reduce + _dispatch_table[types.MappingProxyType] = _mappingproxy_reduce + _dispatch_table[weakref.WeakSet] = _weakset_reduce + _dispatch_table[typing.TypeVar] = _typevar_reduce + + dispatch_table = ChainMap(_dispatch_table, copyreg.dispatch_table) # function reducers are defined as instance methods of CloudPickler # objects, as they rely on a CloudPickler attribute (globals_ref) diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index 656f793b9..ad6a9c9a6 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -2244,6 +2244,24 @@ def f(a: int) -> str: f1 = pickle_depickle(f, protocol=self.protocol) assert f1.__annotations__ == f.__annotations__ + def test_always_use_up_to_date_copyreg(self): + # test that updates of copyreg.dispatch_table are taken in account by + # cloudpickle + import copyreg + try: + class MyClass: + pass + + def reduce_myclass(x): + return MyClass, (), {'custom_reduce': True} + + copyreg.dispatch_table[MyClass] = reduce_myclass + my_obj = MyClass() + depickled_myobj = pickle_depickle(my_obj, protocol=self.protocol) + assert hasattr(depickled_myobj, 'custom_reduce') + finally: + copyreg.dispatch_table.pop(MyClass) + class Protocol2CloudPickleTest(CloudPickleTest): From 52746b950ec143c8280b3820fead3eccf42d72ff Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 17 Jun 2020 20:57:01 +0200 Subject: [PATCH 36/89] don't (always) catch deprecation warnings --- tests/test_backward_compat.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_backward_compat.py b/tests/test_backward_compat.py index cdbc9b5ce..6ac7f39a2 100644 --- a/tests/test_backward_compat.py +++ b/tests/test_backward_compat.py @@ -10,13 +10,18 @@ best-effort initiative. """ import pickle +import sys import pytest from .generate_old_pickles import PICKLE_DIRECTORY -def load_obj(filename, check_deprecation_warning=True): +def load_obj(filename, check_deprecation_warning='auto'): + if check_deprecation_warning == 'auto': + # pickles files generated with cloudpickle_fast.py on old versions of + # coudpickle with Python < 3.8 use non-deprecated reconstructors. + check_deprecation_warning == sys.version_info < (3, 8) try: f = open(str(PICKLE_DIRECTORY / filename), "rb") if check_deprecation_warning: From b531b1450b7858c2a123c2033f6d66c05c0e2a20 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 17 Jun 2020 20:57:46 +0200 Subject: [PATCH 37/89] fixup! don't (always) catch deprecation warnings --- tests/test_backward_compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_backward_compat.py b/tests/test_backward_compat.py index 6ac7f39a2..24a50c2fc 100644 --- a/tests/test_backward_compat.py +++ b/tests/test_backward_compat.py @@ -21,7 +21,7 @@ def load_obj(filename, check_deprecation_warning='auto'): if check_deprecation_warning == 'auto': # pickles files generated with cloudpickle_fast.py on old versions of # coudpickle with Python < 3.8 use non-deprecated reconstructors. - check_deprecation_warning == sys.version_info < (3, 8) + check_deprecation_warning = (sys.version_info < (3, 8)) try: f = open(str(PICKLE_DIRECTORY / filename), "rb") if check_deprecation_warning: From e85ab4d4f2fd00e7f9adf20cccb533eba923271a Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 17 Jun 2020 21:29:45 +0200 Subject: [PATCH 38/89] debug ci --- tests/generate_old_pickles.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/generate_old_pickles.py b/tests/generate_old_pickles.py index cb27c6f25..376937031 100644 --- a/tests/generate_old_pickles.py +++ b/tests/generate_old_pickles.py @@ -70,6 +70,9 @@ class DynamicEnum(IntEnum): # supported since cloudpickle 1.4 T = TypeVar("T") + # debug what's happening in the CI + print(T.__module__) + class MyClass(Generic[T]): def __init__(self, attribute: T): self.attribute = attribute From 2a82a419f450eae29828304b9e189d60d9c3a8c3 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 17 Jun 2020 21:31:57 +0200 Subject: [PATCH 39/89] fixup! debug ci --- tests/generate_old_pickles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/generate_old_pickles.py b/tests/generate_old_pickles.py index 376937031..c78872250 100644 --- a/tests/generate_old_pickles.py +++ b/tests/generate_old_pickles.py @@ -71,7 +71,7 @@ class DynamicEnum(IntEnum): T = TypeVar("T") # debug what's happening in the CI - print(T.__module__) + raise ValueError(T.__module__) class MyClass(Generic[T]): def __init__(self, attribute: T): From 21c82b9d4185a1abc5d9e301a9142483e2f5dd90 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 17 Jun 2020 21:33:41 +0200 Subject: [PATCH 40/89] fixup! debug ci --- tests/generate_old_pickles.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/generate_old_pickles.py b/tests/generate_old_pickles.py index c78872250..6617582d0 100644 --- a/tests/generate_old_pickles.py +++ b/tests/generate_old_pickles.py @@ -9,6 +9,7 @@ cloudpickle files. """ from pathlib import Path +import sys from enum import IntEnum from types import ModuleType @@ -71,7 +72,8 @@ class DynamicEnum(IntEnum): T = TypeVar("T") # debug what's happening in the CI - raise ValueError(T.__module__) + if sys.version_info[:2] == (3, 6): + raise ValueError(T.__module__) class MyClass(Generic[T]): def __init__(self, attribute: T): From 4fb4ca5670f89fbd9a5fbca9cdd8d6f54625c690 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Thu, 18 Jun 2020 10:39:00 +0200 Subject: [PATCH 41/89] fixup! debug ci --- tests/generate_old_pickles.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/generate_old_pickles.py b/tests/generate_old_pickles.py index 6617582d0..dc4d4f1a8 100644 --- a/tests/generate_old_pickles.py +++ b/tests/generate_old_pickles.py @@ -73,7 +73,8 @@ class DynamicEnum(IntEnum): # debug what's happening in the CI if sys.version_info[:2] == (3, 6): - raise ValueError(T.__module__) + from cloudpickle.cloudpickle import _lookup_module_and_qualname + raise ValueError(_lookup_module_and_qualname(T, T.__name__)) class MyClass(Generic[T]): def __init__(self, attribute: T): From 5909c3813fc046ba846fd87830724cfc169c5691 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Sun, 21 Jun 2020 12:04:58 +0200 Subject: [PATCH 42/89] remove debug leftovers --- tests/generate_old_pickles.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/generate_old_pickles.py b/tests/generate_old_pickles.py index dc4d4f1a8..cb27c6f25 100644 --- a/tests/generate_old_pickles.py +++ b/tests/generate_old_pickles.py @@ -9,7 +9,6 @@ cloudpickle files. """ from pathlib import Path -import sys from enum import IntEnum from types import ModuleType @@ -71,11 +70,6 @@ class DynamicEnum(IntEnum): # supported since cloudpickle 1.4 T = TypeVar("T") - # debug what's happening in the CI - if sys.version_info[:2] == (3, 6): - from cloudpickle.cloudpickle import _lookup_module_and_qualname - raise ValueError(_lookup_module_and_qualname(T, T.__name__)) - class MyClass(Generic[T]): def __init__(self, attribute: T): self.attribute = attribute From 282239013f0cf1985a68f43e87ec890886818152 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Sun, 21 Jun 2020 12:12:37 +0200 Subject: [PATCH 43/89] separate pickles per python version --- tests/generate_old_pickles.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/generate_old_pickles.py b/tests/generate_old_pickles.py index cb27c6f25..2c42eb19d 100644 --- a/tests/generate_old_pickles.py +++ b/tests/generate_old_pickles.py @@ -8,15 +8,18 @@ active cloudpickle branch to make sure that cloudpickle is able to depickle old cloudpickle files. """ -from pathlib import Path +import sys +from pathlib import Path from enum import IntEnum from types import ModuleType from typing import TypeVar, Generic import cloudpickle -PICKLE_DIRECTORY = Path(__file__).parent / "old_pickles" +PYTHON_VERSION = "{}.{}".format(sys.version_info.major, sys.version_info.minor) + +PICKLE_DIRECTORY = Path(__file__).parent / "old_pickles" / PYTHON_VERSION def dump_obj(obj, filename): From d2399436c6d65788dcc3dd89f098d5d1d6a9bcb2 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Sun, 21 Jun 2020 12:14:23 +0200 Subject: [PATCH 44/89] fixup! separate pickles per python version --- tests/generate_old_pickles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/generate_old_pickles.py b/tests/generate_old_pickles.py index 2c42eb19d..7ac6c8c97 100644 --- a/tests/generate_old_pickles.py +++ b/tests/generate_old_pickles.py @@ -37,7 +37,7 @@ def nested_func(b): if __name__ == "__main__": - PICKLE_DIRECTORY.mkdir() + PICKLE_DIRECTORY.mkdir(parents=True) # simple dynamic function def simple_func(x: int, y=1): From 8deef74d87dc7e5365fd5b82223374894f7e1ecc Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Sun, 21 Jun 2020 17:11:39 +0200 Subject: [PATCH 45/89] fixup! separate pickles per python version --- tests/generate_old_pickles.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/generate_old_pickles.py b/tests/generate_old_pickles.py index 7ac6c8c97..e6ddee5ae 100644 --- a/tests/generate_old_pickles.py +++ b/tests/generate_old_pickles.py @@ -17,9 +17,11 @@ import cloudpickle -PYTHON_VERSION = "{}.{}".format(sys.version_info.major, sys.version_info.minor) +PYTHON_INFO = "{}_{}{}".format( + sys.implementation.name, sys.version_info.major, sys.version_info.minor +) -PICKLE_DIRECTORY = Path(__file__).parent / "old_pickles" / PYTHON_VERSION +PICKLE_DIRECTORY = Path(__file__).parent / "old_pickles" / PYTHON_INFO def dump_obj(obj, filename): From bb3436f291973d31b6eda36b8213f82d8d777f88 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Sun, 21 Jun 2020 17:16:01 +0200 Subject: [PATCH 46/89] add some pickles generated with cloudpickle 1.4.1 --- .../cpython_35/class_with_type_hints.pkl | Bin 0 -> 937 bytes .../cpython_35/function_with_type_hints.pkl | Bin 0 -> 1121 bytes tests/old_pickles/cpython_35/simple_class.pkl | Bin 0 -> 684 bytes tests/old_pickles/cpython_35/simple_enum.pkl | Bin 0 -> 225 bytes tests/old_pickles/cpython_35/simple_func.pkl | Bin 0 -> 485 bytes tests/old_pickles/cpython_35/simple_module.pkl | Bin 0 -> 473 bytes .../cpython_36/class_with_type_hints.pkl | Bin 0 -> 944 bytes .../cpython_36/function_with_type_hints.pkl | Bin 0 -> 1124 bytes tests/old_pickles/cpython_36/simple_class.pkl | Bin 0 -> 681 bytes tests/old_pickles/cpython_36/simple_enum.pkl | Bin 0 -> 225 bytes tests/old_pickles/cpython_36/simple_func.pkl | Bin 0 -> 485 bytes tests/old_pickles/cpython_36/simple_module.pkl | Bin 0 -> 473 bytes .../cpython_37/class_with_type_hints.pkl | Bin 0 -> 822 bytes .../cpython_37/function_with_type_hints.pkl | Bin 0 -> 991 bytes tests/old_pickles/cpython_37/simple_class.pkl | Bin 0 -> 681 bytes tests/old_pickles/cpython_37/simple_enum.pkl | Bin 0 -> 225 bytes tests/old_pickles/cpython_37/simple_func.pkl | Bin 0 -> 485 bytes tests/old_pickles/cpython_37/simple_module.pkl | Bin 0 -> 473 bytes .../cpython_38/class_with_type_hints.pkl | Bin 0 -> 846 bytes .../cpython_38/function_with_type_hints.pkl | Bin 0 -> 1017 bytes tests/old_pickles/cpython_38/simple_class.pkl | Bin 0 -> 705 bytes tests/old_pickles/cpython_38/simple_enum.pkl | Bin 0 -> 279 bytes tests/old_pickles/cpython_38/simple_func.pkl | Bin 0 -> 533 bytes tests/old_pickles/cpython_38/simple_module.pkl | Bin 0 -> 513 bytes .../pypy_35/class_with_type_hints.pkl | Bin 0 -> 914 bytes .../pypy_35/function_with_type_hints.pkl | Bin 0 -> 1145 bytes tests/old_pickles/pypy_35/simple_class.pkl | Bin 0 -> 721 bytes tests/old_pickles/pypy_35/simple_enum.pkl | Bin 0 -> 237 bytes tests/old_pickles/pypy_35/simple_func.pkl | Bin 0 -> 509 bytes tests/old_pickles/pypy_35/simple_module.pkl | Bin 0 -> 494 bytes 30 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/old_pickles/cpython_35/class_with_type_hints.pkl create mode 100644 tests/old_pickles/cpython_35/function_with_type_hints.pkl create mode 100644 tests/old_pickles/cpython_35/simple_class.pkl create mode 100644 tests/old_pickles/cpython_35/simple_enum.pkl create mode 100644 tests/old_pickles/cpython_35/simple_func.pkl create mode 100644 tests/old_pickles/cpython_35/simple_module.pkl create mode 100644 tests/old_pickles/cpython_36/class_with_type_hints.pkl create mode 100644 tests/old_pickles/cpython_36/function_with_type_hints.pkl create mode 100644 tests/old_pickles/cpython_36/simple_class.pkl create mode 100644 tests/old_pickles/cpython_36/simple_enum.pkl create mode 100644 tests/old_pickles/cpython_36/simple_func.pkl create mode 100644 tests/old_pickles/cpython_36/simple_module.pkl create mode 100644 tests/old_pickles/cpython_37/class_with_type_hints.pkl create mode 100644 tests/old_pickles/cpython_37/function_with_type_hints.pkl create mode 100644 tests/old_pickles/cpython_37/simple_class.pkl create mode 100644 tests/old_pickles/cpython_37/simple_enum.pkl create mode 100644 tests/old_pickles/cpython_37/simple_func.pkl create mode 100644 tests/old_pickles/cpython_37/simple_module.pkl create mode 100644 tests/old_pickles/cpython_38/class_with_type_hints.pkl create mode 100644 tests/old_pickles/cpython_38/function_with_type_hints.pkl create mode 100644 tests/old_pickles/cpython_38/simple_class.pkl create mode 100644 tests/old_pickles/cpython_38/simple_enum.pkl create mode 100644 tests/old_pickles/cpython_38/simple_func.pkl create mode 100644 tests/old_pickles/cpython_38/simple_module.pkl create mode 100644 tests/old_pickles/pypy_35/class_with_type_hints.pkl create mode 100644 tests/old_pickles/pypy_35/function_with_type_hints.pkl create mode 100644 tests/old_pickles/pypy_35/simple_class.pkl create mode 100644 tests/old_pickles/pypy_35/simple_enum.pkl create mode 100644 tests/old_pickles/pypy_35/simple_func.pkl create mode 100644 tests/old_pickles/pypy_35/simple_module.pkl diff --git a/tests/old_pickles/cpython_35/class_with_type_hints.pkl b/tests/old_pickles/cpython_35/class_with_type_hints.pkl new file mode 100644 index 0000000000000000000000000000000000000000..6885eca4b3b2679c2bd6d9906071867911de7760 GIT binary patch literal 937 zcmah{O;6iE5H%1;070v^Dyk|FM{-G>57OkEL#0NfQb6jd8m-skc$JNvde@*-P;r5Z zYHw_>{XzW;J@eGR?!O;A4rk2YYK?K!))w z%$P<{W0a`V1h`~M*;{)Md*&HT*!W*>W=+qgA{kk;`34ir1V2K}D9eBJ7IH#cU#+i6D41`iIq z%}y(7cLTrfzwSnUhqqgPvlCfuhxW=|7yk>Q0uZJgzz&qzZ~2I9}{Yh)8hcs^z$bQ-w6s28{YZ$%fI z(8OFUZ6}O+0-JwXC{!NFG+A#9A#9-P)DX_!m2aocQ@?Npv2oHCiC z3*8}~-MbQ09?XN4{XmvtNs?;j^mY(&HGZN@DHL26fFE-v9XV?rFT7T}pO{&piBtJf z5fJa@dPpO@^B-o4+K8W$%RpyHE@UdEo3E9hpMU)bSG`N~5?H|JN(&ANj)=x70%trX zz)d(qHqJ0osL5aN>2OTDTVhQEc1UR^Mi4NC1=4e~LFyGUIN-KfFUpI_R8k6&tNj56 C0BU9c literal 0 HcmV?d00001 diff --git a/tests/old_pickles/cpython_35/function_with_type_hints.pkl b/tests/old_pickles/cpython_35/function_with_type_hints.pkl new file mode 100644 index 0000000000000000000000000000000000000000..554f24686b26b632533f68d1f01a3e9327d92227 GIT binary patch literal 1121 zcmah|U279T6y0rNo3^wH7K&I(@x`_eF<<=%;uk@R4UNzi1tko#v$MHtCcANGwl&g9 zL0YNITgN};U+~%Ap?5ZEEh4zE%gpS(_ndp~nUnmtS9vFU+GbhLGZCkLKV)6Oi`Ujh z-ysQyfuB&tiR^O0RU8o}sFdcyEQ71(NRM{^^-qthK|hQ-Mo(|?h$kW2;fi9Gp4b^Q zvI>~KNLa#c@jgvxkE1!?V?<>pw^!FR0LXv`eB&XMB~$MGF!&$LdBo}>yD@=|4^ zbzX827+5_sdQowyV&U6SJ|n>{0BAnDom?p^1)&Jt`#1r3*d}XeyLMgsJY~wcU6x_EW(P zZ0K3EolFx>4yZ`E4V_1yUAdCT)Efp%(?(0tD2f%edRqush1*i+{g9~)@O?^!C1>=l zk=Ke>6Vo}IIF`?60kD{=T^!-1|9B#?4VXih5tVRsA(en9Ukg7z{rv9Va8C4HLOnKA zI%Fi`9i(x<38M@UaGD%)+D&*Mxk7)v#?2nCE|4`2*doPA*dZR3JVTn-Q>b1hg9YBu zlUez&*B6+Aied- z|M&2rGMW@TMENFRLBgXJJZ`}g*np=kc-A)1K!?Cvn1%&i7}-`Dof}@u;U$7MAX0da RCAOdqFIun-ZRE_0zX3jYs-^${ literal 0 HcmV?d00001 diff --git a/tests/old_pickles/cpython_35/simple_class.pkl b/tests/old_pickles/cpython_35/simple_class.pkl new file mode 100644 index 0000000000000000000000000000000000000000..8d3366da147b9c3d7fe4f1ce98ab9189a920ceb0 GIT binary patch literal 684 zcmah{O>Yx15Ou@ShE^?yMyj|}J*JnCh6tMT-iVY#+bbH`YbR^gv6tAUic};nX)nHE zegMA|#+wClK*I7$=<6bN|bB9qE3LVgc8nZ zC=|6eevAE*gdQwu^)FxOGB1^r+QyK%buBO;9No%gtwi1=D4&OY!#;|_p=%Sr#Bcz> zO#u*RHe3wp+iQBw_|;@Y$9yyOxW_&Ked7#ERFMUqLIPSBOITbn2jWtnpc_&lImdrvwGM)^GaMHVy#npxLqpq5_2u zL9Q-Sycv5~oP0uEr+71!_He`l&CJY;kDt<`kYZq#VrpoTYLIGSVParxU}=_=YG!F- mo|KrHW|@>|ImNGJO3;+rDLq_3P5CLMIjKNx8N9_ML3#kZH&ROg literal 0 HcmV?d00001 diff --git a/tests/old_pickles/cpython_35/simple_func.pkl b/tests/old_pickles/cpython_35/simple_func.pkl new file mode 100644 index 0000000000000000000000000000000000000000..ee6525bb344bc239153a92b10faacbb5b7ecb7ca GIT binary patch literal 485 zcmah_u};G<5KY<&A_|4AVqtCv%12})QisBdPHy5hR-8E0Hc+WZEQRGZj-TUa;F4CT z8z))1_k4Qy?!EQCzQdp`9H!d%tX65E<>a6z9Kb?p4GUkTPML~6X_Nx^qU1X61R;&bj`eSUxL#?5eSlh|g;+`enqmaa z5l#QUI?q<3Y-hPG1s?9jTf&p#`Hz_5QE4)-CFPem z;5@BgNq7?4OUr(Sa1b$AKi>^odf@9sRT(FmteY+QDMtHHEn!zxj$^gh%T(%G6OUf# T#xUg<*v(AJ^1VIwZZ`P^-&4KG literal 0 HcmV?d00001 diff --git a/tests/old_pickles/cpython_35/simple_module.pkl b/tests/old_pickles/cpython_35/simple_module.pkl new file mode 100644 index 0000000000000000000000000000000000000000..ddbbcb02eb26e855765bb738316ec4242acb3588 GIT binary patch literal 473 zcmYjN%TB{E5KPhvLMw_m^%vSpzW{N`jYv5Zu4v^ZPGiN1Lu><;io~Vm;v36v@k_8t z3+hUicE?Wi<2DPb+)NlB~s9KC8|cX20zhF{@6-oR&sp9 znL7u_R0@EJ8v#hgGT!!R4A8K&WNY5T5I{HlP)IpV@AzWi{2U9BS2vyUz z8L?MTv66!>`QeLbVlQE3g*2iDvuSvDSQx2{KXhP+OPs|)?9mJ2Cj_#1Q37M*G%i)a&?fX{SV>_|e<^Pge0SGe=VEf8!`GJ4%%5NUD8^K|-)xscXv^yPt zzt!f4jYDkPzRSL(dMg0KEG#5arm2y^L^`{QRA{yR+j4d-IEqn{E|o8+^cGH(uY( zm(pt7P@B%6smi(k2VZ=hv9%P-EMQU*b*2_##0seu0aru8wR5hK{* z>nc`Vaq88~)28TYo#gjNte}cu8mjCEQW{IsOfyHftCs90kCmx?Dwr%m%%hp|>HxDn znU%r$eM^NtHVY7Ed2EvnvjHsNQ}zUh6k`HDL*R@jNVVV`*&xSAp(Y`{Bjbc_x6}u1 zjJAI#Y0Q=5?um0{xriZP3QN~B(lY2AN$X4{6!fk3@ym}F?-#r)vs@Mz!;vIF>Z|@6 DJ2h-j literal 0 HcmV?d00001 diff --git a/tests/old_pickles/cpython_36/function_with_type_hints.pkl b/tests/old_pickles/cpython_36/function_with_type_hints.pkl new file mode 100644 index 0000000000000000000000000000000000000000..69fe362b957f53390c6da71f8e2418c377c24975 GIT binary patch literal 1124 zcmah||4S4>6yMcTPbCu)BodYA2L-XSyS?)uin>DTa03-0Qis{yclXv?cdwmU?Lt8) z4P$0N8p|A83+G>U!^3K2R)Tr!r!?>;0UT6TM?5uSD0@e{%a@mGu~MygYpt?Z zDpy(+zvS?;16AT?U7SitChX=O0^1} zSFY9TPN`b+*URfr)2_)rrG68P9VS5*0cENpIXsL9W*CQ(Oqz3wwZbT3tt|Fsn8apK zI=y{^SAKQ9lb3~&q$*}DI#b0)62Oje&Z+5AbEs+2JJf76CQj@Vi&=rSZE5#ht8p#~ zHZPMVR22k7EsZM|TFpxkQaogB1HZvIi2~LqS*&(@8W)>y8bg+?mzT|AuYb~Gwwq+c zhTq|Bvz}}Ai-{Vu)Piwnt9Cw;iN_;B(5YeoEuKYEP~k!l`qCUuk>o^{ z0z2YS2IgwC53lK1FwJRJHB3y=)hx;Hw^*6Tynd+Ad!#fT$BEiGnoQuoeQL(Z^ivDKjwBvtTQ#9 zHxCcH5f!Al@ct4$z~yTp43iM1rFCK**eCXZJ!^fo`*OhXoO}DDR(6ksjwLnBgFrWT z{-zo$1A_1|K{bwZ0-w0}lakLe|}Wl_`f-iVY#%N32h>)EVX$6jK)C{mHQ1TMZ| zuKX&DHw*0n3GZ4OduBZEy_qlRkDr62TmHC|+VNJDjl}5#JNOr{#(Klo%p#ZuC0ebZ zl+2j$JxuDvk731{fBF2da6($4Oz^4Qv`7j5@J6g!iFx-x{n+p8;schJT-)$D_yYj0 zOMtL2{-nHmGdr(lm)Y6XMRu80Gx~}P&a>+LHJ)V{7B*Z{*2IqhR+>ABl<9|bU_$k# zP{NW)K#!mjQbOfaX@yoj{Ac&@I~QG3>{IBm!yn+oTyuOES!}83WImWDv)f)Z=-U50brb zdK!tn{Y5LXI|OK1*|24#w}|`{TSdEm9uLH-K{?3$aPH9@OO5=xMCM-!h7mAQ%YMF<=$2M b=$P#C{rJHkOab~f`BC%f9+9)x^iO{SGr0(t literal 0 HcmV?d00001 diff --git a/tests/old_pickles/cpython_36/simple_enum.pkl b/tests/old_pickles/cpython_36/simple_enum.pkl new file mode 100644 index 0000000000000000000000000000000000000000..7351bda2a293c0cd9c245aee9a34be7bdb5d9f4b GIT binary patch literal 225 zcmYj}Jr9B~7=-;G@heUa#?8@*7!l}fWH1^B66dD2B@jw6PzDDR7q@rw^{@GRL|MG) za?jn}t=GHN2&P=+mAE84Rnq?R2s2v9_$mr!BrQ{^q{%hqij^fkaS^v*LNk{BF{!KU zi@^Fh=MBjM}r~W5Qg~S`RfMO|u$qIZc{lR3 zXHQ_^gtS6|-L%|um}I)-H=)_moW^-CkNoT9S(ZFSPjMPuM>N~g1mj1-ctfv*=aw65 zE@8zLUo*==%Mvs@gxHX4+CLK3^ z|3RZmCplFq4uTP?&ahwV!jDqeOB;+^bYBhK`a_vg+OiOPSXdon@{_M2>G`VFiBy~h9m literal 0 HcmV?d00001 diff --git a/tests/old_pickles/cpython_36/simple_module.pkl b/tests/old_pickles/cpython_36/simple_module.pkl new file mode 100644 index 0000000000000000000000000000000000000000..1403335533a1e6dd1b5a3c078c4218bb0628a2f2 GIT binary patch literal 473 zcmYk2!A`?442Iiouxb-Soc0RaWlw;(^hTu~23KTdZMsHHo6;n}q#E z%FrxDihui;?eA^)_3e9yaI;h@n>8XWC10NW(OqTRnpGkNZP%h|RBP}P-SntdDzlQ~ zE6!aAK%E=_i-8*fWGbb9;wAtMOG~!tN{Jf-NX0VV9x^iNhWCwrj)mu@kP9gxw{>cS zs%hGs)Jv#X$w8OAFOf{@1+1-*M$}-o4eu5UBbD*{F4^M>=W!5w^ny5w!kxGCXWpHM z$sQNZe?jMO@trK!+O(qH++Z<|gDCKPTr3Q^Z+|H0vSO4JXT`vNkk%bK+DNr#Qd8o> zjOWZsqcQgP=!Tg{O{Y%@qP8uECnhb|bocomDuOfYIMFmWAVyT0!3pvXblSMc9NRyPDE(II>^>+3pR4iX7710dTk_o0| z0aOsB+z)q}2etz%viz6NwqnDD=7n;$sn;dL1bg^lJ*$)L5y+BZSy`bqW14HW@(~9Z z%8Lw*?^wYqPSJ=k&$ckW&JC$NwryOyMgQT!_Tq6ko<=DdPp0fVjKXM`QZkK3@sLL8 zG-OV@D|e4~Hf!HDZUYFC;8Wlh8)uJ*))kM9_Mx_^m`GiIcJJ&f&6HMeA@kb#nMskr zcCKnru3g@nf1K?F4cG5`V()+9@$8NPNoxk=UWL5M2UwQL=r`BO6 zWK4weg0~Wu5+k(2!p($J?Tsh6Wy(|xZiq0v&$6oC|P6}OJ+_*pc3ChIV zP~|>5drXQ#YU1VgX!%**QMUEP5K$MgFLoO@0NBmRwwGU@S~B_y#|E=5WZ_dO16d(}V+ zf}~Q|H`o8n|3_z&)QSi$EX>Z%%s2CW^EvDv3!TsFdcLDTAwTks%%Y>rWT_j0?qMX>?wVM+_76wYTHWELj&xGGbUZ zNzvP9iYqp>5z83L<32iHv6!ViKqtc7n8VmQr!=+L+U*%{`{F{E{Ccb2Y)7=-XtLc_ zyVb5n0d2Nxel=)E%@#At9GY{yQ*(xO2?>*c5aUT*+V^X$5$rYE`whBF_kva{^cf~q z!>CrTRqMvHxo3#BLdY-)Gr=%Z7w|!OyfAsp6`I)P8i}|NB+BAI@g%lwqW9(%MYFHV zi5C2jOL7XGBxF6y)D%I>-KDOBcd6UytW4aAL!8h#a*Q`Mx=58uxt}Q;s`HYG$iU9I z(RUS-N@7Hh$Jl)lwj@09;3?YAJm^3Pp2K^1;laye i1FumkID+f21CI95LTrAm(TYLC!`h11fSYX!!f}h-iTErdEOSNp8I`WeB{)kYa6}>e*nNV z1qch{&*(H`SLM5D$>{9zVwz=>l9jX|*~Qz*ESpWj!iH%wP@eF+_Q_yfG3YsPOQi!C-C&3p3%|K2>$2X}jS z2_!7JNy1_q{0n03S`^O4P@ftu%TS%&2Y+h0vE~gdx#F#!S~A#aZpO_fRKxjY-b?nv z`FSMv{uiFe?hv3Mv?j|)ZxQ(^vWj+f952YK<^8|%?x9obU@3J$q=|y}883;G)?ne3 z37iBz9^nQUx90GfNXMgVeFMUqLIPSBOITbn2jWtnpc_&lImdrvwGM)^GaMHVy#npxLqpq5_2u zL9Q-Sycv5~oP0uEr+71!_He`l&CJY;kDt<`kdkU?YLsGPY+_`ZY+_+)nPh5_nwDZ{ mkz#I{Vr*b+ImNGJO3;+rDLq_3P5CLMIjKNx8N9_ML3#kUPEq{; literal 0 HcmV?d00001 diff --git a/tests/old_pickles/cpython_37/simple_func.pkl b/tests/old_pickles/cpython_37/simple_func.pkl new file mode 100644 index 0000000000000000000000000000000000000000..a6b82cd59b775835d80730d0603a7bba8ea3ccc7 GIT binary patch literal 485 zcmah_u};G<5KY<&A_|2q8*@8QJ|Y{DIuurPauc_);@F|Kfl45;6qegKevY33y8+aV zlPtY^XT5v(UIrhZakMK;3#nb%ilUZ$x`zozuoP0l(y77~S`RfMO|u$qIZc{lR3 zXHQ_^gtS6|-L%|um}I)-H=)_moW^-CkNoT9S(ZFSPjMPuM>N~g1mj1-ctfv*=aw65 zE@8zLUo*==%Mvs@gxHX4+CLK3^ z|3RZmCplFq4uTP?&ahwV!jDqeOB;+^bYBhK`a_vg+OiOPSXdon@{_M2>G`VFiBy~h9m literal 0 HcmV?d00001 diff --git a/tests/old_pickles/cpython_37/simple_module.pkl b/tests/old_pickles/cpython_37/simple_module.pkl new file mode 100644 index 0000000000000000000000000000000000000000..1403335533a1e6dd1b5a3c078c4218bb0628a2f2 GIT binary patch literal 473 zcmYk2!A`?442Iiouxb-Soc0RaWlw;(^hTu~23KTdZMsHHo6;n}q#E z%FrxDihui;?eA^)_3e9yaI;h@n>8XWC10NW(OqTRnpGkNZP%h|RBP}P-SntdDzlQ~ zE6!aAK%E=_i-8*fWGbb9;wAtMOG~!tN{Jf-NX0VV9x^iNhWCwrj)mu@kP9gxw{>cS zs%hGs)Jv#X$w8OAFOf{@1+1-*M$}-o4eu5UBbD*{F4^M>=W!5w^ny5w!kxGCXWpHM z$sQNZe?jMO@trK!+O(qH++Z<|gDCKPTr3Q^Z+|H0vSO4JXT`vNkk%bK+DNr#Qd8o> zjOWZsqcQgP=!Tg{O{Y%@qP8uECnhb|bocomDuOfYIMFmWAVyT0!3pvXbl-OA814 z=KB9SoAj`6W(mxEJ3BMqH}mW6pV=L!{`6^-=2cvXXqmF%fo=3tSmBa^T(Xp@JcB4j zDa~(lt~+5RQYA8J^pRQ@j6i+-c|EHG^aQVT@DH4c}4t6eD-yj@)?czaM)sx+bvk`sD%5L&}h#R7O8;n1WT0!va#v9d;cTT^Xl^XSR&FF1@skpKVy literal 0 HcmV?d00001 diff --git a/tests/old_pickles/cpython_38/function_with_type_hints.pkl b/tests/old_pickles/cpython_38/function_with_type_hints.pkl new file mode 100644 index 0000000000000000000000000000000000000000..269639184331a403c3c59e86054e099a364f9593 GIT binary patch literal 1017 zcmah{ZHv=D5KgYAy{m7BCmx)lcL>s=)YNNQD}qSXL#h$(%6*ZtY?5s!*EFfwjaq0y zkQNU1o9o~7H#nQL+HYnF%se|gGtV>g{nD@B1ta@eej2VKm_+iq{0^~M{$1I^DP7ob15ePQY5tlJE zhz1BAvxp^uhai~pT3$xy`9hN{zu8qz<>EpfST=3;8%?)Hj~aEm)@U{D7P|F1s~veQ z+fzcFs0IFWHB<7yvs*UxY16jcrx@O(p4G5wmS1l@tJ^-cl%wP2-8Gu)lu4g*q2!I6 zDd9|TL4{@q`!l3nB4iZ%X~+msatj?AL`yS>0)b{mXDJR@DN}k+N3Ku1TBteJFT(@5 z)sB7k59w@l3_Kfn+U?TRoEpUNjZ;H8b0xQ_5J}*s*a>=@oP~V_j~7bb6O0S~gbcKt z*eHoZpX8I|)$v5ZVdrCe!!T8)qIqiyR|pwXZ%7B4gQH1Abfj%K&`v+d$^9>L>NuH{ za7W%G{WS7KfD5uN*z=jL2MsusR|)x?(r{Tw?kpFjO2|?*RGiQ#icxG#!?Bwg5`a#vgL!og;CtLU7J~m!NEcUW% z^W5Asewa8%cm&04X682~%?Z8)OBSf_t4?=qL{MJ|`vN|#6g-2Ma0aimoTpHNeb|(R lAQHHidh?+RZVGJ&UOQS0>=xcSaNJYy4pl6|KF^u4>L0ywd%^$! literal 0 HcmV?d00001 diff --git a/tests/old_pickles/cpython_38/simple_class.pkl b/tests/old_pickles/cpython_38/simple_class.pkl new file mode 100644 index 0000000000000000000000000000000000000000..bc22f06d80c53ba76a016681d64ad36edaa73ce0 GIT binary patch literal 705 zcmah{OK;Oa5Oza>kkTTx5xpSb*c=EUiIVi}V~`ImS2SAN>+F*CquE`dN{!S@a@rfi zU*Y#LYp23DyqEcAt@$3akI#Sq=>+xXE)!MSw35kM@nNe**N2>~Ip{T4+>`|*ifN6% z@XB>#E0vLjMt5xPD^5*!ax3#n@v#0$JaaF~_y; zrC%oFH*|sY83}S*CoJY zaVTXW4V5Brr*IGsPpW%gSsthCn>Vo|{wwj;b;B~{|2oUgGN99;6NbHqqlW;3G`I~g z+M#>JjM>Q8k`hA4nyU=OhdsLYhHIlg!-^MtLqefcDKshTVRet9znq4h;0Ol;@7r5A z0jOBAW-IR@@+JzF`&wLhkjci}|0dfgs-qNV?i?~(Bu3Km+JSxAdOk4_efJvR&N6i% za{bm4hX9A+yea z!jhXleLCHH_kH<$Jv!?L??j_r@llDok>MZ{-a^Z338t2fv|52wuB8(_;J8m1yeOOf zcRz&F&6nbBCHkfe4u0<<&o;Cj;rMy?meKay;RO86N&yhglXWcCY_W{RB8rnJ~1t+Wcs9q35MJPQ8+F41MU literal 0 HcmV?d00001 diff --git a/tests/old_pickles/cpython_38/simple_func.pkl b/tests/old_pickles/cpython_38/simple_func.pkl new file mode 100644 index 0000000000000000000000000000000000000000..df7c83a5db954f528a35dec1ddf5d869900c0493 GIT binary patch literal 533 zcmaixO-sW-5QdY87+SR;XhFntd$9T+a*X7lUS-%Mn0;lv0)R8`HT{7N&- zhVmi$duKJ_QBfIgqVXJH@t(>|SY_J6sBLx-hwhetfcL9Y+pU#f;R#pNNmtCCFub;M62){&DAMeiClAv zdz_gNz#=ICvXMCl$VF!T)LZ~GB-@a+drD0Nzy--^+eNlWHna!&1v1ZE!b+E1l{Khs zL*20GnN+Ksyt?p!OPsNZ31gAd|4b4jp0&VQ9EUq^=TE&C57PtAjsJLVVP5B6T-;t#qL!h8S# literal 0 HcmV?d00001 diff --git a/tests/old_pickles/pypy_35/class_with_type_hints.pkl b/tests/old_pickles/pypy_35/class_with_type_hints.pkl new file mode 100644 index 0000000000000000000000000000000000000000..fd9404f27da40656c56868907ab059abc79ed9b7 GIT binary patch literal 914 zcmah{&2AGh5ZA%>|oEUcYyD_BmvHvWPO& z9MqIcuG0jVM5WvpcQ^^`Q<&lOUvG9pFXkc{JG=9WCp;7EglnWMyLqx0%f8+#Bk&W`)7&QTW| z?M|8|qsC5-|Qr3}E0ls&*M_S(%(+~;A`#zuVH?S~;} zP1a|vUZdS}BcFXq{k8#SX=Eg)Oj{uXMN(WvA~aoT&OL~Qln|SQX^|#=Q1U*!<12r< z#S|JLHC6Gj$Y(2qG~#F8SW?r&VQCo9R~ilmyBBN0MG!y`L_zS@jh4=Cp(f@+qkXh3 zD=y=|oLA0PHCI}_gt6a+Cp?uAtTwLd^F)pr{zb=Q-T~SXo8s7m z9(mLWEzCo$dV{d$Z2Yf}xh^=|9LqFBsfbWpiFk}gYDI<1kzm?8_ehA!WE{>>8t$Jq zR;2rmW#9b%&~}{`NiqAR&OcfPWx`dCa_>lpm?Wu2zm2@@J=!LZm94xtC|7kK6%wrj z6r|Yg>4&>1jXtIQ-inrI%JcBVyV6XIAwk>oT+(b*u>KoWDPs%( literal 0 HcmV?d00001 diff --git a/tests/old_pickles/pypy_35/function_with_type_hints.pkl b/tests/old_pickles/pypy_35/function_with_type_hints.pkl new file mode 100644 index 0000000000000000000000000000000000000000..a60ffc05e9865803d366774d706328ddbd79e4c7 GIT binary patch literal 1145 zcmah|-D(p-6y9w@o3^wH7J^XPB3>jSu}SmO+=vpT(vU6Gf+&<>c6K(iX0jW1W~-4F z1Zkx*caHDjwU6PG_y&4rcWbK%U6^I&%=!Dy_sxgm*RW`%-=VIkGT~9;O#(9Fw032$ z)om10HuYjGDH0>fsfg= z45H993f3E!^GdHSFu6&Bg_+Ou4c2Ln~FtK{A$yx07QaM8Pxel%Sa?{79;c_2oZ;2B(Y&5Eqi5b z?3PfaHworh-kZ5ZYY62-Q46EDmt7`uANaixEscel!-_FQaWF(K7Bn@LL9Ix}6x56b z1EJLVcPi{It6orD0IFA{WvG4)gxls{{%GfnU%sLH zK(74`))ZuYJz^THmUX!e4w3iilhk73_)*b}zSo~ksG?Kte&p=P@v i3HFLDvQ<^er7N^5WKTAGVC*Fe6E?`>_8fpnYrg?t&bJf* literal 0 HcmV?d00001 diff --git a/tests/old_pickles/pypy_35/simple_class.pkl b/tests/old_pickles/pypy_35/simple_class.pkl new file mode 100644 index 0000000000000000000000000000000000000000..cddb4e6452bf9f97392876703cabc5e0393a83b3 GIT binary patch literal 721 zcmaiy&2AGh5P;nfwV_qZB~rx&aZE2kQPY$??~O<~v|Q21yE~hhb?ha!L6KUCOWKQX zm{;PpV7$A~AR%GxO5^eTeKYg5|NTdgHOHUlQoEuSc_neYCkOu$Hn`dr8*ULyg%Yh+ zkV|e%_!dSh=7+H6)xX~SAag=mp-k|7yRDHN{K>6Y*Amkvz`n(y&OTyp$z~VsgFgZ& zbPf;}#=lOkCs)(SEWw=TQ+}OfZ)at~FSFzd%gO9A2@4x;$ZO<}0oJ;363NpK=)#5S zM4^Nwkt#leQb-A^8N1Poc9nE`lMt^Ufrl)rY z?2a*DtYGXmEOx;^=hkjS=4`a~eS@<6OZgD|D~raOH?Ty7O_#MSpq+#n*V_i8t9?FA zd+Z=wTtwV=Kj}o~6M&lM6<+&@ z4F&EOg62cYGg=ViHVB`&bQqrw+os2i{g`~eX#b1`NjB%SHGg&tjLTXd6F$&ZaHX{6 faT;yfPmaqr&~R^I#sFO>{BiZ=sWfMA=o|k6=ZF~_ literal 0 HcmV?d00001 diff --git a/tests/old_pickles/pypy_35/simple_enum.pkl b/tests/old_pickles/pypy_35/simple_enum.pkl new file mode 100644 index 0000000000000000000000000000000000000000..f25ef6075177fb15b64cbb504197c78aee7cb855 GIT binary patch literal 237 zcmY+6I}5@v6oq{h6a^OtadUJMrHR&6+CdbDf^&#TprvUlHXR%U7q`24{e%7zYhCJ@ z&V%pVo!7hZ6O4q;3VDu2qSdVC5ytGIqN}`ct5}{Wt!!qP&^*uaiPNY7LzeQSwkcDj z-wB+`L1$}h@arB3egA5B8jGs=Pq&h3hk=i~CODh@F~D7`fG#WlW5XE2WFbAuDOH}! l3E}SCqr{a9LCEq%SL->!gT)YU2)z=@WT91wMT6XivkwfHR-XU> literal 0 HcmV?d00001 diff --git a/tests/old_pickles/pypy_35/simple_func.pkl b/tests/old_pickles/pypy_35/simple_func.pkl new file mode 100644 index 0000000000000000000000000000000000000000..3885519c586b0292354e82e6cf71e00a0c793a1d GIT binary patch literal 509 zcmah`u};G<5KY<&A_`^9#@r5+kH|)(4uut++{A6IICiLQpi+@o3d?O=f0Q2pyFsWz zNStKp?%8+m-SbOt_Yp_?z_gIsm5nGW$;Ssd;Q;1BN|-xUSfLfY({LH#In=D;U@G2T zdDx?8Fmpm$p}?*i?lDXRB>mGHuH zW6c#TxZ*2jIcQlztIUj>HDTX~x{-V@iSo;=6U8(dSwH{!)5kLnu@BI&qGAgUKod*= z6stoX(wcdJZR%IsqyKTbH$OcLK3+&YW77BnlakMwlh#lkZwY&)C~Uw7-aq4393Gf- z+<5+@T9;08TBg_wLVoRl-BK6+SqcZvgK@LA&Cs21U`lDr!WRVc5-oJ^rYjI6EDG0bL@Iy73XMRBk25 zSDd;CfS4Qr*L`;mkgJUH)QtcdmX&PfTd5lYNX2sA9x4*)hIhoUz;)opPzWiZuytmH zs%hGk&`YRT$w8OAYY|N71uU(QM$}+74fl&hvC8?qFLt=VY0^sqdc7o$qiqmu!(eiU z=?-Tud_fm(a8HtJZCX*UZg4$JdT}oZaW*rgzWt`~m;Jlo^icawM9(>2Nws8BQ{keV z7tBhdF$s6*2D!+L$H(NXwk?MzCN0-w#@Vlb Date: Sun, 21 Jun 2020 17:17:06 +0200 Subject: [PATCH 47/89] stop generating pickles inside the CI --- .github/workflows/testing.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index a4fcc82ee..2b5f067bc 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -68,14 +68,14 @@ jobs: - name: Display Python version shell: bash run: python -c "import sys; print(sys.version)" - - name: Generate old pickles (backward compat) - shell: bash - run: | - git_head=$(git rev-parse HEAD) - cp tests/generate_old_pickles.py tests/_generate_old_pickles.py - git checkout v1.4.1 - python tests/_generate_old_pickles.py - git checkout ${git_head} + # - name: Generate old pickles (backward compat) + # shell: bash + # run: | + # git_head=$(git rev-parse HEAD) + # cp tests/generate_old_pickles.py tests/_generate_old_pickles.py + # git checkout v1.4.1 + # python tests/_generate_old_pickles.py + # git checkout ${git_head} - name: Look for syntax errors/undefined names shell: bash run: | From 3b984920383c028037776b847ab34f31e17580c9 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Sun, 21 Jun 2020 17:29:52 +0200 Subject: [PATCH 48/89] fix use correct pypy version --- .../pypy_35/class_with_type_hints.pkl | Bin 914 -> 0 bytes .../pypy_35/function_with_type_hints.pkl | Bin 1145 -> 0 bytes tests/old_pickles/pypy_35/simple_class.pkl | Bin 721 -> 0 bytes tests/old_pickles/pypy_35/simple_enum.pkl | Bin 237 -> 0 bytes tests/old_pickles/pypy_35/simple_func.pkl | Bin 509 -> 0 bytes tests/old_pickles/pypy_35/simple_module.pkl | Bin 494 -> 0 bytes .../pypy_36/class_with_type_hints.pkl | Bin 0 -> 927 bytes .../pypy_36/function_with_type_hints.pkl | Bin 0 -> 1111 bytes tests/old_pickles/pypy_36/simple_class.pkl | Bin 0 -> 669 bytes tests/old_pickles/pypy_36/simple_enum.pkl | Bin 0 -> 225 bytes tests/old_pickles/pypy_36/simple_func.pkl | Bin 0 -> 454 bytes tests/old_pickles/pypy_36/simple_module.pkl | Bin 0 -> 473 bytes 12 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 tests/old_pickles/pypy_35/class_with_type_hints.pkl delete mode 100644 tests/old_pickles/pypy_35/function_with_type_hints.pkl delete mode 100644 tests/old_pickles/pypy_35/simple_class.pkl delete mode 100644 tests/old_pickles/pypy_35/simple_enum.pkl delete mode 100644 tests/old_pickles/pypy_35/simple_func.pkl delete mode 100644 tests/old_pickles/pypy_35/simple_module.pkl create mode 100644 tests/old_pickles/pypy_36/class_with_type_hints.pkl create mode 100644 tests/old_pickles/pypy_36/function_with_type_hints.pkl create mode 100644 tests/old_pickles/pypy_36/simple_class.pkl create mode 100644 tests/old_pickles/pypy_36/simple_enum.pkl create mode 100644 tests/old_pickles/pypy_36/simple_func.pkl create mode 100644 tests/old_pickles/pypy_36/simple_module.pkl diff --git a/tests/old_pickles/pypy_35/class_with_type_hints.pkl b/tests/old_pickles/pypy_35/class_with_type_hints.pkl deleted file mode 100644 index fd9404f27da40656c56868907ab059abc79ed9b7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 914 zcmah{&2AGh5ZA%>|oEUcYyD_BmvHvWPO& z9MqIcuG0jVM5WvpcQ^^`Q<&lOUvG9pFXkc{JG=9WCp;7EglnWMyLqx0%f8+#Bk&W`)7&QTW| z?M|8|qsC5-|Qr3}E0ls&*M_S(%(+~;A`#zuVH?S~;} zP1a|vUZdS}BcFXq{k8#SX=Eg)Oj{uXMN(WvA~aoT&OL~Qln|SQX^|#=Q1U*!<12r< z#S|JLHC6Gj$Y(2qG~#F8SW?r&VQCo9R~ilmyBBN0MG!y`L_zS@jh4=Cp(f@+qkXh3 zD=y=|oLA0PHCI}_gt6a+Cp?uAtTwLd^F)pr{zb=Q-T~SXo8s7m z9(mLWEzCo$dV{d$Z2Yf}xh^=|9LqFBsfbWpiFk}gYDI<1kzm?8_ehA!WE{>>8t$Jq zR;2rmW#9b%&~}{`NiqAR&OcfPWx`dCa_>lpm?Wu2zm2@@J=!LZm94xtC|7kK6%wrj z6r|Yg>4&>1jXtIQ-inrI%JcBVyV6XIAwk>oT+(b*u>KoWDPs%( diff --git a/tests/old_pickles/pypy_35/function_with_type_hints.pkl b/tests/old_pickles/pypy_35/function_with_type_hints.pkl deleted file mode 100644 index a60ffc05e9865803d366774d706328ddbd79e4c7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1145 zcmah|-D(p-6y9w@o3^wH7J^XPB3>jSu}SmO+=vpT(vU6Gf+&<>c6K(iX0jW1W~-4F z1Zkx*caHDjwU6PG_y&4rcWbK%U6^I&%=!Dy_sxgm*RW`%-=VIkGT~9;O#(9Fw032$ z)om10HuYjGDH0>fsfg= z45H993f3E!^GdHSFu6&Bg_+Ou4c2Ln~FtK{A$yx07QaM8Pxel%Sa?{79;c_2oZ;2B(Y&5Eqi5b z?3PfaHworh-kZ5ZYY62-Q46EDmt7`uANaixEscel!-_FQaWF(K7Bn@LL9Ix}6x56b z1EJLVcPi{It6orD0IFA{WvG4)gxls{{%GfnU%sLH zK(74`))ZuYJz^THmUX!e4w3iilhk73_)*b}zSo~ksG?Kte&p=P@v i3HFLDvQ<^er7N^5WKTAGVC*Fe6E?`>_8fpnYrg?t&bJf* diff --git a/tests/old_pickles/pypy_35/simple_class.pkl b/tests/old_pickles/pypy_35/simple_class.pkl deleted file mode 100644 index cddb4e6452bf9f97392876703cabc5e0393a83b3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 721 zcmaiy&2AGh5P;nfwV_qZB~rx&aZE2kQPY$??~O<~v|Q21yE~hhb?ha!L6KUCOWKQX zm{;PpV7$A~AR%GxO5^eTeKYg5|NTdgHOHUlQoEuSc_neYCkOu$Hn`dr8*ULyg%Yh+ zkV|e%_!dSh=7+H6)xX~SAag=mp-k|7yRDHN{K>6Y*Amkvz`n(y&OTyp$z~VsgFgZ& zbPf;}#=lOkCs)(SEWw=TQ+}OfZ)at~FSFzd%gO9A2@4x;$ZO<}0oJ;363NpK=)#5S zM4^Nwkt#leQb-A^8N1Poc9nE`lMt^Ufrl)rY z?2a*DtYGXmEOx;^=hkjS=4`a~eS@<6OZgD|D~raOH?Ty7O_#MSpq+#n*V_i8t9?FA zd+Z=wTtwV=Kj}o~6M&lM6<+&@ z4F&EOg62cYGg=ViHVB`&bQqrw+os2i{g`~eX#b1`NjB%SHGg&tjLTXd6F$&ZaHX{6 faT;yfPmaqr&~R^I#sFO>{BiZ=sWfMA=o|k6=ZF~_ diff --git a/tests/old_pickles/pypy_35/simple_enum.pkl b/tests/old_pickles/pypy_35/simple_enum.pkl deleted file mode 100644 index f25ef6075177fb15b64cbb504197c78aee7cb855..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 237 zcmY+6I}5@v6oq{h6a^OtadUJMrHR&6+CdbDf^&#TprvUlHXR%U7q`24{e%7zYhCJ@ z&V%pVo!7hZ6O4q;3VDu2qSdVC5ytGIqN}`ct5}{Wt!!qP&^*uaiPNY7LzeQSwkcDj z-wB+`L1$}h@arB3egA5B8jGs=Pq&h3hk=i~CODh@F~D7`fG#WlW5XE2WFbAuDOH}! l3E}SCqr{a9LCEq%SL->!gT)YU2)z=@WT91wMT6XivkwfHR-XU> diff --git a/tests/old_pickles/pypy_35/simple_func.pkl b/tests/old_pickles/pypy_35/simple_func.pkl deleted file mode 100644 index 3885519c586b0292354e82e6cf71e00a0c793a1d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 509 zcmah`u};G<5KY<&A_`^9#@r5+kH|)(4uut++{A6IICiLQpi+@o3d?O=f0Q2pyFsWz zNStKp?%8+m-SbOt_Yp_?z_gIsm5nGW$;Ssd;Q;1BN|-xUSfLfY({LH#In=D;U@G2T zdDx?8Fmpm$p}?*i?lDXRB>mGHuH zW6c#TxZ*2jIcQlztIUj>HDTX~x{-V@iSo;=6U8(dSwH{!)5kLnu@BI&qGAgUKod*= z6stoX(wcdJZR%IsqyKTbH$OcLK3+&YW77BnlakMwlh#lkZwY&)C~Uw7-aq4393Gf- z+<5+@T9;08TBg_wLVoRl-BK6+SqcZvgK@LA&Cs21U`lDr!WRVc5-oJ^rYjI6EDG0bL@Iy73XMRBk25 zSDd;CfS4Qr*L`;mkgJUH)QtcdmX&PfTd5lYNX2sA9x4*)hIhoUz;)opPzWiZuytmH zs%hGk&`YRT$w8OAYY|N71uU(QM$}+74fl&hvC8?qFLt=VY0^sqdc7o$qiqmu!(eiU z=?-Tud_fm(a8HtJZCX*UZg4$JdT}oZaW*rgzWt`~m;Jlo^icawM9(>2Nws8BQ{keV z7tBhdF$s6*2D!+L$H(NXwk?MzCN0-w#@VlbMu3cINHw*y(QFnT6Cq;sp$5 zZjAqr|Czqo7SY5>b~EqI`}KYA`&$3umV@fpZLacCOeG&HZ2on3_B9lk&P2fsf*zwn zlV`vc)7ss;{WP#IVZz4$d$X%%HkDc8?B+Ynu#o%&4Wlf(b}}1O6)1fdc!9q7lof1( zrjTDy0A(ViGBXsOuGqCj0dEEN$-JYFxnPAK!!Dh>(ZSuF-Rt-J?U={C_$cahx}6@2 z!=rXz9EMD^d!aaT#+|wwcU%3hX%qmFa{xEecCX7re%R`^;6^vjt}{clMWykfwysgd|2S`* z-8HCy{D!WRgf@f4{STp}A67;7{no)2W4(e0k*pe@nHA7qS$ s_T;?quDp;5L`-AldPVvLQy}S^>6C(wzyJJHv+Gq~IhiV|MI$wT1D8E#ga7~l literal 0 HcmV?d00001 diff --git a/tests/old_pickles/pypy_36/function_with_type_hints.pkl b/tests/old_pickles/pypy_36/function_with_type_hints.pkl new file mode 100644 index 0000000000000000000000000000000000000000..d310462a0d1c947d57c3c129c2125854dc0bc4ce GIT binary patch literal 1111 zcmZuwT~8B16y0qr6o^P72{95N#s``hpe?QR!Dy^sp|B=E;)~5>Iy2k5blI&tvtSJ* zCI$><-WdOn|4i@f7E$9SySewy+^=)aelC6ADLL8GHLH4og>lM<0ULy3?GM}Ndn6Gs z;t5rP$bkrjiX*~8Dy6wHtKjN+azqFJJJSd;_NT?+EBsjrLK_G|}UWW-iIWAy5SKrP%+B?TIv14fWz<8C*OTE%+wQ4@| zoBmdBbF;D8q<(#?*5Vs=%4^L!-!jVVnR9cIea~b4OV9x)ke*K+^TLi z8?{zd)O)aB*Lb-|5iv(K>KG$BVZPeXy3x{MADqOlo7fkG4KoF#q`hQv=Jrh+)K zgJSf~4PM#N^-jGs2o+YbYhg^6+c6gh);Y(ftDQo}!MD(9x2I3tQ-?U**>{Y0ZuAsY z2wN{z_Np#P5&C}xFN|JNLMpjV`j!Zbb{uk&(=FGAqil!q;GMVag5#P>#qPTO6CbLuyNwY;^-QT|{>9XaObnTcAXyz5J>)T~rs1qF7N2vF!%ya98T`hm?j{ z5j{6n+ZrL`74`JsNH5P}N1sx#=AIOCvPL z+`tn=X$lr4JoVri+Rr^`!!o=80xvyy)iv-2IfwT!gMgIB!jwiA^B*$*eV_|*3fp-% K*zs^?vGxnFjooSl(?ziqE zh-h>b`PtgLH^i7F&#VchF4jyG{{$bs`(T(h`UK`ovPM}YC~VZ%gL>6)Xn{}1X(!t9 z$H#%|=ihh&;az~5X&4!Y>Aw8rUg35yb%-zE>Airq*``$c8{9CFQW+8?-rV-Z a64CU#gBQym0JIEUulo9o$eK(127drldIOsP literal 0 HcmV?d00001 diff --git a/tests/old_pickles/pypy_36/simple_enum.pkl b/tests/old_pickles/pypy_36/simple_enum.pkl new file mode 100644 index 0000000000000000000000000000000000000000..64cbbdddc6ec94d36136d41f74f3be7ace4d96e4 GIT binary patch literal 225 zcmYj}JqyAx7=-;O_=S^$xH&o%D-r7|c94og!8xQ!N=wsJY&tjyE^hDU>tFNtSnblA zj(hIzVZA@CS}^4*FU2L{sgicXBh2Vk#y4?gM$#gcN}60#u2@mv3m0(n;jD#=CxZdr5r)+$7p0QbEFKpou)o$BQquqc literal 0 HcmV?d00001 diff --git a/tests/old_pickles/pypy_36/simple_func.pkl b/tests/old_pickles/pypy_36/simple_func.pkl new file mode 100644 index 0000000000000000000000000000000000000000..1761a387a18ee87832894a106086e35642876bd7 GIT binary patch literal 454 zcmYjNu};G<5KY<&A_|2q8*@8QJ|Y{DIuurPauc_);@F|Kfl5VUDJ-{fd<`E5b_1v< zS$g-*clYkS4nDu)XdjpsQoFJhMJ@UC027X2DWrs@Q-u{;(FcvI2+yHmH3w7kuHqGIKttFU%6bb8CJ&Cu Pzo{qn)4!^jv-9Z>6NIqV literal 0 HcmV?d00001 diff --git a/tests/old_pickles/pypy_36/simple_module.pkl b/tests/old_pickles/pypy_36/simple_module.pkl new file mode 100644 index 0000000000000000000000000000000000000000..1403335533a1e6dd1b5a3c078c4218bb0628a2f2 GIT binary patch literal 473 zcmYk2!A`?442Iiouxb-Soc0RaWlw;(^hTu~23KTdZMsHHo6;n}q#E z%FrxDihui;?eA^)_3e9yaI;h@n>8XWC10NW(OqTRnpGkNZP%h|RBP}P-SntdDzlQ~ zE6!aAK%E=_i-8*fWGbb9;wAtMOG~!tN{Jf-NX0VV9x^iNhWCwrj)mu@kP9gxw{>cS zs%hGs)Jv#X$w8OAFOf{@1+1-*M$}-o4eu5UBbD*{F4^M>=W!5w^ny5w!kxGCXWpHM z$sQNZe?jMO@trK!+O(qH++Z<|gDCKPTr3Q^Z+|H0vSO4JXT`vNkk%bK+DNr#Qd8o> zjOWZsqcQgP=!Tg{O{Y%@qP8uECnhb|bocomDuOfYIMFmWAVyT0!3pvXbl Date: Mon, 29 Jun 2020 18:26:39 +0200 Subject: [PATCH 49/89] Remove commented lines --- .github/workflows/testing.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 2b5f067bc..687ad079f 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -68,14 +68,6 @@ jobs: - name: Display Python version shell: bash run: python -c "import sys; print(sys.version)" - # - name: Generate old pickles (backward compat) - # shell: bash - # run: | - # git_head=$(git rev-parse HEAD) - # cp tests/generate_old_pickles.py tests/_generate_old_pickles.py - # git checkout v1.4.1 - # python tests/_generate_old_pickles.py - # git checkout ${git_head} - name: Look for syntax errors/undefined names shell: bash run: | From d661ce64297bb909370383feafaad2f8ea2adeed Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Tue, 30 Jun 2020 09:22:49 +0200 Subject: [PATCH 50/89] tornado is already in dev-requirements.txt --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 687ad079f..a7004bc50 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -63,7 +63,7 @@ jobs: export - name: Install supported dependencies (only test in Python 3.6) shell: bash - run: python -m pip install typing-extensions tornado + run: python -m pip install typing-extensions if: matrix.python_version == '3.6' - name: Display Python version shell: bash From 4de14b2a9392ddc7f80ac30ccfc47a4315d35fbe Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Tue, 30 Jun 2020 10:44:54 +0200 Subject: [PATCH 51/89] Test nested function in old_pickles --- tests/generate_old_pickles.py | 10 +++++++--- tests/old_pickles/cpython_35/nested_function.pkl | Bin 0 -> 513 bytes tests/old_pickles/cpython_36/nested_function.pkl | Bin 0 -> 513 bytes tests/old_pickles/cpython_37/nested_function.pkl | Bin 0 -> 513 bytes tests/old_pickles/cpython_38/nested_function.pkl | Bin 0 -> 590 bytes tests/test_backward_compat.py | 13 +++++++++---- 6 files changed, 16 insertions(+), 7 deletions(-) create mode 100644 tests/old_pickles/cpython_35/nested_function.pkl create mode 100644 tests/old_pickles/cpython_36/nested_function.pkl create mode 100644 tests/old_pickles/cpython_37/nested_function.pkl create mode 100644 tests/old_pickles/cpython_38/nested_function.pkl diff --git a/tests/generate_old_pickles.py b/tests/generate_old_pickles.py index e6ddee5ae..c5a1d1c44 100644 --- a/tests/generate_old_pickles.py +++ b/tests/generate_old_pickles.py @@ -29,13 +29,13 @@ def dump_obj(obj, filename): cloudpickle.dump(obj, f) -def nested_function_generator(): +def nested_function_factory(): a = 1 - def nested_func(b): + def nested_function(b): return a + b - return nested_func + return nested_function if __name__ == "__main__": @@ -85,3 +85,7 @@ def add(x: MyClass[int], y: MyClass[int]): return MyClass(x.attribute + y.attribute) dump_obj([MyClass, add], "function_with_type_hints.pkl") + + # Locally defined closure + nested_function = nested_function_factory() + dump_obj(nested_function, "nested_function.pkl") diff --git a/tests/old_pickles/cpython_35/nested_function.pkl b/tests/old_pickles/cpython_35/nested_function.pkl new file mode 100644 index 0000000000000000000000000000000000000000..a71d0e425f4f3a59c2a8c75d1d6dee104055f5fd GIT binary patch literal 513 zcmah`u};G<5Ott}h>CV#$i&3bE`bj~EFB|tD6Ht@*iIY8jYDmNDuKjO!EhVqSDCpa z163C+SvtRa_wL>KeenGgvF>0ZwDGbjMWt1`7Y7_cQECl^uZ1g39lpY3#c&L3UMaAZ z>Qe+0dJefSwJU3I+eQ(F@ytl|&^~ry5hhubF}g?D#Vr10?3JO2korVAn7ZqQ&?79CS*Y++wAOuWc@6W*cd~KxH WohLl$CV#C<_xyy97P}v2={op|GNp<2Y>;Cl0j@sss{C1;cHaUuEXp z46V9g$kTqA6$8~^RcONK+Jc_qPA zvU?(!&~wOKschMRUAK}jjHf!6kD=N4xgV!d%IF@Y7t{EQy|EE{V1Bmo6O1yNz$+__ zH8%^{$Q8E|w926~Ws-L7aoEti%y(|m>**1TJVu=86H6Lt^zO&l2WYvd_(B5s8O8t& zuR|Tmnv=t>q`z&^e}vs9Z4P@+7fNSb8EP=jWx<`Yh6H+fDQo}-6rOP_2`^kZY5Wrz z;&@W)+$rgom)HwN{BnrLxfaA6!Cu^8T-J>SCiZ#LXv;(Ho#M0OYRkvlgCwBf!s^vJ Wxl>w@!TaQ3z~foCV#C<_xyy97P}v2={op|GNp<2Y>;Cl0j@sss{C1;cHaUuEXp z46V9g$kTqA6$8~^RcONK+Jc_qPA zvU?(!&~wOKschMRUAK}jjHf!6kD=N4xgV!d%IF@Y7t{EQy|EE{V1Bmo6O1yNz$+__ zH8%^{$Q8E|w926~Ws-L7aoEti%y(|m>**1TJVu=86H6Lt^zO&l2WYvd_(B5s8O8t& zuR|Tmnv=t>q`z&^e}vs9Z4P@+7fNSb8EP=jWx<`Yh6H+fDQo}-6rOP_2`^kZY5Wrz z;&@W)+$rgom)HwN{BnrLxfaA6!Cu^8T-J>SCiZ#LXv;(Ho#M0OYRkvlgCwBf!s^vJ Wxl>w@!TaQ3z~foDZjR*j!#-etf^*Px^86J!*N)XM=!ND9?6XVb|OFWX!ubpx*WAXy>~D0o;-|KIIw;~r|h-nN~^mGulSs44pL;W zkyl~8Kzm-%I?o_CRjNx_VRTDoeY8HK;{#2)OYGYdfSRRKHsJsm+crSOX0CSSjFIb2 znU|vJ{v&JRxCz6B*o7I4Nzm!p-X55JNRTt7(O&G#g5p|fibc4Jo27GFToqk=3h-)} z*jgMOWTbmiE$55yUdWUQ^$;FxAS?i^swCY_OcaG%qU{1q-!h&vBeZM9PUp|OLo8%w z8U#gPk7-$^%}(M+p_G|BIz=z?-GkEe1g^dPAwZdAb6(*KI_4806EfK7n>DV-!|(@O Cz2qPO literal 0 HcmV?d00001 diff --git a/tests/test_backward_compat.py b/tests/test_backward_compat.py index 24a50c2fc..20977a74b 100644 --- a/tests/test_backward_compat.py +++ b/tests/test_backward_compat.py @@ -22,16 +22,16 @@ def load_obj(filename, check_deprecation_warning='auto'): # pickles files generated with cloudpickle_fast.py on old versions of # coudpickle with Python < 3.8 use non-deprecated reconstructors. check_deprecation_warning = (sys.version_info < (3, 8)) - try: - f = open(str(PICKLE_DIRECTORY / filename), "rb") + pickle_filepath = PICKLE_DIRECTORY / filename + if not pickle_filepath.exists(): + pytest.skip("Could not find {}".format(str(pickle_filepath))) + with open(str(pickle_filepath), "rb") as f: if check_deprecation_warning: msg = "A pickle file created using an old" with pytest.warns(UserWarning, match=msg): obj = pickle.load(f) else: obj = pickle.load(f) - finally: - f.close() return obj @@ -90,3 +90,8 @@ def test_complex_function(): c = f(a, b) assert c.attribute == 3 + + +def test_nested_function(): + f = load_obj("nested_function.pkl") + assert f(41) == 42 From a0f8d73287a4533a47a36970f640c2f364fcee4a Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Tue, 30 Jun 2020 10:47:16 +0200 Subject: [PATCH 52/89] Skip failing macos + pypy3 --- .github/workflows/testing.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index a7004bc50..ec56f5005 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -44,6 +44,10 @@ jobs: python_version: 3.6 - os: macos-latest python_version: 3.7 + - os: macos-latest + # numpy triggers: RuntimeError: Polyfit sanity test emitted a + # warning + python_version: "pypy3" runs-on: ${{ matrix.os }} From ca07fce64bfcfd3d0a9a229384ff5f5239d5b908 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Tue, 30 Jun 2020 10:55:38 +0200 Subject: [PATCH 53/89] Trigger CI From 96fe5c02f0c68dc30aa6b026ea0792206144e64f Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Tue, 30 Jun 2020 18:28:28 +0200 Subject: [PATCH 54/89] copy cloudpickle_fast.py inside ray source --- .github/workflows/testing.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index ec56f5005..085c27b52 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -226,6 +226,8 @@ jobs: PROJECT_DIR=$(python -c "import os, ray; print(os.path.dirname(ray.__file__), flush=True)") rm $PROJECT_DIR/cloudpickle/cloudpickle.py cp cloudpickle/cloudpickle.py $PROJECT_DIR/cloudpickle/cloudpickle.py + cp cloudpickle/cloudpickle_fast.py.py $PROJECT_DIR/cloudpickle/cloudpickle_fast.py + cp cloudpickle/__init__.py $PROJECT_DIR/cloudpickle/__init__.py - name: Test the downstream project run: | PROJECT_DIR="$(python -c "import os, ray; print(os.path.dirname(ray.__file__), flush=True)")" From 78602e2c7113c5c351ba8871d3d501fa3f0570e8 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Tue, 30 Jun 2020 18:30:30 +0200 Subject: [PATCH 55/89] fixup! copy cloudpickle_fast.py inside ray source --- .github/workflows/testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 085c27b52..d122fbd67 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -226,7 +226,7 @@ jobs: PROJECT_DIR=$(python -c "import os, ray; print(os.path.dirname(ray.__file__), flush=True)") rm $PROJECT_DIR/cloudpickle/cloudpickle.py cp cloudpickle/cloudpickle.py $PROJECT_DIR/cloudpickle/cloudpickle.py - cp cloudpickle/cloudpickle_fast.py.py $PROJECT_DIR/cloudpickle/cloudpickle_fast.py + cp cloudpickle/cloudpickle_fast.py $PROJECT_DIR/cloudpickle/cloudpickle_fast.py cp cloudpickle/__init__.py $PROJECT_DIR/cloudpickle/__init__.py - name: Test the downstream project run: | From dad293859da9f01733b893145527774d47f5fe5a Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Tue, 30 Jun 2020 18:32:50 +0200 Subject: [PATCH 56/89] fixup! copy cloudpickle_fast.py inside ray source --- .github/workflows/testing.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index d122fbd67..777410b50 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -227,7 +227,6 @@ jobs: rm $PROJECT_DIR/cloudpickle/cloudpickle.py cp cloudpickle/cloudpickle.py $PROJECT_DIR/cloudpickle/cloudpickle.py cp cloudpickle/cloudpickle_fast.py $PROJECT_DIR/cloudpickle/cloudpickle_fast.py - cp cloudpickle/__init__.py $PROJECT_DIR/cloudpickle/__init__.py - name: Test the downstream project run: | PROJECT_DIR="$(python -c "import os, ray; print(os.path.dirname(ray.__file__), flush=True)")" From 4dcbf3ef6eac66c45fa2cc984d8f7a5ab3a75f22 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Tue, 30 Jun 2020 18:58:48 +0200 Subject: [PATCH 57/89] update vendored ray to more recent version --- .github/workflows/testing.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 777410b50..bb72c8d25 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -222,9 +222,11 @@ jobs: - name: Install project and dependencies run: | python -m pip install --upgrade -r dev-requirements.txt - python -m pip install setproctitle psutil ray==0.6.4 + python -m pip install setproctitle psutil ray==0.8.6 PROJECT_DIR=$(python -c "import os, ray; print(os.path.dirname(ray.__file__), flush=True)") rm $PROJECT_DIR/cloudpickle/cloudpickle.py + git clone https://github.com/ray-project/ray.git + cp -R ray/python/ray/tests $PROJECT_DIR/tests cp cloudpickle/cloudpickle.py $PROJECT_DIR/cloudpickle/cloudpickle.py cp cloudpickle/cloudpickle_fast.py $PROJECT_DIR/cloudpickle/cloudpickle_fast.py - name: Test the downstream project From 677a1140adc2ffc9f3575eb989fef4cea09a58f3 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Tue, 30 Jun 2020 19:01:34 +0200 Subject: [PATCH 58/89] install ray outside of cloudpickle --- .github/workflows/testing.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index bb72c8d25..2f6e52fe8 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -225,8 +225,8 @@ jobs: python -m pip install setproctitle psutil ray==0.8.6 PROJECT_DIR=$(python -c "import os, ray; print(os.path.dirname(ray.__file__), flush=True)") rm $PROJECT_DIR/cloudpickle/cloudpickle.py - git clone https://github.com/ray-project/ray.git - cp -R ray/python/ray/tests $PROJECT_DIR/tests + git clone https://github.com/ray-project/ray.git ../ray + cp -R ../ray/python/ray/tests $PROJECT_DIR/tests cp cloudpickle/cloudpickle.py $PROJECT_DIR/cloudpickle/cloudpickle.py cp cloudpickle/cloudpickle_fast.py $PROJECT_DIR/cloudpickle/cloudpickle_fast.py - name: Test the downstream project From cd7cd1a0dbda182b1cfa06dde9305b1b8077821c Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Tue, 30 Jun 2020 19:06:52 +0200 Subject: [PATCH 59/89] don't try to test deleted ray tests --- .github/workflows/testing.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 2f6e52fe8..54c767f59 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -233,8 +233,6 @@ jobs: run: | PROJECT_DIR="$(python -c "import os, ray; print(os.path.dirname(ray.__file__), flush=True)")" COVERAGE_PROCESS_START="$TRAVIS_BUILD_DIR/.coveragerc" PYTHONPATH='.:tests' pytest -r s - pytest -vl $PROJECT_DIR/tests/test_basic.py::test_simple_serialization pytest -vl $PROJECT_DIR/tests/test_basic.py::test_complex_serialization pytest -vl $PROJECT_DIR/tests/test_basic.py::test_ray_recursive_objects pytest -vl $PROJECT_DIR/tests/test_basic.py::test_serialization_final_fallback - pytest -vl $PROJECT_DIR/tests/test_recursion.py From 8eba950e160e8139f56dd0af52eb7d6ef632c0c8 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Tue, 30 Jun 2020 19:12:11 +0200 Subject: [PATCH 60/89] fixup! don't try to test deleted ray tests --- .github/workflows/testing.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 54c767f59..bfd33b097 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -233,6 +233,8 @@ jobs: run: | PROJECT_DIR="$(python -c "import os, ray; print(os.path.dirname(ray.__file__), flush=True)")" COVERAGE_PROCESS_START="$TRAVIS_BUILD_DIR/.coveragerc" PYTHONPATH='.:tests' pytest -r s - pytest -vl $PROJECT_DIR/tests/test_basic.py::test_complex_serialization + pytest -vl $PROJECT_DIR/tests/test_serialization.py::test_simple_serialization + pytest -vl $PROJECT_DIR/tests/test_serialization.py::test_complex_serialization pytest -vl $PROJECT_DIR/tests/test_basic.py::test_ray_recursive_objects - pytest -vl $PROJECT_DIR/tests/test_basic.py::test_serialization_final_fallback + pytest -vl $PROJECT_DIR/tests/test_serialization.py::test_serialization_final_fallback + pytest -vl $PROJECT_DIR/tests/test_basic.py::test_nested_functions From bff0786686387bb12e42836742e49fda6fb0439f Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 21 May 2020 14:40:09 -0700 Subject: [PATCH 61/89] Add `compat` Tries to use `pickle5` for `pickle` if available on older Python versions. --- cloudpickle/compat.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 cloudpickle/compat.py diff --git a/cloudpickle/compat.py b/cloudpickle/compat.py new file mode 100644 index 000000000..453386d05 --- /dev/null +++ b/cloudpickle/compat.py @@ -0,0 +1,14 @@ +import sys + + +if sys.version_info.major == 3 and sys.version_info.minor < 8: + try: + import pickle5 as pickle # noqa: F401 + import pickle5._pickle as _pickle # noqa: F401 + from pickle5._pickle import Pickler # noqa: F401 + except ImportError: + import pickle # noqa: F401 + from pickle import _Pickler as Pickler # noqa: F401 +else: + import pickle # noqa: F401 + from _pickle import Pickler # noqa: F401 From fccb9e32d46be27966288d27b959723711950069 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 21 May 2020 14:40:10 -0700 Subject: [PATCH 62/89] Use `compat` to provide `pickle` or `pickle5` --- cloudpickle/cloudpickle.py | 2 +- cloudpickle/cloudpickle_fast.py | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 0ab6c83f8..8e683e7a6 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -45,7 +45,6 @@ import builtins import dis import opcode -import pickle import platform import sys import types @@ -55,6 +54,7 @@ import typing import warnings +from .compat import pickle from typing import Generic, Union, Tuple, Callable from pickle import _getattribute from importlib._bootstrap import _find_spec diff --git a/cloudpickle/cloudpickle_fast.py b/cloudpickle/cloudpickle_fast.py index 4b52a1e8e..bcc12e2fb 100644 --- a/cloudpickle/cloudpickle_fast.py +++ b/cloudpickle/cloudpickle_fast.py @@ -15,7 +15,6 @@ import io import itertools import logging -import pickle import sys import struct import types @@ -25,6 +24,7 @@ from enum import Enum from collections import ChainMap +from .compat import pickle, Pickler from .cloudpickle import ( _extract_code_globals, _BUILTIN_TYPE_NAMES, DEFAULT_PROTOCOL, _find_imported_submodules, _get_cell_contents, _is_importable, @@ -37,8 +37,8 @@ ) -if sys.version_info >= (3, 8) and not PYPY: - from _pickle import Pickler + +if pickle.HIGHEST_PROTOCOL >= 5 and not PYPY: # Shorthands similar to pickle.dump/pickle.dumps def dump(obj, file, protocol=None, buffer_callback=None): @@ -73,8 +73,6 @@ def dumps(obj, protocol=None, buffer_callback=None): return file.getvalue() else: - from pickle import _Pickler as Pickler - # Shorthands similar to pickle.dump/pickle.dumps def dump(obj, file, protocol=None): """Serialize obj as bytes streamed into file From 0d11d664f261d1545d823e1ce4472ac0661af868 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 21 May 2020 14:40:11 -0700 Subject: [PATCH 63/89] Import `pickle` from `compat` in tests --- tests/cloudpickle_file_test.py | 2 +- tests/cloudpickle_test.py | 17 ++++++++++------- tests/testutils.py | 3 ++- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/tests/cloudpickle_file_test.py b/tests/cloudpickle_file_test.py index 4f05186e3..6f1099a19 100644 --- a/tests/cloudpickle_file_test.py +++ b/tests/cloudpickle_file_test.py @@ -1,7 +1,6 @@ from __future__ import unicode_literals import os -import pickle import shutil import sys import tempfile @@ -10,6 +9,7 @@ import pytest import cloudpickle +from cloudpickle.compat import pickle class CloudPickleFileTests(unittest.TestCase): diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index ad6a9c9a6..209b37e4a 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -9,7 +9,6 @@ import logging import math from operator import itemgetter, attrgetter -import pickle import platform import random import shutil @@ -44,6 +43,8 @@ import cloudpickle from cloudpickle.cloudpickle import _is_importable +from cloudpickle.compat import pickle +from cloudpickle.cloudpickle import _is_dynamic from cloudpickle.cloudpickle import _make_empty_cell, cell_set from cloudpickle.cloudpickle import _extract_class_dict, _whichmodule from cloudpickle.cloudpickle import _lookup_module_and_qualname @@ -521,7 +522,7 @@ def test_module_locals_behavior(self): pickled_func_path = os.path.join(self.tmpdir, 'local_func_g.pkl') child_process_script = ''' - import pickle + from cloudpickle.compat import pickle import gc with open("{pickled_func_path}", 'rb') as f: func = pickle.load(f) @@ -606,7 +607,7 @@ def test_load_dynamic_module_in_grandchild_process(self): child_process_module_file = os.path.join( self.tmpdir, 'dynamic_module_from_child_process.pkl') child_process_script = ''' - import pickle + from cloudpickle.compat import pickle import textwrap import cloudpickle @@ -626,7 +627,7 @@ def test_load_dynamic_module_in_grandchild_process(self): # The script ran by the process created by the child process child_of_child_process_script = """ ''' - import pickle + from cloudpickle.compat import pickle with open('{child_process_module_file}','rb') as fid: mod = pickle.load(fid) ''' """ @@ -681,7 +682,7 @@ def my_small_function(x, y): assert b'math' not in b def test_module_importability(self): - import pickle # decouple this test from global imports + from cloudpickle.compat import pickle import os.path import distutils import distutils.ccompiler @@ -1008,7 +1009,8 @@ def example(): # choose "subprocess" rather than "multiprocessing" because the latter # library uses fork to preserve the parent environment. - command = ("import pickle, base64; " + command = ("import base64; " + "from cloudpickle.compat import pickle; " "pickle.loads(base64.b32decode('" + base64.b32encode(s).decode('ascii') + "'))()") @@ -1029,7 +1031,8 @@ def example(): s = cloudpickle.dumps(example, protocol=self.protocol) - command = ("import pickle, base64; " + command = ("import base64; " + "from cloudpickle.compat import pickle; " "pickle.loads(base64.b32decode('" + base64.b32encode(s).decode('ascii') + "'))()") diff --git a/tests/testutils.py b/tests/testutils.py index 303d0a996..e0276f58c 100644 --- a/tests/testutils.py +++ b/tests/testutils.py @@ -4,7 +4,7 @@ import tempfile import base64 from subprocess import Popen, check_output, PIPE, STDOUT, CalledProcessError -from pickle import loads +from cloudpickle.compat import pickle from contextlib import contextmanager from concurrent.futures import ProcessPoolExecutor @@ -12,6 +12,7 @@ from cloudpickle import dumps from subprocess import TimeoutExpired +loads = pickle.loads TIMEOUT = 60 TEST_GLOBALS = "a test value" From 0522cfbca46644fc98284a6d5f39f87c619886d2 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 21 May 2020 17:27:41 -0700 Subject: [PATCH 64/89] Add pickle5 to the dev requirements --- dev-requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev-requirements.txt b/dev-requirements.txt index 4e26b2106..22b7a6cf6 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -3,6 +3,8 @@ flake8 pytest pytest-cov psutil +# To test on older Python versions +pickle5 >=0.0.10 ; python_version <= '3.7' # To be able to test tornado coroutines tornado # To be able to test numpy specific things From 149b01ecdbf43908eaba0f019f73045b4bdcc773 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 21 May 2020 19:41:46 -0700 Subject: [PATCH 65/89] Always remove `_abc_impl` from `clsdict` --- cloudpickle/cloudpickle_fast.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudpickle/cloudpickle_fast.py b/cloudpickle/cloudpickle_fast.py index bcc12e2fb..15f2735ba 100644 --- a/cloudpickle/cloudpickle_fast.py +++ b/cloudpickle/cloudpickle_fast.py @@ -183,13 +183,13 @@ def _class_getstate(obj): # cache/negative caches populated during isinstance/issubclass # checks, but pickle the list of registered subclasses of obj. clsdict.pop('_abc_cache', None) + clsdict.pop('_abc_impl', None) clsdict.pop('_abc_negative_cache', None) clsdict.pop('_abc_negative_cache_version', None) registry = clsdict.pop('_abc_registry', None) if registry is None: # in Python3.7+, the abc caches and registered subclasses of a # class are bundled into the single _abc_impl attribute - clsdict.pop('_abc_impl', None) (registry, _, _, _) = abc._get_dump(obj) clsdict["_abc_impl"] = [subclass_weakref() From 695cbb8026a3150017136ba5b2e04d22edf210b4 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 21 May 2020 19:43:00 -0700 Subject: [PATCH 66/89] Keep all abstract base class elements --- cloudpickle/cloudpickle.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index 8e683e7a6..b8fa80104 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -439,6 +439,8 @@ def _extract_class_dict(cls): inherited_dict.update(base.__dict__) to_remove = [] for name, value in clsdict.items(): + if name.startswith("_abc"): + continue try: base_value = inherited_dict[name] if value is base_value: From adc12208741d778128dbc59f03f639f782f6c873 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Fri, 29 May 2020 20:33:20 +0200 Subject: [PATCH 67/89] call typing reducers inside reducer_override reducer_override is no longer Python 3.8+-only code --- cloudpickle/cloudpickle_fast.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cloudpickle/cloudpickle_fast.py b/cloudpickle/cloudpickle_fast.py index 15f2735ba..28a690af0 100644 --- a/cloudpickle/cloudpickle_fast.py +++ b/cloudpickle/cloudpickle_fast.py @@ -602,6 +602,11 @@ def reducer_override(self, obj): reducers, such as Exceptions. See https://github.com/cloudpipe/cloudpickle/issues/248 """ + if sys.version_info[:2] < (3, 7) and _is_parametrized_type_hint(obj): # noqa # pragma: no branch + return ( + _create_parametrized_type_hint, + parametrized_type_hint_getinitargs(obj) + ) t = type(obj) try: is_anyclass = issubclass(t, type) From 71dcd2a378b65567fb89a10f69fc58e2c9bbe13d Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Fri, 29 May 2020 20:39:25 +0200 Subject: [PATCH 68/89] revert older temptative fixes --- cloudpickle/cloudpickle.py | 2 -- cloudpickle/cloudpickle_fast.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py index b8fa80104..8e683e7a6 100644 --- a/cloudpickle/cloudpickle.py +++ b/cloudpickle/cloudpickle.py @@ -439,8 +439,6 @@ def _extract_class_dict(cls): inherited_dict.update(base.__dict__) to_remove = [] for name, value in clsdict.items(): - if name.startswith("_abc"): - continue try: base_value = inherited_dict[name] if value is base_value: diff --git a/cloudpickle/cloudpickle_fast.py b/cloudpickle/cloudpickle_fast.py index 28a690af0..be97d3965 100644 --- a/cloudpickle/cloudpickle_fast.py +++ b/cloudpickle/cloudpickle_fast.py @@ -183,13 +183,13 @@ def _class_getstate(obj): # cache/negative caches populated during isinstance/issubclass # checks, but pickle the list of registered subclasses of obj. clsdict.pop('_abc_cache', None) - clsdict.pop('_abc_impl', None) clsdict.pop('_abc_negative_cache', None) clsdict.pop('_abc_negative_cache_version', None) registry = clsdict.pop('_abc_registry', None) if registry is None: # in Python3.7+, the abc caches and registered subclasses of a # class are bundled into the single _abc_impl attribute + clsdict.pop('_abc_impl', None) (registry, _, _, _) = abc._get_dump(obj) clsdict["_abc_impl"] = [subclass_weakref() From e4fc3a0e95b34a821921cb555afd86be57884251 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Fri, 29 May 2020 20:44:48 +0200 Subject: [PATCH 69/89] skip numpy + pickle5 test on Python 3.5 --- tests/cloudpickle_test.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index 209b37e4a..73bd0f6dc 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -2089,6 +2089,10 @@ def __getattr__(self, name): with pytest.raises(pickle.PicklingError, match='recursion'): cloudpickle.dumps(a) + @pytest.mark.skipif( + sys.version_info < (3, 6), + reason='numpy does not support pickle protocol 5 on Python 3.5' + ) def test_out_of_band_buffers(self): if self.protocol < 5: pytest.skip("Need Pickle Protocol 5 or later") From e41b4dda214b97d45aa1e397304be6f5e5817a20 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Fri, 29 May 2020 21:01:03 +0200 Subject: [PATCH 70/89] don't try to install pickle5 on PYPY --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 22b7a6cf6..25e731b89 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -4,7 +4,7 @@ pytest pytest-cov psutil # To test on older Python versions -pickle5 >=0.0.10 ; python_version <= '3.7' +pickle5 >=0.0.10 ; python_version <= '3.7' and python_implementation == 'CPython' # To be able to test tornado coroutines tornado # To be able to test numpy specific things From 46886438068b3374cf241065d83df6a4d87b14ae Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Wed, 10 Jun 2020 11:23:28 -0700 Subject: [PATCH 71/89] Simplify `version_info` check --- cloudpickle/compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudpickle/compat.py b/cloudpickle/compat.py index 453386d05..b362db55c 100644 --- a/cloudpickle/compat.py +++ b/cloudpickle/compat.py @@ -1,7 +1,7 @@ import sys -if sys.version_info.major == 3 and sys.version_info.minor < 8: +if sys.version_info < (3, 8): try: import pickle5 as pickle # noqa: F401 import pickle5._pickle as _pickle # noqa: F401 From 5bfb9f3739c11a6dcfb38016c90f2d9a265aaa46 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Wed, 10 Jun 2020 17:48:35 -0700 Subject: [PATCH 72/89] Tweak pickle5 imports --- cloudpickle/compat.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cloudpickle/compat.py b/cloudpickle/compat.py index b362db55c..aab0015d5 100644 --- a/cloudpickle/compat.py +++ b/cloudpickle/compat.py @@ -4,8 +4,7 @@ if sys.version_info < (3, 8): try: import pickle5 as pickle # noqa: F401 - import pickle5._pickle as _pickle # noqa: F401 - from pickle5._pickle import Pickler # noqa: F401 + from pickle5 import _Pickler as Pickler # noqa: F401 except ImportError: import pickle # noqa: F401 from pickle import _Pickler as Pickler # noqa: F401 From 794cd9df365ab327cd8a22445611220ade2ee333 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Tue, 30 Jun 2020 19:24:48 +0200 Subject: [PATCH 73/89] empty commit to trigger the ci From b9ccea7f75b5b0a0de666fea4ff21380b7743625 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Tue, 30 Jun 2020 19:27:25 +0200 Subject: [PATCH 74/89] add compat to vendored cloudpickle in ray --- .github/workflows/testing.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index bfd33b097..b8d3ed462 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -228,6 +228,7 @@ jobs: git clone https://github.com/ray-project/ray.git ../ray cp -R ../ray/python/ray/tests $PROJECT_DIR/tests cp cloudpickle/cloudpickle.py $PROJECT_DIR/cloudpickle/cloudpickle.py + cp cloudpickle/compat.py $PROJECT_DIR/cloudpickle/compat.py cp cloudpickle/cloudpickle_fast.py $PROJECT_DIR/cloudpickle/cloudpickle_fast.py - name: Test the downstream project run: | From afec159db81d9eb004936dad0929db986ace5ae0 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Tue, 30 Jun 2020 19:30:15 +0200 Subject: [PATCH 75/89] remove unused import --- tests/cloudpickle_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index 73bd0f6dc..9265b510f 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -44,7 +44,6 @@ import cloudpickle from cloudpickle.cloudpickle import _is_importable from cloudpickle.compat import pickle -from cloudpickle.cloudpickle import _is_dynamic from cloudpickle.cloudpickle import _make_empty_cell, cell_set from cloudpickle.cloudpickle import _extract_class_dict, _whichmodule from cloudpickle.cloudpickle import _lookup_module_and_qualname From 1dc8dd8dd20c8c2590606afd18d8535945ca4266 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Tue, 30 Jun 2020 19:42:21 +0200 Subject: [PATCH 76/89] use latest pickle5 with ray --- .github/workflows/testing.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index b8d3ed462..4609584d1 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -222,6 +222,7 @@ jobs: - name: Install project and dependencies run: | python -m pip install --upgrade -r dev-requirements.txt + python -m pip install pickle5 python -m pip install setproctitle psutil ray==0.8.6 PROJECT_DIR=$(python -c "import os, ray; print(os.path.dirname(ray.__file__), flush=True)") rm $PROJECT_DIR/cloudpickle/cloudpickle.py From 9502999d0068e5edcb05b5e5e8a9bbd68f6942d1 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Tue, 30 Jun 2020 20:06:02 +0200 Subject: [PATCH 77/89] restor CloudPickler's dispatch attribute --- cloudpickle/cloudpickle_fast.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cloudpickle/cloudpickle_fast.py b/cloudpickle/cloudpickle_fast.py index be97d3965..1e9c9d169 100644 --- a/cloudpickle/cloudpickle_fast.py +++ b/cloudpickle/cloudpickle_fast.py @@ -549,6 +549,9 @@ def dump(self, obj): raise if pickle.HIGHEST_PROTOCOL >= 5: + # left for backward compat + dispatch = dispatch_table + # Implementation of the reducer_override callback, in order to # efficiently serialize dynamic functions and classes by subclassing # the C-implemented Pickler. From 6530ff639680c21a720e998e29ce1e8f5882d293 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Tue, 30 Jun 2020 20:12:50 +0200 Subject: [PATCH 78/89] use C-implemented pickler if pickle5 is installed --- cloudpickle/compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudpickle/compat.py b/cloudpickle/compat.py index aab0015d5..afa285f62 100644 --- a/cloudpickle/compat.py +++ b/cloudpickle/compat.py @@ -4,7 +4,7 @@ if sys.version_info < (3, 8): try: import pickle5 as pickle # noqa: F401 - from pickle5 import _Pickler as Pickler # noqa: F401 + from pickle5 import Pickler # noqa: F401 except ImportError: import pickle # noqa: F401 from pickle import _Pickler as Pickler # noqa: F401 From 7aabd2a22b678b2b31e456d2ac670a042bcd7b54 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Tue, 30 Jun 2020 20:28:05 +0200 Subject: [PATCH 79/89] mention the overloaded meaning of `dispatch` --- cloudpickle/cloudpickle_fast.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cloudpickle/cloudpickle_fast.py b/cloudpickle/cloudpickle_fast.py index 1e9c9d169..8a0093e2e 100644 --- a/cloudpickle/cloudpickle_fast.py +++ b/cloudpickle/cloudpickle_fast.py @@ -549,7 +549,15 @@ def dump(self, obj): raise if pickle.HIGHEST_PROTOCOL >= 5: - # left for backward compat + # `Cloudpickle.dispatch` is only left for backward compatibility - note + # that when using protocol 5, `CloudPickler.dispatch` is not an + # extension of `Pickler.dispatch` dictionary, because CloudPickler + # subclasses the C-implemented Pickler, which does not expose a + # `dispatch` attribute. Earlier version of the protocol 5 CloudPickler + # used `CloudPickler.dispatch` as a class-level attribute storing all + # reducers implemented by cloudpickle, but the attribute name was not a + # great choice given the meaning of `Cloudpickler.dispatch` when + # `CloudPickler` extends the pure-python pickler. dispatch = dispatch_table # Implementation of the reducer_override callback, in order to From cc5efb2f235cb5fcec69c2affcccffaa7c759dd8 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Tue, 30 Jun 2020 12:59:08 -0700 Subject: [PATCH 80/89] Drop `pickle5` from CI (already in dev reqs) --- .github/workflows/testing.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 4609584d1..b8d3ed462 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -222,7 +222,6 @@ jobs: - name: Install project and dependencies run: | python -m pip install --upgrade -r dev-requirements.txt - python -m pip install pickle5 python -m pip install setproctitle psutil ray==0.8.6 PROJECT_DIR=$(python -c "import os, ray; print(os.path.dirname(ray.__file__), flush=True)") rm $PROJECT_DIR/cloudpickle/cloudpickle.py From ec3468ed32d742e376afade27842f264e8e4174e Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Tue, 30 Jun 2020 13:02:06 -0700 Subject: [PATCH 81/89] Require NumPy 1.18.5 for testing This includes pickle5 support for Python 3.5. So make sure we get this version for testing. --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 25e731b89..033fba80c 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -9,7 +9,7 @@ pickle5 >=0.0.10 ; python_version <= '3.7' and python_implementation == 'CPython tornado # To be able to test numpy specific things # but do not build numpy from source on Python nightly -numpy; python_version <= '3.8' +numpy >=1.18.5; python_version <= '3.8' # Code coverage uploader for Travis: codecov coverage From d58877457c2e56f2d866e412c86717edc5c8d55b Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Tue, 30 Jun 2020 13:03:18 -0700 Subject: [PATCH 82/89] Require pickle5 0.0.11 Has a fix for the module name `PickleBuffer` is in. --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 033fba80c..d1cf1be14 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -4,7 +4,7 @@ pytest pytest-cov psutil # To test on older Python versions -pickle5 >=0.0.10 ; python_version <= '3.7' and python_implementation == 'CPython' +pickle5 >=0.0.11 ; python_version <= '3.7' and python_implementation == 'CPython' # To be able to test tornado coroutines tornado # To be able to test numpy specific things From 7b3f1a7db738392c8684b75067edcc6a2321da41 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Tue, 30 Jun 2020 13:04:07 -0700 Subject: [PATCH 83/89] Include NumPy test on Python 3.5 As NumPy 1.18.5 supports `pickle5` on Python 3.5 and we require that NumPy version for testing, include this out-of-band buffers test on Python 3.5 as well. --- tests/cloudpickle_test.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py index 7ecdae144..ff5d03d94 100644 --- a/tests/cloudpickle_test.py +++ b/tests/cloudpickle_test.py @@ -2088,10 +2088,6 @@ def __getattr__(self, name): with pytest.raises(pickle.PicklingError, match='recursion'): cloudpickle.dumps(a) - @pytest.mark.skipif( - sys.version_info < (3, 6), - reason='numpy does not support pickle protocol 5 on Python 3.5' - ) def test_out_of_band_buffers(self): if self.protocol < 5: pytest.skip("Need Pickle Protocol 5 or later") From 8178a2ce9d2248a634202b39ab5ec6219eca9145 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Tue, 30 Jun 2020 13:35:30 -0700 Subject: [PATCH 84/89] Add blank line --- cloudpickle/compat.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudpickle/compat.py b/cloudpickle/compat.py index afa285f62..2b183e7b7 100644 --- a/cloudpickle/compat.py +++ b/cloudpickle/compat.py @@ -1,6 +1,7 @@ import sys + if sys.version_info < (3, 8): try: import pickle5 as pickle # noqa: F401 From dc8bd28d62fcd54f37f70a3cb16906454eee55c3 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Tue, 30 Jun 2020 13:35:38 -0700 Subject: [PATCH 85/89] Revert "Add blank line" This reverts commit 8178a2ce9d2248a634202b39ab5ec6219eca9145. --- cloudpickle/compat.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cloudpickle/compat.py b/cloudpickle/compat.py index 2b183e7b7..afa285f62 100644 --- a/cloudpickle/compat.py +++ b/cloudpickle/compat.py @@ -1,7 +1,6 @@ import sys - if sys.version_info < (3, 8): try: import pickle5 as pickle # noqa: F401 From 8a890f5d4ee587f1e2714b992cd429c1c6e8fe4f Mon Sep 17 00:00:00 2001 From: jakirkham Date: Tue, 30 Jun 2020 15:20:27 -0700 Subject: [PATCH 86/89] Fix typo Co-authored-by: Pierre Glaser --- cloudpickle/cloudpickle_fast.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudpickle/cloudpickle_fast.py b/cloudpickle/cloudpickle_fast.py index 9db79961f..52ebfd3c2 100644 --- a/cloudpickle/cloudpickle_fast.py +++ b/cloudpickle/cloudpickle_fast.py @@ -553,7 +553,7 @@ def dump(self, obj): # that when using protocol 5, `CloudPickler.dispatch` is not an # extension of `Pickler.dispatch` dictionary, because CloudPickler # subclasses the C-implemented Pickler, which does not expose a - # `dispatch` attribute. Earlier version of the protocol 5 CloudPickler + # `dispatch` attribute. Earlier versions of the protocol 5 CloudPickler # used `CloudPickler.dispatch` as a class-level attribute storing all # reducers implemented by cloudpickle, but the attribute name was not a # great choice given the meaning of `Cloudpickler.dispatch` when From 5cdb5e1a2c4db8a09f775a5c1aca23546bcde014 Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 1 Jul 2020 10:09:05 +0200 Subject: [PATCH 87/89] Update cloudpickle/cloudpickle_fast.py --- cloudpickle/cloudpickle_fast.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudpickle/cloudpickle_fast.py b/cloudpickle/cloudpickle_fast.py index 52ebfd3c2..e8e46b88f 100644 --- a/cloudpickle/cloudpickle_fast.py +++ b/cloudpickle/cloudpickle_fast.py @@ -549,7 +549,7 @@ def dump(self, obj): raise if pickle.HIGHEST_PROTOCOL >= 5: - # `Cloudpickle.dispatch` is only left for backward compatibility - note + # `CloudPickler.dispatch` is only left for backward compatibility - note # that when using protocol 5, `CloudPickler.dispatch` is not an # extension of `Pickler.dispatch` dictionary, because CloudPickler # subclasses the C-implemented Pickler, which does not expose a From 268778b3f0f28cc5e478e78af7bbe37415ab099b Mon Sep 17 00:00:00 2001 From: Pierre Glaser Date: Wed, 1 Jul 2020 17:27:24 +0200 Subject: [PATCH 88/89] CI trigger From f17b31a420d1019f2bbd1c35cd8ee9f97daf68a6 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 1 Jul 2020 17:43:47 +0200 Subject: [PATCH 89/89] Update CHANGES.md --- CHANGES.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index df02e6a69..bbd26464a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,6 +5,10 @@ importable modules. ([issue #360](https://github.com/cloudpipe/cloudpickle/issues/354)) +- Add optional dependency on `pickle5` to get improved performance on + Python 3.6 and 3.7. + ([PR #370](https://github.com/cloudpipe/cloudpickle/pull/370)) + 1.4.1 =====