diff --git a/dill/__init__.py b/dill/__init__.py index 76419283..7f974b6c 100644 --- a/dill/__init__.py +++ b/dill/__init__.py @@ -289,7 +289,7 @@ """ from ._dill import ( - dump, dumps, load, loads, dump_module, load_module, load_module_vars, + dump, dumps, load, loads, dump_module, load_module, load_module_asdict, dump_session, load_session, Pickler, Unpickler, register, copy, pickle, pickles, check, HIGHEST_PROTOCOL, DEFAULT_PROTOCOL, PicklingError, UnpicklingError, HANDLE_FMODE, CONTENTS_FMODE, FILE_FMODE, PickleError, diff --git a/dill/_dill.py b/dill/_dill.py index 1cdd6bcc..80293399 100644 --- a/dill/_dill.py +++ b/dill/_dill.py @@ -17,7 +17,7 @@ """ __all__ = [ 'dump', 'dumps', 'load', 'loads', 'dump_module', 'load_module', - 'load_module_vars', 'dump_session', 'load_session', 'Pickler', 'Unpickler', + 'load_module_asdict', 'dump_session', 'load_session', 'Pickler', 'Unpickler', 'register', 'copy', 'pickle', 'pickles', 'check', 'HIGHEST_PROTOCOL', 'DEFAULT_PROTOCOL', 'PicklingError', 'UnpicklingError', 'HANDLE_FMODE', 'CONTENTS_FMODE', 'FILE_FMODE', 'PickleError', 'PickleWarning', @@ -408,7 +408,7 @@ def _restore_modules(unpickler, main_module): def dump_module( filename = str(TEMPDIR/'session.pkl'), main: Optional[Union[ModuleType, str]] = None, - imported_byref: bool = False, + refimported: bool = False, **kwds ) -> None: """Pickle the current state of :py:mod:`__main__` or another module to a file. @@ -421,16 +421,16 @@ def dump_module( :py:class:`~types.ModuleType`, can also be saved and restored thereafter. Parameters: - filename: a path-like object or a writable stream - main: a module object or an importable module name - imported_byref: if `True`, imported objects in the module's namespace + filename: a path-like object or a writable stream. + main: a module object or an importable module name. + refimported: if `True`, all imported objects in the module's namespace are saved by reference. *Note:* this is different from the ``byref`` option of other "dump" functions and is not affected by ``settings['byref']``. - **kwds: extra keyword arguments passed to :py:class:`Pickler()` + **kwds: extra keyword arguments passed to :py:class:`Pickler()`. Raises: - :py:exc:`PicklingError`: if pickling fails + :py:exc:`PicklingError`: if pickling fails. Examples: - Save current session state: @@ -444,30 +444,30 @@ def dump_module( >>> m.var = 'new value' >>> dill.dump_module('my_mod_session.pkl', main='my_mod') - - Save the state of an non-importable, runtime-created module: + - Save the state of a non-importable, runtime-created module: >>> from types import ModuleType >>> runtime = ModuleType('runtime') >>> runtime.food = ['bacon', 'eggs', 'spam'] >>> runtime.process_food = m.process_food - >>> dill.dump_module('runtime_session.pkl', main=runtime, byref=True) + >>> dill.dump_module('runtime_session.pkl', main=runtime, refimported=True) *Changed in version 0.3.6:* the function ``dump_session()`` was renamed to ``dump_module()``. *Changed in version 0.3.6:* the parameter ``byref`` was renamed to - ``imported_byref``. + ``refimported``. """ if 'byref' in kwds: warnings.warn( - "The parameter 'byref' was renamed to 'imported_byref', use this" + "The parameter 'byref' was renamed to 'refimported', use this" " instead. Note: the underlying dill.Pickler do accept a 'byref'" " argument, but it has no effect on session saving.", PendingDeprecationWarning ) - if imported_byref: - raise ValueError("both 'imported_byref' and 'byref' arguments were used.") - imported_byref = kwds.pop('byref') + if refimported: + raise ValueError("both 'refimported' and 'byref' arguments were used.") + refimported = kwds.pop('byref') from .settings import settings protocol = settings['protocol'] if main is None: main = _main_module @@ -478,7 +478,7 @@ def dump_module( try: pickler = Pickler(file, protocol, **kwds) pickler._original_main = main - if imported_byref: + if refimported: main = _stash_modules(main) pickler._main = main #FIXME: dill.settings are disabled pickler._byref = False # disable pickling by name reference @@ -494,8 +494,8 @@ def dump_module( # Backward compatibility. def dump_session(filename=str(TEMPDIR/'session.pkl'), main=None, byref=False, **kwds): - warnings.warn("dump_session() was renamed to dump_module().", PendingDeprecationWarning) - dump_module(filename, main, imported_byref=byref, **kwds) + warnings.warn("dump_session() was renamed to dump_module()", PendingDeprecationWarning) + dump_module(filename, main, refimported=byref, **kwds) dump_session.__doc__ = dump_module.__doc__ class _PeekableReader: @@ -562,7 +562,8 @@ def load_module( main: Union[ModuleType, str] = None, **kwds ) -> Optional[ModuleType]: - """Update :py:mod:`__main__` or another module with the state from the session file. + """Update :py:mod:`__main__` or another module with the state from the + session file. Restore the interpreter session (the built-in module :py:mod:`__main__`) or the state of another module from a pickle file created by the function @@ -574,15 +575,18 @@ def load_module( after it's updated. Parameters: - filename: a path-like object or a readable stream - main: an importable module name or a module object (optional) - **kwds: extra keyword arguments passed to :py:class:`Unpickler()` + filename: a path-like object or a readable stream. + main: an importable module name or a module object. + **kwds: extra keyword arguments passed to :py:class:`Unpickler()`. Raises: - :py:exc:`UnpicklingError`: if unpickling fails + :py:exc:`UnpicklingError`: if unpickling fails. + :py:exc:`ValueError`: if the ``main`` argument and the session file's + module are incompatible. Returns: - the restored module if different from :py:mod:`__main__` + The restored module if it's different from :py:mod:`__main__` and + wasn't passed as the ``main`` argument. Examples: - Load a saved session state: @@ -608,10 +612,23 @@ def load_module( >>> runtime in sys.modules.values() False + - Update the state of a non-importable, runtime-created module: + + >>> from types import ModuleType + >>> runtime = ModuleType('runtime') + >>> runtime.food = ['pizza', 'burger'] + >>> dill.load_module('runtime_session.pkl', main=runtime) + >>> runtime.food + ['bacon', 'eggs', 'spam'] + + *Changed in version 0.3.6:* the function ``load_session()`` was renamed to + ``load_module()``. + See also: - :py:func:`load_module_vars` to load the contents of a saved session (from - :py:mod:`__main__` or any importable module) into a dictionary. + :py:func:`load_module_asdict` to load the contents of a saved session + (from :py:mod:`__main__` or any importable module) into a dictionary. """ + main_arg = main if hasattr(filename, 'read'): file = filename else: @@ -636,7 +653,8 @@ def load_module( if not isinstance(main, ModuleType): raise ValueError("%r is not a module" % main) unpickler._main = main - main = unpickler._main + else: + main = unpickler._main # Check against the pickle's main. is_main_imported = _is_imported_module(main) @@ -645,14 +663,20 @@ def load_module( if is_runtime_mod: pickle_main = pickle_main.partition('.')[-1] if is_runtime_mod and is_main_imported: - raise UnpicklingError("can't restore non-imported module %r into an imported one" \ - % pickle_main) + raise ValueError( + "can't restore non-imported module %r into an imported one" + % pickle_main + ) if not is_runtime_mod and not is_main_imported: - raise UnpicklingError("can't restore imported module %r into a non-imported one" \ - % pickle_main) + raise ValueError( + "can't restore imported module %r into a non-imported one" + % pickle_main + ) if main.__name__ != pickle_main: - raise UnpicklingError("can't restore module %r into module %r" \ - % (pickle_main, main.__name__)) + raise ValueError( + "can't restore module %r into module %r" + % (pickle_main, main.__name__) + ) # This is for find_class() to be able to locate it. if not is_main_imported: @@ -669,7 +693,7 @@ def load_module( pass assert module is main _restore_modules(unpickler, module) - if module is not _main_module: + if not (module is _main_module or module is main_arg): return module # Backward compatibility. @@ -678,7 +702,7 @@ def load_session(filename=str(TEMPDIR/'session.pkl'), main=None, **kwds): load_module(filename, main, **kwds) load_session.__doc__ = load_module.__doc__ -def load_module_vars( +def load_module_asdict( filename = str(TEMPDIR/'session.pkl'), update: bool = False, **kwds @@ -686,6 +710,12 @@ def load_module_vars( """ Load the contents of a module from a session file into a dictionary. + ``load_module_asdict()`` does the equivalent of this function:: + + lambda filename: vars(load_module(filename)).copy() + + but without changing the original module. + The loaded module's origin is stored in the ``__session__`` attribute. Parameters: @@ -697,6 +727,13 @@ def load_module_vars( Raises: :py:exc:`UnpicklingError`: if unpickling fails + Returns: + A copy of the restored module's dictionary. + + Note: + If the ``update`` option is used, the original module will be loaded if + it wasn't yet. + Example: >>> import dill >>> alist = [1, 2, 3] @@ -704,7 +741,7 @@ def load_module_vars( >>> dill.dump_module() >>> anum = 0 >>> new_var = 'spam' - >>> main_vars = dill.load_module_vars() + >>> main_vars = dill.load_module_asdict() >>> main_vars['__name__'], main_vars['__session__'] ('__main__', '/tmp/session.pkl') >>> main_vars is globals() # loaded objects don't reference current global variables @@ -719,7 +756,7 @@ def load_module_vars( False """ if 'main' in kwds: - raise TypeError("'main' is an invalid keyword argument for load_module_vars()") + raise TypeError("'main' is an invalid keyword argument for load_module_asdict()") if hasattr(filename, 'read'): file = filename else: @@ -730,11 +767,14 @@ def load_module_vars( old_main = sys.modules.get(main_name) main = ModuleType(main_name) if update: + if old_main is None: + old_main = _import_module(main_name) main.__dict__.update(old_main.__dict__) - main.__builtins__ = __builtin__ + else: + main.__builtins__ = __builtin__ sys.modules[main_name] = main load_module(file, **kwds) - main.__session__ = filename if isinstance(filename, str) else repr(filename) + main.__session__ = str(filename) finally: if not hasattr(filename, 'read'): # if newly opened file file.close() diff --git a/dill/tests/test_session.py b/dill/tests/test_session.py index 6c507034..8f687934 100644 --- a/dill/tests/test_session.py +++ b/dill/tests/test_session.py @@ -13,25 +13,25 @@ import dill -session_file = os.path.join(os.path.dirname(__file__), 'session-byref-%s.pkl') +session_file = os.path.join(os.path.dirname(__file__), 'session-refimported-%s.pkl') ################### # Child process # ################### -def _error_line(error, obj, imported_byref): +def _error_line(error, obj, refimported): import traceback line = traceback.format_exc().splitlines()[-2].replace('[obj]', '['+repr(obj)+']') - return "while testing (with imported_byref=%s): %s" % (imported_byref, line.lstrip()) + return "while testing (with refimported=%s): %s" % (refimported, line.lstrip()) if __name__ == '__main__' and len(sys.argv) >= 3 and sys.argv[1] == '--child': # Test session loading in a fresh interpreter session. - imported_byref = (sys.argv[2] == 'True') - dill.load_module(session_file % imported_byref) + refimported = (sys.argv[2] == 'True') + dill.load_module(session_file % refimported) - def test_modules(imported_byref): + def test_modules(refimported): # FIXME: In this test setting with CPython 3.7, 'calendar' is not included - # in sys.modules, independent of the value of imported_byref. Tried to + # in sys.modules, independent of the value of refimported. Tried to # run garbage collection just before loading the session with no luck. It # fails even when preceding them with 'import calendar'. Needed to run # these kinds of tests in a supbrocess. Failing test sample: @@ -45,16 +45,16 @@ def test_modules(imported_byref): for obj in ('Calendar', 'isleap'): assert globals()[obj] is sys.modules['calendar'].__dict__[obj] assert __main__.day_name.__module__ == 'calendar' - if imported_byref: + if refimported: assert __main__.day_name is calendar.day_name assert __main__.complex_log is cmath.log except AssertionError as error: - error.args = (_error_line(error, obj, imported_byref),) + error.args = (_error_line(error, obj, refimported),) raise - test_modules(imported_byref) + test_modules(refimported) sys.exit() #################### @@ -118,7 +118,7 @@ def _clean_up_cache(module): atexit.register(_clean_up_cache, local_mod) -def _test_objects(main, globals_copy, imported_byref): +def _test_objects(main, globals_copy, refimported): try: main_dict = __main__.__dict__ global Person, person, Calendar, CalendarSubclass, cal, selfref @@ -144,13 +144,13 @@ def _test_objects(main, globals_copy, imported_byref): assert selfref is __main__ except AssertionError as error: - error.args = (_error_line(error, obj, imported_byref),) + error.args = (_error_line(error, obj, refimported),) raise -def test_session_main(imported_byref): +def test_session_main(refimported): """test dump/load_module() for __main__, both in this process and in a subprocess""" extra_objects = {} - if imported_byref: + if refimported: # Test unpickleable imported object in main. from sys import flags extra_objects['flags'] = flags @@ -158,22 +158,22 @@ def test_session_main(imported_byref): with TestNamespace(**extra_objects) as ns: try: # Test session loading in a new session. - dill.dump_module(session_file % imported_byref, imported_byref=imported_byref) + dill.dump_module(session_file % refimported, refimported=refimported) from dill.tests.__main__ import python, shell, sp - error = sp.call([python, __file__, '--child', str(imported_byref)], shell=shell) + error = sp.call([python, __file__, '--child', str(refimported)], shell=shell) if error: sys.exit(error) finally: try: - os.remove(session_file % imported_byref) + os.remove(session_file % refimported) except OSError: pass # Test session loading in the same session. session_buffer = BytesIO() - dill.dump_module(session_buffer, imported_byref=imported_byref) + dill.dump_module(session_buffer, refimported=refimported) session_buffer.seek(0) dill.load_module(session_buffer) - ns.backup['_test_objects'](__main__, ns.backup, imported_byref) + ns.backup['_test_objects'](__main__, ns.backup, refimported) def test_session_other(): """test dump/load_module() for a module other than __main__""" @@ -206,17 +206,17 @@ def test_runtime_module(): mod.__dill_imported, mod.__dill_imported_as, mod.__dill_imported_top_level, file=sys.stderr) - # This is also for code coverage, tests the use case of dump_module(imported_byref=True) + # This is also for code coverage, tests the use case of dump_module(refimported=True) # without imported objects in the namespace. It's a contrived example because # even dill can't be in it. This should work after fixing #462. session_buffer = BytesIO() - dill.dump_module(session_buffer, main=runtime, imported_byref=True) + dill.dump_module(session_buffer, main=runtime, refimported=True) session_dump = session_buffer.getvalue() # Pass a new runtime created module with the same name. runtime = ModuleType(modname) # empty - returned_mod = dill.load_module(BytesIO(session_dump), main=runtime) - assert returned_mod is runtime + return_val = dill.load_module(BytesIO(session_dump), main=runtime) + assert return_val is None assert runtime.__name__ == modname assert runtime.x == 42 assert runtime not in sys.modules.values() @@ -228,7 +228,7 @@ def test_runtime_module(): assert runtime.x == 42 assert runtime not in sys.modules.values() -def test_load_module_vars(): +def test_load_module_asdict(): with TestNamespace(): session_buffer = BytesIO() dill.dump_module(session_buffer) @@ -239,7 +239,7 @@ def test_load_module_vars(): globals_state = globals().copy() session_buffer.seek(0) - main_vars = dill.load_module_vars(session_buffer) + main_vars = dill.load_module_asdict(session_buffer) assert main_vars is not globals() assert globals() == globals_state @@ -252,8 +252,8 @@ def test_load_module_vars(): assert 'empty' in main_vars if __name__ == '__main__': - test_session_main(imported_byref=False) - test_session_main(imported_byref=True) + test_session_main(refimported=False) + test_session_main(refimported=True) test_session_other() test_runtime_module() - test_load_module_vars() + test_load_module_asdict() diff --git a/docs/source/dill.rst b/docs/source/dill.rst index 9061863c..31d41c91 100644 --- a/docs/source/dill.rst +++ b/docs/source/dill.rst @@ -11,7 +11,7 @@ dill module :special-members: :show-inheritance: :imported-members: -.. :exclude-members: + :exclude-members: dump_session, load_session detect module -------------