From 40ce17ee798fe33355fa079b91d410d257b95f69 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Wed, 12 Jul 2023 08:01:25 -0600 Subject: [PATCH 1/8] Switch from dill to cloudpickle Primarily for consistency with our other developments --- pyiron_base/jobs/datamining.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyiron_base/jobs/datamining.py b/pyiron_base/jobs/datamining.py index 27cc533eb..66f9a7aa5 100644 --- a/pyiron_base/jobs/datamining.py +++ b/pyiron_base/jobs/datamining.py @@ -4,7 +4,7 @@ import codecs from datetime import datetime -import dill as pickle +import cloudpickle import json import numpy as np import os @@ -34,11 +34,11 @@ def _to_pickle(hdf, key, value): - hdf[key] = codecs.encode(pickle.dumps(value), "base64").decode() + hdf[key] = codecs.encode(cloudpickle.dumps(value), "base64").decode() def _from_pickle(hdf, key): - return pickle.loads(codecs.decode(hdf[key].encode(), "base64")) + return cloudpickle.loads(codecs.decode(hdf[key].encode(), "base64")) def get_job_id(job): From 04ff1af47f629fda4b83ef89f41b3511f2064fa8 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Wed, 12 Jul 2023 11:15:06 -0600 Subject: [PATCH 2/8] Update environment.yml --- .ci_support/environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci_support/environment.yml b/.ci_support/environment.yml index d348b59f5..7ba107995 100644 --- a/.ci_support/environment.yml +++ b/.ci_support/environment.yml @@ -4,7 +4,7 @@ dependencies: - coveralls - coverage - codacy-coverage -- dill =0.3.6 +- cloudpickle =2.2.1 - gitpython =3.1.31 - h5io =0.1.8 - h5py =3.9.0 From 62b38c201bd38e747516c9953bfe519d6e17a9a0 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Wed, 12 Jul 2023 11:15:32 -0600 Subject: [PATCH 3/8] Update setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 25052ec4a..06980e863 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ keywords='pyiron', packages=find_packages(exclude=["*tests*", "*docs*", "*binder*", "*conda*", "*notebooks*", "*.ci_support*"]), install_requires=[ - 'dill==0.3.6', + 'cloudpickle==2.2.1', 'gitpython==3.1.31', 'h5io==0.1.8', 'h5py==3.9.0', From 5fafa8d6aceac2eaa34c2d57fa93973963fb4c37 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Sun, 17 Dec 2023 11:23:27 +0100 Subject: [PATCH 4/8] update dependencies --- .ci_support/environment-old.yml | 2 +- .ci_support/environment.yml | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.ci_support/environment-old.yml b/.ci_support/environment-old.yml index 127b09785..6b0ad257b 100644 --- a/.ci_support/environment-old.yml +++ b/.ci_support/environment-old.yml @@ -1,7 +1,7 @@ channels: - conda-forge dependencies: -- dill =0.3.1.1 +- cloudpickle =2.0.0 - gitpython =3.1.23 - h5io =0.1.2 - h5py =3.6.0 diff --git a/.ci_support/environment.yml b/.ci_support/environment.yml index 67186c0d2..84cc96bce 100644 --- a/.ci_support/environment.yml +++ b/.ci_support/environment.yml @@ -3,7 +3,7 @@ channels: dependencies: - coverage - codacy-coverage -- cloudpickle =2.2.1 +- cloudpickle =3.0.0 - gitpython =3.1.40 - h5io =0.1.9 - h5py =3.10.0 diff --git a/pyproject.toml b/pyproject.toml index 1231fc790..f986ba51c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ classifiers = [ "Programming Language :: Python :: 3.11", ] dependencies = [ - "dill==0.3.7", + "cloudpickle==3.0.0", "gitpython==3.1.40", "h5io==0.1.9", "h5py==3.10.0", From ee7cdda347d39555ba3f3c9d4121a39a77a6cbf3 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Sun, 17 Dec 2023 11:25:24 +0100 Subject: [PATCH 5/8] fix setup.py --- setup.py | 53 +++-------------------------------------------------- 1 file changed, 3 insertions(+), 50 deletions(-) diff --git a/setup.py b/setup.py index e49e79b3c..edb08a34d 100644 --- a/setup.py +++ b/setup.py @@ -1,55 +1,8 @@ -""" -Setuptools based setup module -""" -from setuptools import setup, find_packages +from setuptools import setup + import versioneer setup( - name='pyiron_base', version=versioneer.get_version(), - description='pyiron_base - an integrated development environment (IDE) for computational science.', - long_description='http://pyiron.org', - - url='https://github.com/pyiron/pyiron_base', - author='Max-Planck-Institut für Eisenforschung GmbH - Computational Materials Design (CM) Department', - author_email='janssen@mpie.de', - license='BSD', - - classifiers=['Development Status :: 5 - Production/Stable', - 'Topic :: Scientific/Engineering :: Physics', - 'License :: OSI Approved :: BSD License', - 'Intended Audience :: Science/Research', - 'Operating System :: OS Independent', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11' - ], - - keywords='pyiron', - packages=find_packages(exclude=["*tests*", "*docs*", "*binder*", "*conda*", "*notebooks*", "*.ci_support*"]), - install_requires=[ - 'cloudpickle==2.2.1', - 'gitpython==3.1.32', - 'h5io==0.1.8', - 'h5py==3.9.0', - 'jinja2==3.1.2', - 'numpy==1.24.3', - 'pandas==2.0.3', - 'pint==0.22', - 'psutil==5.9.5', - 'pyfileindex==0.0.11', - 'pysqa==0.1.0', - 'sqlalchemy==2.0.19', - 'tables==3.8.0', - 'tqdm==4.66.1', - 'traitlets==5.9.0', - ], cmdclass=versioneer.get_cmdclass(), - - entry_points={ - "console_scripts": [ - 'pyiron=pyiron_base.cli:main' - ] - } - ) +) \ No newline at end of file From a962c849aa0f93f22c0a79c5d462352e6359e1cc Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Sun, 17 Dec 2023 11:55:39 +0100 Subject: [PATCH 6/8] Add backwards compatibility tests --- pyiron_base/jobs/datamining.py | 6 +++- .../test_dill_cloudpickle_compatibility.py | 33 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 tests/backwards/test_dill_cloudpickle_compatibility.py diff --git a/pyiron_base/jobs/datamining.py b/pyiron_base/jobs/datamining.py index 66f9a7aa5..e8e955907 100644 --- a/pyiron_base/jobs/datamining.py +++ b/pyiron_base/jobs/datamining.py @@ -38,7 +38,11 @@ def _to_pickle(hdf, key, value): def _from_pickle(hdf, key): - return cloudpickle.loads(codecs.decode(hdf[key].encode(), "base64")) + try: + return cloudpickle.loads(codecs.decode(hdf[key].encode(), "base64")) + except ModuleNotFoundError: + import dill + return dill.loads(codecs.decode(hdf[key].encode(), "base64")) def get_job_id(job): diff --git a/tests/backwards/test_dill_cloudpickle_compatibility.py b/tests/backwards/test_dill_cloudpickle_compatibility.py new file mode 100644 index 000000000..61b2e6161 --- /dev/null +++ b/tests/backwards/test_dill_cloudpickle_compatibility.py @@ -0,0 +1,33 @@ +import unittest +from pyiron_base.jobs.datamining import _from_pickle + +dill_str_1 = 'gASV9AIAAAAAAAB9lCiMCmdldF9qb2JfaWSUiIwJZ2V0X2luY2FylImMCWdldF9zaWdtYZSJjBln\nZXRfdG90YWxfbnVtYmVyX29mX2F0b21zlImMDGdldF9lbGVtZW50c5SJjBVnZXRfY29udmVyZ2Vu\nY2VfY2hlY2uUiYwVZ2V0X251bWJlcl9vZl9zcGVjaWVzlImMGWdldF9udW1iZXJfb2ZfaW9uaWNf\nc3RlcHOUiYwKZ2V0X2lzbWVhcpSJjAlnZXRfZW5jdXSUiYwKZ2V0X25fa3B0c5SJjA5nZXRfbl9l\ncXVfa3B0c5SJjCRnZXRfbnVtYmVyX29mX2ZpbmFsX2VsZWN0cm9uaWNfc3RlcHOUiYwUZ2V0X21h\nam9yaXR5X3NwZWNpZXOUiYwMZ2V0X2pvYl9uYW1llImMDmdldF9lbmVyZ3lfdG90lImMDmdldF9l\nbmVyZ3lfcG90lImMD2dldF9lbmVyZ3lfZnJlZZSJjA5nZXRfZW5lcmd5X2ludJSJjBdnZXRfZW5l\ncmd5X3RvdF9wZXJfYXRvbZSJjBdnZXRfZW5lcmd5X3BvdF9wZXJfYXRvbZSJjBhnZXRfZW5lcmd5\nX2ZyZWVfcGVyX2F0b22UiYwXZ2V0X2VuZXJneV9pbnRfcGVyX2F0b22UiYwQZ2V0X2VfY29udl9s\nZXZlbJSJjAxnZXRfZl9zdGF0ZXOUiYwKZ2V0X2VfYmFuZJSJjB5nZXRfbWFqb3JpdHlfY3J5c3Rh\nbF9zdHJ1Y3R1cmWUiYwaZ2V0X2VxdWlsaWJyaXVtX3BhcmFtZXRlcnOUiYwNZ2V0X3N0cnVjdHVy\nZZSJjApnZXRfZm9yY2VzlImMFmdldF9tYWduZXRpY19zdHJ1Y3R1cmWUiYwRZ2V0X2F2ZXJhZ2Vf\nd2F2ZXOUiYwPZ2V0X3BsYW5lX3dhdmVzlImMDmdldF9la2luX2Vycm9ylImMCmdldF92b2x1bWWU\niYwTZ2V0X3ZvbHVtZV9wZXJfYXRvbZSJdS4=\n' +dill_str_2 = 'gASVIQUAAAAAAAB9lCiMBGFfZXGUjApkaWxsLl9kaWxslIwQX2NyZWF0ZV9mdW5jdGlvbpSTlCho\nAowMX2NyZWF0ZV9jb2RllJOUKEMAlEsBSwBLAEsBSwNLE0MMiAGIAHwAgwFpAVMAlE6FlCmMA2pv\nYpSFlIxCL1VzZXJzL2phbnNzZW4vcHJvamVjdHMvcHlpcm9uX2Jhc2UvcHlpcm9uX2Jhc2Uvam9i\ncy9kYXRhbWluaW5nLnB5lIwIPGxhbWJkYT6US1lDAgwAlIwEaXRlbZSMA2tleZSGlCl0lFKUY3B5\naXJvbl9iYXNlLmpvYnMuZGF0YW1pbmluZwpfX2RpY3RfXwpoDU5oAowMX2NyZWF0ZV9jZWxslJOU\nToWUUpRoFU6FlFKUhpR0lFKUfZR9lCiMD19fYW5ub3RhdGlvbnNfX5R9lIwMX19xdWFsbmFtZV9f\nlIwvRnVuY3Rpb25Db250YWluZXIuX19zZXRpdGVtX18uPGxvY2Fscz4uPGxhbWJkYT6UdYaUYowI\nYnVpbHRpbnOUjAdnZXRhdHRylJOUjARkaWxslIwFX2RpbGyUk5SMCF9zZXRhdHRylGgkjAdzZXRh\ndHRylJOUh5RSlGgZjA1jZWxsX2NvbnRlbnRzlGgBh5RSMGguaBdoL2gEKGgGKEMCAAGUSwFLAEsA\nSwFLAktDQwx8AGQBGQBkAhMAUwCUTowZb3V0cHV0L2VxdWlsaWJyaXVtX3ZvbHVtZZRHP9VVVVVV\nVVWHlCloCoWUjE4vdmFyL2ZvbGRlcnMvOXAvcnp0eXYwNmQweHY0aDI2Y3l2OG5ydzNtMDAwMGdx\nL1QvaXB5a2VybmVsXzUwODA2LzE1NTkyNTk0MzEucHmUjBVnZXRfbGF0dGljZV9wYXJhbWV0ZXKU\nSwJDAgwBlCkpdJRSlGNfX2J1aWx0aW5fXwpfX21haW5fXwpoN05OdJRSlH2UfZRoH32Uc4aUYoeU\nUjCMDGJ1bGtfbW9kdWx1c5RoBChoE2NweWlyb25fYmFzZS5qb2JzLmRhdGFtaW5pbmcKX19kaWN0\nX18KaA1OaBVOhZRSlGgVToWUUpSGlHSUUpR9lH2UKGgffZRoIWgidYaUYmguaEZoL2hCh5RSMGgu\naERoL2gEKGgGKEMCAAGUSwFLAEsASwFLAktDQwh8AGQBGQBTAJROjB9vdXRwdXQvZXF1aWxpYnJp\ndW1fYnVsa19tb2R1bHVzlIaUKWgKhZRoNowGZ2V0X2JtlEsGQwIIAZQpKXSUUpRjX19idWlsdGlu\nX18KX19tYWluX18KaFROTnSUUpR9lH2UaB99lHOGlGKHlFIwjAlwb3RlbnRpYWyUaAQoaBNjcHlp\ncm9uX2Jhc2Uuam9icy5kYXRhbWluaW5nCl9fZGljdF9fCmgNTmgVToWUUpRoFU6FlFKUhpR0lFKU\nfZR9lChoH32UaCFoInWGlGJoLmhjaC9oX4eUUjBoLmhhaC9oBChoBihDBAABFAGUSwFLAEsASwJL\nBEtDQxx8AGoAoAF8AGQBGQBkAhkAoQF9AXwBZAMZAFMAlChOjAlvdXRwdXQvaWSUSwCMFGlucHV0\nL3BvdGVudGlhbC9OYW1llHSUjAdwcm9qZWN0lIwHaW5zcGVjdJSGlGgKjAVjaGlsZJSGlGg2jAdn\nZXRfcG90lEsKQwQUAQgBlCkpdJRSlGNfX2J1aWx0aW5fXwpfX21haW5fXwpodk5OdJRSlH2UfZRo\nH32Uc4aUYoeUUjB1Lg==\n' + + +try: + import dill + + skip_dill_test = False +except ImportError: + skip_dill_test = True + + +class TestDillCloudpickleComptaibility(unittest.TestCase): + def test_1(self): + output = _from_pickle(hdf={"test": dill_str_1}, key="test") + self.assertEqual(len(output.keys()), 36) + self.assertTrue(output['get_job_id']) + for k in output.keys(): + if k != 'get_job_id': + self.assertFalse(output[k]) + + @unittest.skipIf( + skip_dill_test, "dill is not installed, so the dill tests are skipped." + ) + def test_2(self): + output = _from_pickle(hdf={"test": dill_str_2}, key="test") + self.assertEqual(len(output.keys()), 3) + self.assertEqual(list(output.keys()), ['a_eq', 'bulk_modulus', 'potential']) + for k in output.keys(): + self.assertTrue(callable(output[k])) From 3e01016f83e47059db61e568f097a93df67435e2 Mon Sep 17 00:00:00 2001 From: pyiron-runner Date: Sun, 17 Dec 2023 10:57:48 +0000 Subject: [PATCH 7/8] Format black --- pyiron_base/jobs/datamining.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pyiron_base/jobs/datamining.py b/pyiron_base/jobs/datamining.py index e8e955907..214adad09 100644 --- a/pyiron_base/jobs/datamining.py +++ b/pyiron_base/jobs/datamining.py @@ -42,6 +42,7 @@ def _from_pickle(hdf, key): return cloudpickle.loads(codecs.decode(hdf[key].encode(), "base64")) except ModuleNotFoundError: import dill + return dill.loads(codecs.decode(hdf[key].encode(), "base64")) From d07b0f7fcdfe3a0ae5cf4d05a02687e5a8c328ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Mon, 18 Dec 2023 17:27:54 +0100 Subject: [PATCH 8/8] Fix the pickle protocol version --- pyiron_base/jobs/datamining.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyiron_base/jobs/datamining.py b/pyiron_base/jobs/datamining.py index 214adad09..f035b89f7 100644 --- a/pyiron_base/jobs/datamining.py +++ b/pyiron_base/jobs/datamining.py @@ -34,7 +34,9 @@ def _to_pickle(hdf, key, value): - hdf[key] = codecs.encode(cloudpickle.dumps(value), "base64").decode() + hdf[key] = codecs.encode( + cloudpickle.dumps(obj=value, protocol=5, buffer_callback=None), "base64" + ).decode() def _from_pickle(hdf, key):