From bade3abbcb1baf10cdea3e497dca630bc1654a81 Mon Sep 17 00:00:00 2001 From: Marvin Poul Date: Tue, 20 Aug 2024 16:30:50 +0200 Subject: [PATCH 01/29] Only pass known keys to Server dataclass --- pyiron_base/jobs/job/extension/server/generic.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pyiron_base/jobs/job/extension/server/generic.py b/pyiron_base/jobs/job/extension/server/generic.py index a08fc7cc8..df4b4f44d 100644 --- a/pyiron_base/jobs/job/extension/server/generic.py +++ b/pyiron_base/jobs/job/extension/server/generic.py @@ -8,7 +8,7 @@ import numbers import socket from concurrent.futures import Executor, Future -from dataclasses import asdict +from dataclasses import asdict, fields from typing import Union from pyiron_snippets.deprecate import deprecate @@ -564,7 +564,6 @@ def view_queues(): def _to_dict(self): self._data.run_mode = self._run_mode.mode return asdict(self._data) - return server_dict def _from_dict(self, obj_dict, version=None): # backwards compatibility @@ -578,9 +577,11 @@ def _from_dict(self, obj_dict, version=None): if "additional_arguments" not in obj_dict.keys(): obj_dict["additional_arguments"] = {} - # Reload dataclass - for key in ["NAME", "TYPE", "OBJECT", "VERSION", "DICT_VERSION"]: - if key in obj_dict.keys(): + # Reload dataclass and discard unknown keys + server_fields = tuple(f.name for f in fields(ServerDataClass)) + # force tuple otherwise dict complains about changing size + for key in tuple(obj_dict): + if key not in server_fields: del obj_dict[key] self._data = ServerDataClass(**obj_dict) self._run_mode = Runmode(mode=self._data.run_mode) From 3b27693f4e6c185ca9c59dbcd897d56171260696 Mon Sep 17 00:00:00 2001 From: Marvin Poul Date: Tue, 20 Aug 2024 16:36:31 +0200 Subject: [PATCH 02/29] Backwards compatible access to executable name during instantiate --- pyiron_base/jobs/job/extension/executable.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyiron_base/jobs/job/extension/executable.py b/pyiron_base/jobs/job/extension/executable.py index 034d40fa4..b2417d744 100644 --- a/pyiron_base/jobs/job/extension/executable.py +++ b/pyiron_base/jobs/job/extension/executable.py @@ -208,7 +208,11 @@ def executable_path(self, new_path): @classmethod def instantiate(cls, obj_dict: dict, version: str = None) -> "Self": - return cls(codename=obj_dict["name"]) + try: + codename = obj_dict["name"] + except KeyError: + codename = obj_dict["executable"]["name"] + return cls(codename=codename) def _to_dict(self): return asdict(self.storage) From 90f61c8f15331aeb16aa65a86127aabdb497acc6 Mon Sep 17 00:00:00 2001 From: Marvin Poul Date: Tue, 20 Aug 2024 17:23:19 +0200 Subject: [PATCH 03/29] Add backwards compatibility when restoring server/executable --- pyiron_base/jobs/job/generic.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/pyiron_base/jobs/job/generic.py b/pyiron_base/jobs/job/generic.py index aa2a85426..a9107f624 100644 --- a/pyiron_base/jobs/job/generic.py +++ b/pyiron_base/jobs/job/generic.py @@ -1178,9 +1178,20 @@ def _from_dict(self, obj_dict, version=None): self._type_from_dict(type_dict=obj_dict) if "import_directory" in obj_dict.keys(): self._import_directory = obj_dict["import_directory"] - self._server = obj_dict["server"] + # Backwards compatibility: Previously server and executable were stored + # as plain dicts, but now they are dicts with additional info so that + # HasDict can load them automatically. + # We need to check whether that was possible with the instance check + # below and if not, call from_dict ourselves. + if isinstance(server := obj_dict["server"], Server): + self._server = server + else: + self._server.from_dict(server) if "executable" in obj_dict.keys() and obj_dict["executable"] is not None: - self._executable = obj_dict["executable"] + if isinstance(executable := obj_dict["executable"], Executable): + self._executable = executable + else: + self.executable.from_dict(executable) input_dict = obj_dict["input"] if "generic_dict" in input_dict.keys(): generic_dict = input_dict["generic_dict"] From 702e292fb0a4b63603fcada60c9c541951bd314e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Thu, 22 Aug 2024 15:08:14 +0200 Subject: [PATCH 04/29] Fix project mv --- pyiron_base/jobs/job/util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyiron_base/jobs/job/util.py b/pyiron_base/jobs/job/util.py index bbb66b377..1bc03a995 100644 --- a/pyiron_base/jobs/job/util.py +++ b/pyiron_base/jobs/job/util.py @@ -95,12 +95,12 @@ def _get_project_for_copy(job, project, new_job_name): ): file_project = project.project hdf5_project = project.project_hdf5.open(new_job_name) - elif isinstance(project, job.project.__class__): + elif isinstance(job.project, project.__class__): file_project = project hdf5_project = job.project_hdf5.__class__( project=project, file_name=new_job_name, h5_path="/" + new_job_name ) - elif isinstance(project, job.project_hdf5.__class__): + elif isinstance(job.project_hdf5, project.__class__): file_project = project.project hdf5_project = project.open(new_job_name) elif project is None: From 509ebaecaa47297e740330e229c646993c3d6a98 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Thu, 22 Aug 2024 17:46:51 +0200 Subject: [PATCH 05/29] Fix windows paths --- pyiron_base/project/archiving/import_archive.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pyiron_base/project/archiving/import_archive.py b/pyiron_base/project/archiving/import_archive.py index 07d6c4255..775cc5dd3 100644 --- a/pyiron_base/project/archiving/import_archive.py +++ b/pyiron_base/project/archiving/import_archive.py @@ -1,4 +1,5 @@ import os +import posixpath import tarfile import tempfile from shutil import copytree @@ -57,8 +58,8 @@ def import_jobs(project_instance, archive_directory): pr_import = project_instance.open(os.curdir) df["project"] = [ - os.path.normpath( - os.path.join(pr_import.project_path, os.path.relpath(p, common_path)) + posixpath.normpath( + posixpath.join(pr_import.project_path, posixpath.relpath(p, common_path)) ) + "/" for p in df["project"].values @@ -105,8 +106,8 @@ def transfer_files(origin_path: str, project_path: str): str: Common path. """ df = get_dataframe(origin_path=origin_path) - common_path = os.path.commonpath(list(df["project"])) - copytree(os.path.join(origin_path, common_path), project_path, dirs_exist_ok=True) + common_path = posixpath.commonpath(list(df["project"])) + copytree(posixpath.join(origin_path, common_path), project_path, dirs_exist_ok=True) return df, common_path From 8bfaf29a421caf672a402b66d8bbab1567207810 Mon Sep 17 00:00:00 2001 From: samwaseda Date: Fri, 23 Aug 2024 08:18:25 +0000 Subject: [PATCH 06/29] add old tar files --- tests/static/pack/export.csv | 2 ++ tests/static/pack/test_pack.tar.gz | Bin 0 -> 3280 bytes tests/unit/archiving/test_import.py | 19 ++++++++++++++++++- 3 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 tests/static/pack/export.csv create mode 100644 tests/static/pack/test_pack.tar.gz diff --git a/tests/static/pack/export.csv b/tests/static/pack/export.csv new file mode 100644 index 000000000..2a0997cb5 --- /dev/null +++ b/tests/static/pack/export.csv @@ -0,0 +1,2 @@ +,id,status,chemicalformula,job,subjob,project,timestart,timestop,totalcputime,computer,hamilton,hamversion,parentid,masterid +0,0,finished,,toy,/toy,test_pack/my_project,2024-08-22 16:10:26.556984,,,pyiron@7720454e9ac5#1,ToyJob,0.4,, diff --git a/tests/static/pack/test_pack.tar.gz b/tests/static/pack/test_pack.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..5742bc3f6095d3f96fcdc74eb4758cd8066174e4 GIT binary patch literal 3280 zcmXw3cRU-~`;Kbu?Os(%QPkd}wf7!XqjqYG88d3PwMVU*r7lI$3aUoTTZ!7#-g^W= z5kwN?*ZZsQ`^Pz-&w0-Kyytn}&v_mslaZY`_PX981iAzS0>KVWo+5z`{(yre%5X+a zcG9Cch57WHPWq~WzbPL5lrb!*DCe4O)&;&QGpmo(nc#haYV4623n^uvN_)_lgdqwC z$EPuxwX1*IS>wNBqJb|pXTNj&DsuHu!30r_8G~mn3?DY>dZ~owQ>UZ}Kht5~# z4CNXmKkJ#7yN))to(6v&Ue>65ioO({^6rNvcB$NeifT0~O8f&8Bxa(h!$|hPSIWRz!rLG3&1ZIRgH>d}u;(}DBbhLw*gv2mpc%(mTao#xxbQ4) zhktMfn!v9FM7ci02JE8_Kc^QBCw+{a&YWJOx=@MO(|E#WYYjS%rXO7l)Wpml;N*;&WNRCrIsDHn%f`iIvX=_LQO>rw zo|ryv26D!3G5Qahwtb>Mdcg0oB#H`I!^Ohk7&5>Iebi^}eg7 zi-cT~$4o2t2l`J_HB0M6{G97`ob$NT7`WUKYHAOeM3p3_^qTFR#}Z=SX$ie0d_PE%veQojENT=K-y7zRKHHKT`2)kPKW==Zgn z1+3g-L}RormRnpVR{RIgu=}zjIlbmY3sOsFk#@}VpAm_l-gh#~8K0!^96n6qQuih9 zu5VR7mztcGn%}?V@yiRWDh|vq4uln-(kbtjM2`0u3pzdt8csc=^>w-|6TB*;O+x&- z1^EAy`jDIl9YW-;&K43ITU zbEwdAjyLssbryP@v~mVnoX77bWz!x+VUwvJv3D8bThI+nHnq!V;Mw{P!~PlB`-v|b zaE)7Sr4`LJB9iRV?zte;2H~oYjo7=Sm;DmWvm_c{D{`orA2TrU78m#YY zNkjk05orr}J$|`gdmw?_if|C@>NxqFsJ{{pt?Ke0d5%YUR9ip?_=i1F^2{Bl+}I5# zY833Bzwc#IW6sX0`*1Bq6kF|Ag6&-<{giVpfcT2wlJu9ThM|v7$|~v+q4Tg_`DBw5Oewn$vZBvWHxQ zEQ4|>c!nNl!9l)+^HjA4SoD1+pPUyJ;+p`>`bFNmN0F}7b(Fj38B+jxIJ1I3kfoA^ z16@f(!6LnSQ6$_D;bijgv@~(B_+D~ixDdCr?K+a(WxtG`8e(kar{Y8pnOm+?%Zh~J zZL{doOsy5BL24R^0-4!bzT*-8eYYGw^la9s+$+0n1ICHSzDXkiDHNKVUf(qV%!_G?2aR=hfF5%fVgkil|wwXQQ3%(e3iV z+dj{BM;y*FaYWsb51bXSejeo%zN&6xZN-*jAGLxHk{$CX75#nRtE_oFA8$E#VNVyN zJj24?K<4}om2g#=S0sb}dXxerqfe}`uVHtl7>)<&oP11F?O(nzGawt4-ORQTpfGTK zQ@`IVA^yZn2c|Dl5wnY?>^TUIquQR?uaher&L?+#-QMJoII&hJ_3c&1u2z_->iUb3 z;kQ3{QI26FDc=qESUw9kHnGKAK#!dq9Sv?Q^X1uA8%C z3i5&ezVN~3-rP=L$d{?WMZrR#R|T!t*~sLT1-)Fz<0S3cg8i^ioc&387MY}JcNAMW zvz!}6fZi;v>Q)8zj#mM`&|F|P%~^u?(Xjj*0)A+7F9PzVH#L`+MY*8*-f{d>RU$w% zO%xwro0rME#IK5*cnO1@m5Cc z_U&U{$~AjZXh4-^VNG9K?QF>qlq!T&#U}$o@@f~LQ@TWSbp05Osr&~3P$8B3Lkj@B zUIE!Gz{bQV{Jh*{j=d~LKW{hF_ok9mI7H-V!*S~-$?y-k(4Z0-0N`J2H?Pnb?M)`# zWjLM);pUz=C|UTorw}{^-Pd86N6aS{pvGhxzOyzhOvw`Al)>|5ki|;qYn=b8*;vAz z?0EKbrLK~dyicM`KfJ*W(RMrvnn4$HBF*mlS-TcxQ&O+BxA+tNbkUiM-;>YtB)zj| z6Q)b7ZS^qUA*>*cqyfplemVQRXwmUgkK>9ldh91)h7l{(>je9`XIwD17&D}el(NEw zr5ts(=8~I~&1@DM54fPKd_Hnk`rhzWD({ywNAEkp@DG-pDt8;hQy_|TMZ#&Bf5Cos z{ZAJd8ao{4{N`h`X#dX%(?TxJEY>Z?_^49n%xcNTN<>R--cpIiq60)yvhGJHEPx5r z30l-shJk+D_%{2aIqlBV(56j(S5DkD==uWjqpQR!fvYH-wa3eWI#(u4ksxQM!`hiY z@5Ej@@+elgHw3X>g2XH`99bpES%kRJrGIC5#(H3ta1FA|Olcqn0CxV}`Mv%Z(W}~+ zisO;zSQiza4eGYE^&R5mc3&QZCHq&GX0v_DiW(=A842S)&V8&W&^O4q zf}T(FbUcAA`!M}# zJb$umiUvcawNa-i;woACM+sX zsq5!6lNS=KDJ~0zm-Fe&K8BU)0lF84(otEMc?hFJid}))gd)5DY+Q({8j?+chmBux;h7!?)r;A5BxT~S7*&q zM3Y?J^xYL=HTjG*%F<+CJicZPIQkbP_^=>|%A6bk7~x!J!|&Wk1&~GNDc@-1xS&>! z`uDE!v1bR8_NM~;lgIoAi;qh;=xVsP?1cksmyCGuYGXrrEXSslFsZrwlN Date: Fri, 23 Aug 2024 08:20:55 +0000 Subject: [PATCH 07/29] save current state --- tests/unit/archiving/test_import.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit/archiving/test_import.py b/tests/unit/archiving/test_import.py index fd942fd63..c2645bb22 100644 --- a/tests/unit/archiving/test_import.py +++ b/tests/unit/archiving/test_import.py @@ -192,7 +192,6 @@ def test_import_old_tar(self): pr = Project("old_tar") pr.unpack(origin_path="test_pack.tar.gz") job = pr.load("toy") - print(job) pr.remove(enable=True, enforce=True) os.remove("test_pack.tar.gz") os.remove("export.csv") From 9dd4157e85f969e84d52eba9c88b794a9c19a854 Mon Sep 17 00:00:00 2001 From: samwaseda Date: Fri, 23 Aug 2024 08:38:16 +0000 Subject: [PATCH 08/29] stash changes to see whether the tests still fail --- tests/unit/archiving/test_import.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/tests/unit/archiving/test_import.py b/tests/unit/archiving/test_import.py index c2645bb22..ff1509b78 100644 --- a/tests/unit/archiving/test_import.py +++ b/tests/unit/archiving/test_import.py @@ -180,22 +180,6 @@ def test_backwards_compatibility(self): with self.assertRaises(ValueError): self.imp_pr.unpack(origin_path=self.arch_dir_comp, csv_file_name="ahoy.csv") - def test_import_old_tar(self): - copytree( - os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "../../static/pack", - ), - ".", - dirs_exist_ok=True, - ) - pr = Project("old_tar") - pr.unpack(origin_path="test_pack.tar.gz") - job = pr.load("toy") - pr.remove(enable=True, enforce=True) - os.remove("test_pack.tar.gz") - os.remove("export.csv") - if __name__ == "__main__": unittest.main() From fa80cd4e05c477fa1c62039e8d545ab4475e7684 Mon Sep 17 00:00:00 2001 From: samwaseda Date: Fri, 23 Aug 2024 08:41:55 +0000 Subject: [PATCH 09/29] restore changes --- tests/unit/archiving/test_import.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/unit/archiving/test_import.py b/tests/unit/archiving/test_import.py index ff1509b78..c2645bb22 100644 --- a/tests/unit/archiving/test_import.py +++ b/tests/unit/archiving/test_import.py @@ -180,6 +180,22 @@ def test_backwards_compatibility(self): with self.assertRaises(ValueError): self.imp_pr.unpack(origin_path=self.arch_dir_comp, csv_file_name="ahoy.csv") + def test_import_old_tar(self): + copytree( + os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "../../static/pack", + ), + ".", + dirs_exist_ok=True, + ) + pr = Project("old_tar") + pr.unpack(origin_path="test_pack.tar.gz") + job = pr.load("toy") + pr.remove(enable=True, enforce=True) + os.remove("test_pack.tar.gz") + os.remove("export.csv") + if __name__ == "__main__": unittest.main() From 69820dec79fa3856225decac7c3929fb1911c3d3 Mon Sep 17 00:00:00 2001 From: samwaseda Date: Fri, 23 Aug 2024 08:43:40 +0000 Subject: [PATCH 10/29] separate class --- tests/unit/archiving/test_import.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/archiving/test_import.py b/tests/unit/archiving/test_import.py index c2645bb22..d18333ee1 100644 --- a/tests/unit/archiving/test_import.py +++ b/tests/unit/archiving/test_import.py @@ -180,6 +180,8 @@ def test_backwards_compatibility(self): with self.assertRaises(ValueError): self.imp_pr.unpack(origin_path=self.arch_dir_comp, csv_file_name="ahoy.csv") + +class TestUnpackingBackwardsCompatibility(PyironTestCase): def test_import_old_tar(self): copytree( os.path.join( @@ -191,7 +193,6 @@ def test_import_old_tar(self): ) pr = Project("old_tar") pr.unpack(origin_path="test_pack.tar.gz") - job = pr.load("toy") pr.remove(enable=True, enforce=True) os.remove("test_pack.tar.gz") os.remove("export.csv") From 5b5223a34d0fe12c153a8e66a82968b5a653ab4d Mon Sep 17 00:00:00 2001 From: samwaseda Date: Fri, 23 Aug 2024 08:48:22 +0000 Subject: [PATCH 11/29] add loading --- tests/unit/archiving/test_import.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/archiving/test_import.py b/tests/unit/archiving/test_import.py index d18333ee1..8c5525526 100644 --- a/tests/unit/archiving/test_import.py +++ b/tests/unit/archiving/test_import.py @@ -192,6 +192,7 @@ def test_import_old_tar(self): dirs_exist_ok=True, ) pr = Project("old_tar") + job = pr.load("toy") pr.unpack(origin_path="test_pack.tar.gz") pr.remove(enable=True, enforce=True) os.remove("test_pack.tar.gz") From 16b4e18fbc6bf2460abc0b889bf619d957402a5c Mon Sep 17 00:00:00 2001 From: samwaseda Date: Fri, 23 Aug 2024 09:04:33 +0000 Subject: [PATCH 12/29] change order --- tests/unit/archiving/test_import.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/archiving/test_import.py b/tests/unit/archiving/test_import.py index 8c5525526..1dfc5713d 100644 --- a/tests/unit/archiving/test_import.py +++ b/tests/unit/archiving/test_import.py @@ -192,8 +192,8 @@ def test_import_old_tar(self): dirs_exist_ok=True, ) pr = Project("old_tar") - job = pr.load("toy") pr.unpack(origin_path="test_pack.tar.gz") + job = pr.load("toy") pr.remove(enable=True, enforce=True) os.remove("test_pack.tar.gz") os.remove("export.csv") From e9abb0ba7a88ed9e8159b570ca3a5c7bedb903e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Jan=C3=9Fen?= Date: Fri, 23 Aug 2024 12:02:32 +0200 Subject: [PATCH 13/29] Get executable keys from dataclass fields Following @pmrv suggestion in https://github.com/pyiron/pyiron_base/pull/1611 --- pyiron_base/jobs/job/extension/executable.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/pyiron_base/jobs/job/extension/executable.py b/pyiron_base/jobs/job/extension/executable.py index 034d40fa4..5603904c2 100644 --- a/pyiron_base/jobs/job/extension/executable.py +++ b/pyiron_base/jobs/job/extension/executable.py @@ -3,7 +3,7 @@ # Distributed under the terms of "New BSD License", see the LICENSE file. import os -from dataclasses import asdict +from dataclasses import asdict, fields from pyiron_snippets.resources import ExecutableResolver @@ -214,14 +214,7 @@ def _to_dict(self): return asdict(self.storage) def _from_dict(self, obj_dict, version=None): - data_container_keys = [ - "version", - "name", - "operation_system_nt", - "executable", - "mpi", - "accepted_return_codes", - ] + data_container_keys = tuple(f.name for f in fields(ExecutableDataClass)) executable_class_dict = {} # Backwards compatibility; dict state used to be nested one level deeper if "executable" in obj_dict.keys() and isinstance(obj_dict["executable"], dict): From 0c0c1eb038da75efc3c7d913bb27a3c85b160b64 Mon Sep 17 00:00:00 2001 From: samwaseda Date: Fri, 23 Aug 2024 10:40:14 +0000 Subject: [PATCH 14/29] also check inputs --- tests/unit/archiving/test_import.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/archiving/test_import.py b/tests/unit/archiving/test_import.py index 1dfc5713d..ccea2e619 100644 --- a/tests/unit/archiving/test_import.py +++ b/tests/unit/archiving/test_import.py @@ -194,6 +194,8 @@ def test_import_old_tar(self): pr = Project("old_tar") pr.unpack(origin_path="test_pack.tar.gz") job = pr.load("toy") + self.assertEqual(job.job_name, "toy") + self.assertEqual(job.input.data_in, 100) pr.remove(enable=True, enforce=True) os.remove("test_pack.tar.gz") os.remove("export.csv") From f6587f5030c6b97de41ef875a10bbcdc2536e09f Mon Sep 17 00:00:00 2001 From: samwaseda Date: Fri, 23 Aug 2024 12:30:11 +0000 Subject: [PATCH 15/29] align things that I didn't change --- pyiron_base/jobs/job/util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyiron_base/jobs/job/util.py b/pyiron_base/jobs/job/util.py index 1bc03a995..bbb66b377 100644 --- a/pyiron_base/jobs/job/util.py +++ b/pyiron_base/jobs/job/util.py @@ -95,12 +95,12 @@ def _get_project_for_copy(job, project, new_job_name): ): file_project = project.project hdf5_project = project.project_hdf5.open(new_job_name) - elif isinstance(job.project, project.__class__): + elif isinstance(project, job.project.__class__): file_project = project hdf5_project = job.project_hdf5.__class__( project=project, file_name=new_job_name, h5_path="/" + new_job_name ) - elif isinstance(job.project_hdf5, project.__class__): + elif isinstance(project, job.project_hdf5.__class__): file_project = project.project hdf5_project = project.open(new_job_name) elif project is None: From 7d375813a37d9c132a54b46fe1f65755625556c3 Mon Sep 17 00:00:00 2001 From: samwaseda Date: Fri, 23 Aug 2024 12:35:19 +0000 Subject: [PATCH 16/29] Make a backward change made by Jan temporarily, to see whether the Windows tests fail (I'll revert this commit as soon as I confirm it) --- pyiron_base/project/archiving/import_archive.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyiron_base/project/archiving/import_archive.py b/pyiron_base/project/archiving/import_archive.py index 775cc5dd3..51ffeaad4 100644 --- a/pyiron_base/project/archiving/import_archive.py +++ b/pyiron_base/project/archiving/import_archive.py @@ -58,8 +58,8 @@ def import_jobs(project_instance, archive_directory): pr_import = project_instance.open(os.curdir) df["project"] = [ - posixpath.normpath( - posixpath.join(pr_import.project_path, posixpath.relpath(p, common_path)) + os.path.normpath( + os.path.join(pr_import.project_path, os.path.relpath(p, common_path)) ) + "/" for p in df["project"].values @@ -106,8 +106,8 @@ def transfer_files(origin_path: str, project_path: str): str: Common path. """ df = get_dataframe(origin_path=origin_path) - common_path = posixpath.commonpath(list(df["project"])) - copytree(posixpath.join(origin_path, common_path), project_path, dirs_exist_ok=True) + common_path = os.path.commonpath(list(df["project"])) + copytree(os.path.join(origin_path, common_path), project_path, dirs_exist_ok=True) return df, common_path From 5989c1896a43398586485be6ac9363e8c1057be4 Mon Sep 17 00:00:00 2001 From: samwaseda Date: Fri, 23 Aug 2024 12:40:21 +0000 Subject: [PATCH 17/29] I don't understand this one change --- pyiron_base/jobs/job/util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyiron_base/jobs/job/util.py b/pyiron_base/jobs/job/util.py index bbb66b377..1bc03a995 100644 --- a/pyiron_base/jobs/job/util.py +++ b/pyiron_base/jobs/job/util.py @@ -95,12 +95,12 @@ def _get_project_for_copy(job, project, new_job_name): ): file_project = project.project hdf5_project = project.project_hdf5.open(new_job_name) - elif isinstance(project, job.project.__class__): + elif isinstance(job.project, project.__class__): file_project = project hdf5_project = job.project_hdf5.__class__( project=project, file_name=new_job_name, h5_path="/" + new_job_name ) - elif isinstance(project, job.project_hdf5.__class__): + elif isinstance(job.project_hdf5, project.__class__): file_project = project.project hdf5_project = project.open(new_job_name) elif project is None: From 672d26e04be0102a55f095ae10a201d10cf0dd7a Mon Sep 17 00:00:00 2001 From: samwaseda Date: Fri, 23 Aug 2024 12:54:07 +0000 Subject: [PATCH 18/29] add output check --- tests/unit/archiving/test_import.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/archiving/test_import.py b/tests/unit/archiving/test_import.py index ccea2e619..859a34dad 100644 --- a/tests/unit/archiving/test_import.py +++ b/tests/unit/archiving/test_import.py @@ -196,6 +196,7 @@ def test_import_old_tar(self): job = pr.load("toy") self.assertEqual(job.job_name, "toy") self.assertEqual(job.input.data_in, 100) + self.assertEqual(job.output.data_out, 101) pr.remove(enable=True, enforce=True) os.remove("test_pack.tar.gz") os.remove("export.csv") From 29fa89c21126d679db76c73f57fcc05f7d883ba3 Mon Sep 17 00:00:00 2001 From: samwaseda Date: Fri, 23 Aug 2024 12:58:16 +0000 Subject: [PATCH 19/29] replace . by os.getcwd --- tests/unit/archiving/test_import.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/archiving/test_import.py b/tests/unit/archiving/test_import.py index 859a34dad..4ccc93d0e 100644 --- a/tests/unit/archiving/test_import.py +++ b/tests/unit/archiving/test_import.py @@ -188,7 +188,7 @@ def test_import_old_tar(self): os.path.dirname(os.path.abspath(__file__)), "../../static/pack", ), - ".", + os.getcwd(), dirs_exist_ok=True, ) pr = Project("old_tar") From fe25269c99363e1e5fb07188c0ac3ac83ea5c2fa Mon Sep 17 00:00:00 2001 From: Marvin Poul Date: Fri, 23 Aug 2024 15:05:03 +0200 Subject: [PATCH 20/29] Revert changes to DataContainer storage format PR #1364 changed saved the contents of the container in a data subgroup, but that breaks backwards compat and is not actually necessary for the TemplateJob, so revert it here. --- pyiron_base/storage/datacontainer.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pyiron_base/storage/datacontainer.py b/pyiron_base/storage/datacontainer.py index 46346c063..0fe17aed0 100644 --- a/pyiron_base/storage/datacontainer.py +++ b/pyiron_base/storage/datacontainer.py @@ -1027,13 +1027,17 @@ def to(v): return data def _to_dict(self): - return {"data": self.to_builtin(), "READ_ONLY": self.read_only} + data = self.to_builtin() + data["READ_ONLY"] = self.read_only + return data def _from_dict(self, obj_dict, version=None): + self.read_only = obj_dict.pop("READ_ONLY", False) + for key in _internal_hdf_nodes: + obj_dict.pop(key, None) with self.unlocked(): self.clear() - self.update(obj_dict["data"], wrap=True) - self.read_only = obj_dict.get("READ_ONLY", False) + self.update(obj_dict) HDFStub.register(DataContainer, lambda h, g: h[g].to_object(lazy=True)) From 103302e7754a0392051f8eea82f76b762dedd434 Mon Sep 17 00:00:00 2001 From: samwaseda Date: Fri, 23 Aug 2024 13:10:19 +0000 Subject: [PATCH 21/29] Revert "Make a backward change made by Jan temporarily, to see whether the Windows tests fail (I'll revert this commit as soon as I confirm it)" This reverts commit 7d375813a37d9c132a54b46fe1f65755625556c3. --- pyiron_base/project/archiving/import_archive.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyiron_base/project/archiving/import_archive.py b/pyiron_base/project/archiving/import_archive.py index 51ffeaad4..775cc5dd3 100644 --- a/pyiron_base/project/archiving/import_archive.py +++ b/pyiron_base/project/archiving/import_archive.py @@ -58,8 +58,8 @@ def import_jobs(project_instance, archive_directory): pr_import = project_instance.open(os.curdir) df["project"] = [ - os.path.normpath( - os.path.join(pr_import.project_path, os.path.relpath(p, common_path)) + posixpath.normpath( + posixpath.join(pr_import.project_path, posixpath.relpath(p, common_path)) ) + "/" for p in df["project"].values @@ -106,8 +106,8 @@ def transfer_files(origin_path: str, project_path: str): str: Common path. """ df = get_dataframe(origin_path=origin_path) - common_path = os.path.commonpath(list(df["project"])) - copytree(os.path.join(origin_path, common_path), project_path, dirs_exist_ok=True) + common_path = posixpath.commonpath(list(df["project"])) + copytree(posixpath.join(origin_path, common_path), project_path, dirs_exist_ok=True) return df, common_path From 7f694c6809642d7018d69b3f329d9765087ec9c3 Mon Sep 17 00:00:00 2001 From: Marvin Poul Date: Fri, 23 Aug 2024 17:45:27 +0200 Subject: [PATCH 22/29] Handle list in to_dict --- pyiron_base/storage/datacontainer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyiron_base/storage/datacontainer.py b/pyiron_base/storage/datacontainer.py index 0fe17aed0..4adfa55e4 100644 --- a/pyiron_base/storage/datacontainer.py +++ b/pyiron_base/storage/datacontainer.py @@ -1028,6 +1028,8 @@ def to(v): def _to_dict(self): data = self.to_builtin() + if isinstance(data, list): + data = {str(i): v for i, v in enumerate(data)} data["READ_ONLY"] = self.read_only return data From dbd263a9eb68674ae99d0f8d1a96aa5142776578 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Aug 2024 04:53:33 +0000 Subject: [PATCH 23/29] Bump executorlib from 0.0.1 to 0.0.2 Bumps [executorlib](https://github.com/pyiron/executorlib) from 0.0.1 to 0.0.2. - [Release notes](https://github.com/pyiron/executorlib/releases) - [Commits](https://github.com/pyiron/executorlib/compare/executorlib-0.0.1...executorlib-0.0.2) --- updated-dependencies: - dependency-name: executorlib dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index cd064fd2c..8a590e0ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ classifiers = [ ] dependencies = [ "cloudpickle==3.0.0", - "executorlib==0.0.1", + "executorlib==0.0.2", "h5io_browser==0.1.0", "h5py==3.11.0", "numpy==2.1.0", From 697ebd28487e86e115a0381b3ce3aaac56d8b0fc Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 26 Aug 2024 04:53:49 +0000 Subject: [PATCH 24/29] [dependabot skip] Update environment --- .ci_support/environment-docs.yml | 2 +- .ci_support/environment-mini.yml | 2 +- .ci_support/environment.yml | 2 +- binder/environment.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.ci_support/environment-docs.yml b/.ci_support/environment-docs.yml index 0b6f4e3cc..4063f5d2a 100644 --- a/.ci_support/environment-docs.yml +++ b/.ci_support/environment-docs.yml @@ -19,7 +19,7 @@ dependencies: - psutil =6.0.0 - pyfileindex =0.0.27 - pyiron_snippets =0.1.4 -- executorlib =0.0.1 +- executorlib =0.0.2 - pysqa =0.1.21 - pytables =3.10.1 - sqlalchemy =2.0.32 diff --git a/.ci_support/environment-mini.yml b/.ci_support/environment-mini.yml index aafc597ba..65e3cd8f0 100644 --- a/.ci_support/environment-mini.yml +++ b/.ci_support/environment-mini.yml @@ -10,7 +10,7 @@ dependencies: - psutil =6.0.0 - pyfileindex =0.0.27 - pyiron_snippets =0.1.3 -- executorlib =0.0.1 +- executorlib =0.0.2 - pysqa =0.1.21 - pytables =3.10.1 - sqlalchemy =2.0.32 diff --git a/.ci_support/environment.yml b/.ci_support/environment.yml index de400c55a..d58fc18d9 100644 --- a/.ci_support/environment.yml +++ b/.ci_support/environment.yml @@ -17,7 +17,7 @@ dependencies: - psutil =6.0.0 - pyfileindex =0.0.27 - pyiron_snippets =0.1.4 -- executorlib =0.0.1 +- executorlib =0.0.2 - pysqa =0.1.21 - pytables =3.10.1 - sqlalchemy =2.0.32 diff --git a/binder/environment.yml b/binder/environment.yml index aae94e696..afe8e4750 100644 --- a/binder/environment.yml +++ b/binder/environment.yml @@ -16,7 +16,7 @@ dependencies: - psutil =6.0.0 - pyfileindex =0.0.27 - pyiron_snippets =0.1.4 -- executorlib =0.0.1 +- executorlib =0.0.2 - pysqa =0.1.21 - pytables =3.10.1 - sqlalchemy =2.0.32 From d10d227e6c3d0d944976b091a1597eed7b3a1bc2 Mon Sep 17 00:00:00 2001 From: Marvin Poul Date: Mon, 26 Aug 2024 10:36:13 +0200 Subject: [PATCH 25/29] Add split_children_dict to undo join_children_dict HasDict.to_dict uses join_children_dict to enable more efficient writing to HDF. When reading from HDF this does not matter, because h5io_browser returns the dictionaries correctly, but when trying something like >>> data = obj.to_dict() >>> obj.from_dict(data) objects can choke on the joined keys, so now from_dict splits the keys automatically in case it finds any. --- pyiron_base/interfaces/has_dict.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/pyiron_base/interfaces/has_dict.py b/pyiron_base/interfaces/has_dict.py index e3eb01634..91fcabebf 100644 --- a/pyiron_base/interfaces/has_dict.py +++ b/pyiron_base/interfaces/has_dict.py @@ -16,6 +16,7 @@ """ from abc import ABC, abstractmethod +from collections import defaultdict from typing import Any from pyiron_base.interfaces.has_hdf import HasHDF @@ -122,6 +123,9 @@ def load(inner_dict): return {k: load(v) for k, v in inner_dict.items()} return create_from_dict(inner_dict) + obj_dict = self._split_children_dict(obj_dict) + if version is None: + version = obj_dict.get("DICT_VERSION", None) self._from_dict({k: load(v) for k, v in obj_dict.items()}, version) @abstractmethod @@ -208,9 +212,26 @@ def _join_children_dict(children: dict[str, dict[str, Any]]) -> dict[str, Any]: return { "/".join((k1, k2)): v2 for k1, v1 in children.items() - for k2, v2 in v2.items() + for k2, v2 in v1.items() } + @staticmethod + def _split_children_dict(obj_dict: dict[str, Any]) -> dict[str, Any | dict[str, Any]]: + """ + Undoes _join_children_dict. + """ + subs = defaultdict(dict) + plain = {} + for k, v in obj_dict.items(): + if "/" not in k: + plain[k] = v + continue + root, k = k.split("/", maxsplit=1) + subs[root][k] = v + # using update keeps type stability, i.e. we always return a plain dict + plain.update(subs) + return plain + class HasHDFfromDict(HasHDF, HasDict): """ From a0175499b0975a42f93b5ce3543bedeb6a2e5fe7 Mon Sep 17 00:00:00 2001 From: Marvin Poul Date: Mon, 26 Aug 2024 10:37:10 +0200 Subject: [PATCH 26/29] Save key order in DataContainer.to_dict Also let the generic code from HasDict take care of the conversion of children. --- pyiron_base/interfaces/has_dict.py | 2 +- pyiron_base/storage/datacontainer.py | 19 +++++++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/pyiron_base/interfaces/has_dict.py b/pyiron_base/interfaces/has_dict.py index 91fcabebf..22e6e8e12 100644 --- a/pyiron_base/interfaces/has_dict.py +++ b/pyiron_base/interfaces/has_dict.py @@ -55,7 +55,7 @@ def create_from_dict(obj_dict): type_field = obj_dict["TYPE"] module_path, class_name = _extract_module_class_name(type_field) class_object = _import_class(module_path, class_name) - version = obj_dict.get("VERSION", None) + version = obj_dict.get("DICT_VERSION", None) obj = class_object.instantiate(obj_dict, version) obj.from_dict(obj_dict, version) return obj diff --git a/pyiron_base/storage/datacontainer.py b/pyiron_base/storage/datacontainer.py index 4adfa55e4..bba832713 100644 --- a/pyiron_base/storage/datacontainer.py +++ b/pyiron_base/storage/datacontainer.py @@ -40,6 +40,7 @@ "HDF_VERSION", "DICT_VERSION", "READ_ONLY", + "KEY_ORDER", ] @@ -827,6 +828,7 @@ def _on_unlock(self): class DataContainer(DataContainerBase, HasHDF, HasDict): + __dict_version__ = "0.2.0" __doc__ = f"""{DataContainerBase.__doc__} If instantiated with the argument `lazy=True`, data read from HDF5 later via :method:`.from_hdf` are not actually @@ -1027,19 +1029,28 @@ def to(v): return data def _to_dict(self): - data = self.to_builtin() - if isinstance(data, list): - data = {str(i): v for i, v in enumerate(data)} + # stringify keys in case we are acting like a list + data = {str(k): v for k, v in dict(self).items()} + order = list(data) data["READ_ONLY"] = self.read_only + data["KEY_ORDER"] = order return data def _from_dict(self, obj_dict, version=None): + if version == "0.2.0": + order = obj_dict.pop("KEY_ORDER") + else: + order = None self.read_only = obj_dict.pop("READ_ONLY", False) for key in _internal_hdf_nodes: obj_dict.pop(key, None) with self.unlocked(): self.clear() - self.update(obj_dict) + if order is not None: + for key in order: + self[key] = obj_dict[key] + else: + self.update(obj_dict) HDFStub.register(DataContainer, lambda h, g: h[g].to_object(lazy=True)) From eadd9b0de10584f72b011d04c4c0e4f88e9fd28a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 26 Aug 2024 08:37:23 +0000 Subject: [PATCH 27/29] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- pyiron_base/interfaces/has_dict.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyiron_base/interfaces/has_dict.py b/pyiron_base/interfaces/has_dict.py index 22e6e8e12..5e619df9b 100644 --- a/pyiron_base/interfaces/has_dict.py +++ b/pyiron_base/interfaces/has_dict.py @@ -216,7 +216,9 @@ def _join_children_dict(children: dict[str, dict[str, Any]]) -> dict[str, Any]: } @staticmethod - def _split_children_dict(obj_dict: dict[str, Any]) -> dict[str, Any | dict[str, Any]]: + def _split_children_dict( + obj_dict: dict[str, Any], + ) -> dict[str, Any | dict[str, Any]]: """ Undoes _join_children_dict. """ From c3890693274ffd1f59c3a7da55cb69cf419e2284 Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Mon, 26 Aug 2024 11:46:10 +0200 Subject: [PATCH 28/29] Update unittests_old.yml --- .github/workflows/unittests_old.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unittests_old.yml b/.github/workflows/unittests_old.yml index 530e10dd4..c8cb5716c 100644 --- a/.github/workflows/unittests_old.yml +++ b/.github/workflows/unittests_old.yml @@ -13,7 +13,7 @@ jobs: - name: Setup Mambaforge uses: conda-incubator/setup-miniconda@v3 with: - python-version: '3.9' + python-version: '3.10' miniforge-variant: Mambaforge channels: conda-forge channel-priority: strict From 3ccfa335cb299081f028db31560ebfc59aed8cda Mon Sep 17 00:00:00 2001 From: Jan Janssen Date: Mon, 26 Aug 2024 11:46:24 +0200 Subject: [PATCH 29/29] Update pyproject.toml --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8a590e0ea..6118082bd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,6 @@ classifiers = [ "License :: OSI Approved :: BSD License", "Intended Audience :: Science/Research", "Operating System :: OS Independent", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12",