From c76b1127e876effb69a66319709dc21bad17b530 Mon Sep 17 00:00:00 2001 From: Omry Yadan Date: Tue, 18 May 2021 15:42:50 -0700 Subject: [PATCH 1/3] Attempt to fix loading of simple OmegaConf pickled configs stored in 2.0 --- MANIFEST.in | 2 +- omegaconf/basecontainer.py | 12 ++++++++++++ tests/data/2.0.6.pickle | Bin 0 -> 610 bytes tests/data/2.1.0.rc1.pickle | Bin 0 -> 638 bytes tests/data/load.py | 8 ++++++++ tests/data/save.py | 8 ++++++++ tests/test_serialization.py | 17 ++++++++++++++++- 7 files changed, 45 insertions(+), 2 deletions(-) create mode 100644 tests/data/2.0.6.pickle create mode 100644 tests/data/2.1.0.rc1.pickle create mode 100644 tests/data/load.py create mode 100644 tests/data/save.py diff --git a/MANIFEST.in b/MANIFEST.in index 5204b8d8c..35b834539 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,3 @@ include LICENSE include requirements/base.txt -recursive-include tests *.py +recursive-include tests *.py *.pickle diff --git a/omegaconf/basecontainer.py b/omegaconf/basecontainer.py index cb67cd3cc..311552981 100644 --- a/omegaconf/basecontainer.py +++ b/omegaconf/basecontainer.py @@ -103,7 +103,19 @@ def __setstate__(self, d: Dict[str, Any]) -> None: if isinstance(self, DictConfig): key_type = d["_metadata"].key_type + + # backward compatibility to load OmegaConf 2.0 configs + if key_type is None: + key_type = Any + d["_metadata"].key_type = key_type + element_type = d["_metadata"].element_type + + # backward compatibility to load OmegaConf 2.0 configs + if element_type is None: + element_type = Any + d["_metadata"].element_type = element_type + ref_type = d["_metadata"].ref_type if is_container_annotation(ref_type): if is_generic_dict(ref_type): diff --git a/tests/data/2.0.6.pickle b/tests/data/2.0.6.pickle new file mode 100644 index 0000000000000000000000000000000000000000..27f25e7dd3ed60083eadb738df4da5e85a0c2e8b GIT binary patch literal 610 zcmZWmzfZzY5Jp;{QUYQ~OjP3FA`63~o6!yJ;NoPO*Y=fr`e;e}O2R;*g9*cJJpW&> z6ewbc=I-VDarfP)`SV(>tUv6<6Fz2s9EHO`_$n{NSYz{%nunrU>A{y?=o>tCCtR_B zDW~Y7CI)HR0S{Rw)kYfH-};42NoA|v z;*w8zv~|loY`9aF&z+f=)qYrB%Bk%5M)h&3n%m75)A0SX{c^g_w-=^9cL{wsvOEV zTlO{d^DR7$o$^>w9B|61L1Q^olG#qZj@G}^fn8d`-CwKqh_`6EGdk6%nR#jEvGsrW E3!UTW8~^|S literal 0 HcmV?d00001 diff --git a/tests/data/2.1.0.rc1.pickle b/tests/data/2.1.0.rc1.pickle new file mode 100644 index 0000000000000000000000000000000000000000..4b23a3fe21dc13eb487b51be94660923da4ebaaa GIT binary patch literal 638 zcmZuuO;5r=5Jk&J1yjUeqA^j42lc|mqbH*`l!JHEY|Fx|E?v@YNjQ+`!32`IvHo9Y zTLc64u$`HmdGGC;kK%W-TgX0asv#e+P(}TAPlVbAVqmcPNXbKWPnVM_d^M_os z9@EUAx1I}F!VNa4TQd>y`1!YKHR#5?@9W8k+RF435ivOT(Znna)>QDuL!DoD(N!ZY zRK%p2q4Ulsri-P%WP^khU^R>USSfAh=*2uya?E2tWFZiCEmTqx7R@J=^?0A9Qrl-) zA~#5d)WTjBm$fhRLUm%v*}=noF8Pp0OF7`B0apAGi>X9PT%Y{WHYPDU>S zV0Q_%0ou)=I)(qBycNV1w5{QbrNduOh?H_jl@-YAa literal 0 HcmV?d00001 diff --git a/tests/data/load.py b/tests/data/load.py new file mode 100644 index 000000000..e8b9ae509 --- /dev/null +++ b/tests/data/load.py @@ -0,0 +1,8 @@ +import pickle +import sys + +from omegaconf import OmegaConf + +with open(f"{sys.argv[1]}.pickle", mode="rb") as fp: + cfg = pickle.load(fp) + assert cfg == OmegaConf.create({"a": [{"b": 10}]}) diff --git a/tests/data/save.py b/tests/data/save.py new file mode 100644 index 000000000..8243bd55f --- /dev/null +++ b/tests/data/save.py @@ -0,0 +1,8 @@ +import pickle + +from omegaconf import OmegaConf, __version__ + +cfg = OmegaConf.create({"a": [{"b": 10}]}) + +with open(f"{__version__}.pickle", mode="wb") as fp: + pickle.dump(cfg, fp) diff --git a/tests/test_serialization.py b/tests/test_serialization.py index 36ae8ef64..8760eac81 100644 --- a/tests/test_serialization.py +++ b/tests/test_serialization.py @@ -65,7 +65,7 @@ def test_load_from_invalid() -> None: ({"foo": 10, "bar": "${foo}"}, False, None, str), ({"foo": 10, "bar": "${foo}"}, False, None, pathlib.Path), ({"foo": 10, "bar": "${foo}"}, False, {"foo": 10, "bar": 10}, str), - ([u"שלום"], False, None, str), + (["שלום"], False, None, str), ], ) class TestSaveLoad: @@ -257,3 +257,18 @@ def test_pickle_flags_consistency() -> None: cfg2._set_flag("test", None) assert cfg2._get_flag("test") is None assert cfg2._get_node("a")._get_flag("test") is None + + +@mark.parametrize( + "version", + [ + "2.0.6", + "2.1.0.rc1", + ], +) +def test_pickle_backward_compatibility(version: str) -> None: + base = os.path.dirname(__file__) + path = os.path.abspath(f"{base}/data/{version}.pickle") + with open(path, mode="rb") as fp: + cfg = pickle.load(fp) + assert cfg == OmegaConf.create({"a": [{"b": 10}]}) From 72b79a2d3239c82dcb25dc4d2521095e8e72e3a7 Mon Sep 17 00:00:00 2001 From: Omry Yadan Date: Tue, 18 May 2021 18:02:33 -0700 Subject: [PATCH 2/3] Update tests/test_serialization.py Co-authored-by: Olivier Delalleau <507137+odelalleau@users.noreply.github.com> --- tests/test_serialization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_serialization.py b/tests/test_serialization.py index 8760eac81..eeb6709cb 100644 --- a/tests/test_serialization.py +++ b/tests/test_serialization.py @@ -268,7 +268,7 @@ def test_pickle_flags_consistency() -> None: ) def test_pickle_backward_compatibility(version: str) -> None: base = os.path.dirname(__file__) - path = os.path.abspath(f"{base}/data/{version}.pickle") + path = os.path.abspath(os.path.join(base), "data", f"{version}.pickle") with open(path, mode="rb") as fp: cfg = pickle.load(fp) assert cfg == OmegaConf.create({"a": [{"b": 10}]}) From 80547f11fd25b3294ca33e8aee44f28d4c9eaa0f Mon Sep 17 00:00:00 2001 From: Omry Yadan Date: Tue, 18 May 2021 18:40:13 -0700 Subject: [PATCH 3/3] fixed test --- tests/test_serialization.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_serialization.py b/tests/test_serialization.py index eeb6709cb..522b37c4f 100644 --- a/tests/test_serialization.py +++ b/tests/test_serialization.py @@ -267,8 +267,7 @@ def test_pickle_flags_consistency() -> None: ], ) def test_pickle_backward_compatibility(version: str) -> None: - base = os.path.dirname(__file__) - path = os.path.abspath(os.path.join(base), "data", f"{version}.pickle") + path = Path(__file__).parent / "data" / f"{version}.pickle" with open(path, mode="rb") as fp: cfg = pickle.load(fp) assert cfg == OmegaConf.create({"a": [{"b": 10}]})