to_container: replace exclude_structured_configs with SCMode (omry#552)

* to_container: replace exclude_structured_configs with SCMode

Closes omry#548

Co-authored-by: Omry Yadan <[email protected]>

* Add note about `structured_config_mode` to docs

* revise documentation

* Update OmegaConf.to_container docstring

* add news/548.api_change

docs/source/usage.rst

@@ -740,6 +740,22 @@ If resolve is set to True, interpolations will be resolved during conversion.
     >>> print(resolved)
     {'foo': 'bar', 'foo2': 'bar'}
 
+You can customize the treatment of **OmegaConf.to_container()** for
+Structured Config nodes using the `structured_config_mode` option.
+By default, Structured Config nodes are converted to plain dict.
+Using **structured_config_mode=SCMode.DICT_CONFIG** causes such nodes to remain
+as DictConfig, allowing attribute style access on the resulting node.
+
+.. doctest::
+
+    >>> from omegaconf import SCMode
+    >>> conf = OmegaConf.create({"structured_config": MyConfig})
+    >>> container = OmegaConf.to_container(conf, structured_config_mode=SCMode.DICT_CONFIG)
+    >>> print(container)
+    {'structured_config': {'port': 80, 'host': 'localhost'}}
+    >>> assert type(container) is dict
+    >>> assert type(container["structured_config"]) is DictConfig
+    >>> assert container["structured_config"].port == 80
 
 OmegaConf.select
 ^^^^^^^^^^^^^^^^

news/548.api_change

@@ -0,0 +1 @@
+OmegaConf.to_container now takes a `structured_config_mode` keyword argument. Setting `structured_config_mode=SCMode.DICT_CONFIG` causes `to_container` to not convert Structured Config objects to python dicts (it leaves them as DictConfig objects).

omegaconf/__init__.py

@@ -1,4 +1,4 @@
-from .base import Container, DictKeyType, Node
+from .base import Container, DictKeyType, Node, SCMode
 from .dictconfig import DictConfig
 from .errors import (
     KeyValidationError,
@@ -42,6 +42,7 @@
     "DictKeyType",
     "OmegaConf",
     "Resolver",
+    "SCMode",
     "flag_override",
     "read_write",
     "open_dict",

omegaconf/base.py

@@ -615,3 +615,8 @@ def _invalidate_flags_cache(self) -> None:
 
     def _has_ref_type(self) -> bool:
         return self._metadata.ref_type is not Any
+
+
+class SCMode(Enum):
+    DICT = 1  # convert to plain dict
+    DICT_CONFIG = 2  # Keep as OmegaConf DictConfig

omegaconf/basecontainer.py

@@ -25,7 +25,7 @@
     is_primitive_type,
     is_structured_config,
 )
-from .base import Container, ContainerMetadata, DictKeyType, Node
+from .base import Container, ContainerMetadata, DictKeyType, Node, SCMode
 from .errors import MissingMandatoryValue, ReadonlyConfigError, ValidationError
 
 if TYPE_CHECKING:
@@ -192,7 +192,7 @@ def _to_content(
         conf: Container,
         resolve: bool,
         enum_to_str: bool = False,
-        exclude_structured_configs: bool = False,
+        structured_config_mode: SCMode = SCMode.DICT,
     ) -> Union[None, Any, str, Dict[DictKeyType, Any], List[Any]]:
         from .dictconfig import DictConfig
         from .listconfig import ListConfig
@@ -214,7 +214,10 @@ def convert(val: Node) -> Any:
         elif conf._is_missing():
             return "???"
         elif isinstance(conf, DictConfig):
-            if conf._metadata.object_type is not None and exclude_structured_configs:
+            if (
+                conf._metadata.object_type is not None
+                and structured_config_mode == SCMode.DICT_CONFIG
+            ):
                 return conf
 
             retdict: Dict[str, Any] = {}
@@ -234,7 +237,7 @@ def convert(val: Node) -> Any:
                         node,
                         resolve=resolve,
                         enum_to_str=enum_to_str,
-                        exclude_structured_configs=exclude_structured_configs,
+                        structured_config_mode=structured_config_mode,
                     )
                 else:
                     retdict[key] = convert(node)
@@ -254,7 +257,7 @@ def convert(val: Node) -> Any:
                         node,
                         resolve=resolve,
                         enum_to_str=enum_to_str,
-                        exclude_structured_configs=exclude_structured_configs,
+                        structured_config_mode=structured_config_mode,
                     )
                     retlist.append(item)
                 else:

omegaconf/omegaconf.py

@@ -47,7 +47,7 @@
     is_tuple_annotation,
     type_str,
 )
-from .base import Container, Node
+from .base import Container, Node, SCMode
 from .basecontainer import BaseContainer
 from .errors import (
     ConfigKeyError,
@@ -579,17 +579,34 @@ def to_container(
         *,
         resolve: bool = False,
         enum_to_str: bool = False,
-        exclude_structured_configs: bool = False,
+        structured_config_mode: SCMode = SCMode.DICT,
+        exclude_structured_configs: Optional[bool] = None,
     ) -> Union[Dict[DictKeyType, Any], List[Any], None, str]:
         """
         Resursively converts an OmegaConf config to a primitive container (dict or list).
         :param cfg: the config to convert
         :param resolve: True to resolve all values
         :param enum_to_str: True to convert Enum keys and values to strings
-        :param exclude_structured_configs: If True, do not convert Structured Configs
-               (DictConfigs backed by a dataclass)
+        :param structured_config_mode: Specify how Structured Configs (DictConfigs backed by a dataclass) are handled.
+            By default (`structured_config_mode=SCMode.DICT`) structured configs are converted to plain dicts.
+            If `structured_config_mode=SCMode.DICT_CONFIG`, structured config nodes will remain as DictConfig.
+        :param exclude_structured_configs: (DEPRECATED) If true, do not convert structured configs.
+            Equivalent to `structured_config_mode=SCMode.DICT_CONFIG`.
         :return: A dict or a list representing this config as a primitive container.
         """
+        if exclude_structured_configs is not None:
+            warnings.warn(
+                dedent(
+                    """\
+                The exclude_structured_configs argument to to_container is deprecated.
+                See https://github.com/omry/omegaconf/issues/548 for migration instructions.
+                """
+                ),
+                UserWarning,
+                stacklevel=2,
+            )
+            if exclude_structured_configs is True:
+                structured_config_mode = SCMode.DICT_CONFIG
         if not OmegaConf.is_config(cfg):
             raise ValueError(
                 f"Input cfg is not an OmegaConf config object ({type_str(type(cfg))})"
@@ -599,7 +616,7 @@ def to_container(
             cfg,
             resolve=resolve,
             enum_to_str=enum_to_str,
-            exclude_structured_configs=exclude_structured_configs,
+            structured_config_mode=structured_config_mode,
         )
 
     @staticmethod

tests/test_to_container.py

@@ -2,9 +2,9 @@
 from enum import Enum
 from typing import Any, Dict, List
 
-from pytest import mark, param, raises
+from pytest import fixture, mark, param, raises, warns
 
-from omegaconf import DictConfig, ListConfig, OmegaConf
+from omegaconf import DictConfig, ListConfig, OmegaConf, SCMode
 from tests import Color, User
 
 
@@ -39,32 +39,57 @@ def assert_container_with_primitives(item: Any) -> None:
 
 
 @mark.parametrize(
-    "cfg,ex_false,ex_true",
+    "src,ex_dict,ex_dict_config,key",
     [
         param(
             {"user": User(age=7, name="Bond")},
             {"user": {"name": "Bond", "age": 7}},
             {"user": User(age=7, name="Bond")},
+            "user",
+            id="structured-inside-dict",
         ),
         param(
             [1, User(age=7, name="Bond")],
             [1, {"name": "Bond", "age": 7}],
             [1, User(age=7, name="Bond")],
-        ),
-        param(
-            {"users": [User(age=1, name="a"), User(age=2, name="b")]},
-            {"users": [{"age": 1, "name": "a"}, {"age": 2, "name": "b"}]},
-            {"users": [User(age=1, name="a"), User(age=2, name="b")]},
+            1,
+            id="structured-inside-list",
         ),
     ],
 )
-def test_exclude_structured_configs(cfg: Any, ex_false: Any, ex_true: Any) -> None:
-    cfg = OmegaConf.create(cfg)
-    ret1 = OmegaConf.to_container(cfg, exclude_structured_configs=False)
-    assert ret1 == ex_false
+class TestSCMode:
+    @fixture
+    def cfg(self, src: Any) -> Any:
+        return OmegaConf.create(src)
+
+    def test_exclude_structured_configs_default(
+        self, cfg: Any, ex_dict: Any, ex_dict_config: Any, key: Any
+    ) -> None:
+        ret = OmegaConf.to_container(cfg)
+        assert ret == ex_dict
+        assert isinstance(ret[key], dict)
+
+    def test_scmode_dict(
+        self, cfg: Any, ex_dict: Any, ex_dict_config: Any, key: Any
+    ) -> None:
+        ret = OmegaConf.to_container(cfg, structured_config_mode=SCMode.DICT)
+        assert ret == ex_dict
+        assert isinstance(ret[key], dict)
+
+        with warns(UserWarning):
+            ret = OmegaConf.to_container(cfg, exclude_structured_configs=False)
+        assert ret == ex_dict
+
+    def test_scmode_dict_config(
+        self, cfg: Any, ex_dict: Any, ex_dict_config: Any, key: Any
+    ) -> None:
+        ret = OmegaConf.to_container(cfg, structured_config_mode=SCMode.DICT_CONFIG)
+        assert ret == ex_dict_config
+        assert isinstance(ret[key], DictConfig)
 
-    ret1 = OmegaConf.to_container(cfg, exclude_structured_configs=True)
-    assert ret1 == ex_true
+        with warns(UserWarning):
+            ret = OmegaConf.to_container(cfg, exclude_structured_configs=True)
+        assert ret == ex_dict_config
 
 
 @mark.parametrize(