Add new resolver oc.create

docs/notebook/Tutorial.ipynb

@@ -821,14 +821,19 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Strings may be converted using ``oc.decode``:\n",
+    "With ``oc.decode``, strings can be converted into their corresponding data types using the OmegaConf grammar.\n",
+    "This grammar recognizes typical data types like ``bool``, ``int``, ``float``, ``dict`` and ``list``,\n",
+    "e.g. ``\"true\"``, ``\"1\"``, ``\"1e-3\"``, ``\"{a: b}\"``, ``\"[a, b, c]\"``.\n",
+    "It will also resolve interpolations like ``\"${foo}\"``, returning the corresponding value of the node.\n",
     "\n",
-    "- Primitive values (e.g., ``\"true\"``, ``\"1\"``, ``\"1e-3\"``) are automatically converted to their corresponding type (bool, int, float)\n",
-    "- Dictionaries and lists (e.g., ``\"{a: b}\"``, ``\"[a, b, c]\"``)  are returned as transient config nodes (DictConfig and ListConfig)\n",
-    "- Interpolations (e.g., ``\"${foo}\"``) are automatically resolved\n",
-    "- ``None`` is the only valid non-string input to ``oc.decode`` (returning ``None`` in that case)\n",
+    "Note that:\n",
     "\n",
-    "This can be useful for instance to parse environment variables:"
+    "- When providing as input to ``oc.decode`` a string that is meant to be decoded into another string, in general\n",
+    "  the input string should be quoted (since only a subset of characters are allowed by the grammar in unquoted\n",
+    "  strings). For instance, a proper string interpolation could be: ``\"'Hi! My name is: ${name}'\"`` (with extra quotes).\n",
+    "- ``None`` (written as ``null`` in the grammar) is the only valid non-string input to ``oc.decode`` (returning ``None`` in that case)\n",
+    "\n",
+    "This resolver can be useful for instance to parse environment variables:"
    ]
   },
   {

docs/source/custom_resolvers.rst

@@ -60,19 +60,6 @@ simply use quotes to bypass character limitations in strings.
     'Hello, World'
 
 
-Custom resolvers can return lists or dictionaries, that are automatically converted into DictConfig and ListConfig:
-
-.. doctest::
-
-    >>> OmegaConf.register_new_resolver(
-    ...     "min_max", lambda *a: {"min": min(a), "max": max(a)}
-    ... )
-    >>> c = OmegaConf.create({'stats': '${min_max: -1, 3, 2, 5, -10}'})
-    >>> assert isinstance(c.stats, DictConfig)
-    >>> c.stats.min, c.stats.max
-    (-10, 5)
-
-
 You can take advantage of nested interpolations to perform custom operations over variables:
 
 .. doctest::
@@ -213,6 +200,29 @@ The following example falls back to default passwords when ``DB_PASSWORD`` is no
     >>> show(cfg.database.password3)
     type: NoneType, value: None
 
+
+.. _oc.create:
+
+oc.create
+^^^^^^^^^
+
+``oc.create`` may be used for dynamic generation of config nodes
+(typically from Python ``dict`` / ``list`` objects or YAML strings, similar to :ref:`OmegaConf.create<creating>`).
+The following example combines ``oc.create`` with ``oc.decode`` and ``oc.env`` to generate
+a sub-config from an environment variable:
+
+.. doctest::
+
+    >>> cfg = OmegaConf.create(
+    ...     {
+    ...         "model": "${oc.create:${oc.decode:${oc.env:MODEL}}}",
+    ...     }
+    ... )
+    >>> os.environ["MODEL"] = "{name: my_model, layer_size: [100, 200]}"
+    >>> show(cfg.model.layer_size)
+    type: ListConfig, value: [100, 200]
+
+
 .. _oc.deprecated:
 
 oc.deprecated
@@ -245,14 +255,18 @@ It takes two parameters:
 oc.decode
 ^^^^^^^^^
 
-Strings may be converted using ``oc.decode``:
+With ``oc.decode``, strings can be converted into their corresponding data types using the OmegaConf grammar.
+This grammar recognizes typical data types like ``bool``, ``int``, ``float``, ``dict`` and ``list``,
+e.g. ``"true"``, ``"1"``, ``"1e-3"``, ``"{a: b}"``, ``"[a, b, c]"``.
+
+Note that:
 
-- Primitive values (e.g., ``"true"``, ``"1"``, ``"1e-3"``) are automatically converted to their corresponding type (bool, int, float)
-- Dictionaries and lists (e.g., ``"{a: b}"``, ``"[a, b, c]"``)  are returned as transient config nodes (DictConfig and ListConfig)
-- Interpolations (e.g., ``"${foo}"``) are automatically resolved
-- ``None`` is the only valid non-string input to ``oc.decode`` (returning ``None`` in that case)
+- When providing as input to ``oc.decode`` a string that is meant to be decoded into another string, in general
+  the input string should be quoted (since only a subset of characters are allowed by the grammar in unquoted
+  strings). For instance: ``"'Hello, world!'"`` (with extra quotes).
+- ``None`` (written as ``null`` in the grammar) is the only valid non-string input to ``oc.decode`` (returning ``None`` in that case)
 
-This can be useful for instance to parse environment variables:
+This resolver can be useful for instance to parse environment variables:
 
 .. doctest::
 
@@ -269,8 +283,8 @@ This can be useful for instance to parse environment variables:
     >>> show(cfg.database.port)  # converted to int
     type: int, value: 3308
     >>> os.environ["DB_NODES"] = "[host1, host2, host3]"
-    >>> show(cfg.database.nodes)  # converted to a ListConfig
-    type: ListConfig, value: ['host1', 'host2', 'host3']
+    >>> show(cfg.database.nodes)  # converted to a Python list
+    type: list, value: ['host1', 'host2', 'host3']
     >>> show(cfg.database.timeout)  # keeping `None` as is
     type: NoneType, value: None
     >>> os.environ["DB_TIMEOUT"] = "${.port}"

docs/source/usage.rst

@@ -22,6 +22,8 @@ Just pip install::
 
 OmegaConf requires Python 3.6 and newer.
 
+.. _creating:
+
 Creating
 --------
 You can create OmegaConf objects from multiple sources.
@@ -401,6 +403,7 @@ Built-in resolvers
 ^^^^^^^^^^^^^^^^^^
 OmegaConf comes with a set of built-in custom resolvers:
 
+* :ref:`oc.create`: Dynamically generating config nodes
 * :ref:`oc.decode`: Parsing an input string using interpolation grammar
 * :ref:`oc.deprecated`: Deprecate a key in your config
 * :ref:`oc.env`: Accessing environment variables

news/488.api_change

@@ -1 +1 @@
-When resolving an interpolation of a typed config value, the interpolated value is validated and possibly converted based on the node's type.
+When resolving an interpolation of a config value with a primitive type, the interpolated value is validated and possibly converted based on the node's type.

news/645.feature

@@ -0,0 +1 @@
+The new built-in resolver `oc.create` can be used to dynamically generate config nodes

omegaconf/base.py

@@ -15,6 +15,7 @@
     _is_missing_value,
     format_and_raise,
     get_value_kind,
+    is_primitive_type,
     split_key,
 )
 from .errors import (
@@ -23,7 +24,6 @@
     InterpolationResolutionError,
     InterpolationToMissingValueError,
     InterpolationValidationError,
-    KeyValidationError,
     MissingMandatoryValue,
     UnsupportedInterpolationType,
     ValidationError,
@@ -552,10 +552,14 @@ def _wrap_interpolation_result(
         throw_on_resolution_failure: bool,
     ) -> Optional["Node"]:
         from .basecontainer import BaseContainer
+        from .nodes import AnyNode
         from .omegaconf import _node_wrap
 
         assert parent is None or isinstance(parent, BaseContainer)
-        try:
+
+        if is_primitive_type(type(resolved)):
+            # Primitive types get wrapped using `_node_wrap()`, ensuring value is
+            # validated and potentially converted.
             wrapped = _node_wrap(
                 type_=value._metadata.ref_type,
                 parent=parent,
@@ -564,19 +568,14 @@ def _wrap_interpolation_result(
                 key=key,
                 ref_type=value._metadata.ref_type,
             )
-        except (KeyValidationError, ValidationError) as e:
-            if throw_on_resolution_failure:
-                self._format_and_raise(
-                    key=key,
-                    value=resolved,
-                    cause=e,
-                    type_override=InterpolationValidationError,
-                )
-            return None
-        # Since we created a new node on the fly, future changes to this node are
-        # likely to be lost. We thus set the "readonly" flag to `True` to reduce
-        # the risk of accidental modifications.
-        wrapped._set_flag("readonly", True)
+        else:
+            # Other objects get wrapped into an `AnyNode` with `allow_objects` set
+            # to True.
+            wrapped = AnyNode(
+                value=resolved, key=key, parent=None, flags={"allow_objects": True}
+            )
+            wrapped._set_parent(parent)
+
         return wrapped
 
     def _validate_not_dereferencing_to_parent(self, node: Node, target: Node) -> None:

omegaconf/nodes.py

@@ -111,11 +111,14 @@ def __init__(
         value: Any = None,
         key: Any = None,
         parent: Optional[Container] = None,
+        flags: Optional[Dict[str, bool]] = None,
     ):
         super().__init__(
             parent=parent,
             value=value,
-            metadata=Metadata(ref_type=Any, object_type=None, key=key, optional=True),
+            metadata=Metadata(
+                ref_type=Any, object_type=None, key=key, optional=True, flags=flags
+            ),
         )
 
     def _validate_and_convert_impl(self, value: Any) -> Any:
@@ -145,12 +148,17 @@ def __init__(
         key: Any = None,
         parent: Optional[Container] = None,
         is_optional: bool = True,
+        flags: Optional[Dict[str, bool]] = None,
     ):
         super().__init__(
             parent=parent,
             value=value,
             metadata=Metadata(
-                key=key, optional=is_optional, ref_type=str, object_type=str
+                key=key,
+                optional=is_optional,
+                ref_type=str,
+                object_type=str,
+                flags=flags,
             ),
         )
 
@@ -174,12 +182,17 @@ def __init__(
         key: Any = None,
         parent: Optional[Container] = None,
         is_optional: bool = True,
+        flags: Optional[Dict[str, bool]] = None,
     ):
         super().__init__(
             parent=parent,
             value=value,
             metadata=Metadata(
-                key=key, optional=is_optional, ref_type=int, object_type=int
+                key=key,
+                optional=is_optional,
+                ref_type=int,
+                object_type=int,
+                flags=flags,
             ),
         )
 
@@ -206,12 +219,17 @@ def __init__(
         key: Any = None,
         parent: Optional[Container] = None,
         is_optional: bool = True,
+        flags: Optional[Dict[str, bool]] = None,
     ):
         super().__init__(
             parent=parent,
             value=value,
             metadata=Metadata(
-                key=key, optional=is_optional, ref_type=float, object_type=float
+                key=key,
+                optional=is_optional,
+                ref_type=float,
+                object_type=float,
+                flags=flags,
             ),
         )
 
@@ -255,12 +273,17 @@ def __init__(
         key: Any = None,
         parent: Optional[Container] = None,
         is_optional: bool = True,
+        flags: Optional[Dict[str, bool]] = None,
     ):
         super().__init__(
             parent=parent,
             value=value,
             metadata=Metadata(
-                key=key, optional=is_optional, ref_type=bool, object_type=bool
+                key=key,
+                optional=is_optional,
+                ref_type=bool,
+                object_type=bool,
+                flags=flags,
             ),
         )
 
@@ -307,6 +330,7 @@ def __init__(
         key: Any = None,
         parent: Optional[Container] = None,
         is_optional: bool = True,
+        flags: Optional[Dict[str, bool]] = None,
     ):
         if not isinstance(enum_type, type) or not issubclass(enum_type, Enum):
             raise ValidationError(
@@ -320,7 +344,11 @@ def __init__(
             parent=parent,
             value=value,
             metadata=Metadata(
-                key=key, optional=is_optional, ref_type=enum_type, object_type=enum_type
+                key=key,
+                optional=is_optional,
+                ref_type=enum_type,
+                object_type=enum_type,
+                flags=flags,
             ),
         )
 

omegaconf/omegaconf.py

@@ -94,6 +94,7 @@ def SI(interpolation: str) -> Any:
 def register_default_resolvers() -> None:
     from omegaconf.resolvers import env, oc
 
+    OmegaConf.register_new_resolver("oc.create", oc.create)
     OmegaConf.register_new_resolver("oc.decode", oc.decode)
     OmegaConf.register_new_resolver("oc.deprecated", oc.deprecated)
     OmegaConf.register_new_resolver("oc.env", oc.env)

omegaconf/resolvers/oc/__init__.py

@@ -5,11 +5,20 @@
 
 from omegaconf import Container, Node
 from omegaconf._utils import _DEFAULT_MARKER_, _get_value
+from omegaconf.basecontainer import BaseContainer
 from omegaconf.errors import ConfigKeyError
 from omegaconf.grammar_parser import parse
 from omegaconf.resolvers.oc import dict
 
 
+def create(obj: Any, _parent_: Container) -> Any:
+    """Create a config object from `obj`, similar to `OmegaConf.create`"""
+    from omegaconf import OmegaConf
+
+    assert isinstance(_parent_, BaseContainer)
+    return OmegaConf.create(obj, parent=_parent_)
+
+
 def env(key: str, default: Any = _DEFAULT_MARKER_) -> Optional[str]:
     """
     :param key: Environment variable key
@@ -85,6 +94,7 @@ def deprecated(
 
 
 __all__ = [
+    "create",
     "decode",
     "deprecated",
     "dict",