Introduce new oc.env and oc.decode resolvers

docs/notebook/Tutorial.ipynb

@@ -698,7 +698,8 @@
    "source": [
     "## Environment variable interpolation\n",
     "\n",
-    "Environment variable interpolation is also supported."
+    "Environment variable interpolation is also supported.\n",
+    "An environment variable is always returned as a string."
    ]
   },
   {
@@ -733,8 +734,8 @@
      "output_type": "stream",
      "text": [
       "user:\n",
-      "  name: ${env:USER}\n",
-      "  home: /home/${env:USER}\n",
+      "  name: ${oc.env:USER}\n",
+      "  home: /home/${oc.env:USER}\n",
       "\n"
      ]
     }
@@ -773,7 +774,10 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "You can specify a default value to use in case the environment variable is not defined. The following example sets `abc123` as the the default value when `DB_PASSWORD` is not defined."
+    "You can specify a default value to use in case the environment variable is not defined.\n",
+    "This default value can be a string or ``null`` (representing Python ``None``). Passing a default with a different type will result in an error.\n",
+    "\n",
+    "The following example sets default database passwords when ``DB_PASSWORD`` is not defined:"
    ]
   },
   {
@@ -782,27 +786,43 @@
    "metadata": {},
    "outputs": [
     {
-     "data": {
-      "text/plain": [
-       "'abc123'"
-      ]
-     },
-     "execution_count": 25,
-     "metadata": {},
-     "output_type": "execute_result"
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "'abc123'\n",
+      "'12345'\n"
+     ]
     }
    ],
    "source": [
     "os.environ.pop('DB_PASSWORD', None)  # ensure env variable does not exist\n",
-    "cfg = OmegaConf.create({'database': {'password': '${env:DB_PASSWORD,abc123}'}})\n",
-    "cfg.database.password"
+    "cfg = OmegaConf.create(\n",
+    "    {\n",
+    "        \"database\": {\n",
+    "            \"password1\": \"${oc.env:DB_PASSWORD,abc123}\",  # the string 'abc123'\n",
+    "            \"password2\": \"${oc.env:DB_PASSWORD,'12345'}\",  # the string '12345'\n",
+    "        },\n",
+    "    }\n",
+    ")\n",
+    "print(repr(cfg.database.password1))\n",
+    "print(repr(cfg.database.password2))"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Environment variables are parsed when they are recognized as valid quantities that may be evaluated (e.g., int, float, dict, list):"
+    "## Decoding strings with interpolations"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "You can automatically convert a string to its corresponding type (e.g., bool, int, float, dict, list) using `oc.decode` (which can even resolve interpolations).\n",
+    "This resolver also accepts ``None`` as input, in which case it returns ``None``.\n",
+    "\n",
+    "This can be useful for instance to parse environment variables:"
    ]
   },
   {
@@ -814,25 +834,34 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "3308\n",
-      "['host1', 'host2', 'host3']\n",
-      "'a%#@~{}$*&^?/<'\n"
+      "port (int): 3308\n",
+      "nodes (list): ['host1', 'host2', 'host3']\n",
+      "timeout (missing variable): None\n",
+      "timeout (interpolation): 3308\n"
      ]
     }
    ],
    "source": [
-    "cfg = OmegaConf.create({'database': {'password': '${env:DB_PASSWORD,abc123}',\n",
-    "                                    'user': 'someuser',\n",
-    "                                    'port': '${env:DB_PORT,3306}',\n",
-    "                                    'nodes': '${env:DB_NODES,[]}'}})\n",
+    "cfg = OmegaConf.create(\n",
+    "    {\n",
+    "        \"database\": {\n",
+    "            \"port\": '${oc.decode:${oc.env:DB_PORT}}',\n",
+    "            \"nodes\": '${oc.decode:${oc.env:DB_NODES,null}}',\n",
+    "            \"timeout\": '${oc.decode:${oc.env:DB_TIMEOUT,null}}',\n",
+    "        }\n",
+    "    }\n",
+    ")\n",
+    "\n",
+    "os.environ[\"DB_PORT\"] = \"3308\"  # integer\n",
+    "os.environ[\"DB_NODES\"] = \"[host1, host2, host3]\"  # list\n",
+    "os.environ.pop(\"DB_TIMEOUT\", None)  # unset variable\n",
     "\n",
-    "os.environ[\"DB_PORT\"] = '3308'  # integer\n",
-    "os.environ[\"DB_NODES\"] = '[host1, host2, host3]'  # list\n",
-    "os.environ[\"DB_PASSWORD\"] = 'a%#@~{}$*&^?/<'  # string\n",
+    "print(\"port (int):\", repr(cfg.database.port))\n",
+    "print(\"nodes (list):\", repr(cfg.database.nodes))\n",
+    "print(\"timeout (missing variable):\", repr(cfg.database.timeout))\n",
     "\n",
-    "print(repr(cfg.database.port))\n",
-    "print(repr(cfg.database.nodes))\n",
-    "print(repr(cfg.database.password))"
+    "os.environ[\"DB_TIMEOUT\"] = \"${.port}\"\n",
+    "print(\"timeout (interpolation):\", repr(cfg.database.timeout))"
    ]
   },
   {

docs/source/env_interpolation.yaml

@@ -1,3 +1,3 @@
 user:
-  name: ${env:USER}
-  home: /home/${env:USER}
+  name: ${oc.env:USER}
+  home: /home/${oc.env:USER}

docs/source/usage.rst

@@ -6,6 +6,8 @@
     import tempfile
     import pickle
     os.environ['USER'] = 'omry'
+    # ensures that DB_TIMEOUT is not set in the doc.
+    os.environ.pop('DB_TIMEOUT', None)
 
 .. testsetup:: loaded
 
@@ -334,20 +336,17 @@ Example:
 .. doctest::
 
     >>> conf = OmegaConf.load('source/config_interpolation.yaml')
+    >>> def show(x):
+    ...     print(f"type: {type(x).__name__}, value: {repr(x)}")
     >>> # Primitive interpolation types are inherited from the reference
-    >>> conf.client.server_port
-    80
-    >>> type(conf.client.server_port).__name__
-    'int'
-    >>> conf.client.description
-    'Client of http://localhost:80/'
-
-    >>> # Composite interpolation types are always string
-    >>> conf.client.url
-    'http://localhost:80/'
-    >>> type(conf.client.url).__name__
-    'str'
-
+    >>> show(conf.client.server_port)
+    type: int, value: 80
+    >>> # String interpolations concatenate fragments into a string
+    >>> show(conf.client.url)
+    type: str, value: 'http://localhost:80/'
+    >>> # Relative interpolation example
+    >>> show(conf.client.description)
+    type: str, value: 'Client of http://localhost:80/'
 
 Interpolations may be nested, enabling more advanced behavior like dynamically selecting a sub-config:
 
@@ -386,7 +385,7 @@ Interpolated nodes can be any node in the config, not just leaf nodes:
 Environment variable interpolation
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Environment variable interpolation is also supported.
+Access to environment variables is supported using ``oc.env``:
 
 Input YAML file:
 
@@ -402,36 +401,59 @@ Input YAML file:
     '/home/omry'
 
 You can specify a default value to use in case the environment variable is not defined.
-The following example sets `abc123` as the the default value when `DB_PASSWORD` is not defined.
+This default value can be a string or ``null`` (representing Python ``None``). Passing a default with a different type will result in an error.
+The following example falls back to default passwords when ``DB_PASSWORD`` is not defined:
 
 .. doctest::
 
-    >>> cfg = OmegaConf.create({
-    ...       'database': {'password': '${env:DB_PASSWORD,abc123}'}
-    ... })
-    >>> cfg.database.password
+    >>> cfg = OmegaConf.create(
+    ...     {
+    ...         "database": {
+    ...             "password1": "${oc.env:DB_PASSWORD,abc123}",
+    ...             "password2": "${oc.env:DB_PASSWORD,'12345'}",
+    ...         },
+    ...     }
+    ... )
+    >>> cfg.database.password1  # the string 'abc123'
     'abc123'
+    >>> cfg.database.password2  # the string '12345'
+    '12345'
+
 
-Environment variables are parsed when they are recognized as valid quantities that
-may be evaluated (e.g., int, float, dict, list):
+Decoding strings with interpolations
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Strings may be converted using ``oc.decode``:
+
+- Primitive values (e.g., ``"true"``, ``"1"``, ``"1e-3"``) are automatically converted to their corresponding type (bool, int, float)
+- Dictionaries and lists (e.g., ``"{a: b}"``, ``"[a, b, c]"``)  are returned as transient config nodes (DictConfig and ListConfig)
+- Interpolations (e.g., ``"${foo}"``) are automatically resolved
+- ``None`` is the only valid non-string input to ``oc.decode`` (returning ``None`` in that case)
+
+This can be useful for instance to parse environment variables:
 
 .. doctest::
 
-    >>> cfg = OmegaConf.create({
-    ...       'database': {'password': '${env:DB_PASSWORD,abc123}',
-    ...                    'user': 'someuser',
-    ...                    'port': '${env:DB_PORT,3306}',
-    ...                    'nodes': '${env:DB_NODES,[]}'}
-    ... })
-    >>> os.environ["DB_PORT"] = '3308'
-    >>> cfg.database.port  # converted to int
-    3308
-    >>> os.environ["DB_NODES"] = '[host1, host2, host3]'
-    >>> cfg.database.nodes  # converted to list
-    ['host1', 'host2', 'host3']
-    >>> os.environ["DB_PASSWORD"] = 'a%#@~{}$*&^?/<'
-    >>> cfg.database.password  # kept as a string
-    'a%#@~{}$*&^?/<'
+    >>> cfg = OmegaConf.create(
+    ...     {
+    ...         "database": {
+    ...             "port": '${oc.decode:${oc.env:DB_PORT}}',
+    ...             "nodes": '${oc.decode:${oc.env:DB_NODES}}',
+    ...             "timeout": '${oc.decode:${oc.env:DB_TIMEOUT,null}}',
+    ...         }
+    ...     }
+    ... )
+    >>> os.environ["DB_PORT"] = "3308"
+    >>> show(cfg.database.port)  # converted to int
+    type: int, value: 3308
+    >>> os.environ["DB_NODES"] = "[host1, host2, host3]"
+    >>> show(cfg.database.nodes)  # converted to a ListConfig
+    type: ListConfig, value: ['host1', 'host2', 'host3']
+    >>> show(cfg.database.timeout)  # keeping `None` as is
+    type: NoneType, value: None
+    >>> os.environ["DB_TIMEOUT"] = "${.port}"
+    >>> show(cfg.database.timeout)  # resolving interpolation
+    type: int, value: 3308
 
 
 Custom interpolations
@@ -762,12 +784,11 @@ If resolve is set to True, interpolations will be resolved during conversion.
     >>> conf = OmegaConf.create({"foo": "bar", "foo2": "${foo}"})
     >>> assert type(conf) == DictConfig
     >>> primitive = OmegaConf.to_container(conf)
-    >>> assert type(primitive) == dict
-    >>> print(primitive)
-    {'foo': 'bar', 'foo2': '${foo}'}
+    >>> show(primitive)
+    type: dict, value: {'foo': 'bar', 'foo2': '${foo}'}
     >>> resolved = OmegaConf.to_container(conf, resolve=True)
-    >>> print(resolved)
-    {'foo': 'bar', 'foo2': 'bar'}
+    >>> show(resolved)
+    type: dict, value: {'foo': 'bar', 'foo2': 'bar'}
 
 You can customize the treatment of **OmegaConf.to_container()** for
 Structured Config nodes using the `structured_config_mode` option.
@@ -780,11 +801,10 @@ as DictConfig, allowing attribute style access on the resulting node.
     >>> from omegaconf import SCMode
     >>> conf = OmegaConf.create({"structured_config": MyConfig})
     >>> container = OmegaConf.to_container(conf, structured_config_mode=SCMode.DICT_CONFIG)
-    >>> print(container)
-    {'structured_config': {'port': 80, 'host': 'localhost'}}
-    >>> assert type(container) is dict
-    >>> assert type(container["structured_config"]) is DictConfig
-    >>> assert container["structured_config"].port == 80
+    >>> show(container)
+    type: dict, value: {'structured_config': {'port': 80, 'host': 'localhost'}}
+    >>> show(container["structured_config"])
+    type: DictConfig, value: {'port': 80, 'host': 'localhost'}
 
 OmegaConf.select
 ^^^^^^^^^^^^^^^^

news/445.feature.1

@@ -1 +1 @@
-Add ability to nest interpolations, e.g. ${foo.${bar}}}, ${env:{$var1},${var2}}, or ${${func}:x1,x2}
+Add ability to nest interpolations, e.g. ${foo.${bar}}}, ${oc.env:{$var1},${var2}}, or ${${func}:x1,x2}

news/573.api_change

@@ -0,0 +1 @@
+The `env` resolver is deprecated in favor of `oc.env`, which keeps the string representation of environment variables, does not cache the resulting value, and handles "null" as default value.

news/574.feature

@@ -0,0 +1 @@
+New resolver `oc.decode` that can be used to automatically convert a string to bool, int, float, dict, list, etc.

omegaconf/_utils.py

@@ -63,6 +63,18 @@
 _CMP_TYPES = {t: i for i, t in enumerate([float, int, bool, str, type(None)])}
 
 
+class Marker:
+    def __init__(self, desc: str):
+        self.desc = desc
+
+    def __repr__(self) -> str:
+        return self.desc
+
+
+# To be used as default value when `None` is not an option.
+_DEFAULT_MARKER_: Any = Marker("_DEFAULT_MARKER_")
+
+
 class OmegaConfDumper(yaml.Dumper):  # type: ignore
     str_representer_added = False
 
@@ -404,6 +416,12 @@ def get_value_kind(
         return ValueKind.VALUE
 
 
+# DEPRECATED: remove in 2.2
+def is_bool(st: str) -> bool:
+    st = str.lower(st)
+    return st == "true" or st == "false"
+
+
 def is_float(st: str) -> bool:
     try:
         float(st)
@@ -420,6 +438,20 @@ def is_int(st: str) -> bool:
         return False
 
 
+# DEPRECATED: remove in 2.2
+def decode_primitive(s: str) -> Any:
+    if is_bool(s):
+        return str.lower(s) == "true"
+
+    if is_int(s):
+        return int(s)
+
+    if is_float(s):
+        return float(s)
+
+    return s
+
+
 def is_primitive_list(obj: Any) -> bool:
     from .base import Container
 

omegaconf/base.py

@@ -9,6 +9,7 @@
 from antlr4 import ParserRuleContext
 
 from ._utils import (
+    _DEFAULT_MARKER_,
     ValueKind,
     _get_value,
     _is_missing_value,
@@ -32,8 +33,6 @@
 
 DictKeyType = Union[str, int, Enum, float, bool]
 
-_MARKER_ = object()
-
 
 @dataclass
 class Metadata:
@@ -155,8 +154,8 @@ def _get_flag(self, flag: str) -> Optional[bool]:
         if cache is None:
             cache = self.__dict__["_flags_cache"] = {}
 
-        ret = cache.get(flag, _MARKER_)
-        if ret is _MARKER_:
+        ret = cache.get(flag, _DEFAULT_MARKER_)
+        if ret is _DEFAULT_MARKER_:
             ret = self._get_flag_no_cache(flag)
             cache[flag] = ret
         assert ret is None or isinstance(ret, bool)