From 6cf2827b6af0ea172cd2dfe37429d17f4b36256d Mon Sep 17 00:00:00 2001 From: Omry Yadan Date: Mon, 15 Mar 2021 17:48:23 -0700 Subject: [PATCH 1/4] Added pydevd extension to setup.py --- setup.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3145f83c6..714883cb0 100644 --- a/setup.py +++ b/setup.py @@ -50,7 +50,13 @@ tests_require=["pytest"], url="https://github.com/omry/omegaconf", keywords="yaml configuration config", - packages=["omegaconf", "omegaconf.grammar", "omegaconf.grammar.gen"], + packages=[ + "omegaconf", + "omegaconf.grammar", + "omegaconf.grammar.gen", + "pydevd_plugins", + "pydevd_plugins.extensions", + ], python_requires=">=3.6", classifiers=[ "Programming Language :: Python :: 3.6", From d3b1476226ae7ab0fa2d234fe646c07c17e3c962 Mon Sep 17 00:00:00 2001 From: Omry Yadan Date: Sun, 14 Mar 2021 14:14:52 -0700 Subject: [PATCH 2/4] pydev debugger plugin --- docs/source/usage.rst | 16 ++ news/214.feature | 1 + pydevd_plugins/__init__.py | 6 + pydevd_plugins/extensions/__init__.py | 6 + .../extensions/pydevd_plugin_omegaconf.py | 114 ++++++++ requirements/dev.txt | 1 + tests/test_pydev_resolver_plugin.py | 254 ++++++++++++++++++ 7 files changed, 398 insertions(+) create mode 100644 news/214.feature create mode 100644 pydevd_plugins/__init__.py create mode 100644 pydevd_plugins/extensions/__init__.py create mode 100644 pydevd_plugins/extensions/pydevd_plugin_omegaconf.py create mode 100644 tests/test_pydev_resolver_plugin.py diff --git a/docs/source/usage.rst b/docs/source/usage.rst index a4e2e096f..834a66a36 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -858,3 +858,19 @@ Creates a copy of a DictConfig that contains only specific keys. a: b: 10 + +Debugger integration +^^^^^^^^^^^^^^^^^^^^ +OmegaConf is packaged with a PyDev.Debugger extension which enables better debugging experience in PyCharm, +VSCode and other `PyDev.Debugger `_ powered IDEs. + +The debugger extension enables OmegaConf-aware object inspection: + - providing information about interpolations. + - properly handling missing values (``???``). + +The plugin comes in two flavors: + - USER: Default behavior, useful when debugging your OmegaConf objects. + - DEV: Useful when debugging OmegaConf itself, shows the exact data model of OmegaConf. + +The default flavor is ``USER``. You can select which flavor to use using the environment variable ``OC_PYDEVD_RESOLVER``, +Which takes the possible values ``USER``, ``DEV`` and ``DISABLE``. diff --git a/news/214.feature b/news/214.feature new file mode 100644 index 000000000..38bfa08ed --- /dev/null +++ b/news/214.feature @@ -0,0 +1 @@ +New pydevd resolver plugin for easier debugging diff --git a/pydevd_plugins/__init__.py b/pydevd_plugins/__init__.py new file mode 100644 index 000000000..3a973c9d5 --- /dev/null +++ b/pydevd_plugins/__init__.py @@ -0,0 +1,6 @@ +try: + __import__("pkg_resources").declare_namespace(__name__) +except ImportError: # pragma: no cover + import pkgutil + + __path__ = pkgutil.extend_path(__path__, __name__) # type: ignore diff --git a/pydevd_plugins/extensions/__init__.py b/pydevd_plugins/extensions/__init__.py new file mode 100644 index 000000000..3a973c9d5 --- /dev/null +++ b/pydevd_plugins/extensions/__init__.py @@ -0,0 +1,6 @@ +try: + __import__("pkg_resources").declare_namespace(__name__) +except ImportError: # pragma: no cover + import pkgutil + + __path__ = pkgutil.extend_path(__path__, __name__) # type: ignore diff --git a/pydevd_plugins/extensions/pydevd_plugin_omegaconf.py b/pydevd_plugins/extensions/pydevd_plugin_omegaconf.py new file mode 100644 index 000000000..675731edf --- /dev/null +++ b/pydevd_plugins/extensions/pydevd_plugin_omegaconf.py @@ -0,0 +1,114 @@ +# based on https://github.com/fabioz/PyDev.Debugger/tree/main/pydevd_plugins/extensions +import os +import sys +from functools import lru_cache +from typing import Any, Dict, Sequence + +from _pydevd_bundle.pydevd_extension_api import ( # type: ignore + StrPresentationProvider, + TypeResolveProvider, +) + + +@lru_cache(maxsize=128) +def find_mod_attr(mod_name: str, attr: str) -> Any: + mod = sys.modules.get(mod_name) + return getattr(mod, attr, None) + + +class OmegaConfDeveloperResolver(object): + def can_provide(self, type_object: Any, type_name: str) -> bool: + Node = find_mod_attr("omegaconf", "Node") + return Node is not None and issubclass(type_object, Node) + + def resolve(self, obj: Any, attribute: str) -> Any: + return getattr(obj, attribute) + + def get_dictionary(self, obj: Any) -> Any: + return obj.__dict__ + + +class OmegaConfUserResolver(StrPresentationProvider): # type: ignore + def can_provide(self, type_object: Any, type_name: str) -> bool: + Node = find_mod_attr("omegaconf", "Node") + return Node is not None and issubclass(type_object, Node) + + def resolve(self, obj: Any, attribute: Any) -> Any: + if isinstance(obj, Sequence): + if isinstance(attribute, str): + attribute = int(attribute) + val = obj.__dict__["_content"][attribute] + else: + val = obj.__dict__["_content"][attribute] + + return val + + def _is_simple_value(self, val: Any) -> bool: + ValueNode = find_mod_attr("omegaconf", "ValueNode") + return ( + isinstance(val, ValueNode) + and not val._is_missing() + and not val._is_interpolation() + and not val._is_none() + ) + + def get_dictionary(self, obj: Any) -> Dict[str, Any]: + ListConfig = find_mod_attr("omegaconf", "ListConfig") + DictConfig = find_mod_attr("omegaconf", "DictConfig") + Node = find_mod_attr("omegaconf", "Node") + IRE = find_mod_attr("omegaconf.errors", "InterpolationResolutionError") + + if isinstance(obj, Node): + try: + obj = obj._dereference_node() + if obj._is_none() or obj._is_missing(): + return {} + except IRE: + obj = None + + if isinstance(obj, DictConfig): + d = {} + for k, v in obj.__dict__["_content"].items(): + if self._is_simple_value(v): + v = v._value() + d[k] = v + elif isinstance(obj, ListConfig): + d = {} + for idx, v in enumerate(obj.__dict__["_content"]): + if self._is_simple_value(v): + v = v._value() + d[str(idx)] = v + else: + d = {} + + return d + + def get_str(self, val: Any) -> str: + IRE = find_mod_attr("omegaconf.errors", "InterpolationResolutionError") + + if val._is_missing(): + return "??? " + if val._is_interpolation(): + try: + dr = val._dereference_node() + except IRE as e: + dr = f"ERR: {e}" + return f"{val._value()} -> {dr}" + else: + return f"{val}" + + +# OC_PYDEVD_RESOLVER env can take: +# DISABLE: Do not install a pydevd resolver +# USER: Install a resolver for OmegaConf users (default) +# DEV: Install a resolver for OmegaConf developers. Shows underlying data-model in the debugger. +resolver = os.environ.get("OC_PYDEVD_RESOLVER", "USER").upper() +if resolver != "DISABLE": # pragma: no cover + if resolver == "USER": + TypeResolveProvider.register(OmegaConfUserResolver) + elif resolver == "DEV": + TypeResolveProvider.register(OmegaConfDeveloperResolver) + else: + sys.stderr.write( + f"OmegaConf pydev plugin: Not installing. Unknown mode {resolver}. Supported one of [USER, DEV, DISABLE]\n" + ) diff --git a/requirements/dev.txt b/requirements/dev.txt index d1b6b208b..c18b9b73a 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -14,3 +14,4 @@ pytest-mock sphinx towncrier twine +pydevd \ No newline at end of file diff --git a/tests/test_pydev_resolver_plugin.py b/tests/test_pydev_resolver_plugin.py new file mode 100644 index 000000000..58e93b571 --- /dev/null +++ b/tests/test_pydev_resolver_plugin.py @@ -0,0 +1,254 @@ +import builtins +from typing import Any + +from pytest import fixture, mark, param + +from omegaconf import ( + AnyNode, + BooleanNode, + Container, + DictConfig, + EnumNode, + FloatNode, + IntegerNode, + ListConfig, + Node, + OmegaConf, + StringNode, + ValueNode, +) +from omegaconf._utils import type_str +from pydevd_plugins.extensions.pydevd_plugin_omegaconf import ( + OmegaConfDeveloperResolver, + OmegaConfUserResolver, +) +from tests import Color + + +@fixture +def resolver() -> Any: + yield OmegaConfUserResolver() + + +@mark.parametrize( + ("obj", "expected"), + [ + # nodes + param(AnyNode(10), {}, id="any:10"), + param(StringNode("foo"), {}, id="str:foo"), + param(IntegerNode(10), {}, id="int:10"), + param(FloatNode(3.14), {}, id="float:3.14"), + param(BooleanNode(True), {}, id="bool:True"), + param(EnumNode(enum_type=Color, value=Color.RED), {}, id="Color:Color.RED"), + # nodes are never returning a dictionary + param(AnyNode("${foo}", parent=DictConfig({"foo": 10})), {}, id="any:10"), + param(AnyNode("${foo}", parent=DictConfig({"foo": 10})), {}, id="any:10"), + # DictConfig + param(DictConfig({"a": 10}), {"a": AnyNode(10)}, id="dict"), + param( + DictConfig({"a": 10, "b": "${a}"}), + {"a": AnyNode(10), "b": AnyNode("${a}")}, + id="dict:interpolation_value", + ), + param( + DictConfig({"a": 10, "b": "${zzz}"}), + {"a": AnyNode(10), "b": AnyNode("${zzz}")}, + id="dict:interpolation_value_error", + ), + param( + DictConfig({"a": 10, "b": "foo_${a}"}), + {"a": AnyNode(10), "b": AnyNode("foo_${a}")}, + id="dict:str_interpolation_value", + ), + param(DictConfig("${zzz}"), {}, id="dict:inter_error"), + # ListConfig + param( + ListConfig(["a", "b"]), {"0": AnyNode("a"), "1": AnyNode("b")}, id="list" + ), + param( + ListConfig(["${1}", 10]), + {"0": AnyNode("${1}"), "1": AnyNode(10)}, + id="list:interpolation_value", + ), + param(ListConfig("${zzz}"), {}, id="list:inter_error"), + ], +) +def test_get_dictionary_node(resolver: Any, obj: Any, expected: Any) -> None: + res = resolver.get_dictionary(obj) + assert res == expected + + +@mark.parametrize( + ("obj", "attribute", "expected"), + [ + # dictconfig + param(DictConfig({"a": 10}), "a", AnyNode(10), id="dict"), + param( + DictConfig({"a": DictConfig(None)}), + "a", + DictConfig(None), + id="dict:none", + ), + # listconfig + param(ListConfig([10]), 0, AnyNode(10), id="list"), + param(ListConfig(["???"]), 0, AnyNode("???"), id="list"), + ], +) +def test_resolve( + resolver: Any, + obj: Any, + attribute: str, + expected: Any, +) -> None: + res = resolver.resolve(obj, attribute) + assert res == expected + assert type(res) is type(expected) + + +@mark.parametrize( + ("obj", "attribute", "expected"), + [ + param( + OmegaConf.create({"a": 10, "inter": "${a}"}), "inter", {}, id="dict:inter" + ), + param( + OmegaConf.create({"missing": "???"}), + "missing", + {}, + id="dict:missing_value", + ), + param( + OmegaConf.create({"none": None}), + "none", + {}, + id="dict:none_value", + ), + param( + OmegaConf.create({"none": DictConfig(None)}), + "none", + {}, + id="dict:none_dictconfig_value", + ), + param( + OmegaConf.create({"missing": DictConfig("???")}), + "missing", + {}, + id="dict:missing_dictconfig_value", + ), + param( + OmegaConf.create({"missing": DictConfig("???")}), + "missing", + {}, + id="dict:missing_dictconfig_value", + ), + param( + OmegaConf.create({"a": {"b": 10}, "b": DictConfig("${a}")}), + "b", + {"b": 10}, + id="dict:interpolation_dictconfig_value", + ), + ], +) +def test_get_dictionary_dictconfig( + resolver: Any, + obj: Any, + attribute: str, + expected: Any, +) -> None: + field = resolver.resolve(obj, attribute) + res = resolver.get_dictionary(field) + assert res == expected + assert type(res) is type(expected) + + +@mark.parametrize( + ("obj", "attribute", "expected"), + [ + param(OmegaConf.create(["${.1}", 10]), "0", {}, id="list:inter_value"), + param( + OmegaConf.create({"a": ListConfig(None)}), + "a", + {}, + id="list:none_listconfig_value", + ), + param( + OmegaConf.create({"a": ListConfig("???")}), + "a", + {}, + id="list:missing_listconfig_value", + ), + param( + OmegaConf.create({"a": [1, 2], "b": ListConfig("${a}")}), + "b", + {"0": 1, "1": 2}, + id="list:interpolationn_listconfig_value", + ), + ], +) +def test_get_dictionary_listconfig( + resolver: Any, + obj: Any, + attribute: str, + expected: Any, +) -> None: + field = resolver.resolve(obj, attribute) + res = resolver.get_dictionary(field) + assert res == expected + assert type(res) is type(expected) + + +@mark.parametrize("resolver", [OmegaConfUserResolver(), OmegaConfDeveloperResolver()]) +@mark.parametrize( + ("type_", "expected"), + [ + # containers + (Container, True), + (DictConfig, True), + (ListConfig, True), + # nodes + (Node, True), + (ValueNode, True), + (AnyNode, True), + (IntegerNode, True), + (FloatNode, True), + (StringNode, True), + (BooleanNode, True), + # not covering some other things. + (builtins.int, False), + (dict, False), + (list, False), + ], +) +def test_can_provide(resolver: Any, type_: Any, expected: bool) -> None: + assert resolver.can_provide(type_, type_str(type_)) == expected + + +@mark.parametrize( + ("obj", "expected"), + [ + (AnyNode(10), "10"), + (AnyNode("???"), "??? "), + ( + AnyNode("${foo}", parent=OmegaConf.create({})), + "${foo} -> ERR: Interpolation key 'foo' not found", + ), + (AnyNode("${foo}", parent=OmegaConf.create({"foo": 10})), "${foo} -> 10"), + ( + DictConfig("${foo}", parent=OmegaConf.create({"foo": {"a": 10}})), + "${foo} -> {'a': 10}", + ), + ( + ListConfig("${foo}", parent=OmegaConf.create({"foo": [1, 2]})), + "${foo} -> [1, 2]", + ), + ], +) +def test_get_str(resolver: Any, obj: Any, expected: str) -> None: + assert resolver.get_str(obj) == expected + + +def test_dev_resolver() -> None: + resolver = OmegaConfDeveloperResolver() + cfg = OmegaConf.create({"foo": 10}) + assert resolver.resolve(cfg, "_metadata") is cfg.__dict__["_metadata"] + assert resolver.get_dictionary(cfg) is cfg.__dict__ From 1458c7531ccbe0398b09e9fbe4843bfe75a33ada Mon Sep 17 00:00:00 2001 From: Omry Yadan Date: Tue, 16 Mar 2021 12:34:48 -0700 Subject: [PATCH 3/4] Addressing comments --- news/214.feature | 2 +- .../extensions/pydevd_plugin_omegaconf.py | 15 ++++++--------- tests/test_pydev_resolver_plugin.py | 14 ++++---------- 3 files changed, 11 insertions(+), 20 deletions(-) diff --git a/news/214.feature b/news/214.feature index 38bfa08ed..4feb86e2b 100644 --- a/news/214.feature +++ b/news/214.feature @@ -1 +1 @@ -New pydevd resolver plugin for easier debugging +New PyDev.Debugger resolver plugin for easier debugging diff --git a/pydevd_plugins/extensions/pydevd_plugin_omegaconf.py b/pydevd_plugins/extensions/pydevd_plugin_omegaconf.py index 675731edf..74a500ab7 100644 --- a/pydevd_plugins/extensions/pydevd_plugin_omegaconf.py +++ b/pydevd_plugins/extensions/pydevd_plugin_omegaconf.py @@ -34,12 +34,9 @@ def can_provide(self, type_object: Any, type_name: str) -> bool: return Node is not None and issubclass(type_object, Node) def resolve(self, obj: Any, attribute: Any) -> Any: - if isinstance(obj, Sequence): - if isinstance(attribute, str): - attribute = int(attribute) - val = obj.__dict__["_content"][attribute] - else: - val = obj.__dict__["_content"][attribute] + if isinstance(obj, Sequence) and isinstance(attribute, str): + attribute = int(attribute) + val = obj.__dict__["_content"][attribute] return val @@ -47,9 +44,9 @@ def _is_simple_value(self, val: Any) -> bool: ValueNode = find_mod_attr("omegaconf", "ValueNode") return ( isinstance(val, ValueNode) + and not val._is_none() and not val._is_missing() and not val._is_interpolation() - and not val._is_none() ) def get_dictionary(self, obj: Any) -> Dict[str, Any]: @@ -60,11 +57,11 @@ def get_dictionary(self, obj: Any) -> Dict[str, Any]: if isinstance(obj, Node): try: - obj = obj._dereference_node() + obj = obj._dereference_node(throw_on_resolution_failure=False) if obj._is_none() or obj._is_missing(): return {} except IRE: - obj = None + return {} if isinstance(obj, DictConfig): d = {} diff --git a/tests/test_pydev_resolver_plugin.py b/tests/test_pydev_resolver_plugin.py index 58e93b571..a361b85d1 100644 --- a/tests/test_pydev_resolver_plugin.py +++ b/tests/test_pydev_resolver_plugin.py @@ -41,8 +41,7 @@ def resolver() -> Any: param(BooleanNode(True), {}, id="bool:True"), param(EnumNode(enum_type=Color, value=Color.RED), {}, id="Color:Color.RED"), # nodes are never returning a dictionary - param(AnyNode("${foo}", parent=DictConfig({"foo": 10})), {}, id="any:10"), - param(AnyNode("${foo}", parent=DictConfig({"foo": 10})), {}, id="any:10"), + param(AnyNode("${foo}", parent=DictConfig({"foo": 10})), {}, id="any:inter_10"), # DictConfig param(DictConfig({"a": 10}), {"a": AnyNode(10)}, id="dict"), param( @@ -91,7 +90,7 @@ def test_get_dictionary_node(resolver: Any, obj: Any, expected: Any) -> None: ), # listconfig param(ListConfig([10]), 0, AnyNode(10), id="list"), - param(ListConfig(["???"]), 0, AnyNode("???"), id="list"), + param(ListConfig(["???"]), 0, AnyNode("???"), id="list:missing_item"), ], ) def test_resolve( @@ -135,12 +134,6 @@ def test_resolve( {}, id="dict:missing_dictconfig_value", ), - param( - OmegaConf.create({"missing": DictConfig("???")}), - "missing", - {}, - id="dict:missing_dictconfig_value", - ), param( OmegaConf.create({"a": {"b": 10}, "b": DictConfig("${a}")}), "b", @@ -181,7 +174,7 @@ def test_get_dictionary_dictconfig( OmegaConf.create({"a": [1, 2], "b": ListConfig("${a}")}), "b", {"0": 1, "1": 2}, - id="list:interpolationn_listconfig_value", + id="list:interpolation_listconfig_value", ), ], ) @@ -213,6 +206,7 @@ def test_get_dictionary_listconfig( (FloatNode, True), (StringNode, True), (BooleanNode, True), + (EnumNode, True), # not covering some other things. (builtins.int, False), (dict, False), From 131fca49e5ea8e1804fada976b06ecdc82236ffa Mon Sep 17 00:00:00 2001 From: Omry Yadan Date: Tue, 16 Mar 2021 14:45:14 -0700 Subject: [PATCH 4/4] rebase --- pydevd_plugins/extensions/pydevd_plugin_omegaconf.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pydevd_plugins/extensions/pydevd_plugin_omegaconf.py b/pydevd_plugins/extensions/pydevd_plugin_omegaconf.py index 74a500ab7..e9a815e27 100644 --- a/pydevd_plugins/extensions/pydevd_plugin_omegaconf.py +++ b/pydevd_plugins/extensions/pydevd_plugin_omegaconf.py @@ -53,14 +53,10 @@ def get_dictionary(self, obj: Any) -> Dict[str, Any]: ListConfig = find_mod_attr("omegaconf", "ListConfig") DictConfig = find_mod_attr("omegaconf", "DictConfig") Node = find_mod_attr("omegaconf", "Node") - IRE = find_mod_attr("omegaconf.errors", "InterpolationResolutionError") if isinstance(obj, Node): - try: - obj = obj._dereference_node(throw_on_resolution_failure=False) - if obj._is_none() or obj._is_missing(): - return {} - except IRE: + obj = obj._dereference_node(throw_on_resolution_failure=False) + if obj is None or obj._is_none() or obj._is_missing(): return {} if isinstance(obj, DictConfig):