From 5f2d94c4f96db79eebad30d80979f61a6ffd6be7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Noord?= <13665637+DanielNoord@users.noreply.github.com> Date: Wed, 18 Dec 2024 23:55:02 +0100 Subject: [PATCH 1/3] Add test for importing `collections.abc` --- tests/brain/test_brain.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/brain/test_brain.py b/tests/brain/test_brain.py index 14d4bd5d8b..610dcd05c9 100644 --- a/tests/brain/test_brain.py +++ b/tests/brain/test_brain.py @@ -192,6 +192,22 @@ def check_metaclass_is_abc(node: nodes.ClassDef): class CollectionsBrain(unittest.TestCase): + def test_collections_abc_is_importable(self) -> None: + """ + Test that we can import `collections.abc`. + + The collections.abc has gone through various formats of being frozen. Therefore, we ensure + that we can still import it (correctly). + """ + import_node = builder.extract_node("import collections.abc") + assert isinstance(import_node, nodes.Import) + imported_module = import_node.do_import_module(import_node.names[0][0]) + # Make sure that the file we have imported is actually the submodule of collections and + # not the `abc` module. (Which would happen if you call `importlib.util.find_spec("abc")` + # instead of `importlib.util.find_spec("collections.abc")`) + assert isinstance(imported_module.file, str) + assert "collections" in imported_module.file + def test_collections_object_not_subscriptable(self) -> None: """ Test that unsubscriptable types are detected From 8631740c1fd339409695d8346e9ed17bdd53c890 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Noord?= <13665637+DanielNoord@users.noreply.github.com> Date: Thu, 19 Dec 2024 00:29:18 +0100 Subject: [PATCH 2/3] Fix issue with importing of frozen submodules --- ChangeLog | 4 ++- astroid/brain/brain_collections.py | 11 +++--- astroid/const.py | 1 - astroid/interpreter/_import/spec.py | 52 +++++++++++++++++++++++------ 4 files changed, 49 insertions(+), 19 deletions(-) diff --git a/ChangeLog b/ChangeLog index 046da2fe7e..3424a6830a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -33,7 +33,8 @@ What's New in astroid 3.3.7? ============================ Release date: TBA -* Fix inability to import `collections.abc` in python 3.13.1. +* Fix inability to import `collections.abc` in python 3.13.1. The reported fix in astroid 3.3.6 + did not actually fix this issue. Closes pylint-dev/pylint#10112 @@ -43,6 +44,7 @@ What's New in astroid 3.3.6? Release date: 2024-12-08 * Fix inability to import `collections.abc` in python 3.13.1. + _It was later found that this did not resolve the linked issue. It was fixed in astroid 3.3.7_ Closes pylint-dev/pylint#10112 diff --git a/astroid/brain/brain_collections.py b/astroid/brain/brain_collections.py index 462c85add2..94944e67ad 100644 --- a/astroid/brain/brain_collections.py +++ b/astroid/brain/brain_collections.py @@ -8,7 +8,7 @@ from astroid.brain.helpers import register_module_extender from astroid.builder import AstroidBuilder, extract_node, parse -from astroid.const import PY313_0, PY313_PLUS +from astroid.const import PY313_PLUS from astroid.context import InferenceContext from astroid.exceptions import AttributeInferenceError from astroid.manager import AstroidManager @@ -20,8 +20,7 @@ def _collections_transform(): return parse( - (" import _collections_abc as abc" if PY313_PLUS and not PY313_0 else "") - + """ + """ class defaultdict(dict): default_factory = None def __missing__(self, key): pass @@ -33,7 +32,7 @@ def __getitem__(self, key): return default_factory ) -def _collections_abc_313_0_transform() -> nodes.Module: +def _collections_abc_313_transform() -> nodes.Module: """See https://github.com/python/cpython/pull/124735""" return AstroidBuilder(AstroidManager()).string_build( "from _collections_abc import *" @@ -133,7 +132,7 @@ def register(manager: AstroidManager) -> None: ClassDef, easy_class_getitem_inference, _looks_like_subscriptable ) - if PY313_0: + if PY313_PLUS: register_module_extender( - manager, "collections.abc", _collections_abc_313_0_transform + manager, "collections.abc", _collections_abc_313_transform ) diff --git a/astroid/const.py b/astroid/const.py index a10c0f4a2b..c010818063 100644 --- a/astroid/const.py +++ b/astroid/const.py @@ -9,7 +9,6 @@ PY311_PLUS = sys.version_info >= (3, 11) PY312_PLUS = sys.version_info >= (3, 12) PY313_PLUS = sys.version_info >= (3, 13) -PY313_0 = sys.version_info[:3] == (3, 13, 0) WIN32 = sys.platform == "win32" diff --git a/astroid/interpreter/_import/spec.py b/astroid/interpreter/_import/spec.py index e0c54d4a8a..e34a9b493f 100644 --- a/astroid/interpreter/_import/spec.py +++ b/astroid/interpreter/_import/spec.py @@ -134,32 +134,62 @@ def find_module( processed: tuple[str, ...], submodule_path: tuple[str, ...] | None, ) -> ModuleSpec | None: - if submodule_path is not None: - search_paths = list(submodule_path) - elif modname in sys.builtin_module_names: + # Although we should be able to use `find_spec` this doesn't work on PyPy for builtins. + # Therefore, we use the `builtin_module_nams` heuristic for these. + if submodule_path is None and modname in sys.builtin_module_names: return ModuleSpec( name=modname, location=None, type=ModuleType.C_BUILTIN, ) - else: - try: - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", category=UserWarning) - spec = importlib.util.find_spec(modname) + + # sys.stdlib_module_names was added in Python 3.10 + if PY310_PLUS: + # If the module is a stdlib module, check whether this is a frozen module. Note that + # `find_spec` actually imports the module, so we want to make sure we only run this code + # for stuff this is can be expected to be frozen. For now this is only stdlib. + if modname in sys.stdlib_module_names or ( + processed and processed[0] in sys.stdlib_module_names + ): + spec = importlib.util.find_spec(".".join((*processed, modname))) if ( spec and spec.loader # type: ignore[comparison-overlap] # noqa: E501 is importlib.machinery.FrozenImporter ): - # No need for BuiltinImporter; builtins handled above return ModuleSpec( name=modname, location=getattr(spec.loader_state, "filename", None), type=ModuleType.PY_FROZEN, ) - except ValueError: - pass + else: + # NOTE: This is broken code. It doesn't work on Python 3.13+ where submodules can also + # be frozen. However, we don't want to worry about this and we don't want to break + # support for older versions of Python. This is just copy-pasted from the old non + # working version to at least have no functional behaviour change on <=3.10. + # It can be removed after 3.10 is no longer supported in favour of the logic above. + if submodule_path is None: # pylint: disable=else-if-used + try: + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=UserWarning) + spec = importlib.util.find_spec(modname) + if ( + spec + and spec.loader # type: ignore[comparison-overlap] # noqa: E501 + is importlib.machinery.FrozenImporter + ): + # No need for BuiltinImporter; builtins handled above + return ModuleSpec( + name=modname, + location=getattr(spec.loader_state, "filename", None), + type=ModuleType.PY_FROZEN, + ) + except ValueError: + pass + + if submodule_path is not None: + search_paths = list(submodule_path) + else: search_paths = sys.path suffixes = (".py", ".pyi", importlib.machinery.BYTECODE_SUFFIXES[0]) From 37a5ca314134e8d0cf14947c960ebce172e5976a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20van=20Noord?= <13665637+DanielNoord@users.noreply.github.com> Date: Thu, 19 Dec 2024 12:56:12 +0100 Subject: [PATCH 3/3] Update spec.py Co-authored-by: Pierre Sassoulas --- astroid/interpreter/_import/spec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/astroid/interpreter/_import/spec.py b/astroid/interpreter/_import/spec.py index e34a9b493f..457a71cbdc 100644 --- a/astroid/interpreter/_import/spec.py +++ b/astroid/interpreter/_import/spec.py @@ -147,7 +147,7 @@ def find_module( if PY310_PLUS: # If the module is a stdlib module, check whether this is a frozen module. Note that # `find_spec` actually imports the module, so we want to make sure we only run this code - # for stuff this is can be expected to be frozen. For now this is only stdlib. + # for stuff that can be expected to be frozen. For now this is only stdlib. if modname in sys.stdlib_module_names or ( processed and processed[0] in sys.stdlib_module_names ):