From 9144da04ba24422fb52972ee32e8958d133c6907 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Tue, 8 Aug 2023 10:36:06 -0400 Subject: [PATCH 01/29] Refactor future_annotations into a DefUseChains() param. --- beniget/beniget.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/beniget/beniget.py b/beniget/beniget.py index 50f035b..bfaa69c 100644 --- a/beniget/beniget.py +++ b/beniget/beniget.py @@ -290,9 +290,10 @@ class DefUseChains(ast.NodeVisitor): One instance of DefUseChains is only suitable to analyse one AST Module in it's lifecycle. """ - def __init__(self, filename=None): + def __init__(self, filename=None, future_annotations=False): """ - filename: str, included in error messages if specified + - future_annotations: bool, PEP 563 compatible mode """ self.chains = {} self.locals = defaultdict(list) @@ -339,7 +340,7 @@ def __init__(self, filename=None): # attributes set in visit_Module self.module = None - self.future_annotations = False + self.future_annotations = future_annotations # ## helpers From aa7f6bc4bea183289d32301f2805a6f735bd3709 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Tue, 8 Aug 2023 12:31:24 -0400 Subject: [PATCH 02/29] Stubs: Add support for forward references in generic arguments of base classes. --- beniget/beniget.py | 30 ++++++++++++++++++++++++++---- tests/test_chains.py | 22 +++++++++++++++++++--- 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/beniget/beniget.py b/beniget/beniget.py index bfaa69c..7f76bf3 100644 --- a/beniget/beniget.py +++ b/beniget/beniget.py @@ -290,15 +290,19 @@ class DefUseChains(ast.NodeVisitor): One instance of DefUseChains is only suitable to analyse one AST Module in it's lifecycle. """ - def __init__(self, filename=None, future_annotations=False): + def __init__(self, filename=None, future_annotations=False, is_stub=False): """ - filename: str, included in error messages if specified - - future_annotations: bool, PEP 563 compatible mode + - future_annotations: bool, PEP 563 mode + - is_stub: bool, stub module semantics mode, implies future_annotations=True. + When the module is a stub, there is no need for quoting to do a forward reference + inside a type alias, typevar bound argument or the generic part of base classes. """ self.chains = {} self.locals = defaultdict(list) self.filename = filename + self.is_stub = is_stub # deep copy of builtins, to remain reentrant self._builtins = {k: Def(v) for k, v in Builtins.items()} @@ -340,7 +344,7 @@ def __init__(self, filename=None, future_annotations=False): # attributes set in visit_Module self.module = None - self.future_annotations = future_annotations + self.future_annotations = is_stub or future_annotations # ## helpers @@ -717,12 +721,30 @@ def visit_FunctionDef(self, node, step=DeclarationStep): visit_AsyncFunctionDef = visit_FunctionDef + def _link_stubs_generic_base(self, dclass, node, dvalue, dslice): + # Mirrors self.visit_Subscript + dnode = self.chains.setdefault(node, Def(node)) + dvalue.add_user(dnode) + dslice.add_user(dnode) + dnode.add_user(dclass) + def visit_ClassDef(self, node): dnode = self.chains.setdefault(node, Def(node)) self.locals[self._scopes[-1]].append(dnode) for base in node.bases: - self.visit(base).add_user(dnode) + if self.is_stub and isinstance(base, ast.Subscript): + # special treatment for generic arguments of base classes in stub modules + # so they can contain forward-references. + dvalue = self.visit(base.value) + self._defered_annotations[-1].append(( + base.slice, list(self._scopes), + # lambda argument defaults being bound at definition time, + # so they won't be overriden by next loop iteration + lambda dslice, base=base, dvalue=dvalue: + self._link_stubs_generic_base(dnode, base, dvalue, dslice))) + else: + self.visit(base).add_user(dnode) for keyword in node.keywords: self.visit(keyword.value).add_user(dnode) for decorator in node.decorator_list: diff --git a/tests/test_chains.py b/tests/test_chains.py index e9d33f8..1c70233 100644 --- a/tests/test_chains.py +++ b/tests/test_chains.py @@ -24,7 +24,7 @@ def captured_output(): class TestDefUseChains(TestCase): - def checkChains(self, code, ref, strict=True): + def checkChains(self, code, ref, strict=True, is_stub=False): class StrictDefUseChains(beniget.DefUseChains): def unbound_identifier(self, name, node): raise RuntimeError( @@ -35,9 +35,9 @@ def unbound_identifier(self, name, node): node = ast.parse(code) if strict: - c = StrictDefUseChains() + c = StrictDefUseChains(is_stub=is_stub) else: - c = beniget.DefUseChains() + c = beniget.DefUseChains(is_stub=is_stub) c.visit(node) self.assertEqual(c.dump_chains(node), ref) return node, c @@ -1129,6 +1129,22 @@ class A: 'A -> (A -> (), A -> ())'], # good strict=False ) + + @skipIf(sys.version_info.major < 3, "Python 3 semantics") + def test_stubs_generic_base_forward_ref(self): + code = ''' +Thing = object +class _ScandirIterator(str, int, Thing[_ScandirIterator[F]], object): + ... +F = object +''' + self.checkChains( + code, + ['Thing -> (Thing -> (Subscript -> (_ScandirIterator -> (_ScandirIterator -> (Subscript -> ((#2)))))))', + '_ScandirIterator -> (_ScandirIterator -> (Subscript -> (Subscript -> ((#0)))))', + 'F -> (F -> (Subscript -> (Subscript -> (_ScandirIterator -> (_ScandirIterator -> ((#2)))))))'], + is_stub=True + ) class TestUseDefChains(TestCase): def checkChains(self, code, ref): From 16b0ba357291727dbac71a85bef107ddd4fae142 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Tue, 8 Aug 2023 13:10:49 -0400 Subject: [PATCH 03/29] Stubs: Add support for explicit type aliases --- beniget/beniget.py | 24 ++++++++++++++++++++++-- tests/test_chains.py | 27 +++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/beniget/beniget.py b/beniget/beniget.py index 7f76bf3..c4c9372 100644 --- a/beniget/beniget.py +++ b/beniget/beniget.py @@ -780,20 +780,40 @@ def visit_Assign(self, node): self.visit(node.value) for target in node.targets: self.visit(target) + + def _is_TypeAlias_annotation(self, annotation): + if isinstance(annotation, ast.Name): + return annotation.id=='TypeAlias' + if isinstance(annotation, ast.Attribute): + if isinstance(annotation.value, ast.Name): + if annotation.value.id in {'typing', 'typing_extensions', 't'}: + return annotation.attr=='TypeAlias' + return False def visit_AnnAssign(self, node): - if node.value: + visit_value = True + if (self.is_stub and node.value and + self._is_TypeAlias_annotation(node.annotation)): + # support for PEP 613 – Explicit Type Aliases + # BUT an untyped global expression 'x=int' will NOT be considered a type alias. + visit_value = False + self._defered_annotations[-1].append( + (node.value, list(self._scopes), + lambda dvalue:dvalue.add_user(dtarget))) + elif node.value: dvalue = self.visit(node.value) + if not self.future_annotations: dannotation = self.visit(node.annotation) else: self._defered_annotations[-1].append( (node.annotation, list(self._scopes), lambda d:dtarget.add_user(d))) + dtarget = self.visit(node.target) if not self.future_annotations: dtarget.add_user(dannotation) - if node.value: + if node.value and visit_value: dvalue.add_user(dtarget) def visit_AugAssign(self, node): diff --git a/tests/test_chains.py b/tests/test_chains.py index 1c70233..e59697a 100644 --- a/tests/test_chains.py +++ b/tests/test_chains.py @@ -1146,6 +1146,33 @@ class _ScandirIterator(str, int, Thing[_ScandirIterator[F]], object): is_stub=True ) + self.checkChains( + code, + ['Thing -> (Thing -> (Subscript -> (_ScandirIterator -> ())))', + '_ScandirIterator -> ()', + 'F -> ()'], + strict=False, + ) + + @skipIf(sys.version_info.major < 3, "Python 3 semantics") + def test_stubs_forward_ref(self): + code = ''' +import TypeAlias +LiteralValue: TypeAlias = list[LiteralValue]|object +''' + self.checkChains( + code, + ['TypeAlias -> (TypeAlias -> ())', + 'LiteralValue -> (LiteralValue -> (Subscript -> (BinOp -> ((#0)))), TypeAlias -> ())'], + is_stub=True + ) + self.checkChains( + code, + ['TypeAlias -> (TypeAlias -> ())', + 'LiteralValue -> (TypeAlias -> ())'], + strict=False, + ) + class TestUseDefChains(TestCase): def checkChains(self, code, ref): class StrictDefUseChains(beniget.DefUseChains): From cda3429b21b09105aa3d327eb735a76b93397524 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Tue, 8 Aug 2023 13:14:04 -0400 Subject: [PATCH 04/29] fix import --- tests/test_chains.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_chains.py b/tests/test_chains.py index e59697a..28ce920 100644 --- a/tests/test_chains.py +++ b/tests/test_chains.py @@ -1157,7 +1157,7 @@ class _ScandirIterator(str, int, Thing[_ScandirIterator[F]], object): @skipIf(sys.version_info.major < 3, "Python 3 semantics") def test_stubs_forward_ref(self): code = ''' -import TypeAlias +from typing import TypeAlias LiteralValue: TypeAlias = list[LiteralValue]|object ''' self.checkChains( From 58eb450b96842e52a04f689eb56fcd0ad7242035 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Tue, 8 Aug 2023 14:57:52 -0400 Subject: [PATCH 05/29] Stubs: Add support for forward references in TypeVar arguments --- beniget/beniget.py | 51 +++++++++++++++++++++++++++++++------------- tests/test_chains.py | 22 +++++++++++++++++++ 2 files changed, 58 insertions(+), 15 deletions(-) diff --git a/beniget/beniget.py b/beniget/beniget.py index c4c9372..cc235aa 100644 --- a/beniget/beniget.py +++ b/beniget/beniget.py @@ -781,19 +781,10 @@ def visit_Assign(self, node): for target in node.targets: self.visit(target) - def _is_TypeAlias_annotation(self, annotation): - if isinstance(annotation, ast.Name): - return annotation.id=='TypeAlias' - if isinstance(annotation, ast.Attribute): - if isinstance(annotation.value, ast.Name): - if annotation.value.id in {'typing', 'typing_extensions', 't'}: - return annotation.attr=='TypeAlias' - return False - def visit_AnnAssign(self, node): visit_value = True - if (self.is_stub and node.value and - self._is_TypeAlias_annotation(node.annotation)): + if (self.is_stub and node.value and _is_typing_name( + node.annotation, 'TypeAlias')): # support for PEP 613 – Explicit Type Aliases # BUT an untyped global expression 'x=int' will NOT be considered a type alias. visit_value = False @@ -1147,10 +1138,23 @@ def visit_Compare(self, node): def visit_Call(self, node): dnode = self.chains.setdefault(node, Def(node)) self.visit(node.func).add_user(dnode) - for arg in node.args: - self.visit(arg).add_user(dnode) - for kw in node.keywords: - self.visit(kw.value).add_user(dnode) + if self.is_stub and _is_typing_name(node.func, 'TypeVar'): + # In stubs, constraints and bound argument + # of TypeVar() can be forward references. + current_scopes = list(self._scopes) + for arg in node.args: + self._defered_annotations[-1].append( + (arg, current_scopes, + lambda darg:darg.add_user(dnode))) + for kw in node.keywords: + self._defered_annotations[-1].append( + (kw.value, current_scopes, + lambda dkw:dkw.add_user(dnode))) + else: + for arg in node.args: + self.visit(arg).add_user(dnode) + for kw in node.keywords: + self.visit(kw.value).add_user(dnode) return dnode visit_Repr = visit_Await @@ -1306,6 +1310,23 @@ def _iter_arguments(args): if args.kwarg: yield args.kwarg +def _is_name(expr, name, modules=()): + """ + Returns True if the expression matches: + - Name(id=) or + - Attribue(value=Name(id=), attr=) + """ + if isinstance(expr, ast.Name): + return expr.id==name + if isinstance(expr, ast.Attribute): + if isinstance(expr.value, ast.Name): + if expr.value.id in modules: + return expr.attr==name + return False + +def _is_typing_name(expr, name): + return _is_name(expr, name, {'typing', 'typing_extensions', 't'}) + def lookup_annotation_name_defs(name, heads, locals_map): r""" Simple identifier -> defs resolving. diff --git a/tests/test_chains.py b/tests/test_chains.py index 28ce920..49fa038 100644 --- a/tests/test_chains.py +++ b/tests/test_chains.py @@ -1172,6 +1172,28 @@ def test_stubs_forward_ref(self): 'LiteralValue -> (TypeAlias -> ())'], strict=False, ) + + @skipIf(sys.version_info.major < 3, "Python 3 semantics") + def test_stubs_typevar_forward_ref(self): + code = ''' +from typing import TypeVar +AnyStr = TypeVar('AnyStr', F, bound=ast.AST) +import ast +F = object +''' + self.checkChains( + code, + ['TypeVar -> (TypeVar -> (Call -> ()))', + 'AnyStr -> ()', + 'ast -> (ast -> (Attribute -> (Call -> ())))', + 'F -> (F -> (Call -> ()))'], + is_stub=True + ) + self.checkChains( + code, + ['TypeVar -> (TypeVar -> (Call -> ()))', 'AnyStr -> ()', 'ast -> ()', 'F -> ()'], + strict=False, + ) class TestUseDefChains(TestCase): def checkChains(self, code, ref): From ac9b5c44001073261e2c37b272fffb4c963a7eab Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Tue, 8 Aug 2023 15:33:06 -0400 Subject: [PATCH 06/29] Stubs: The whole base expression should be processed later. This also simplifies the callback. --- beniget/beniget.py | 33 +++++++++++++-------------------- tests/test_chains.py | 12 +++++++++--- 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/beniget/beniget.py b/beniget/beniget.py index cc235aa..1da3fbb 100644 --- a/beniget/beniget.py +++ b/beniget/beniget.py @@ -721,32 +721,25 @@ def visit_FunctionDef(self, node, step=DeclarationStep): visit_AsyncFunctionDef = visit_FunctionDef - def _link_stubs_generic_base(self, dclass, node, dvalue, dslice): - # Mirrors self.visit_Subscript - dnode = self.chains.setdefault(node, Def(node)) - dvalue.add_user(dnode) - dslice.add_user(dnode) - dnode.add_user(dclass) - def visit_ClassDef(self, node): dnode = self.chains.setdefault(node, Def(node)) self.locals[self._scopes[-1]].append(dnode) - for base in node.bases: - if self.is_stub and isinstance(base, ast.Subscript): - # special treatment for generic arguments of base classes in stub modules - # so they can contain forward-references. - dvalue = self.visit(base.value) + if self.is_stub: + # special treatment for base classes in stub modules + # so they can contain forward-references. + currentscopes = list(self._scopes) + for base in node.bases: self._defered_annotations[-1].append(( - base.slice, list(self._scopes), - # lambda argument defaults being bound at definition time, - # so they won't be overriden by next loop iteration - lambda dslice, base=base, dvalue=dvalue: - self._link_stubs_generic_base(dnode, base, dvalue, dslice))) - else: + base, currentscopes, lambda dbase: dbase.add_user(dnode))) + for keyword in node.keywords: + self._defered_annotations[-1].append(( + keyword.value, currentscopes, lambda dkeyword: dkeyword.add_user(dnode))) + else: + for base in node.bases: self.visit(base).add_user(dnode) - for keyword in node.keywords: - self.visit(keyword.value).add_user(dnode) + for keyword in node.keywords: + self.visit(keyword.value).add_user(dnode) for decorator in node.decorator_list: self.visit(decorator).add_user(dnode) diff --git a/tests/test_chains.py b/tests/test_chains.py index 49fa038..68cf5c0 100644 --- a/tests/test_chains.py +++ b/tests/test_chains.py @@ -1136,13 +1136,17 @@ def test_stubs_generic_base_forward_ref(self): Thing = object class _ScandirIterator(str, int, Thing[_ScandirIterator[F]], object): ... -F = object +class C(F, k=H): + ... +F = H = object ''' self.checkChains( code, ['Thing -> (Thing -> (Subscript -> (_ScandirIterator -> (_ScandirIterator -> (Subscript -> ((#2)))))))', '_ScandirIterator -> (_ScandirIterator -> (Subscript -> (Subscript -> ((#0)))))', - 'F -> (F -> (Subscript -> (Subscript -> (_ScandirIterator -> (_ScandirIterator -> ((#2)))))))'], + 'C -> ()', + 'F -> (F -> (Subscript -> (Subscript -> (_ScandirIterator -> (_ScandirIterator -> ((#2)))))), F -> (C -> ()))', + 'H -> (H -> (C -> ()))'], is_stub=True ) @@ -1150,7 +1154,9 @@ class _ScandirIterator(str, int, Thing[_ScandirIterator[F]], object): code, ['Thing -> (Thing -> (Subscript -> (_ScandirIterator -> ())))', '_ScandirIterator -> ()', - 'F -> ()'], + 'C -> ()', + 'F -> ()', + 'H -> ()'], strict=False, ) From cc142e138111b3e0522a484cc90c8af0e0b99a40 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Tue, 8 Aug 2023 15:35:32 -0400 Subject: [PATCH 07/29] Fix doc --- beniget/beniget.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beniget/beniget.py b/beniget/beniget.py index 1da3fbb..7255568 100644 --- a/beniget/beniget.py +++ b/beniget/beniget.py @@ -296,7 +296,7 @@ def __init__(self, filename=None, future_annotations=False, is_stub=False): - future_annotations: bool, PEP 563 mode - is_stub: bool, stub module semantics mode, implies future_annotations=True. When the module is a stub, there is no need for quoting to do a forward reference - inside a type alias, typevar bound argument or the generic part of base classes. + inside a type alias, typevar bound argument or a classdef base expression. """ self.chains = {} self.locals = defaultdict(list) From f46d07529c671667738465d0e31a55f82a334644 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Tue, 8 Aug 2023 15:40:09 -0400 Subject: [PATCH 08/29] Fix SyntaxError: Non-ASCII character --- beniget/beniget.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beniget/beniget.py b/beniget/beniget.py index 7255568..9fdd7a9 100644 --- a/beniget/beniget.py +++ b/beniget/beniget.py @@ -778,7 +778,7 @@ def visit_AnnAssign(self, node): visit_value = True if (self.is_stub and node.value and _is_typing_name( node.annotation, 'TypeAlias')): - # support for PEP 613 – Explicit Type Aliases + # support for PEP 613 - Explicit Type Aliases # BUT an untyped global expression 'x=int' will NOT be considered a type alias. visit_value = False self._defered_annotations[-1].append( From 4c6952adeb799757745a7e03f49a3f6cca1d79fd Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Mon, 18 Sep 2023 19:05:40 -0400 Subject: [PATCH 09/29] Add import parsing analysis. --- README.rst | 3 +- beniget/imports.py | 101 ++++++++++++++++++++++++++++++++++++++++++ tests/test_imports.py | 58 ++++++++++++++++++++++++ 3 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 beniget/imports.py create mode 100644 tests/test_imports.py diff --git a/README.rst b/README.rst index 19ce623..82f5c40 100644 --- a/README.rst +++ b/README.rst @@ -11,11 +11,12 @@ Python3. API --- -Basically Beniget provides three analyse: +Basically Beniget provides four analyses: - ``beniget.Ancestors`` that maps each node to the list of enclosing nodes; - ``beniget.DefUseChains`` that maps each node to the list of definition points in that node; - ``beniget.UseDefChains`` that maps each node to the list of possible definition of that node. +- ``beniget.ImportParser`` that maps each import alias node to their resolved orginin name and module. See sample usages and/or run ``pydoc beniget`` for more information :-). diff --git a/beniget/imports.py b/beniget/imports.py new file mode 100644 index 0000000..fd2ebfc --- /dev/null +++ b/beniget/imports.py @@ -0,0 +1,101 @@ +import gast as ast +import sys + +class ImportInfo: + """ + Complement an `ast.alias` node with resolved + origin module and name of the locally bound name. + + :note: `orgname` will be ``*`` for wildcard imports. + """ + __slots__ = 'orgmodule', 'orgname' + + def __init__(self, orgmodule, orgname) -> None: + """ + :param orgmodule: str + :param orgname: str or None + """ + self.orgmodule = orgmodule + self.orgname = orgname + +_alias_needs_lineno = sys.implementation.name == 'cpython' and sys.version_info < (3,10) + +# The MIT License (MIT) +# Copyright (c) 2017 Jelle Zijlstra +# Adapted from the project typeshed_client. +class ImportParser(ast.NodeVisitor): + """ + Transform import statements into a mapping from `ast.alias` to `ImportInfo`. + One instance of `ImportParser` can be used to parse all imports in a given module. + + Call to `visit` will parse the given import node into a mapping of aliases to `ImportInfo`. + """ + + def __init__(self, modname, *, is_package) -> None: + self._modname = tuple(modname.split(".")) + self._is_package = is_package + self._result = {} + + def generic_visit(self, node): + raise TypeError('unexpected node type: {}'.format(type(node))) + + def visit_Import(self, node): + self._result.clear() + for al in node.names: + if al.asname: + self._result[al] = ImportInfo(orgmodule=al.name) + else: + # Here, we're not including information + # regarding the submodules imported - if there is one. + # This is because this analysis map the names bounded by imports, + # not the dependencies. + self._result[al] = ImportInfo(orgmodule=al.name.split(".", 1)[0]) + + # This seems to be the most resonable place to fix the ast.alias node not having + # proper line number information on python3.9 and before. + if _alias_needs_lineno: + al.lineno = node.lineno + + return self._result + + def visit_ImportFrom(self, node): + self._result.clear() + current_module = self._modname + + if node.module is None: + module = () + else: + module = tuple(node.module.split(".")) + + if not node.level: + source_module = module + else: + # parse relative imports + if node.level == 1: + if self._is_package: + relative_module = current_module + else: + relative_module = current_module[:-1] + else: + if self._is_package: + relative_module = current_module[: 1 - node.level] + else: + relative_module = current_module[: -node.level] + + if not relative_module: + # We don't raise errors when an relative import makes no sens, + # we simply pad the name with dots. + relative_module = ("",) * node.level + + source_module = relative_module + module + + for alias in node.names: + self._result[alias] = ImportInfo( + orgmodule=".".join(source_module), orgname=alias.name + ) + + # fix the ast.alias node not having proper line number on python3.9 and before. + if _alias_needs_lineno: + alias.lineno = node.lineno + + return self._result diff --git a/tests/test_imports.py b/tests/test_imports.py new file mode 100644 index 0000000..32cf6b3 --- /dev/null +++ b/tests/test_imports.py @@ -0,0 +1,58 @@ +import gast as ast +from unittest import TestCase +from textwrap import dedent + +from beniget.imports import ImportParser +from beniget.beniget import Def + +class TestImportParser(TestCase): + + def test_import_parser(self): + code = ''' + import mod2 + import pack.subpack + import pack.subpack as a + from mod2 import _k as k, _l as l, _m as m + from pack.subpack.stuff import C + ''' + expected = [{'mod2':('mod2', None)}, + {'pack':('pack', None)}, + {'a':('pack.subpack', None)}, + {'k':('mod2','_k'), + 'l':('mod2','_l'), + 'm':('mod2','_m')}, + {'C':('pack.subpack.stuff','C')},] + parser = ImportParser('mod1', is_package=False) + node = ast.parse(dedent(code)) + assert len(expected)==len(node.body) + for import_node, expected_names in zip(node.body, expected): + assert isinstance(import_node, (ast.Import, ast.ImportFrom)) + for al,(orgmodule, orgname) in parser.visit(import_node).items(): + assert Def(al).name() in expected_names + expected_orgmodule, expected_orgname = expected_names[Def(al).name()] + assert orgmodule == expected_orgmodule + assert orgname == expected_orgname + ran=True + assert ran + + def test_import_parser_relative(self): + code = ''' + from ...mod2 import bar as b + from .pack import foo + from ......error import x + ''' + expected = [{'b':('top.mod2','bar')}, + {'foo':('top.subpack.other.pack','foo')}, + {'x': ('......error', 'x')}] + parser = ImportParser('top.subpack.other', is_package=True) + node = ast.parse(dedent(code)) + assert len(expected)==len(node.body) + for import_node, expected_names in zip(node.body, expected): + assert isinstance(import_node, (ast.Import, ast.ImportFrom)) + for al,(orgmodule, orgname) in parser.visit(import_node).items(): + assert Def(al).name() in expected_names + expected_orgmodule, expected_orgname = expected_names[Def(al).name()] + assert orgmodule == expected_orgmodule + assert orgname == expected_orgname + ran=True + assert ran \ No newline at end of file From 4001c7c6039c8a305d6c4a88876788221247278e Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Tue, 19 Sep 2023 23:12:34 -0400 Subject: [PATCH 10/29] Improve the understanding of imports. Better implementation of is_typing_name(). --- beniget/__init__.py | 1 + beniget/beniget.py | 180 ++++++++++++++++++++++++++++++++++++------ beniget/imports.py | 15 +++- tests/test_chains.py | 101 +++++++++++++++++++++++- tests/test_imports.py | 12 +-- 5 files changed, 273 insertions(+), 36 deletions(-) diff --git a/beniget/__init__.py b/beniget/__init__.py index 8e35334..1bd2fea 100644 --- a/beniget/__init__.py +++ b/beniget/__init__.py @@ -1,3 +1,4 @@ from __future__ import absolute_import from beniget.version import __version__ from beniget.beniget import Ancestors, DefUseChains, UseDefChains +from beniget.imports import ImportParser, ImportInfo diff --git a/beniget/beniget.py b/beniget/beniget.py index 9fdd7a9..8037a95 100644 --- a/beniget/beniget.py +++ b/beniget/beniget.py @@ -1,9 +1,12 @@ -from collections import defaultdict, OrderedDict +from collections import defaultdict, OrderedDict, deque from contextlib import contextmanager import sys +import os.path import gast as ast +from beniget.imports import ImportParser + # TODO: remove me when python 2 is not supported anymore class _ordered_set(object): def __init__(self, elements=None): @@ -267,6 +270,64 @@ def collect_locals(node): visitor.generic_visit(node) return visitor.Locals +def posixpath_splitparts(path): + """ + Split a POSIX filename in parts. + + >>> posixpath_splitparts('typing.pyi') + ('typing.pyi',) + + >>> posixpath_splitparts('/var/lib/config.ini') + ('var', 'lib', 'config.ini') + + >>> posixpath_splitparts('/var/lib/config/') + ('var', 'lib', 'config') + + >>> posixpath_splitparts('c:/dir/config.ini') + ('c:', 'dir', 'config.ini') + """ + sep = '/' + r = deque(path.split(sep)) + # make sure the parts doesn't + # start or ends with a separator or empty string. + while r and r[0] in (sep, ''): + r.popleft() + while r and r[-1] in (sep, ''): + r.pop() + return tuple(r) + +def potential_module_names(filename: str): + """ + Returns a tuple of potential module + names deducted from the filename. + + >>> potential_module_names('/var/lib/config.py') + ('var.lib.config', 'lib.config', 'config') + >>> potential_module_names('git-repos/pydoctor/pydoctor/driver.py') + ('pydoctor.pydoctor.driver', 'pydoctor.driver', 'driver') + >>> potential_module_names('git-repos/pydoctor/pydoctor/__init__.py') + ('pydoctor.pydoctor', 'pydoctor') + """ + parts = posixpath_splitparts(filename) + mod = os.path.splitext(parts[-1])[0] + if mod == '__init__': + parts = parts[:-1] + else: + parts = parts[:-1] + (mod,) + + names = [] + len_parts = len(parts) + for i in range(len_parts): + p = parts[i:] + if not p or any(not all(sb.isidentifier() + for sb in s.split('.')) for s in p): + # the path cannot be converted to a module name + # because there are unallowed caracters. + continue + names.append('.'.join(p)) + + return tuple(names) or ('',) + class DefUseChains(ast.NodeVisitor): """ @@ -290,19 +351,61 @@ class DefUseChains(ast.NodeVisitor): One instance of DefUseChains is only suitable to analyse one AST Module in it's lifecycle. """ - def __init__(self, filename=None, future_annotations=False, is_stub=False): + def __init__(self, + filename=None, + *, + modname=None, + future_annotations=False, + is_stub=False): """ - - filename: str, included in error messages if specified + - filename: str, POSIX-like path pointing to the source file, + you can use `Path.as_posix` to ensure the value has proper format. + It's recommended to either provide the filename of the source + relative to the root of the package or provide both + a module name and a filename. + Included in error messages and used as part of the import resolving. + - modname: str, fully qualified name of the module we're analysing. + Required to have a correct parsing of relative imports. + A module name may end with '.__init__' to indicate the module is a package. - future_annotations: bool, PEP 563 mode - is_stub: bool, stub module semantics mode, implies future_annotations=True. - When the module is a stub, there is no need for quoting to do a forward reference - inside a type alias, typevar bound argument or a classdef base expression. + When the module is a stub file, there is no need for quoting to do a forward reference + inside: + - annotations (like PEP 563 mode) + - `TypeAlias`` values + - ``TypeVar()`` call arguments + - classe base expressions, keywords and decorators + - function decorators """ self.chains = {} self.locals = defaultdict(list) + # mapping from ast.alias to their ImportInfo. + self.imports = {} self.filename = filename self.is_stub = is_stub + + # determine module name, we provide some flexibility: + # - The module name is not required to have correct parsing when the + # filename is a relative filename that starts at the package root. + # - We deduce whether the module is a package from module name or filename + # if they ends with __init__. + # - The module name doesn't have to be provided to use _is_qualname() + # if filename is provided. + is_package = False + if filename and posixpath_splitparts(filename)[-1].split('.')[0] == '__init__': + is_package = True + if modname: + if modname.endswith('.__init__'): + modname = modname[:-9] # strip __init__ + is_package = True + self._modnames = (modname, ) + elif filename: + self._modnames = potential_module_names(filename) + else: + self._modnames = ('', ) + self.modname = next(iter(self._modnames)) + self.is_package = is_package # deep copy of builtins, to remain reentrant self._builtins = {k: Def(v) for k, v in Builtins.items()} @@ -342,6 +445,8 @@ def __init__(self, filename=None, future_annotations=False, is_stub=False): # dead code levels, it's non null for code that cannot be executed self._deadcode = 0 + self._import_parser = ImportParser(self.modname, is_package=self.is_package) + # attributes set in visit_Module self.module = None self.future_annotations = is_stub or future_annotations @@ -349,6 +454,48 @@ def __init__(self, filename=None, future_annotations=False, is_stub=False): # ## helpers # + + def _is_qualname(self, expr, qnames): + """ + Returns True if - one of - the expression's definition(s) matches + one of the given qualified names. + + The expression definition is looked up with + `lookup_annotation_name_defs`. + """ + + if isinstance(expr, ast.Name): + try: + defs = lookup_annotation_name_defs( + expr.id, self._scopes, self.locals) + except Exception: + return False + + for d in defs: + if isinstance(d.node, ast.alias): + # the symbol is an imported name + import_alias = self.imports[d.node].target() + if any(import_alias == n for n in qnames): + return True + elif any('{}.{}'.format(mod, d.name()) in qnames for mod in self._modnames): + # the symbol is a localy defined name + return True + else: + # localy defined name, but module name doesn't match + break + + elif isinstance(expr, ast.Attribute): + for n in qnames: + mod, _, _name = n.rpartition('.') + if mod and expr.attr == _name: + if self._is_qualname(expr.value, set((mod,))): + return True + return False + + def _is_typing_name(self, expr, name): + return self._is_qualname(expr, set(('typing.{}'.format(name), + 'typing_extensions.{}'.format(name)))) + def _dump_locals(self, node, only_live=False): """ Like `dump_definitions` but returns the result grouped by symbol name and it includes linenos. @@ -776,7 +923,7 @@ def visit_Assign(self, node): def visit_AnnAssign(self, node): visit_value = True - if (self.is_stub and node.value and _is_typing_name( + if (self.is_stub and node.value and self._is_typing_name( node.annotation, 'TypeAlias')): # support for PEP 613 - Explicit Type Aliases # BUT an untyped global expression 'x=int' will NOT be considered a type alias. @@ -976,6 +1123,7 @@ def visit_Import(self, node): base = alias.name.split(".", 1)[0] self.set_definition(alias.asname or base, dalias) self.locals[self._scopes[-1]].append(dalias) + self.imports.update(self._import_parser.visit(node)) def visit_ImportFrom(self, node): for alias in node.names: @@ -985,6 +1133,7 @@ def visit_ImportFrom(self, node): else: self.set_definition(alias.asname or alias.name, dalias) self.locals[self._scopes[-1]].append(dalias) + self.imports.update(self._import_parser.visit(node)) def visit_Exec(self, node): dnode = self.chains.setdefault(node, Def(node)) @@ -1131,7 +1280,7 @@ def visit_Compare(self, node): def visit_Call(self, node): dnode = self.chains.setdefault(node, Def(node)) self.visit(node.func).add_user(dnode) - if self.is_stub and _is_typing_name(node.func, 'TypeVar'): + if self.is_stub and self._is_typing_name(node.func, 'TypeVar'): # In stubs, constraints and bound argument # of TypeVar() can be forward references. current_scopes = list(self._scopes) @@ -1303,23 +1452,6 @@ def _iter_arguments(args): if args.kwarg: yield args.kwarg -def _is_name(expr, name, modules=()): - """ - Returns True if the expression matches: - - Name(id=) or - - Attribue(value=Name(id=), attr=) - """ - if isinstance(expr, ast.Name): - return expr.id==name - if isinstance(expr, ast.Attribute): - if isinstance(expr.value, ast.Name): - if expr.value.id in modules: - return expr.attr==name - return False - -def _is_typing_name(expr, name): - return _is_name(expr, name, {'typing', 'typing_extensions', 't'}) - def lookup_annotation_name_defs(name, heads, locals_map): r""" Simple identifier -> defs resolving. diff --git a/beniget/imports.py b/beniget/imports.py index fd2ebfc..f513a66 100644 --- a/beniget/imports.py +++ b/beniget/imports.py @@ -10,13 +10,22 @@ class ImportInfo: """ __slots__ = 'orgmodule', 'orgname' - def __init__(self, orgmodule, orgname) -> None: + def __init__(self, orgmodule, orgname=None) -> None: """ :param orgmodule: str :param orgname: str or None """ self.orgmodule = orgmodule self.orgname = orgname + + def target(self) -> str: + """ + Returns the qualified name of the the imported symbol. + """ + if self.orgname: + return f"{self.orgmodule}.{self.orgname}" + else: + return self.orgmodule _alias_needs_lineno = sys.implementation.name == 'cpython' and sys.version_info < (3,10) @@ -56,7 +65,7 @@ def visit_Import(self, node): if _alias_needs_lineno: al.lineno = node.lineno - return self._result + return self._result.copy() def visit_ImportFrom(self, node): self._result.clear() @@ -98,4 +107,4 @@ def visit_ImportFrom(self, node): if _alias_needs_lineno: alias.lineno = node.lineno - return self._result + return self._result.copy() diff --git a/tests/test_chains.py b/tests/test_chains.py index 68cf5c0..2d9df6e 100644 --- a/tests/test_chains.py +++ b/tests/test_chains.py @@ -24,7 +24,8 @@ def captured_output(): class TestDefUseChains(TestCase): - def checkChains(self, code, ref, strict=True, is_stub=False): + def checkChains(self, code, ref, strict=True, is_stub=False, + filename=None, modname=None): class StrictDefUseChains(beniget.DefUseChains): def unbound_identifier(self, name, node): raise RuntimeError( @@ -35,9 +36,12 @@ def unbound_identifier(self, name, node): node = ast.parse(code) if strict: - c = StrictDefUseChains(is_stub=is_stub) + c = StrictDefUseChains(is_stub=is_stub, + filename=filename, modname=modname) else: - c = beniget.DefUseChains(is_stub=is_stub) + c = beniget.DefUseChains(is_stub=is_stub, + filename=filename, modname=modname) + c.visit(node) self.assertEqual(c.dump_chains(node), ref) return node, c @@ -1172,6 +1176,12 @@ def test_stubs_forward_ref(self): 'LiteralValue -> (LiteralValue -> (Subscript -> (BinOp -> ((#0)))), TypeAlias -> ())'], is_stub=True ) + self.checkChains( + code.replace('typing', 'typing_extensions'), + ['TypeAlias -> (TypeAlias -> ())', + 'LiteralValue -> (LiteralValue -> (Subscript -> (BinOp -> ((#0)))), TypeAlias -> ())'], + is_stub=True + ) self.checkChains( code, ['TypeAlias -> (TypeAlias -> ())', @@ -1201,6 +1211,71 @@ def test_stubs_typevar_forward_ref(self): strict=False, ) + # c.TypeVar is not recognized as being typing.TypeVar. + self.checkChains( + code.replace('from typing import TypeVar', 'from c import TypeVar'), + ['TypeVar -> (TypeVar -> (Call -> ()))', 'AnyStr -> ()', 'ast -> ()', 'F -> ()'], + is_stub=True, + strict=False, + ) + + @skipIf(sys.version_info.major < 3, "Python 3 semantics") + def test_stubs_typevar_typing_pyi(self): + code = ''' +class TypeVar: pass +AnyStr = TypeVar('AnyStr', F) +F = object +''' + self.checkChains( + code, + ['TypeVar -> (TypeVar -> (Call -> ()))', + 'AnyStr -> ()', + 'F -> (F -> (Call -> ()))'], + is_stub=True, + filename='typing.pyi', + ) + self.checkChains( + code, + ['TypeVar -> (TypeVar -> (Call -> ()))', + 'AnyStr -> ()', + 'F -> (F -> (Call -> ()))'], + is_stub=True, + modname='typing', + ) + self.checkChains( + code, + ['TypeVar -> (TypeVar -> (Call -> ()))', + 'AnyStr -> ()', + 'F -> (F -> (Call -> ()))'], + is_stub=True, + filename='/home/dev/projects/typeshed_client/typeshed/typing.pyi', + ) + + # When geniget doesn't know we're analysing the typing module, it cannot link + # TypeVar to typing.TypeVar, so this special stub semantics doesn't apply. + self.checkChains( + code, + ['TypeVar -> (TypeVar -> (Call -> ()))', 'AnyStr -> ()', 'F -> ()'], + is_stub=True, + strict=False, + ) + + @skipIf(sys.version_info.major < 3, "Python 3 semantics") + def test_stubs_typealias_typing_pyi(self): + code = ''' +TypeAlias: object +LiteralValue: TypeAlias = list[LiteralValue]|object +''' + self.checkChains( + code, + ['TypeAlias -> (object -> (), TypeAlias -> ())', + 'LiteralValue -> (LiteralValue -> (Subscript -> (BinOp -> ((#0)))), TypeAlias -> ())'], + is_stub=True, + filename='typing.pyi', + ) + + # TODO: add tests for decorators. + class TestUseDefChains(TestCase): def checkChains(self, code, ref): class StrictDefUseChains(beniget.DefUseChains): @@ -1225,3 +1300,23 @@ def test_simple_expression(self): def test_call(self): code = "from foo import bar; bar(1, 2)" self.checkChains(code, "Call <- {Constant, Constant, bar}, bar <- {bar}") + +class TestDefUseChainsUnderstandsFilename(TestCase): + + def test_potential_module_names(self): + from beniget.beniget import potential_module_names + self.assertEqual(potential_module_names('/var/lib/config.py'), + ('var.lib.config', 'lib.config', 'config')) + self.assertEqual(potential_module_names('git-repos/pydoctor/pydoctor/driver.py'), + ('pydoctor.pydoctor.driver', 'pydoctor.driver', 'driver')) + self.assertEqual(potential_module_names('git-repos/pydoctor/pydoctor/__init__.py'), + ('pydoctor.pydoctor', 'pydoctor')) + + def test_def_use_chains_init(self): + self.assertEqual(beniget.DefUseChains( + 'typing.pyi').modname, 'typing') + self.assertEqual(beniget.DefUseChains( + 'beniget/beniget.py').modname, 'beniget.beniget') + self.assertEqual(beniget.DefUseChains( + '/root/repos/beniget/beniget/beniget.py', + modname='beniget.beniget').modname, 'beniget.beniget') \ No newline at end of file diff --git a/tests/test_imports.py b/tests/test_imports.py index 32cf6b3..5fabd2f 100644 --- a/tests/test_imports.py +++ b/tests/test_imports.py @@ -27,11 +27,11 @@ def test_import_parser(self): assert len(expected)==len(node.body) for import_node, expected_names in zip(node.body, expected): assert isinstance(import_node, (ast.Import, ast.ImportFrom)) - for al,(orgmodule, orgname) in parser.visit(import_node).items(): + for al,i in parser.visit(import_node).items(): assert Def(al).name() in expected_names expected_orgmodule, expected_orgname = expected_names[Def(al).name()] - assert orgmodule == expected_orgmodule - assert orgname == expected_orgname + assert i.orgmodule == expected_orgmodule + assert i.orgname == expected_orgname ran=True assert ran @@ -49,10 +49,10 @@ def test_import_parser_relative(self): assert len(expected)==len(node.body) for import_node, expected_names in zip(node.body, expected): assert isinstance(import_node, (ast.Import, ast.ImportFrom)) - for al,(orgmodule, orgname) in parser.visit(import_node).items(): + for al,i in parser.visit(import_node).items(): assert Def(al).name() in expected_names expected_orgmodule, expected_orgname = expected_names[Def(al).name()] - assert orgmodule == expected_orgmodule - assert orgname == expected_orgname + assert i.orgmodule == expected_orgmodule + assert i.orgname == expected_orgname ran=True assert ran \ No newline at end of file From 5ba1718c515d7da0002434953d93a75546e9f72a Mon Sep 17 00:00:00 2001 From: tristanlatr <19967168+tristanlatr@users.noreply.github.com> Date: Wed, 20 Sep 2023 12:05:16 -0400 Subject: [PATCH 11/29] Update README.rst --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 82f5c40..86d650c 100644 --- a/README.rst +++ b/README.rst @@ -16,7 +16,7 @@ Basically Beniget provides four analyses: - ``beniget.Ancestors`` that maps each node to the list of enclosing nodes; - ``beniget.DefUseChains`` that maps each node to the list of definition points in that node; - ``beniget.UseDefChains`` that maps each node to the list of possible definition of that node. -- ``beniget.ImportParser`` that maps each import alias node to their resolved orginin name and module. +- ``beniget.ImportParser`` that maps each import alias node to their resolved origin name and module. See sample usages and/or run ``pydoc beniget`` for more information :-). From 2a1522ac19f00595fdbe1925d2c6ef5181cda38c Mon Sep 17 00:00:00 2001 From: tristanlatr <19967168+tristanlatr@users.noreply.github.com> Date: Thu, 21 Sep 2023 11:07:38 -0400 Subject: [PATCH 12/29] We don't need to explicitely specify is_stubs=True if the filename endswith .pyi --- README.rst | 2 +- beniget/beniget.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 86d650c..6347eaa 100644 --- a/README.rst +++ b/README.rst @@ -16,7 +16,7 @@ Basically Beniget provides four analyses: - ``beniget.Ancestors`` that maps each node to the list of enclosing nodes; - ``beniget.DefUseChains`` that maps each node to the list of definition points in that node; - ``beniget.UseDefChains`` that maps each node to the list of possible definition of that node. -- ``beniget.ImportParser`` that maps each import alias node to their resolved origin name and module. +- ``beniget.ImportParser`` that maps import alias node to their resolved origin name and module. See sample usages and/or run ``pydoc beniget`` for more information :-). diff --git a/beniget/beniget.py b/beniget/beniget.py index ccbedc5..bee2447 100644 --- a/beniget/beniget.py +++ b/beniget/beniget.py @@ -383,7 +383,7 @@ def __init__(self, self.imports = {} self.filename = filename - self.is_stub = is_stub + self.is_stub = is_stub or filename is not None and filename.endswith('.pyi') # determine module name, we provide some flexibility: # - The module name is not required to have correct parsing when the @@ -449,7 +449,7 @@ def __init__(self, # attributes set in visit_Module self.module = None - self.future_annotations = is_stub or future_annotations + self.future_annotations = self.is_stub or future_annotations # ## helpers From a41b095c9fddd10e9830620d798c9b1c4523d643 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Thu, 21 Sep 2023 11:19:18 -0400 Subject: [PATCH 13/29] Add a few tests --- beniget/beniget.py | 16 ++++++++-------- tests/test_chains.py | 24 ++++++++++++++++++++++-- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/beniget/beniget.py b/beniget/beniget.py index bee2447..c912092 100644 --- a/beniget/beniget.py +++ b/beniget/beniget.py @@ -365,17 +365,17 @@ def __init__(self, a module name and a filename. Included in error messages and used as part of the import resolving. - modname: str, fully qualified name of the module we're analysing. - Required to have a correct parsing of relative imports. A module name may end with '.__init__' to indicate the module is a package. - future_annotations: bool, PEP 563 mode - is_stub: bool, stub module semantics mode, implies future_annotations=True. - When the module is a stub file, there is no need for quoting to do a forward reference - inside: - - annotations (like PEP 563 mode) - - `TypeAlias`` values - - ``TypeVar()`` call arguments - - classe base expressions, keywords and decorators - - function decorators + It will auotmatically be enabled if the filename endswith '.pyi'. + When the module is a stub file, there is no need for quoting to do a forward reference + inside: + - annotations (like PEP 563 mode) + - `TypeAlias`` values + - ``TypeVar()`` call arguments + - classe base expressions, keywords and decorators + - function decorators """ self.chains = {} self.locals = defaultdict(list) diff --git a/tests/test_chains.py b/tests/test_chains.py index c9eabaa..0cc9af2 100644 --- a/tests/test_chains.py +++ b/tests/test_chains.py @@ -1456,11 +1456,31 @@ def test_potential_module_names(self): self.assertEqual(potential_module_names('git-repos/pydoctor/pydoctor/__init__.py'), ('pydoctor.pydoctor', 'pydoctor')) - def test_def_use_chains_init(self): + def test_def_use_chains_init_modname(self): self.assertEqual(beniget.DefUseChains( 'typing.pyi').modname, 'typing') self.assertEqual(beniget.DefUseChains( 'beniget/beniget.py').modname, 'beniget.beniget') self.assertEqual(beniget.DefUseChains( '/root/repos/beniget/beniget/beniget.py', - modname='beniget.beniget').modname, 'beniget.beniget') \ No newline at end of file + modname='beniget.__init__').modname, 'beniget') + + def test_def_use_chains_init_is_stub(self): + self.assertEqual(beniget.DefUseChains( + 'typing.pyi').is_stub, True) + self.assertEqual(beniget.DefUseChains( + 'beniget/beniget.py').is_stub, False) + self.assertEqual(beniget.DefUseChains( + '/root/repos/beniget/beniget/beniget.pyi').is_stub, True) + self.assertEqual(beniget.DefUseChains( + 'beniget/beniget.py', is_stub=True).is_stub, True) + + def test_def_use_chains_init_is_package(self): + self.assertEqual(beniget.DefUseChains( + 'typing.pyi').is_package, False) + self.assertEqual(beniget.DefUseChains( + 'beniget/beniget.py').is_package, False) + self.assertEqual(beniget.DefUseChains( + '/root/repos/beniget/beniget/__init__.pyi').is_package, True) + self.assertEqual(beniget.DefUseChains( + 'beniget/beniget/', modname='beniget.__init__').is_package, True) \ No newline at end of file From 6c084811735b715181db9040d90e00b27b347be2 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Thu, 21 Sep 2023 11:27:19 -0400 Subject: [PATCH 14/29] We don't need to special case the alias nodes since we're setting a lineno when we need it now. --- tests/test_definitions.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/tests/test_definitions.py b/tests/test_definitions.py index e4702c7..5d8ba9f 100644 --- a/tests/test_definitions.py +++ b/tests/test_definitions.py @@ -500,10 +500,7 @@ def test_BothLive(self): else: _PY37PLUS = False ''' - if sys.version_info>=(3,10): - self.checkLiveLocals(code, ["sys:2", "_PY37PLUS:4,6"], ["sys:2", "_PY37PLUS:4,6"]) - else: - self.checkLiveLocals(code, ["sys:None", "_PY37PLUS:4,6"], ["sys:None", "_PY37PLUS:4,6"]) + self.checkLiveLocals(code, ["sys:2", "_PY37PLUS:4,6"], ["sys:2", "_PY37PLUS:4,6"]) def test_BuiltinNameRedefConditional(self): code = ''' @@ -516,12 +513,8 @@ class ExceptionGroup(Exception): def exceptions(self): pass ''' - if sys.version_info>=(3,10): - self.checkLiveLocals(code, ['sys:2', 'property:3', 'ExceptionGroup:6'], + self.checkLiveLocals(code, ['sys:2', 'property:3', 'ExceptionGroup:6'], ['sys:2', 'property:3', 'ExceptionGroup:6']) - else: - self.checkLiveLocals(code, ['sys:None', 'property:3', 'ExceptionGroup:6'], - ['sys:None', 'property:3', 'ExceptionGroup:6']) def test_loop_body_might_not_run(self): code = """ From 56d28b1c3d272726b5dc4b5698e7967b50bcea29 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Thu, 21 Sep 2023 11:29:48 -0400 Subject: [PATCH 15/29] remove annotation --- beniget/beniget.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beniget/beniget.py b/beniget/beniget.py index c912092..a551b84 100644 --- a/beniget/beniget.py +++ b/beniget/beniget.py @@ -296,7 +296,7 @@ def posixpath_splitparts(path): r.pop() return tuple(r) -def potential_module_names(filename: str): +def potential_module_names(filename): """ Returns a tuple of potential module names deducted from the filename. From c82791155666e93e7e25e916b35f54d950d82272 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Thu, 21 Sep 2023 11:32:27 -0400 Subject: [PATCH 16/29] Don't use keyword-only params --- beniget/beniget.py | 1 - 1 file changed, 1 deletion(-) diff --git a/beniget/beniget.py b/beniget/beniget.py index a551b84..0bed89e 100644 --- a/beniget/beniget.py +++ b/beniget/beniget.py @@ -353,7 +353,6 @@ class DefUseChains(ast.NodeVisitor): def __init__(self, filename=None, - *, modname=None, future_annotations=False, is_stub=False): From 7767b64919d4cab33768184c61a8aa8e2ffe49d1 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Thu, 21 Sep 2023 11:35:05 -0400 Subject: [PATCH 17/29] Add docs --- beniget/beniget.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/beniget/beniget.py b/beniget/beniget.py index 0bed89e..c48e1b2 100644 --- a/beniget/beniget.py +++ b/beniget/beniget.py @@ -332,9 +332,11 @@ def potential_module_names(filename): class DefUseChains(ast.NodeVisitor): """ Module visitor that gathers two kinds of informations: - - locals: Dict[node, List[Def]], a mapping between a node and the list + - locals: dict[node, list[Def]], a mapping between a node and the list of variable defined in this node, - - chains: Dict[node, Def], a mapping between nodes and their chains. + - chains: dict[node, Def], a mapping between nodes and their chains. + - imports: dict[node, ImportInfo], a mapping between import aliases + and their resolved target. >>> import gast as ast >>> module = ast.parse("from b import c, d; c()") From c2debd861d026bf7df0857a715529376d044e42c Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Thu, 21 Sep 2023 11:38:32 -0400 Subject: [PATCH 18/29] Move the import parsing code into the beniget.py file. --- beniget/__init__.py | 4 +- beniget/beniget.py | 109 ++++++++++++++++++++++++++++++++++++++++- beniget/imports.py | 110 ------------------------------------------ tests/test_imports.py | 3 +- 4 files changed, 110 insertions(+), 116 deletions(-) delete mode 100644 beniget/imports.py diff --git a/beniget/__init__.py b/beniget/__init__.py index 1bd2fea..842f8e1 100644 --- a/beniget/__init__.py +++ b/beniget/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from beniget.version import __version__ -from beniget.beniget import Ancestors, DefUseChains, UseDefChains -from beniget.imports import ImportParser, ImportInfo +from beniget.beniget import (Ancestors, DefUseChains, UseDefChains, Def, + ImportParser, ImportInfo) diff --git a/beniget/beniget.py b/beniget/beniget.py index c48e1b2..619aa3d 100644 --- a/beniget/beniget.py +++ b/beniget/beniget.py @@ -5,8 +5,6 @@ import gast as ast -from beniget.imports import ImportParser - # TODO: remove me when python 2 is not supported anymore class _ordered_set(object): def __init__(self, elements=None): @@ -88,6 +86,113 @@ def parentFunction(self, node): def parentStmt(self, node): return self.parentInstance(node, ast.stmt) +class ImportInfo: + """ + Complement an `ast.alias` node with resolved + origin module and name of the locally bound name. + + :note: `orgname` will be ``*`` for wildcard imports. + """ + __slots__ = 'orgmodule', 'orgname' + + def __init__(self, orgmodule, orgname=None) -> None: + """ + :param orgmodule: str + :param orgname: str or None + """ + self.orgmodule = orgmodule + self.orgname = orgname + + def target(self) -> str: + """ + Returns the qualified name of the the imported symbol. + """ + if self.orgname: + return f"{self.orgmodule}.{self.orgname}" + else: + return self.orgmodule + +_alias_needs_lineno = sys.implementation.name == 'cpython' and sys.version_info < (3,10) + +# The MIT License (MIT) +# Copyright (c) 2017 Jelle Zijlstra +# Adapted from the project typeshed_client. +class ImportParser(ast.NodeVisitor): + """ + Transform import statements into a mapping from `ast.alias` to `ImportInfo`. + One instance of `ImportParser` can be used to parse all imports in a given module. + + Call to `visit` will parse the given import node into a mapping of aliases to `ImportInfo`. + """ + + def __init__(self, modname, *, is_package) -> None: + self._modname = tuple(modname.split(".")) + self._is_package = is_package + self._result = {} + + def generic_visit(self, node): + raise TypeError('unexpected node type: {}'.format(type(node))) + + def visit_Import(self, node): + self._result.clear() + for al in node.names: + if al.asname: + self._result[al] = ImportInfo(orgmodule=al.name) + else: + # Here, we're not including information + # regarding the submodules imported - if there is one. + # This is because this analysis map the names bounded by imports, + # not the dependencies. + self._result[al] = ImportInfo(orgmodule=al.name.split(".", 1)[0]) + + # This seems to be the most resonable place to fix the ast.alias node not having + # proper line number information on python3.9 and before. + if _alias_needs_lineno: + al.lineno = node.lineno + + return self._result.copy() + + def visit_ImportFrom(self, node): + self._result.clear() + current_module = self._modname + + if node.module is None: + module = () + else: + module = tuple(node.module.split(".")) + + if not node.level: + source_module = module + else: + # parse relative imports + if node.level == 1: + if self._is_package: + relative_module = current_module + else: + relative_module = current_module[:-1] + else: + if self._is_package: + relative_module = current_module[: 1 - node.level] + else: + relative_module = current_module[: -node.level] + + if not relative_module: + # We don't raise errors when an relative import makes no sens, + # we simply pad the name with dots. + relative_module = ("",) * node.level + + source_module = relative_module + module + + for alias in node.names: + self._result[alias] = ImportInfo( + orgmodule=".".join(source_module), orgname=alias.name + ) + + # fix the ast.alias node not having proper line number on python3.9 and before. + if _alias_needs_lineno: + alias.lineno = node.lineno + + return self._result.copy() class Def(object): """ diff --git a/beniget/imports.py b/beniget/imports.py deleted file mode 100644 index f513a66..0000000 --- a/beniget/imports.py +++ /dev/null @@ -1,110 +0,0 @@ -import gast as ast -import sys - -class ImportInfo: - """ - Complement an `ast.alias` node with resolved - origin module and name of the locally bound name. - - :note: `orgname` will be ``*`` for wildcard imports. - """ - __slots__ = 'orgmodule', 'orgname' - - def __init__(self, orgmodule, orgname=None) -> None: - """ - :param orgmodule: str - :param orgname: str or None - """ - self.orgmodule = orgmodule - self.orgname = orgname - - def target(self) -> str: - """ - Returns the qualified name of the the imported symbol. - """ - if self.orgname: - return f"{self.orgmodule}.{self.orgname}" - else: - return self.orgmodule - -_alias_needs_lineno = sys.implementation.name == 'cpython' and sys.version_info < (3,10) - -# The MIT License (MIT) -# Copyright (c) 2017 Jelle Zijlstra -# Adapted from the project typeshed_client. -class ImportParser(ast.NodeVisitor): - """ - Transform import statements into a mapping from `ast.alias` to `ImportInfo`. - One instance of `ImportParser` can be used to parse all imports in a given module. - - Call to `visit` will parse the given import node into a mapping of aliases to `ImportInfo`. - """ - - def __init__(self, modname, *, is_package) -> None: - self._modname = tuple(modname.split(".")) - self._is_package = is_package - self._result = {} - - def generic_visit(self, node): - raise TypeError('unexpected node type: {}'.format(type(node))) - - def visit_Import(self, node): - self._result.clear() - for al in node.names: - if al.asname: - self._result[al] = ImportInfo(orgmodule=al.name) - else: - # Here, we're not including information - # regarding the submodules imported - if there is one. - # This is because this analysis map the names bounded by imports, - # not the dependencies. - self._result[al] = ImportInfo(orgmodule=al.name.split(".", 1)[0]) - - # This seems to be the most resonable place to fix the ast.alias node not having - # proper line number information on python3.9 and before. - if _alias_needs_lineno: - al.lineno = node.lineno - - return self._result.copy() - - def visit_ImportFrom(self, node): - self._result.clear() - current_module = self._modname - - if node.module is None: - module = () - else: - module = tuple(node.module.split(".")) - - if not node.level: - source_module = module - else: - # parse relative imports - if node.level == 1: - if self._is_package: - relative_module = current_module - else: - relative_module = current_module[:-1] - else: - if self._is_package: - relative_module = current_module[: 1 - node.level] - else: - relative_module = current_module[: -node.level] - - if not relative_module: - # We don't raise errors when an relative import makes no sens, - # we simply pad the name with dots. - relative_module = ("",) * node.level - - source_module = relative_module + module - - for alias in node.names: - self._result[alias] = ImportInfo( - orgmodule=".".join(source_module), orgname=alias.name - ) - - # fix the ast.alias node not having proper line number on python3.9 and before. - if _alias_needs_lineno: - alias.lineno = node.lineno - - return self._result.copy() diff --git a/tests/test_imports.py b/tests/test_imports.py index 5fabd2f..f8fb3ea 100644 --- a/tests/test_imports.py +++ b/tests/test_imports.py @@ -2,8 +2,7 @@ from unittest import TestCase from textwrap import dedent -from beniget.imports import ImportParser -from beniget.beniget import Def +from beniget import Def, ImportParser class TestImportParser(TestCase): From 596c4dc070ca1de557421a723b1aca5c0afb1ef5 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Thu, 21 Sep 2023 11:40:01 -0400 Subject: [PATCH 19/29] Remove annotation --- beniget/beniget.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beniget/beniget.py b/beniget/beniget.py index 619aa3d..455dee1 100644 --- a/beniget/beniget.py +++ b/beniget/beniget.py @@ -95,7 +95,7 @@ class ImportInfo: """ __slots__ = 'orgmodule', 'orgname' - def __init__(self, orgmodule, orgname=None) -> None: + def __init__(self, orgmodule, orgname=None): """ :param orgmodule: str :param orgname: str or None From c6d795af09488ccb5fff2a36f259aa816b6e4299 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Thu, 21 Sep 2023 15:44:29 -0400 Subject: [PATCH 20/29] remove annotation --- beniget/beniget.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/beniget/beniget.py b/beniget/beniget.py index 455dee1..5a52415 100644 --- a/beniget/beniget.py +++ b/beniget/beniget.py @@ -103,9 +103,9 @@ def __init__(self, orgmodule, orgname=None): self.orgmodule = orgmodule self.orgname = orgname - def target(self) -> str: + def target(self): """ - Returns the qualified name of the the imported symbol. + Returns the qualified name of the the imported symbol, str. """ if self.orgname: return f"{self.orgmodule}.{self.orgname}" @@ -125,7 +125,7 @@ class ImportParser(ast.NodeVisitor): Call to `visit` will parse the given import node into a mapping of aliases to `ImportInfo`. """ - def __init__(self, modname, *, is_package) -> None: + def __init__(self, modname, *, is_package): self._modname = tuple(modname.split(".")) self._is_package = is_package self._result = {} From 2c9aeef331d1c69e164a7111edc4d2557a856e0f Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Thu, 21 Sep 2023 15:47:27 -0400 Subject: [PATCH 21/29] remove f-string --- beniget/beniget.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beniget/beniget.py b/beniget/beniget.py index 5a52415..2e21811 100644 --- a/beniget/beniget.py +++ b/beniget/beniget.py @@ -108,7 +108,7 @@ def target(self): Returns the qualified name of the the imported symbol, str. """ if self.orgname: - return f"{self.orgmodule}.{self.orgname}" + return "{}.{}".format(self.orgmodule, self.orgname) else: return self.orgmodule From 7e798996db68fa1cd47d77d5ed1622c0a5800167 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Thu, 21 Sep 2023 15:51:44 -0400 Subject: [PATCH 22/29] remove keyword-only arguments --- beniget/beniget.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beniget/beniget.py b/beniget/beniget.py index 2e21811..ded774f 100644 --- a/beniget/beniget.py +++ b/beniget/beniget.py @@ -125,7 +125,7 @@ class ImportParser(ast.NodeVisitor): Call to `visit` will parse the given import node into a mapping of aliases to `ImportInfo`. """ - def __init__(self, modname, *, is_package): + def __init__(self, modname, is_package=False): self._modname = tuple(modname.split(".")) self._is_package = is_package self._result = {} From 0fc304e8483c6ea9d7595c3a4c8bb695d58775be Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Thu, 21 Sep 2023 16:16:59 -0400 Subject: [PATCH 23/29] Also defer decorators. --- beniget/beniget.py | 21 +++++++++++++++------ tests/test_chains.py | 26 +++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/beniget/beniget.py b/beniget/beniget.py index ded774f..9dd0cab 100644 --- a/beniget/beniget.py +++ b/beniget/beniget.py @@ -472,7 +472,8 @@ def __init__(self, Included in error messages and used as part of the import resolving. - modname: str, fully qualified name of the module we're analysing. A module name may end with '.__init__' to indicate the module is a package. - - future_annotations: bool, PEP 563 mode + - future_annotations: bool, PEP 563 mode. + It will auotmatically be enabled if the module has ``from __future__ import annotations``. - is_stub: bool, stub module semantics mode, implies future_annotations=True. It will auotmatically be enabled if the filename endswith '.pyi'. When the module is a stub file, there is no need for quoting to do a forward reference @@ -976,8 +977,13 @@ def visit_FunctionDef(self, node, step=DeclarationStep): self.visit(kw_default).add_user(dnode) for default in node.args.defaults: self.visit(default).add_user(dnode) - for decorator in node.decorator_list: - self.visit(decorator) + if self.is_stub: + for decorator in node.decorator_list: + self._defered_annotations[-1].append(( + decorator, currentscopes, None)) + else: + for decorator in node.decorator_list: + self.visit(decorator) if not self.future_annotations and node.returns: self.visit(node.returns) @@ -1004,7 +1010,7 @@ def visit_ClassDef(self, node): self.add_to_locals(node.name, dnode) if self.is_stub: - # special treatment for base classes in stub modules + # special treatment for classes in stub modules # so they can contain forward-references. currentscopes = list(self._scopes) for base in node.bases: @@ -1013,13 +1019,16 @@ def visit_ClassDef(self, node): for keyword in node.keywords: self._defered_annotations[-1].append(( keyword.value, currentscopes, lambda dkeyword: dkeyword.add_user(dnode))) + for decorator in node.decorator_list: + self._defered_annotations[-1].append(( + decorator, currentscopes, lambda ddecorator: ddecorator.add_user(dnode))) else: for base in node.bases: self.visit(base).add_user(dnode) for keyword in node.keywords: self.visit(keyword.value).add_user(dnode) - for decorator in node.decorator_list: - self.visit(decorator).add_user(dnode) + for decorator in node.decorator_list: + self.visit(decorator).add_user(dnode) with self.ScopeContext(node): self.set_definition("__class__", Def("__class__")) diff --git a/tests/test_chains.py b/tests/test_chains.py index 0cc9af2..3503bd0 100644 --- a/tests/test_chains.py +++ b/tests/test_chains.py @@ -1418,7 +1418,31 @@ def test_stubs_typealias_typing_pyi(self): filename='typing.pyi', ) - # TODO: add tests for decorators. + @skipIf(sys.version_info.major < 3, "Python 3 semantics") + def test_stubs_class_decorators(self): + code = ''' +@dataclass_transform +class Thing: + x: int +def dataclass_transform(f):... +''' + self.checkChains( + code, + ['Thing -> ()', 'dataclass_transform -> (dataclass_transform -> (Thing -> ()))'], + filename='some.pyi') + + @skipIf(sys.version_info.major < 3, "Python 3 semantics") + def test_stubs_function_decorators(self): + code = ''' +class Thing: + @property + def x(self) -> int:... +def property(f):... +''' + self.checkChains( + code, + ['Thing -> ()', 'property -> (property -> ())'], + filename='some.pyi') class TestUseDefChains(TestCase): def checkChains(self, code, ref): From 1e29f56b822c089df08f57218ae747853b0f285f Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Thu, 21 Sep 2023 16:20:16 -0400 Subject: [PATCH 24/29] Fix python implementation check for python2 --- beniget/beniget.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/beniget/beniget.py b/beniget/beniget.py index 9dd0cab..d341410 100644 --- a/beniget/beniget.py +++ b/beniget/beniget.py @@ -1,6 +1,7 @@ from collections import defaultdict, OrderedDict, deque from contextlib import contextmanager import sys +import platform import os.path import gast as ast @@ -112,7 +113,7 @@ def target(self): else: return self.orgmodule -_alias_needs_lineno = sys.implementation.name == 'cpython' and sys.version_info < (3,10) +_alias_needs_lineno = platform.python_implementation().lower() == 'cpython' and sys.version_info < (3,10) # The MIT License (MIT) # Copyright (c) 2017 Jelle Zijlstra From 09fe6506a24ddd836c7a0eb33abaa6f0d5b39c60 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Thu, 21 Sep 2023 16:28:55 -0400 Subject: [PATCH 25/29] Fix missing str.isidentifier function on python2 --- beniget/beniget.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/beniget/beniget.py b/beniget/beniget.py index d341410..579c01e 100644 --- a/beniget/beniget.py +++ b/beniget/beniget.py @@ -402,6 +402,15 @@ def posixpath_splitparts(path): r.pop() return tuple(r) +if sys.version_info < (3,0): + import tokenize, re + identifier_regex = re.compile(tokenize.Name + r'\Z') + + def isidentifier(string): + return identifier_regex.match(string) is not None +else: + isidentifier = str.isidentifier + def potential_module_names(filename): """ Returns a tuple of potential module @@ -425,7 +434,7 @@ def potential_module_names(filename): len_parts = len(parts) for i in range(len_parts): p = parts[i:] - if not p or any(not all(sb.isidentifier() + if not p or any(not all(isidentifier(sb) for sb in s.split('.')) for s in p): # the path cannot be converted to a module name # because there are unallowed caracters. From 1748ee3753c8629cc52118e8848a5090b3014010 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Fri, 10 Nov 2023 21:48:16 -0500 Subject: [PATCH 26/29] Use a function to parse imports (instead of a class) --- beniget/__init__.py | 2 +- beniget/beniget.py | 69 +++++++++++++++++++------------------------ tests/test_imports.py | 10 +++---- 3 files changed, 35 insertions(+), 46 deletions(-) diff --git a/beniget/__init__.py b/beniget/__init__.py index 842f8e1..aa2d3c4 100644 --- a/beniget/__init__.py +++ b/beniget/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from beniget.version import __version__ from beniget.beniget import (Ancestors, DefUseChains, UseDefChains, Def, - ImportParser, ImportInfo) + parse_import, ImportInfo) diff --git a/beniget/beniget.py b/beniget/beniget.py index 579c01e..78c474b 100644 --- a/beniget/beniget.py +++ b/beniget/beniget.py @@ -118,44 +118,37 @@ def target(self): # The MIT License (MIT) # Copyright (c) 2017 Jelle Zijlstra # Adapted from the project typeshed_client. -class ImportParser(ast.NodeVisitor): +def parse_import(node, modname, is_package=False): """ - Transform import statements into a mapping from `ast.alias` to `ImportInfo`. - One instance of `ImportParser` can be used to parse all imports in a given module. - - Call to `visit` will parse the given import node into a mapping of aliases to `ImportInfo`. + Parse the given import node into a mapping of aliases to `ImportInfo`. + + :param node: the import node. + :param str modname: the name of the module. + :param bool is_package: whether the module is a package. + :rtype: dict[ast.alias, ImportInfo] """ + result = {} - def __init__(self, modname, is_package=False): - self._modname = tuple(modname.split(".")) - self._is_package = is_package - self._result = {} - - def generic_visit(self, node): - raise TypeError('unexpected node type: {}'.format(type(node))) + + # This seems to be the most resonable place to fix the ast.alias node not having + # proper line number information on python3.9 and before. + if _alias_needs_lineno: + for alias in node.names: + alias.lineno = node.lineno - def visit_Import(self, node): - self._result.clear() + if isinstance(node, ast.Import): for al in node.names: if al.asname: - self._result[al] = ImportInfo(orgmodule=al.name) + result[al] = ImportInfo(orgmodule=al.name) else: # Here, we're not including information # regarding the submodules imported - if there is one. # This is because this analysis map the names bounded by imports, # not the dependencies. - self._result[al] = ImportInfo(orgmodule=al.name.split(".", 1)[0]) - - # This seems to be the most resonable place to fix the ast.alias node not having - # proper line number information on python3.9 and before. - if _alias_needs_lineno: - al.lineno = node.lineno - - return self._result.copy() - - def visit_ImportFrom(self, node): - self._result.clear() - current_module = self._modname + result[al] = ImportInfo(orgmodule=al.name.split(".", 1)[0]) + + elif isinstance(node, ast.ImportFrom): + current_module = tuple(modname.split(".")) if node.module is None: module = () @@ -167,12 +160,12 @@ def visit_ImportFrom(self, node): else: # parse relative imports if node.level == 1: - if self._is_package: + if is_package: relative_module = current_module else: relative_module = current_module[:-1] else: - if self._is_package: + if is_package: relative_module = current_module[: 1 - node.level] else: relative_module = current_module[: -node.level] @@ -185,15 +178,15 @@ def visit_ImportFrom(self, node): source_module = relative_module + module for alias in node.names: - self._result[alias] = ImportInfo( + result[alias] = ImportInfo( orgmodule=".".join(source_module), orgname=alias.name ) - - # fix the ast.alias node not having proper line number on python3.9 and before. - if _alias_needs_lineno: - alias.lineno = node.lineno - return self._result.copy() + else: + raise TypeError('unexpected node type: {}'.format(type(node))) + + return result + class Def(object): """ @@ -562,8 +555,6 @@ def __init__(self, # dead code levels, it's non null for code that cannot be executed self._deadcode = 0 - self._import_parser = ImportParser(self.modname, is_package=self.is_package) - # attributes set in visit_Module self.module = None self.future_annotations = self.is_stub or future_annotations @@ -1279,7 +1270,7 @@ def visit_Import(self, node): base = alias.name.split(".", 1)[0] self.set_definition(alias.asname or base, dalias) self.add_to_locals(alias.asname or base, dalias) - self.imports.update(self._import_parser.visit(node)) + self.imports.update(parse_import(node, self.modname, is_package=self.is_package)) def visit_ImportFrom(self, node): for alias in node.names: @@ -1289,7 +1280,7 @@ def visit_ImportFrom(self, node): else: self.set_definition(alias.asname or alias.name, dalias) self.add_to_locals(alias.asname or alias.name, dalias) - self.imports.update(self._import_parser.visit(node)) + self.imports.update(parse_import(node, self.modname, is_package=self.is_package)) def visit_Exec(self, node): dnode = self.chains.setdefault(node, Def(node)) diff --git a/tests/test_imports.py b/tests/test_imports.py index f8fb3ea..611ea0a 100644 --- a/tests/test_imports.py +++ b/tests/test_imports.py @@ -2,11 +2,11 @@ from unittest import TestCase from textwrap import dedent -from beniget import Def, ImportParser +from beniget import Def, parse_import class TestImportParser(TestCase): - def test_import_parser(self): + def test_import_parsing(self): code = ''' import mod2 import pack.subpack @@ -21,12 +21,11 @@ def test_import_parser(self): 'l':('mod2','_l'), 'm':('mod2','_m')}, {'C':('pack.subpack.stuff','C')},] - parser = ImportParser('mod1', is_package=False) node = ast.parse(dedent(code)) assert len(expected)==len(node.body) for import_node, expected_names in zip(node.body, expected): assert isinstance(import_node, (ast.Import, ast.ImportFrom)) - for al,i in parser.visit(import_node).items(): + for al,i in parse_import(import_node, 'mod1', is_package=False).items(): assert Def(al).name() in expected_names expected_orgmodule, expected_orgname = expected_names[Def(al).name()] assert i.orgmodule == expected_orgmodule @@ -43,12 +42,11 @@ def test_import_parser_relative(self): expected = [{'b':('top.mod2','bar')}, {'foo':('top.subpack.other.pack','foo')}, {'x': ('......error', 'x')}] - parser = ImportParser('top.subpack.other', is_package=True) node = ast.parse(dedent(code)) assert len(expected)==len(node.body) for import_node, expected_names in zip(node.body, expected): assert isinstance(import_node, (ast.Import, ast.ImportFrom)) - for al,i in parser.visit(import_node).items(): + for al,i in parse_import(import_node, 'top.subpack.other', is_package=True).items(): assert Def(al).name() in expected_names expected_orgmodule, expected_orgname = expected_names[Def(al).name()] assert i.orgmodule == expected_orgmodule From 2b9bf8c6e9f08048750fd326826f327ab08758dc Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Mon, 13 Nov 2023 00:37:57 -0500 Subject: [PATCH 27/29] Update readme --- README.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 6347eaa..a8c06c0 100644 --- a/README.rst +++ b/README.rst @@ -11,12 +11,11 @@ Python3. API --- -Basically Beniget provides four analyses: +Basically Beniget provides three analyses: - ``beniget.Ancestors`` that maps each node to the list of enclosing nodes; - ``beniget.DefUseChains`` that maps each node to the list of definition points in that node; - ``beniget.UseDefChains`` that maps each node to the list of possible definition of that node. -- ``beniget.ImportParser`` that maps import alias node to their resolved origin name and module. See sample usages and/or run ``pydoc beniget`` for more information :-). From 895edae4505c45963bbd2529ec88e1abd401606c Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Sat, 18 Nov 2023 12:14:45 -0500 Subject: [PATCH 28/29] Fix tests --- tests/test_chains.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_chains.py b/tests/test_chains.py index 3979f22..66fc928 100644 --- a/tests/test_chains.py +++ b/tests/test_chains.py @@ -1278,19 +1278,19 @@ def test_stubs_forward_ref(self): self.checkChains( code, ['TypeAlias -> (TypeAlias -> ())', - 'LiteralValue -> (LiteralValue -> (Subscript -> (BinOp -> ((#0)))), TypeAlias -> ())'], + 'LiteralValue -> (LiteralValue -> (Subscript -> (BinOp -> ())))'], is_stub=True ) self.checkChains( code.replace('typing', 'typing_extensions'), ['TypeAlias -> (TypeAlias -> ())', - 'LiteralValue -> (LiteralValue -> (Subscript -> (BinOp -> ((#0)))), TypeAlias -> ())'], + 'LiteralValue -> (LiteralValue -> (Subscript -> (BinOp -> ())))'], is_stub=True ) self.checkChains( code, ['TypeAlias -> (TypeAlias -> ())', - 'LiteralValue -> (TypeAlias -> ())'], + 'LiteralValue -> ()'], strict=False, ) @@ -1370,8 +1370,8 @@ def test_stubs_typealias_typing_pyi(self): ''' self.checkChains( code, - ['TypeAlias -> (object -> (), TypeAlias -> ())', - 'LiteralValue -> (LiteralValue -> (Subscript -> (BinOp -> ((#0)))), TypeAlias -> ())'], + ['TypeAlias -> (TypeAlias -> ())', + 'LiteralValue -> (LiteralValue -> (Subscript -> (BinOp -> ())))'], is_stub=True, filename='typing.pyi', ) From 1eb0c361d7d5b847fa90acdf9f490a1ead38b4f5 Mon Sep 17 00:00:00 2001 From: tristanlatr Date: Sat, 18 Nov 2023 12:24:40 -0500 Subject: [PATCH 29/29] Python2 is not supportted anymore, so remove dead code. --- beniget/beniget.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/beniget/beniget.py b/beniget/beniget.py index e2f47dc..bb49578 100644 --- a/beniget/beniget.py +++ b/beniget/beniget.py @@ -362,15 +362,6 @@ def posixpath_splitparts(path): r.pop() return tuple(r) -if sys.version_info < (3,0): - import tokenize, re - identifier_regex = re.compile(tokenize.Name + r'\Z') - - def isidentifier(string): - return identifier_regex.match(string) is not None -else: - isidentifier = str.isidentifier - def potential_module_names(filename): """ Returns a tuple of potential module @@ -394,7 +385,7 @@ def potential_module_names(filename): len_parts = len(parts) for i in range(len_parts): p = parts[i:] - if not p or any(not all(isidentifier(sb) + if not p or any(not all(sb.isidentifier() for sb in s.split('.')) for s in p): # the path cannot be converted to a module name # because there are unallowed caracters.