Change the way how context caching and scoping is done.

This patch reverts some changes added by a couple of changesets, making
the context caching and scoping to work as before.
The changesets in question are:

 * 41b3bd589da0549ac061bc4c4b5d379cdbb1e10c
   Replace copy_context with some dynamic scoping.

 * 8d3c601
   Remove context.lookupname; make it an argument to infer() when appropriate.

 * partially reverts  048a42c.
   Fix some deep recursion problems.


There were some problems with these changes, which led to horrendous
performance when dealing with multiple inference paths of the same names,
as seen in these Pylint issues:

* https://bitbucket.org/logilab/pylint/issue/395/horrible-performance-related-to-inspect
* https://bitbucket.org/logilab/pylint/issue/465/pylint-hangs-when-using-inspectsignature
* https://bitbucket.org/logilab/pylint/issue/430/pylint-140-execution-time-and-memory

The reverted changes assumed that a context it's only passed to callees
and then destroyed, thus InferenceContext.push always returned another inference context,
with the updated inference path so far. This is wrong, since contexts are sometimes
reused, so the original context, the one before the .push call need to have the same
cache key in its path (this is actually what's happening in these mentioned issues,
the same object is inferred over and over again, but with different contexts).

astroid/bases.py

@@ -58,52 +58,28 @@ def infer(self, context=None):
 
 # Inference ##################################################################
 
-MISSING = object()
-
-
 class InferenceContext(object):
-    __slots__ = ('path', 'callcontext', 'boundnode', 'infered')
-
-    def __init__(self,
-            path=None, callcontext=None, boundnode=None, infered=None):
-        if path is None:
-            self.path = frozenset()
-        else:
-            self.path = path
-        self.callcontext = callcontext
-        self.boundnode = boundnode
-        if infered is None:
-            self.infered = {}
-        else:
-            self.infered = infered
-
-    def push(self, key):
-        # This returns a NEW context with the same attributes, but a new key
-        # added to `path`.  The intention is that it's only passed to callees
-        # and then destroyed; otherwise scope() may not work correctly.
-        # The cache will be shared, since it's the same exact dict.
-        if key in self.path:
-            # End the containing generator
-            raise StopIteration
-
-        return InferenceContext(
-            self.path.union([key]),
-            self.callcontext,
-            self.boundnode,
-            self.infered,
-        )
-
-    @contextmanager
-    def scope(self, callcontext=MISSING, boundnode=MISSING):
-        try:
-            orig = self.callcontext, self.boundnode
-            if callcontext is not MISSING:
-                self.callcontext = callcontext
-            if boundnode is not MISSING:
-                self.boundnode = boundnode
-            yield
-        finally:
-            self.callcontext, self.boundnode = orig
+    __slots__ = ('path', 'lookupname', 'callcontext', 'boundnode', 'infered')
+
+    def __init__(self, path=None, infered=None):
+        self.path = path or set()
+        self.lookupname = None
+        self.callcontext = None
+        self.boundnode = None
+        self.infered = infered or {}
+
+    def push(self, node):
+        name = self.lookupname
+        if (node, name) in self.path:
+            raise StopIteration()
+        self.path.add((node, name))
+
+    def clone(self):
+        # XXX copy lookupname/callcontext ?
+        clone = InferenceContext(self.path, infered=self.infered)
+        clone.callcontext = self.callcontext
+        clone.boundnode = self.boundnode
+        return clone
 
     def cache_generator(self, key, generator):
         results = []
@@ -114,28 +90,38 @@ def cache_generator(self, key, generator):
         self.infered[key] = tuple(results)
         return
 
+    @contextmanager
+    def restore_path(self):
+        path = set(self.path)
+        yield
+        self.path = path
+
+def copy_context(context):
+    if context is not None:
+        return context.clone()
+    else:
+        return InferenceContext()
 
-def _infer_stmts(stmts, context, frame=None, lookupname=None):
+
+def _infer_stmts(stmts, context, frame=None):
     """return an iterator on statements inferred by each statement in <stmts>
     """
     stmt = None
     infered = False
-    if context is None:
+    if context is not None:
+        name = context.lookupname
+        context = context.clone()
+    else:
+        name = None
         context = InferenceContext()
     for stmt in stmts:
         if stmt is YES:
             yield stmt
             infered = True
             continue
-
-        kw = {}
-        infered_name = stmt._infer_name(frame, lookupname)
-        if infered_name is not None:
-            # only returns not None if .infer() accepts a lookupname kwarg
-            kw['lookupname'] = infered_name
-
+        context.lookupname = stmt._infer_name(frame, name)
         try:
-            for infered in stmt.infer(context, **kw):
+            for infered in stmt.infer(context):
                 yield infered
                 infered = True
         except UnresolvableName:
@@ -197,20 +183,21 @@ def igetattr(self, name, context=None):
             context = InferenceContext()
         try:
             # avoid recursively inferring the same attr on the same class
-            new_context = context.push((self._proxied, name))
+
+            context.push((self._proxied, name))
             # XXX frame should be self._proxied, or not ?
-            get_attr = self.getattr(name, new_context, lookupclass=False)
+            get_attr = self.getattr(name, context, lookupclass=False)
             return _infer_stmts(
-                self._wrap_attr(get_attr, new_context),
-                new_context,
+                self._wrap_attr(get_attr, context),
+                context,
                 frame=self,
             )
         except NotFoundError:
             try:
                 # fallback to class'igetattr since it has some logic to handle
                 # descriptors
                 return self._wrap_attr(self._proxied.igetattr(name, context),
-                                            context)
+                                       context)
             except NotFoundError:
                 raise InferenceError(name)
 
@@ -301,9 +288,9 @@ def is_bound(self):
         return True
 
     def infer_call_result(self, caller, context):
-        with context.scope(boundnode=self.bound):
-            for infered in self._proxied.infer_call_result(caller, context):
-                yield infered
+        context = context.clone()
+        context.boundnode = self.bound
+        return self._proxied.infer_call_result(caller, context)
 
 
 class Generator(Instance):
@@ -335,8 +322,7 @@ def wrapped(node, context=None, _func=func, **kwargs):
         """wrapper function handling context"""
         if context is None:
             context = InferenceContext()
-        context = context.push((node, kwargs.get('lookupname')))
-
+        context.push(node)
         yielded = set()
         for res in _func(node, context, **kwargs):
             # unproxy only true instance, not const, tuple, dict...
@@ -409,7 +395,8 @@ def infer(self, context=None, **kwargs):
         if not context:
             return self._infer(context, **kwargs)
 
-        key = (self, kwargs.get('lookupname'), context.callcontext, context.boundnode)
+        key = (self, context.lookupname,
+               context.callcontext, context.boundnode)
         if key in context.infered:
             return iter(context.infered[key])
 

astroid/inference.py

@@ -28,7 +28,7 @@
 from astroid.exceptions import (AstroidError, InferenceError, NoDefault,
                                 NotFoundError, UnresolvableName)
 from astroid.bases import (YES, Instance, InferenceContext,
-                           _infer_stmts, path_wrapper,
+                           _infer_stmts, copy_context, path_wrapper,
                            raise_if_nothing_infered)
 from astroid.protocols import (
     _arguments_infer_argname,
@@ -175,89 +175,92 @@ def infer_name(self, context=None):
 
         if not stmts:
             raise UnresolvableName(self.name)
-    return _infer_stmts(stmts, context, frame, self.name)
+    context = context.clone()
+    context.lookupname = self.name
+    return _infer_stmts(stmts, context, frame)
 nodes.Name._infer = path_wrapper(infer_name)
 nodes.AssName.infer_lhs = infer_name # won't work with a path wrapper
 
 
 def infer_callfunc(self, context=None):
     """infer a CallFunc node by trying to guess what the function returns"""
-    if context is None:
-        context = InferenceContext()
+    callcontext = context.clone()
+    callcontext.callcontext = CallContext(self.args, self.starargs, self.kwargs)
+    callcontext.boundnode = None
     for callee in self.func.infer(context):
-        with context.scope(
-            callcontext=CallContext(self.args, self.starargs, self.kwargs),
-            boundnode=None,
-        ):
-            if callee is YES:
-                yield callee
-                continue
-            try:
-                if hasattr(callee, 'infer_call_result'):
-                    for infered in callee.infer_call_result(self, context):
-                        yield infered
-            except InferenceError:
-                ## XXX log error ?
-                continue
+        if callee is YES:
+            yield callee
+            continue
+        try:
+            if hasattr(callee, 'infer_call_result'):
+                for infered in callee.infer_call_result(self, callcontext):
+                    yield infered
+        except InferenceError:
+            ## XXX log error ?
+            continue
 nodes.CallFunc._infer = path_wrapper(raise_if_nothing_infered(infer_callfunc))
 
 
-def infer_import(self, context=None, asname=True, lookupname=None):
+def infer_import(self, context=None, asname=True):
     """infer an Import node: return the imported module/object"""
-    if lookupname is None:
+    name = context.lookupname
+    if name is None:
         raise InferenceError()
     if asname:
-        yield self.do_import_module(self.real_name(lookupname))
+        yield self.do_import_module(self.real_name(name))
     else:
-        yield self.do_import_module(lookupname)
+        yield self.do_import_module(name)
 nodes.Import._infer = path_wrapper(infer_import)
 
 def infer_name_module(self, name):
     context = InferenceContext()
-    return self.infer(context, asname=False, lookupname=name)
+    context.lookupname = name
+    return self.infer(context, asname=False)
 nodes.Import.infer_name_module = infer_name_module
 
 
-def infer_from(self, context=None, asname=True, lookupname=None):
+def infer_from(self, context=None, asname=True):
     """infer a From nodes: return the imported module/object"""
-    if lookupname is None:
+    name = context.lookupname
+    if name is None:
         raise InferenceError()
     if asname:
-        lookupname = self.real_name(lookupname)
+        name = self.real_name(name)
     module = self.do_import_module()
     try:
-        return _infer_stmts(module.getattr(lookupname, ignore_locals=module is self.root()), context, lookupname=lookupname)
+        context = copy_context(context)
+        context.lookupname = name
+        return _infer_stmts(module.getattr(name, ignore_locals=module is self.root()), context)
     except NotFoundError:
-        raise InferenceError(lookupname)
+        raise InferenceError(name)
 nodes.From._infer = path_wrapper(infer_from)
 
 
 def infer_getattr(self, context=None):
     """infer a Getattr node by using getattr on the associated object"""
-    if not context:
-        context = InferenceContext()
     for owner in self.expr.infer(context):
         if owner is YES:
             yield owner
             continue
         try:
-            with context.scope(boundnode=owner):
-                for obj in owner.igetattr(self.attrname, context):
-                    yield obj
+            context.boundnode = owner
+            for obj in owner.igetattr(self.attrname, context):
+                yield obj
+            context.boundnode = None
         except (NotFoundError, InferenceError):
-            pass
+            context.boundnode = None
         except AttributeError:
             # XXX method / function
-            pass
+            context.boundnode = None
 nodes.Getattr._infer = path_wrapper(raise_if_nothing_infered(infer_getattr))
 nodes.AssAttr.infer_lhs = raise_if_nothing_infered(infer_getattr) # # won't work with a path wrapper
 
 
-def infer_global(self, context=None, lookupname=None):
-    if lookupname is None:
+def infer_global(self, context=None):
+    if context.lookupname is None:
         raise InferenceError()
     try:
-        return _infer_stmts(self.root().getattr(lookupname), context)
+        return _infer_stmts(self.root().getattr(context.lookupname), context)
     except NotFoundError:
         raise InferenceError()
 nodes.Global._infer = path_wrapper(infer_global)
@@ -349,10 +352,11 @@ def infer_binop(self, context=None):
 nodes.BinOp._infer = path_wrapper(infer_binop)
 
 
-def infer_arguments(self, context=None, lookupname=None):
-    if lookupname is None:
+def infer_arguments(self, context=None):
+    name = context.lookupname
+    if name is None:
         raise InferenceError()
-    return _arguments_infer_argname(self, lookupname, context)
+    return _arguments_infer_argname(self, name, context)
 nodes.Arguments._infer = infer_arguments
 
 

astroid/node_classes.py

@@ -24,7 +24,7 @@
 from logilab.common.decorators import cachedproperty
 
 from astroid.exceptions import NoDefault
-from astroid.bases import (NodeNG, Statement, Instance,
+from astroid.bases import (NodeNG, Statement, Instance, InferenceContext,
                            _infer_stmts, YES, BUILTINS)
 from astroid.mixins import (BlockRangeMixIn, AssignTypeMixin,
                             ParentAssignTypeMixin, FromImportMixIn)
@@ -130,7 +130,8 @@ def ilookup(self, name):
         the lookup method
         """
         frame, stmts = self.lookup(name)
-        return _infer_stmts(stmts, None, frame)
+        context = InferenceContext()
+        return _infer_stmts(stmts, context, frame)
 
     def _filter_stmts(self, stmts, frame, offset):
         """filter statements to remove ignorable statements.

astroid/protocols.py

@@ -24,7 +24,7 @@
 
 from astroid.exceptions import InferenceError, NoDefault, NotFoundError
 from astroid.node_classes import unpack_infer
-from astroid.bases import InferenceContext, \
+from astroid.bases import InferenceContext, copy_context, \
      raise_if_nothing_infered, yes_if_nothing_infered, Instance, YES
 from astroid.nodes import const_factory
 from astroid import nodes
@@ -283,8 +283,7 @@ def _arguments_infer_argname(self, name, context):
     # if there is a default value, yield it. And then yield YES to reflect
     # we can't guess given argument value
     try:
-        if context is None:
-            context = InferenceContext()
+        context = copy_context(context)
         for infered in self.default_value(name).infer(context):
             yield infered
         yield YES
@@ -296,6 +295,8 @@ def arguments_assigned_stmts(self, node, context, asspath=None):
     if context.callcontext:
         # reset call context/name
         callcontext = context.callcontext
+        context = copy_context(context)
+        context.callcontext = None
         return callcontext.infer_argument(self.parent, node.name, context)
     return _arguments_infer_argname(self, node.name, context)
 nodes.Arguments.assigned_stmts = arguments_assigned_stmts