Merge pull request #258 from edelvalle/master

Upgrade parso to v0.3.1

Makefile

@@ -13,7 +13,7 @@ dependencies:
 	rm -rf dependencies/
 	mkdir dependencies/
 	$(call get_dependency,jedi,https://github.com/davidhalter/jedi,v0.12.1)
-	$(call get_dependency,parso,https://github.com/davidhalter/parso,v0.3.0)
+	$(call get_dependency,parso,https://github.com/davidhalter/parso,v0.3.1)
 
 clean:
 	find . -name '*.pyc' -delete

dependencies/parso/__init__.py

@@ -43,7 +43,7 @@
 from parso.utils import split_lines, python_bytes_to_unicode
 
 
-__version__ = '0.3.0'
+__version__ = '0.3.1'
 
 
 def parse(code=None, **kwargs):

dependencies/parso/parser.py

@@ -147,7 +147,7 @@ def error_recovery(self, token):
             raise NotImplementedError("Error Recovery is not implemented")
         else:
             type_, value, start_pos, prefix = token
-            error_leaf = tree.ErrorLeaf('TODO %s' % type_, value, start_pos, prefix)
+            error_leaf = tree.ErrorLeaf(type_, value, start_pos, prefix)
             raise ParserSyntaxError('SyntaxError: invalid syntax', error_leaf)
 
     def convert_node(self, nonterminal, children):

dependencies/parso/pgen2/generator.py

@@ -286,7 +286,6 @@ def _make_transition(token_namespace, reserved_syntax_strings, label):
         # Either a keyword or an operator
         assert label[0] in ('"', "'"), label
         assert not label.startswith('"""') and not label.startswith("'''")
-        # TODO use literal_eval instead of a simple eval.
         value = literal_eval(label)
         try:
             return reserved_syntax_strings[value]

dependencies/parso/python/diff.py

@@ -47,6 +47,14 @@ def _flows_finished(pgen_grammar, stack):
     return True
 
 
+def _func_or_class_has_suite(node):
+    if node.type == 'decorated':
+        node = node.children[-1]
+    if node.type in ('async_funcdef', 'async_stmt'):
+        node = node.children[-1]
+    return node.type in ('classdef', 'funcdef') and node.children[-1].type == 'suite'
+
+
 def suite_or_file_input_is_valid(pgen_grammar, stack):
     if not _flows_finished(pgen_grammar, stack):
         return False
@@ -511,7 +519,7 @@ def _copy_nodes(self, tos, nodes, until_line, line_offset):
             # binary search.
             if _get_last_line(node) > until_line:
                 # We can split up functions and classes later.
-                if node.type in ('classdef', 'funcdef') and node.children[-1].type == 'suite':
+                if _func_or_class_has_suite(node):
                     new_nodes.append(node)
                 break
 
@@ -522,23 +530,25 @@ def _copy_nodes(self, tos, nodes, until_line, line_offset):
 
         last_node = new_nodes[-1]
         line_offset_index = -1
-        if last_node.type in ('classdef', 'funcdef'):
-            suite = last_node.children[-1]
-            if suite.type == 'suite':
-                suite_tos = _NodesStackNode(suite)
-                # Don't need to pass line_offset here, it's already done by the
-                # parent.
-                suite_nodes, recursive_tos = self._copy_nodes(
-                    suite_tos, suite.children, until_line, line_offset)
-                if len(suite_nodes) < 2:
-                    # A suite only with newline is not valid.
-                    new_nodes.pop()
-                else:
-                    suite_tos.parent = tos
-                    new_tos = recursive_tos
-                    line_offset_index = -2
+        if _func_or_class_has_suite(last_node):
+            suite = last_node
+            while suite.type != 'suite':
+                suite = suite.children[-1]
+
+            suite_tos = _NodesStackNode(suite)
+            # Don't need to pass line_offset here, it's already done by the
+            # parent.
+            suite_nodes, recursive_tos = self._copy_nodes(
+                suite_tos, suite.children, until_line, line_offset)
+            if len(suite_nodes) < 2:
+                # A suite only with newline is not valid.
+                new_nodes.pop()
+            else:
+                suite_tos.parent = tos
+                new_tos = recursive_tos
+                line_offset_index = -2
 
-        elif (new_nodes[-1].type in ('error_leaf', 'error_node') or
+        elif (last_node.type in ('error_leaf', 'error_node') or
               _is_flow_node(new_nodes[-1])):
             # Error leafs/nodes don't have a defined start/end. Error
             # nodes might not end with a newline (e.g. if there's an
@@ -570,7 +580,7 @@ def _update_tos(self, tree_node):
             self._tos = _NodesStackNode(tree_node, self._tos)
             self._tos.add(list(tree_node.children))
             self._update_tos(tree_node.children[-1])
-        elif tree_node.type in ('classdef', 'funcdef'):
+        elif _func_or_class_has_suite(tree_node):
             self._update_tos(tree_node.children[-1])
 
     def close(self):

dependencies/parso/python/tokenize.py

@@ -448,16 +448,15 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)):
 
             pseudomatch = pseudo_token.match(line, pos)
             if not pseudomatch:                             # scan for tokens
-                if line.endswith('\n'):
-                    new_line = True
                 match = whitespace.match(line, pos)
                 pos = match.end()
                 yield PythonToken(
-                    ERRORTOKEN, line[pos:], (lnum, pos),
+                    ERRORTOKEN, line[pos], (lnum, pos),
                     additional_prefix + match.group(0)
                 )
                 additional_prefix = ''
-                break
+                pos += 1
+                continue
 
             prefix = additional_prefix + pseudomatch.group(1)
             additional_prefix = ''

dependencies/parso/python/tree.py

@@ -537,7 +537,9 @@ def check_python2_nested_param(node):
             if child is None or child == ',':
                 param_children = children[start:end]
                 if param_children:  # Could as well be comma and then end.
-                    if param_children[0] == '*' and param_children[1] == ',' \
+                    if param_children[0] == '*' \
+                            and (len(param_children) == 1
+                                 or param_children[1] == ',') \
                             or check_python2_nested_param(param_children[0]):
                         for p in param_children:
                             p.parent = parent