Merge pull request #140 from Kazy/fix-139-async-for-newline

Fix #139: newlines in async for comprehension

AUTHORS.txt

@@ -51,6 +51,7 @@ Simon Ruggier (@sruggier)
 Élie Gouzien (@ElieGouzien)
 Tim Gates (@timgates42) <[email protected]>
 Batuhan Taskaya (@isidentical) <[email protected]>
+Jocelyn Boullier (@Kazy) <[email protected]>
 
 
 Note: (@user) means a github user name.

conftest.py

@@ -158,6 +158,10 @@ def works_ge_py35(each_version):
     version_info = parse_version_string(each_version)
     return Checker(each_version, version_info >= (3, 5))
 
+@pytest.fixture
+def works_ge_py36(each_version):
+    version_info = parse_version_string(each_version)
+    return Checker(each_version, version_info >= (3, 6))
 
 @pytest.fixture
 def works_ge_py38(each_version):

parso/python/tokenize.py

@@ -260,7 +260,7 @@ def _create_token_collection(version_info):
                            'finally', 'while', 'with', 'return', 'continue',
                            'break', 'del', 'pass', 'global', 'assert')
     if version_info >= (3, 5):
-        ALWAYS_BREAK_TOKENS += ('async', 'nonlocal')
+        ALWAYS_BREAK_TOKENS += ('nonlocal', )
     pseudo_token_compiled = _compile(PseudoToken)
     return TokenCollection(
         pseudo_token_compiled, single_quoted, triple_quoted, endpats,

test/test_pgen2.py

@@ -87,6 +87,39 @@ def test_async_for(works_ge_py35):
     works_ge_py35.parse("async def foo():\n async for a in b: pass")
 
 
+@pytest.mark.parametrize("body", [
+    """[1 async for a in b
+    ]""",
+    """[1 async
+    for a in b
+    ]""",
+    """[
+    1
+    async for a in b
+    ]""",
+    """[
+    1
+    async for a
+    in b
+    ]""",
+    """[
+    1
+    async
+    for
+    a
+    in
+    b
+    ]""",
+    """  [
+    1 async for a in b
+  ]""",
+])
+def test_async_for_comprehension_newline(works_ge_py36, body):
+    # Issue #139
+    works_ge_py36.parse("""async def foo():
+    {}""".format(body))
+
+
 def test_async_with(works_ge_py35):
     works_ge_py35.parse("async def foo():\n async with a: pass")