Implement re-lexing logic for better error recovery

crates/ruff_python_parser/src/lexer.rs

@@ -1306,6 +1306,47 @@ impl<'src> Lexer<'src> {
         }
     }
 
+    /// Re-lex the current token in the context of a logical line.
+    ///
+    /// Returns a boolean indicating that whether the new current token is different than the
+    /// previous current token.
+    ///
+    /// This method is a no-op if the lexer isn't in a parenthesized context.
+    pub(crate) fn re_lex_logical_token(&mut self) -> bool {
+        if self.nesting == 0 {
+            return false;
+        }
+
+        // Reduce the nesting level because the parser recovered from an error inside list parsing.
+        self.nesting -= 1;
+
+        let current_position = self.current_range().start();
+        let reverse_chars = self.source[..current_position.to_usize()].chars().rev();
+        let mut new_position = current_position;
+        let mut has_newline = false;
+
+        for ch in reverse_chars {
+            if is_python_whitespace(ch) {
+                new_position -= ch.text_len();
+            } else if matches!(ch, '\n' | '\r') {
+                has_newline |= true;
+                new_position -= ch.text_len();
+            } else {
+                break;
+            }
+        }
+
+        if new_position != current_position && has_newline {
+            self.cursor = Cursor::new(self.source);
+            self.cursor.skip_bytes(new_position.to_usize());
+            self.state = State::Other;
+            self.next_token();
+            true
+        } else {
+            false
+        }
+    }
+
     #[inline]
     fn token_range(&self) -> TextRange {
         let end = self.offset();

crates/ruff_python_parser/src/parser/mod.rs

@@ -558,7 +558,10 @@ impl<'src> Parser<'src> {
                 }
 
                 self.expect(TokenKind::Comma);
-            } else if recovery_context_kind.list_terminator_kind(self).is_some() {
+            } else if recovery_context_kind
+                .list_terminator_kind(self)
+                .is_some_and(ListTerminatorKind::is_regular)
+            {
                 break;
             } else {
                 // Not a recognised element. Add an error and either skip the token or break
@@ -570,6 +573,7 @@ impl<'src> Parser<'src> {
                 // Run the error recovery: This also handles the case when an element is missing
                 // between two commas: `a,,b`
                 if self.is_enclosing_list_element_or_terminator() {
+                    self.tokens.re_lex_logical_token();
                     break;
                 }
 
@@ -786,6 +790,12 @@ enum ListTerminatorKind {
     ErrorRecovery,
 }
 
+impl ListTerminatorKind {
+    const fn is_regular(self) -> bool {
+        matches!(self, ListTerminatorKind::Regular)
+    }
+}
+
 #[derive(Copy, Clone, Debug)]
 enum RecoveryContextKind {
     /// When parsing a list of statements at the module level i.e., at the top level of a file.

crates/ruff_python_parser/src/token_source.rs

@@ -1,4 +1,4 @@
-use ruff_text_size::{TextRange, TextSize};
+use ruff_text_size::{Ranged, TextRange, TextSize};
 
 use crate::lexer::{Lexer, LexerCheckpoint, LexicalError, Token, TokenFlags, TokenValue};
 use crate::{Mode, TokenKind};
@@ -58,6 +58,23 @@ impl<'src> TokenSource<'src> {
         self.lexer.take_value()
     }
 
+    /// Calls the underying [`re_lex_logical_token`] method on the lexer and updates the token
+    /// vector accordingly.
+    ///
+    /// [`re_lex_logical_token`]: Lexer::re_lex_logical_token
+    pub(crate) fn re_lex_logical_token(&mut self) {
+        if self.lexer.re_lex_logical_token() {
+            let current_start = self.current_range().start();
+            while self
+                .tokens
+                .last()
+                .is_some_and(|last| last.start() >= current_start)
+            {
+                self.tokens.pop();
+            }
+        }
+    }
+
     /// Returns the next non-trivia token without consuming it.
     ///
     /// Use [`peek2`] to get the next two tokens.

crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__arguments__unclosed_0.py.snap

@@ -75,7 +75,9 @@ Module(
 
 
   |
+1 | call(
+  |      ^ Syntax Error: Expected ')', found newline
+2 | 
 3 | def foo():
 4 |     pass
-  |          Syntax Error: unexpected EOF while parsing
   |

crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__arguments__unclosed_1.py.snap

@@ -83,7 +83,9 @@ Module(
 
 
   |
+1 | call(x
+  |       ^ Syntax Error: Expected ')', found newline
+2 | 
 3 | def foo():
 4 |     pass
-  |          Syntax Error: unexpected EOF while parsing
   |

crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__arguments__unclosed_2.py.snap

@@ -83,7 +83,9 @@ Module(
 
 
   |
+1 | call(x,
+  |        ^ Syntax Error: Expected ')', found newline
+2 | 
 3 | def foo():
 4 |     pass
-  |          Syntax Error: unexpected EOF while parsing
   |

crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__dict__missing_closing_brace_2.py.snap

@@ -84,7 +84,9 @@ Module(
 
 
   |
+1 | {x: 1,
+  |       ^ Syntax Error: Expected '}', found newline
+2 | 
 3 | def foo():
 4 |     pass
-  |          Syntax Error: unexpected EOF while parsing
   |

crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__list__missing_closing_bracket_3.py.snap

@@ -82,7 +82,11 @@ Module(
 
 
   |
+2 | # token starts a statement.
+3 | 
+4 | [1, 2
+  |      ^ Syntax Error: Expected ']', found newline
+5 | 
 6 | def foo():
 7 |     pass
-  |          Syntax Error: unexpected EOF while parsing
   |

crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__parenthesized__missing_closing_paren_3.py.snap

@@ -83,7 +83,11 @@ Module(
 
 
   |
+2 | # token starts a statement.
+3 | 
+4 | (1, 2
+  |      ^ Syntax Error: Expected ')', found newline
+5 | 
 6 | def foo():
 7 |     pass
-  |          Syntax Error: unexpected EOF while parsing
   |

crates/ruff_python_parser/tests/snapshots/invalid_syntax@expressions__set__missing_closing_curly_brace_3.py.snap

@@ -81,7 +81,11 @@ Module(
 
 
   |
+2 | # token starts a statement.
+3 | 
+4 | {1, 2
+  |      ^ Syntax Error: Expected '}', found newline
+5 | 
 6 | def foo():
 7 |     pass
-  |          Syntax Error: unexpected EOF while parsing
   |

crates/ruff_python_parser/tests/snapshots/invalid_syntax@from_import_missing_rpar.py.snap

@@ -140,9 +140,9 @@ Module(
 
   |
 1 | from x import (a, b
+  |                    ^ Syntax Error: Expected ')', found newline
 2 | 1 + 1
 3 | from x import (a, b,
-  | ^^^^ Syntax Error: Simple statements must be separated by newlines or semicolons
 4 | 2 + 2
   |
 
@@ -156,6 +156,9 @@ Module(
 
 
   |
+1 | from x import (a, b
+2 | 1 + 1
 3 | from x import (a, b,
+  |                     ^ Syntax Error: Expected ')', found newline
 4 | 2 + 2
   |

crates/ruff_python_parser/tests/snapshots/invalid_syntax@node_range_with_gaps.py.snap

@@ -103,7 +103,7 @@ Module(
   |
 1 | def foo # comment
 2 | def bar(): ...
-  | ^^^ Syntax Error: Expected ')', found 'def'
+  | ^^^ Syntax Error: Expected a parameter or the end of the parameter list
 3 | def baz
   |
 

crates/ruff_python_parser/tests/snapshots/invalid_syntax@with_items_parenthesized_missing_comma.py.snap

@@ -332,9 +332,3 @@ Module(
 5 | with (item1, item2: ...
   |                   ^ Syntax Error: Expected ',', found ':'
   |
-
-
-  |
-4 | with (item1, item2 as f1 item3, item4): ...
-5 | with (item1, item2: ...
-  |