diff --git a/crates/ruff_python_parser/src/lexer.rs b/crates/ruff_python_parser/src/lexer.rs index 47ca855eccec8..744e5212fbc95 100644 --- a/crates/ruff_python_parser/src/lexer.rs +++ b/crates/ruff_python_parser/src/lexer.rs @@ -133,13 +133,24 @@ impl<'src> Lexer<'src> { std::mem::take(&mut self.current_value) } + /// Helper function to push the given error, updating the current range with the error location + /// and return the [`TokenKind::Unknown`] token. + fn push_error(&mut self, error: LexicalError) -> TokenKind { + self.current_range = error.location(); + self.errors.push(error); + TokenKind::Unknown + } + /// Lex the next token. pub fn next_token(&mut self) -> TokenKind { self.cursor.start_token(); self.current_value = TokenValue::None; self.current_flags = TokenFlags::empty(); self.current_kind = self.lex_token(); - self.current_range = self.token_range(); + // For `Unknown` token, the `push_error` method updates the current range. + if !matches!(self.current_kind, TokenKind::Unknown) { + self.current_range = self.token_range(); + } self.current_kind } @@ -236,7 +247,7 @@ impl<'src> Lexer<'src> { } else if !self.cursor.eat_char('\n') { return Some(self.push_error(LexicalError::new( LexicalErrorType::LineContinuationError, - self.token_range(), + TextRange::at(self.offset(), self.cursor.first().text_len()), ))); } indentation = Indentation::root(); @@ -328,7 +339,7 @@ impl<'src> Lexer<'src> { } else if !self.cursor.eat_char('\n') { return Err(LexicalError::new( LexicalErrorType::LineContinuationError, - self.token_range(), + TextRange::at(self.offset(), self.cursor.first().text_len()), )); } } @@ -1464,12 +1475,6 @@ impl<'src> Lexer<'src> { self.token_range().start() } - /// Helper function to push the given error and return the [`TokenKind::Unknown`] token. - fn push_error(&mut self, error: LexicalError) -> TokenKind { - self.errors.push(error); - TokenKind::Unknown - } - /// Creates a checkpoint to which the lexer can later return to using [`Self::rewind`]. pub(crate) fn checkpoint(&self) -> LexerCheckpoint { LexerCheckpoint { diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@re_lexing__line_continuation_1.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@re_lexing__line_continuation_1.py.snap index c00e557392626..b544d9158d39c 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@re_lexing__line_continuation_1.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@re_lexing__line_continuation_1.py.snap @@ -11,10 +11,10 @@ Module( body: [ Expr( StmtExpr { - range: 0..13, + range: 0..14, value: Call( ExprCall { - range: 0..13, + range: 0..14, func: Name( ExprName { range: 0..4, @@ -23,7 +23,7 @@ Module( }, ), arguments: Arguments { - range: 4..13, + range: 4..14, args: [ Name( ExprName { @@ -82,7 +82,7 @@ Module( | 1 | call(a, b, \\\ - | ^^ Syntax Error: unexpected character after line continuation character + | ^ Syntax Error: unexpected character after line continuation character 2 | 3 | def bar(): | @@ -90,7 +90,7 @@ Module( | 1 | call(a, b, \\\ - | ^ Syntax Error: unexpected character after line continuation character + | ^ Syntax Error: unexpected character after line continuation character 2 | 3 | def bar(): |