From 9c1b6ec4119fb3814629cfcd72c5616f516f1720 Mon Sep 17 00:00:00 2001
From: Dhruv Manilawala <dhruvmanila@gmail.com>
Date: Tue, 25 Jun 2024 13:35:24 +0530
Subject: [PATCH] Use correct range to highlight line continuation error
 (#12016)

## Summary

This PR fixes the range highlighted for the line continuation error.

Previously, it would highlight an incorrect range:
```
1 | call(a, b, \\\
  |           ^^ Syntax Error: unexpected character after line continuation character
2 |
3 | def bar():
  |
```

And now:
```
  |
1 | call(a, b, \\\
  |             ^ Syntax Error: unexpected character after line continuation character
2 |
3 | def bar():
  |
```

This is implemented by avoiding to update the token range for the
`Unknown` token which is emitted when there's a lexical error. Instead,
the `push_error` helper method will be responsible to update the range
to the error location.

This actually becomes a requirement which can be seen in follow-up PRs.

## Test Plan

Update and validate the snapshot.
---
 crates/ruff_python_parser/src/lexer.rs        | 23 +++++++++++--------
 ...tax@re_lexing__line_continuation_1.py.snap | 10 ++++----
 2 files changed, 19 insertions(+), 14 deletions(-)

diff --git a/crates/ruff_python_parser/src/lexer.rs b/crates/ruff_python_parser/src/lexer.rs
index 47ca855eccec8..744e5212fbc95 100644
--- a/crates/ruff_python_parser/src/lexer.rs
+++ b/crates/ruff_python_parser/src/lexer.rs
@@ -133,13 +133,24 @@ impl<'src> Lexer<'src> {
         std::mem::take(&mut self.current_value)
     }
 
+    /// Helper function to push the given error, updating the current range with the error location
+    /// and return the [`TokenKind::Unknown`] token.
+    fn push_error(&mut self, error: LexicalError) -> TokenKind {
+        self.current_range = error.location();
+        self.errors.push(error);
+        TokenKind::Unknown
+    }
+
     /// Lex the next token.
     pub fn next_token(&mut self) -> TokenKind {
         self.cursor.start_token();
         self.current_value = TokenValue::None;
         self.current_flags = TokenFlags::empty();
         self.current_kind = self.lex_token();
-        self.current_range = self.token_range();
+        // For `Unknown` token, the `push_error` method updates the current range.
+        if !matches!(self.current_kind, TokenKind::Unknown) {
+            self.current_range = self.token_range();
+        }
         self.current_kind
     }
 
@@ -236,7 +247,7 @@ impl<'src> Lexer<'src> {
                     } else if !self.cursor.eat_char('\n') {
                         return Some(self.push_error(LexicalError::new(
                             LexicalErrorType::LineContinuationError,
-                            self.token_range(),
+                            TextRange::at(self.offset(), self.cursor.first().text_len()),
                         )));
                     }
                     indentation = Indentation::root();
@@ -328,7 +339,7 @@ impl<'src> Lexer<'src> {
                     } else if !self.cursor.eat_char('\n') {
                         return Err(LexicalError::new(
                             LexicalErrorType::LineContinuationError,
-                            self.token_range(),
+                            TextRange::at(self.offset(), self.cursor.first().text_len()),
                         ));
                     }
                 }
@@ -1464,12 +1475,6 @@ impl<'src> Lexer<'src> {
         self.token_range().start()
     }
 
-    /// Helper function to push the given error and return the [`TokenKind::Unknown`] token.
-    fn push_error(&mut self, error: LexicalError) -> TokenKind {
-        self.errors.push(error);
-        TokenKind::Unknown
-    }
-
     /// Creates a checkpoint to which the lexer can later return to using [`Self::rewind`].
     pub(crate) fn checkpoint(&self) -> LexerCheckpoint {
         LexerCheckpoint {
diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@re_lexing__line_continuation_1.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@re_lexing__line_continuation_1.py.snap
index c00e557392626..b544d9158d39c 100644
--- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@re_lexing__line_continuation_1.py.snap
+++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@re_lexing__line_continuation_1.py.snap
@@ -11,10 +11,10 @@ Module(
         body: [
             Expr(
                 StmtExpr {
-                    range: 0..13,
+                    range: 0..14,
                     value: Call(
                         ExprCall {
-                            range: 0..13,
+                            range: 0..14,
                             func: Name(
                                 ExprName {
                                     range: 0..4,
@@ -23,7 +23,7 @@ Module(
                                 },
                             ),
                             arguments: Arguments {
-                                range: 4..13,
+                                range: 4..14,
                                 args: [
                                     Name(
                                         ExprName {
@@ -82,7 +82,7 @@ Module(
 
   |
 1 | call(a, b, \\\
-  |           ^^ Syntax Error: unexpected character after line continuation character
+  |             ^ Syntax Error: unexpected character after line continuation character
 2 | 
 3 | def bar():
   |
@@ -90,7 +90,7 @@ Module(
 
   |
 1 | call(a, b, \\\
-  |             ^ Syntax Error: unexpected character after line continuation character
+  |              ^ Syntax Error: unexpected character after line continuation character
 2 | 
 3 | def bar():
   |