Skip to content

Commit

Permalink
Avoid moving back the lexer for triple-quoted fstring (#11939)
Browse files Browse the repository at this point in the history 
## Summary

This PR avoids moving back the lexer for a triple-quoted f-string during
the re-lexing phase.

The reason this is a problem is that for a triple-quoted f-string the
newlines are part of the f-string itself, specifically they'll be part
of the `FStringMiddle` token. So, if we moved the lexer back, there
would be a `Newline` token whose range would be in between an
`FStringMiddle` token. This creates a panic in downstream usage.

fixes: #11937 

## Test Plan

Add test cases and validate the snapshots.
  • Loading branch information
@dhruvmanila
dhruvmanila authored Jun 20, 2024
1 parent 22733cb commit ed948ea
Show file tree
Hide file tree
Showing 10 changed files with 377 additions and 156 deletions.
8 changes: 1 addition & 7 deletions crates/ruff_python_parser/resources/invalid/re_lex_logical_token.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,10 +54,4 @@ def bar():

if call(f"hello
def bar():
pass


# There are trailing whitespace before the newline character but those whitespaces are
# part of the comment token
f"""hello {x # comment
y = 1
pass
6 changes: 6 additions & 0 deletions crates/ruff_python_parser/resources/invalid/re_lexing/triple_quoted_fstring_1.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# There are trailing whitespace before the newline character but those whitespaces are
# part of the comment token.
# https://github.com/astral-sh/ruff/issues/11929

f"""hello {x # comment
y = 1
6 changes: 6 additions & 0 deletions crates/ruff_python_parser/resources/invalid/re_lexing/triple_quoted_fstring_2.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# The lexer can't be moved back for a triple-quoted f-string because the newlines are
# part of the f-string itself.
# https://github.com/astral-sh/ruff/issues/11937

f'''{foo:.3f
'''
7 changes: 7 additions & 0 deletions crates/ruff_python_parser/resources/invalid/re_lexing/triple_quoted_fstring_3.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
# Here, the nesting level is 2 when the parser is trying to recover from an unclosed `{`
# This test demonstrates that we need to reduce the nesting level when recovering from
# within an f-string but the lexer shouldn't go back.

if call(f'''{x:.3f
'''
pass
11 changes: 11 additions & 0 deletions crates/ruff_python_parser/src/lexer.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1370,6 +1370,12 @@ impl<'src> Lexer<'src> {
// i.e., it recovered from an unclosed parenthesis (`(`, `[`, or `{`).
self.nesting -= 1;

// The lexer can't be moved back for a triple-quoted f-string because the newlines are
// part of the f-string itself, so there is no newline token to be emitted.
if self.current_flags.is_triple_quoted_fstring() {
return false;
}

let mut current_position = self.current_range().start();
let reverse_chars = self.source[..current_position.to_usize()].chars().rev();
let mut newline_position = None;
Expand Down Expand Up @@ -1578,6 +1584,11 @@ impl TokenFlags {
self.intersects(TokenFlags::F_STRING)
}

/// Returns `true` if the token is a triple-quoted f-string.
fn is_triple_quoted_fstring(self) -> bool {
self.contains(TokenFlags::F_STRING | TokenFlags::TRIPLE_QUOTED_STRING)
}

/// Returns `true` if the token is a raw string.
const fn is_raw_string(self) -> bool {
self.intersects(TokenFlags::RAW_STRING)
Expand Down
120 changes: 60 additions & 60 deletions crates/ruff_python_parser/tests/snapshots/invalid_syntax@f_string_unclosed_lbrace.py.snap
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,64 +139,74 @@ Module(
),
Expr(
StmtExpr {
range: 24..29,
range: 24..37,
value: FString(
ExprFString {
range: 24..29,
range: 24..37,
value: FStringValue {
inner: Single(
FString(
FString {
range: 24..29,
elements: [
Expression(
FStringExpressionElement {
range: 26..27,
expression: Name(
ExprName {
range: 27..27,
id: "",
ctx: Invalid,
},
),
debug_text: None,
conversion: None,
format_spec: None,
},
),
],
flags: FStringFlags {
quote_style: Double,
prefix: Regular,
triple_quoted: false,
inner: Concatenated(
[
FString(
FString {
range: 24..29,
elements: [
Expression(
FStringExpressionElement {
range: 26..27,
expression: Name(
ExprName {
range: 27..27,
id: "",
ctx: Invalid,
},
),
debug_text: None,
conversion: None,
format_spec: None,
},
),
],
flags: FStringFlags {
quote_style: Double,
prefix: Regular,
triple_quoted: false,
},
},
},
),
),
FString(
FString {
range: 29..37,
elements: [
Expression(
FStringExpressionElement {
range: 33..34,
expression: Name(
ExprName {
range: 34..34,
id: "",
ctx: Invalid,
},
),
debug_text: None,
conversion: None,
format_spec: None,
},
),
],
flags: FStringFlags {
quote_style: Double,
prefix: Regular,
triple_quoted: true,
},
},
),
],
),
},
},
),
},
),
Expr(
StmtExpr {
range: 33..38,
value: Set(
ExprSet {
range: 33..38,
elts: [
Name(
ExprName {
range: 34..34,
id: "",
ctx: Invalid,
},
),
],
},
),
},
),
],
},
)
Expand Down Expand Up @@ -318,29 +328,19 @@ Module(
|


|
2 | f"{foo!r"
3 | f"{foo="
4 | f"{"
| _____^
5 | | f"""{"""
| |_^ Syntax Error: Expected FStringEnd, found FStringMiddle
|


|
3 | f"{foo="
4 | f"{"
5 | f"""{"""
| ^^^ Syntax Error: Expected a statement
| ^^^^ Syntax Error: Expected FStringEnd, found FStringStart
|


|
3 | f"{foo="
4 | f"{"
5 | f"""{"""
|______^
| ^^^ Syntax Error: Expected an expression
|


Expand Down
94 changes: 5 additions & 89 deletions crates/ruff_python_parser/tests/snapshots/invalid_syntax@re_lex_logical_token.py.snap
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ input_file: crates/ruff_python_parser/resources/invalid/re_lex_logical_token.py
```
Module(
ModModule {
range: 0..1129,
range: 0..979,
body: [
If(
StmtIf {
Expand Down Expand Up @@ -670,53 +670,6 @@ Module(
],
},
),
Expr(
StmtExpr {
range: 1097..1109,
value: FString(
ExprFString {
range: 1097..1109,
value: FStringValue {
inner: Single(
FString(
FString {
range: 1097..1109,
elements: [
Literal(
FStringLiteralElement {
range: 1101..1107,
value: "hello ",
},
),
Expression(
FStringExpressionElement {
range: 1107..1109,
expression: Name(
ExprName {
range: 1108..1109,
id: "x",
ctx: Load,
},
),
debug_text: None,
conversion: None,
format_spec: None,
},
),
],
flags: FStringFlags {
quote_style: Double,
prefix: Regular,
triple_quoted: true,
},
},
),
),
},
},
),
},
),
],
},
)
Expand Down Expand Up @@ -878,45 +831,8 @@ Module(


|
60 | # There are trailing whitespace before the newline character but those whitespaces are
61 | # part of the comment token
62 | f"""hello {x # comment
| Syntax Error: Expected a statement
63 | y = 1
|


|
60 | # There are trailing whitespace before the newline character but those whitespaces are
61 | # part of the comment token
62 | f"""hello {x # comment
| ___________________________^
63 | | y = 1
| |_____^ Syntax Error: f-string: unterminated triple-quoted string
|


|
61 | # part of the comment token
62 | f"""hello {x # comment
63 | y = 1
| ^ Syntax Error: f-string: expecting '}'
|


|
60 | # There are trailing whitespace before the newline character but those whitespaces are
61 | # part of the comment token
62 | f"""hello {x # comment
| ___________________________^
63 | | y = 1
| |_____^ Syntax Error: Expected FStringEnd, found Unknown
|


|
61 | # part of the comment token
62 | f"""hello {x # comment
63 | y = 1
| Syntax Error: Expected a statement
55 | if call(f"hello
56 | def bar():
57 | pass
| Syntax Error: Expected a statement
|
Loading

0 comments on commit ed948ea

Please sign in to comment.