Skip to content

Commit

Permalink
Separate terminator token for f-string elements kind (#11842)
Browse files Browse the repository at this point in the history 
## Summary

This PR separates the terminator token for f-string elements depending
on the context. A list of f-string element can occur either in a regular
f-string or a format spec of an f-string. The terminator token is
different depending on that context.

## Test Plan

`cargo insta test` and verify the updated snapshots.
  • Loading branch information
@dhruvmanila
dhruvmanila authored Jun 12, 2024
1 parent 93973b9 commit a525b4b
Show file tree
Hide file tree
Showing 5 changed files with 73 additions and 44 deletions.
14 changes: 9 additions & 5 deletions crates/ruff_python_parser/src/parser/expression.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ use crate::string::{parse_fstring_literal_element, parse_string_literal, StringT
use crate::token_set::TokenSet;
use crate::{FStringErrorType, Mode, ParseErrorType, TokenKind};

use super::{Parenthesized, RecoveryContextKind};
use super::{FStringElementsKind, Parenthesized, RecoveryContextKind};

/// A token set consisting of a newline or end of file.
const NEWLINE_EOF_SET: TokenSet = TokenSet::new([TokenKind::Newline, TokenKind::EndOfFile]);
Expand Down Expand Up @@ -1307,7 +1307,7 @@ impl<'src> Parser<'src> {
let flags = self.tokens.current_flags().as_any_string_flags();

self.bump(TokenKind::FStringStart);
let elements = self.parse_fstring_elements(flags);
let elements = self.parse_fstring_elements(flags, FStringElementsKind::Regular);

self.expect(TokenKind::FStringEnd);

Expand All @@ -1323,10 +1323,14 @@ impl<'src> Parser<'src> {
/// # Panics
///
/// If the parser isn't positioned at a `{` or `FStringMiddle` token.
fn parse_fstring_elements(&mut self, flags: ast::AnyStringFlags) -> FStringElements {
fn parse_fstring_elements(
&mut self,
flags: ast::AnyStringFlags,
kind: FStringElementsKind,
) -> FStringElements {
let mut elements = vec![];

self.parse_list(RecoveryContextKind::FStringElements, |parser| {
self.parse_list(RecoveryContextKind::FStringElements(kind), |parser| {
let element = match parser.current_token_kind() {
TokenKind::Lbrace => {
FStringElement::Expression(parser.parse_fstring_expression_element(flags))
Expand Down Expand Up @@ -1463,7 +1467,7 @@ impl<'src> Parser<'src> {

let format_spec = if self.eat(TokenKind::Colon) {
let spec_start = self.node_start();
let elements = self.parse_fstring_elements(flags);
let elements = self.parse_fstring_elements(flags, FStringElementsKind::FormatSpec);
Some(Box::new(ast::FStringFormatSpec {
range: self.node_range(spec_start),
elements,
Expand Down
77 changes: 57 additions & 20 deletions crates/ruff_python_parser/src/parser/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -722,6 +722,37 @@ impl WithItemKind {
}
}

#[derive(Debug, PartialEq, Copy, Clone)]
enum FStringElementsKind {
/// The regular f-string elements.
///
/// For example, the `"hello "`, `x`, and `" world"` elements in:
/// ```py
/// f"hello {x:.2f} world"
/// ```
Regular,

/// The f-string elements are part of the format specifier.
///
/// For example, the `.2f` in:
/// ```py
/// f"hello {x:.2f} world"
/// ```
FormatSpec,
}

impl FStringElementsKind {
const fn list_terminator(self) -> TokenKind {
match self {
FStringElementsKind::Regular => TokenKind::FStringEnd,
// test_ok fstring_format_spec_terminator
// f"hello {x:} world"
// f"hello {x:.3f} world"
FStringElementsKind::FormatSpec => TokenKind::Rbrace,
}
}
}

#[derive(Debug, PartialEq, Copy, Clone)]
enum Parenthesized {
/// The elements are parenthesized, e.g., `(a, b)`.
Expand Down Expand Up @@ -819,7 +850,7 @@ enum RecoveryContextKind {

/// When parsing a list of f-string elements which are either literal elements
/// or expressions.
FStringElements,
FStringElements(FStringElementsKind),
}

impl RecoveryContextKind {
Expand Down Expand Up @@ -954,10 +985,8 @@ impl RecoveryContextKind {
p.at(TokenKind::Colon)
}
},
RecoveryContextKind::FStringElements => {
// Tokens other than `FStringEnd` and `}` are for better error recovery
p.at_ts(TokenSet::new([
TokenKind::FStringEnd,
RecoveryContextKind::FStringElements(kind) => {
p.at(kind.list_terminator())
// test_err unterminated_fstring_newline_recovery
// f"hello
// 1 + 1
Expand All @@ -967,15 +996,7 @@ impl RecoveryContextKind {
// 3 + 3
// f"hello {x}
// 4 + 4
TokenKind::Newline,
// When the parser is parsing f-string elements inside format spec,
// the terminator would be `}`.

// test_ok fstring_format_spec_terminator
// f"hello {x:} world"
// f"hello {x:.3f} world"
TokenKind::Rbrace,
]))
|| p.at(TokenKind::Newline)
}
}
}
Expand Down Expand Up @@ -1017,7 +1038,7 @@ impl RecoveryContextKind {
) || p.at_name_or_soft_keyword()
}
RecoveryContextKind::WithItems(_) => p.at_expr(),
RecoveryContextKind::FStringElements => matches!(
RecoveryContextKind::FStringElements(_) => matches!(
p.current_token_kind(),
// Literal element
TokenKind::FStringMiddle
Expand Down Expand Up @@ -1111,9 +1132,14 @@ impl RecoveryContextKind {
"Expected an expression or the end of the with item list".to_string(),
),
},
RecoveryContextKind::FStringElements => ParseErrorType::OtherError(
"Expected an f-string element or the end of the f-string".to_string(),
),
RecoveryContextKind::FStringElements(kind) => match kind {
FStringElementsKind::Regular => ParseErrorType::OtherError(
"Expected an f-string element or the end of the f-string".to_string(),
),
FStringElementsKind::FormatSpec => {
ParseErrorType::OtherError("Expected an f-string element or a '}'".to_string())
}
},
}
}
}
Expand Down Expand Up @@ -1152,6 +1178,7 @@ bitflags! {
const WITH_ITEMS_PARENTHESIZED_EXPRESSION = 1 << 26;
const WITH_ITEMS_UNPARENTHESIZED = 1 << 28;
const F_STRING_ELEMENTS = 1 << 29;
const F_STRING_ELEMENTS_IN_FORMAT_SPEC = 1 << 30;
}
}

Expand Down Expand Up @@ -1204,7 +1231,12 @@ impl RecoveryContext {
}
WithItemKind::Unparenthesized => RecoveryContext::WITH_ITEMS_UNPARENTHESIZED,
},
RecoveryContextKind::FStringElements => RecoveryContext::F_STRING_ELEMENTS,
RecoveryContextKind::FStringElements(kind) => match kind {
FStringElementsKind::Regular => RecoveryContext::F_STRING_ELEMENTS,
FStringElementsKind::FormatSpec => {
RecoveryContext::F_STRING_ELEMENTS_IN_FORMAT_SPEC
}
},
}
}

Expand Down Expand Up @@ -1271,7 +1303,12 @@ impl RecoveryContext {
RecoveryContext::WITH_ITEMS_UNPARENTHESIZED => {
RecoveryContextKind::WithItems(WithItemKind::Unparenthesized)
}
RecoveryContext::F_STRING_ELEMENTS => RecoveryContextKind::FStringElements,
RecoveryContext::F_STRING_ELEMENTS => {
RecoveryContextKind::FStringElements(FStringElementsKind::Regular)
}
RecoveryContext::F_STRING_ELEMENTS_IN_FORMAT_SPEC => {
RecoveryContextKind::FStringElements(FStringElementsKind::FormatSpec)
}
_ => return None,
})
}
Expand Down
20 changes: 4 additions & 16 deletions ..._python_parser/tests/snapshots/invalid_syntax@f_string_lambda_without_parentheses.py.snap
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,15 @@ Module(
body: [
Expr(
StmtExpr {
range: 0..14,
range: 0..16,
value: FString(
ExprFString {
range: 0..14,
range: 0..16,
value: FStringValue {
inner: Single(
FString(
FString {
range: 0..14,
range: 0..16,
elements: [
Expression(
FStringExpressionElement {
Expand Down Expand Up @@ -110,17 +110,5 @@ Module(

|
1 | f"{lambda x: x}"
| ^ Syntax Error: Expected FStringEnd, found '}'
|


|
1 | f"{lambda x: x}"
| ^ Syntax Error: Expected a statement
|


|
1 | f"{lambda x: x}"
| ^ Syntax Error: Expected a statement
| ^ Syntax Error: Expected an f-string element or the end of the f-string
|
4 changes: 2 additions & 2 deletions ...hon_parser/tests/snapshots/invalid_syntax@f_string_unclosed_lbrace_in_format_spec.py.snap
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,15 +116,15 @@ Module(

|
1 | f"hello {x:"
| ^ Syntax Error: f-string: expecting '}'
| ^ Syntax Error: Expected an f-string element or a '}'
2 | f"hello {x:.3f"
|


|
1 | f"hello {x:"
2 | f"hello {x:.3f"
| ^ Syntax Error: f-string: expecting '}'
| ^ Syntax Error: Expected an f-string element or a '}'
|


Expand Down
2 changes: 1 addition & 1 deletion ...ython_parser/tests/snapshots/invalid_syntax@unterminated_fstring_newline_recovery.py.snap
View file
Original file line number Diff line number Diff line change
Expand Up @@ -335,7 +335,7 @@ Module(
4 | 2 + 2
5 | f"hello {x:
6 | 3 + 3
| ^ Syntax Error: Expected an f-string element or the end of the f-string
| ^ Syntax Error: Expected an f-string element or a '}'
7 | f"hello {x}
8 | 4 + 4
|
Expand Down

0 comments on commit a525b4b

Please sign in to comment.