From a525b4be3d55130467805b8e5873d9bc77e5b8b9 Mon Sep 17 00:00:00 2001 From: Dhruv Manilawala Date: Wed, 12 Jun 2024 13:57:35 +0530 Subject: [PATCH] Separate terminator token for f-string elements kind (#11842) ## Summary This PR separates the terminator token for f-string elements depending on the context. A list of f-string element can occur either in a regular f-string or a format spec of an f-string. The terminator token is different depending on that context. ## Test Plan `cargo insta test` and verify the updated snapshots. --- .../src/parser/expression.rs | 14 ++-- crates/ruff_python_parser/src/parser/mod.rs | 77 ++++++++++++++----- ..._string_lambda_without_parentheses.py.snap | 20 +---- ...ing_unclosed_lbrace_in_format_spec.py.snap | 4 +- ...erminated_fstring_newline_recovery.py.snap | 2 +- 5 files changed, 73 insertions(+), 44 deletions(-) diff --git a/crates/ruff_python_parser/src/parser/expression.rs b/crates/ruff_python_parser/src/parser/expression.rs index 38ea35c923e26a..3ca0a44741bd88 100644 --- a/crates/ruff_python_parser/src/parser/expression.rs +++ b/crates/ruff_python_parser/src/parser/expression.rs @@ -18,7 +18,7 @@ use crate::string::{parse_fstring_literal_element, parse_string_literal, StringT use crate::token_set::TokenSet; use crate::{FStringErrorType, Mode, ParseErrorType, TokenKind}; -use super::{Parenthesized, RecoveryContextKind}; +use super::{FStringElementsKind, Parenthesized, RecoveryContextKind}; /// A token set consisting of a newline or end of file. const NEWLINE_EOF_SET: TokenSet = TokenSet::new([TokenKind::Newline, TokenKind::EndOfFile]); @@ -1307,7 +1307,7 @@ impl<'src> Parser<'src> { let flags = self.tokens.current_flags().as_any_string_flags(); self.bump(TokenKind::FStringStart); - let elements = self.parse_fstring_elements(flags); + let elements = self.parse_fstring_elements(flags, FStringElementsKind::Regular); self.expect(TokenKind::FStringEnd); @@ -1323,10 +1323,14 @@ impl<'src> Parser<'src> { /// # Panics /// /// If the parser isn't positioned at a `{` or `FStringMiddle` token. - fn parse_fstring_elements(&mut self, flags: ast::AnyStringFlags) -> FStringElements { + fn parse_fstring_elements( + &mut self, + flags: ast::AnyStringFlags, + kind: FStringElementsKind, + ) -> FStringElements { let mut elements = vec![]; - self.parse_list(RecoveryContextKind::FStringElements, |parser| { + self.parse_list(RecoveryContextKind::FStringElements(kind), |parser| { let element = match parser.current_token_kind() { TokenKind::Lbrace => { FStringElement::Expression(parser.parse_fstring_expression_element(flags)) @@ -1463,7 +1467,7 @@ impl<'src> Parser<'src> { let format_spec = if self.eat(TokenKind::Colon) { let spec_start = self.node_start(); - let elements = self.parse_fstring_elements(flags); + let elements = self.parse_fstring_elements(flags, FStringElementsKind::FormatSpec); Some(Box::new(ast::FStringFormatSpec { range: self.node_range(spec_start), elements, diff --git a/crates/ruff_python_parser/src/parser/mod.rs b/crates/ruff_python_parser/src/parser/mod.rs index 8430407af2ed51..087b1790d8f3a2 100644 --- a/crates/ruff_python_parser/src/parser/mod.rs +++ b/crates/ruff_python_parser/src/parser/mod.rs @@ -722,6 +722,37 @@ impl WithItemKind { } } +#[derive(Debug, PartialEq, Copy, Clone)] +enum FStringElementsKind { + /// The regular f-string elements. + /// + /// For example, the `"hello "`, `x`, and `" world"` elements in: + /// ```py + /// f"hello {x:.2f} world" + /// ``` + Regular, + + /// The f-string elements are part of the format specifier. + /// + /// For example, the `.2f` in: + /// ```py + /// f"hello {x:.2f} world" + /// ``` + FormatSpec, +} + +impl FStringElementsKind { + const fn list_terminator(self) -> TokenKind { + match self { + FStringElementsKind::Regular => TokenKind::FStringEnd, + // test_ok fstring_format_spec_terminator + // f"hello {x:} world" + // f"hello {x:.3f} world" + FStringElementsKind::FormatSpec => TokenKind::Rbrace, + } + } +} + #[derive(Debug, PartialEq, Copy, Clone)] enum Parenthesized { /// The elements are parenthesized, e.g., `(a, b)`. @@ -819,7 +850,7 @@ enum RecoveryContextKind { /// When parsing a list of f-string elements which are either literal elements /// or expressions. - FStringElements, + FStringElements(FStringElementsKind), } impl RecoveryContextKind { @@ -954,10 +985,8 @@ impl RecoveryContextKind { p.at(TokenKind::Colon) } }, - RecoveryContextKind::FStringElements => { - // Tokens other than `FStringEnd` and `}` are for better error recovery - p.at_ts(TokenSet::new([ - TokenKind::FStringEnd, + RecoveryContextKind::FStringElements(kind) => { + p.at(kind.list_terminator()) // test_err unterminated_fstring_newline_recovery // f"hello // 1 + 1 @@ -967,15 +996,7 @@ impl RecoveryContextKind { // 3 + 3 // f"hello {x} // 4 + 4 - TokenKind::Newline, - // When the parser is parsing f-string elements inside format spec, - // the terminator would be `}`. - - // test_ok fstring_format_spec_terminator - // f"hello {x:} world" - // f"hello {x:.3f} world" - TokenKind::Rbrace, - ])) + || p.at(TokenKind::Newline) } } } @@ -1017,7 +1038,7 @@ impl RecoveryContextKind { ) || p.at_name_or_soft_keyword() } RecoveryContextKind::WithItems(_) => p.at_expr(), - RecoveryContextKind::FStringElements => matches!( + RecoveryContextKind::FStringElements(_) => matches!( p.current_token_kind(), // Literal element TokenKind::FStringMiddle @@ -1111,9 +1132,14 @@ impl RecoveryContextKind { "Expected an expression or the end of the with item list".to_string(), ), }, - RecoveryContextKind::FStringElements => ParseErrorType::OtherError( - "Expected an f-string element or the end of the f-string".to_string(), - ), + RecoveryContextKind::FStringElements(kind) => match kind { + FStringElementsKind::Regular => ParseErrorType::OtherError( + "Expected an f-string element or the end of the f-string".to_string(), + ), + FStringElementsKind::FormatSpec => { + ParseErrorType::OtherError("Expected an f-string element or a '}'".to_string()) + } + }, } } } @@ -1152,6 +1178,7 @@ bitflags! { const WITH_ITEMS_PARENTHESIZED_EXPRESSION = 1 << 26; const WITH_ITEMS_UNPARENTHESIZED = 1 << 28; const F_STRING_ELEMENTS = 1 << 29; + const F_STRING_ELEMENTS_IN_FORMAT_SPEC = 1 << 30; } } @@ -1204,7 +1231,12 @@ impl RecoveryContext { } WithItemKind::Unparenthesized => RecoveryContext::WITH_ITEMS_UNPARENTHESIZED, }, - RecoveryContextKind::FStringElements => RecoveryContext::F_STRING_ELEMENTS, + RecoveryContextKind::FStringElements(kind) => match kind { + FStringElementsKind::Regular => RecoveryContext::F_STRING_ELEMENTS, + FStringElementsKind::FormatSpec => { + RecoveryContext::F_STRING_ELEMENTS_IN_FORMAT_SPEC + } + }, } } @@ -1271,7 +1303,12 @@ impl RecoveryContext { RecoveryContext::WITH_ITEMS_UNPARENTHESIZED => { RecoveryContextKind::WithItems(WithItemKind::Unparenthesized) } - RecoveryContext::F_STRING_ELEMENTS => RecoveryContextKind::FStringElements, + RecoveryContext::F_STRING_ELEMENTS => { + RecoveryContextKind::FStringElements(FStringElementsKind::Regular) + } + RecoveryContext::F_STRING_ELEMENTS_IN_FORMAT_SPEC => { + RecoveryContextKind::FStringElements(FStringElementsKind::FormatSpec) + } _ => return None, }) } diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@f_string_lambda_without_parentheses.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@f_string_lambda_without_parentheses.py.snap index 90b9de228fac6d..669ca2dd2ba07d 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@f_string_lambda_without_parentheses.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@f_string_lambda_without_parentheses.py.snap @@ -11,15 +11,15 @@ Module( body: [ Expr( StmtExpr { - range: 0..14, + range: 0..16, value: FString( ExprFString { - range: 0..14, + range: 0..16, value: FStringValue { inner: Single( FString( FString { - range: 0..14, + range: 0..16, elements: [ Expression( FStringExpressionElement { @@ -110,17 +110,5 @@ Module( | 1 | f"{lambda x: x}" - | ^ Syntax Error: Expected FStringEnd, found '}' - | - - - | -1 | f"{lambda x: x}" - | ^ Syntax Error: Expected a statement - | - - - | -1 | f"{lambda x: x}" - | ^ Syntax Error: Expected a statement + | ^ Syntax Error: Expected an f-string element or the end of the f-string | diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@f_string_unclosed_lbrace_in_format_spec.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@f_string_unclosed_lbrace_in_format_spec.py.snap index d7827da843fc47..5060881366ed89 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@f_string_unclosed_lbrace_in_format_spec.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@f_string_unclosed_lbrace_in_format_spec.py.snap @@ -116,7 +116,7 @@ Module( | 1 | f"hello {x:" - | ^ Syntax Error: f-string: expecting '}' + | ^ Syntax Error: Expected an f-string element or a '}' 2 | f"hello {x:.3f" | @@ -124,7 +124,7 @@ Module( | 1 | f"hello {x:" 2 | f"hello {x:.3f" - | ^ Syntax Error: f-string: expecting '}' + | ^ Syntax Error: Expected an f-string element or a '}' | diff --git a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@unterminated_fstring_newline_recovery.py.snap b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@unterminated_fstring_newline_recovery.py.snap index 2807538b954af0..668d3b11464699 100644 --- a/crates/ruff_python_parser/tests/snapshots/invalid_syntax@unterminated_fstring_newline_recovery.py.snap +++ b/crates/ruff_python_parser/tests/snapshots/invalid_syntax@unterminated_fstring_newline_recovery.py.snap @@ -335,7 +335,7 @@ Module( 4 | 2 + 2 5 | f"hello {x: 6 | 3 + 3 - | ^ Syntax Error: Expected an f-string element or the end of the f-string + | ^ Syntax Error: Expected an f-string element or a '}' 7 | f"hello {x} 8 | 4 + 4 |