From 4c7142ba65d168e00370f157a99448f9201763c6 Mon Sep 17 00:00:00 2001 From: Micha Reiser Date: Wed, 7 Feb 2024 17:12:58 -0500 Subject: [PATCH] Reduce `Tok` size by using `Box` instead of `String` --- .../flake8_pytest_style/rules/parametrize.rs | 23 ++- .../rules/flake8_simplify/rules/ast_expr.rs | 2 +- .../flynt/rules/static_join_to_fstring.rs | 3 +- .../rules/invalid_escape_sequence.rs | 2 +- .../pylint/rules/unspecified_encoding.rs | 2 +- .../ruff_linter/src/rules/pyupgrade/fixes.rs | 2 +- .../src/rules/ruff/rules/sequence_sorting.rs | 16 +- crates/ruff_python_ast/src/comparable.rs | 12 +- crates/ruff_python_ast/src/nodes.rs | 16 +- .../ruff_python_formatter/tests/normalizer.rs | 12 +- crates/ruff_python_parser/src/function.rs | 28 +-- crates/ruff_python_parser/src/lexer.rs | 92 +++++---- crates/ruff_python_parser/src/python.lalrpop | 36 ++-- crates/ruff_python_parser/src/python.rs | 192 +++++++++--------- ...exer__tests__invalid_leading_zero_big.snap | 14 +- ...er__tests__invalid_leading_zero_small.snap | 14 +- ...ser__lexer__tests__tet_too_low_dedent.snap | 10 +- .../ruff_python_parser/src/soft_keywords.rs | 2 +- crates/ruff_python_parser/src/string.rs | 18 +- crates/ruff_python_parser/src/token.rs | 12 +- 20 files changed, 260 insertions(+), 248 deletions(-) diff --git a/crates/ruff_linter/src/rules/flake8_pytest_style/rules/parametrize.rs b/crates/ruff_linter/src/rules/flake8_pytest_style/rules/parametrize.rs index d71eb361fce407..daf31d2e2b1f0d 100644 --- a/crates/ruff_linter/src/rules/flake8_pytest_style/rules/parametrize.rs +++ b/crates/ruff_linter/src/rules/flake8_pytest_style/rules/parametrize.rs @@ -257,15 +257,18 @@ fn elts_to_csv(elts: &[Expr], generator: Generator) -> Option { } let node = Expr::from(ast::StringLiteral { - value: elts.iter().fold(String::new(), |mut acc, elt| { - if let Expr::StringLiteral(ast::ExprStringLiteral { value, .. }) = elt { - if !acc.is_empty() { - acc.push(','); + value: elts + .iter() + .fold(String::new(), |mut acc, elt| { + if let Expr::StringLiteral(ast::ExprStringLiteral { value, .. }) = elt { + if !acc.is_empty() { + acc.push(','); + } + acc.push_str(value.to_str()); } - acc.push_str(value.to_str()); - } - acc - }), + acc + }) + .into_boxed_str(), ..ast::StringLiteral::default() }); Some(generator.expr(&node)) @@ -327,7 +330,7 @@ fn check_names(checker: &mut Checker, decorator: &Decorator, expr: &Expr) { .iter() .map(|name| { Expr::from(ast::StringLiteral { - value: (*name).to_string(), + value: (*name).to_string().into_boxed_str(), ..ast::StringLiteral::default() }) }) @@ -360,7 +363,7 @@ fn check_names(checker: &mut Checker, decorator: &Decorator, expr: &Expr) { .iter() .map(|name| { Expr::from(ast::StringLiteral { - value: (*name).to_string(), + value: (*name).to_string().into_boxed_str(), ..ast::StringLiteral::default() }) }) diff --git a/crates/ruff_linter/src/rules/flake8_simplify/rules/ast_expr.rs b/crates/ruff_linter/src/rules/flake8_simplify/rules/ast_expr.rs index 46d41465bb8c68..669be14149ccc5 100644 --- a/crates/ruff_linter/src/rules/flake8_simplify/rules/ast_expr.rs +++ b/crates/ruff_linter/src/rules/flake8_simplify/rules/ast_expr.rs @@ -217,7 +217,7 @@ fn check_os_environ_subscript(checker: &mut Checker, expr: &Expr) { slice.range(), ); let node = ast::StringLiteral { - value: capital_env_var, + value: capital_env_var.into_boxed_str(), unicode: env_var.is_unicode(), ..ast::StringLiteral::default() }; diff --git a/crates/ruff_linter/src/rules/flynt/rules/static_join_to_fstring.rs b/crates/ruff_linter/src/rules/flynt/rules/static_join_to_fstring.rs index 86c77bbb0ed733..bf0ca3d0565a18 100644 --- a/crates/ruff_linter/src/rules/flynt/rules/static_join_to_fstring.rs +++ b/crates/ruff_linter/src/rules/flynt/rules/static_join_to_fstring.rs @@ -72,7 +72,8 @@ fn build_fstring(joiner: &str, joinees: &[Expr]) -> Option { None } }) - .join(joiner), + .join(joiner) + .into_boxed_str(), ..ast::StringLiteral::default() }; return Some(node.into()); diff --git a/crates/ruff_linter/src/rules/pycodestyle/rules/invalid_escape_sequence.rs b/crates/ruff_linter/src/rules/pycodestyle/rules/invalid_escape_sequence.rs index c227c536c7b230..d00728e6fac76e 100644 --- a/crates/ruff_linter/src/rules/pycodestyle/rules/invalid_escape_sequence.rs +++ b/crates/ruff_linter/src/rules/pycodestyle/rules/invalid_escape_sequence.rs @@ -74,7 +74,7 @@ pub(crate) fn invalid_escape_sequence( let Some(range) = indexer.fstring_ranges().innermost(token_range.start()) else { return; }; - (value.as_str(), range.start()) + (value.as_ref(), range.start()) } Tok::String { kind, .. } => { if kind.is_raw() { diff --git a/crates/ruff_linter/src/rules/pylint/rules/unspecified_encoding.rs b/crates/ruff_linter/src/rules/pylint/rules/unspecified_encoding.rs index b6728df6924158..e1dd8284055d86 100644 --- a/crates/ruff_linter/src/rules/pylint/rules/unspecified_encoding.rs +++ b/crates/ruff_linter/src/rules/pylint/rules/unspecified_encoding.rs @@ -110,7 +110,7 @@ fn generate_keyword_fix(checker: &Checker, call: &ast::ExprCall) -> Fix { .generator() .expr(&Expr::StringLiteral(ast::ExprStringLiteral { value: ast::StringLiteralValue::single(ast::StringLiteral { - value: "locale".to_string(), + value: "locale".to_string().into_boxed_str(), unicode: false, range: TextRange::default(), }), diff --git a/crates/ruff_linter/src/rules/pyupgrade/fixes.rs b/crates/ruff_linter/src/rules/pyupgrade/fixes.rs index 59acb3f2ebfdbe..7ae9355e97ddea 100644 --- a/crates/ruff_linter/src/rules/pyupgrade/fixes.rs +++ b/crates/ruff_linter/src/rules/pyupgrade/fixes.rs @@ -21,7 +21,7 @@ pub(crate) fn remove_import_members(contents: &str, members: &[&str]) -> String let last_range = names.last_mut().unwrap(); *last_range = TextRange::new(last_range.start(), range.end()); } else { - if members.contains(&name.as_str()) { + if members.contains(&name.as_ref()) { removal_indices.push(names.len()); } names.push(range); diff --git a/crates/ruff_linter/src/rules/ruff/rules/sequence_sorting.rs b/crates/ruff_linter/src/rules/ruff/rules/sequence_sorting.rs index da082966d1d211..f75fecd730a889 100644 --- a/crates/ruff_linter/src/rules/ruff/rules/sequence_sorting.rs +++ b/crates/ruff_linter/src/rules/ruff/rules/sequence_sorting.rs @@ -559,14 +559,14 @@ fn collect_string_sequence_lines( /// `self` and produces the classification for the line. #[derive(Debug, Default)] struct LineState { - first_item_in_line: Option<(String, TextRange)>, - following_items_in_line: Vec<(String, TextRange)>, + first_item_in_line: Option<(Box, TextRange)>, + following_items_in_line: Vec<(Box, TextRange)>, comment_range_start: Option, comment_in_line: Option, } impl LineState { - fn visit_string_token(&mut self, token_value: String, token_range: TextRange) { + fn visit_string_token(&mut self, token_value: Box, token_range: TextRange) { if self.first_item_in_line.is_none() { self.first_item_in_line = Some((token_value, token_range)); } else { @@ -631,8 +631,8 @@ struct LineWithItems { // For elements in the list, we keep track of the value of the // value of the element as well as the source-code range of the element. // (We need to know the actual value so that we can sort the items.) - first_item: (String, TextRange), - following_items: Vec<(String, TextRange)>, + first_item: (Box, TextRange), + following_items: Vec<(Box, TextRange)>, // For comments, we only need to keep track of the source-code range. trailing_comment_range: Option, } @@ -753,7 +753,7 @@ fn collect_string_sequence_items( /// source-code range of `"a"`. #[derive(Debug)] struct StringSequenceItem { - value: String, + value: Box, preceding_comment_ranges: Vec, element_range: TextRange, // total_range incorporates the ranges of preceding comments @@ -766,7 +766,7 @@ struct StringSequenceItem { impl StringSequenceItem { fn new( - value: String, + value: Box, preceding_comment_ranges: Vec, element_range: TextRange, end_of_line_comments: Option, @@ -787,7 +787,7 @@ impl StringSequenceItem { } } - fn with_no_comments(value: String, element_range: TextRange) -> Self { + fn with_no_comments(value: Box, element_range: TextRange) -> Self { Self::new(value, vec![], element_range, None) } } diff --git a/crates/ruff_python_ast/src/comparable.rs b/crates/ruff_python_ast/src/comparable.rs index b3c7faf116a5cf..bc6327f01dca05 100644 --- a/crates/ruff_python_ast/src/comparable.rs +++ b/crates/ruff_python_ast/src/comparable.rs @@ -631,7 +631,7 @@ pub struct ComparableStringLiteral<'a> { impl<'a> From<&'a ast::StringLiteral> for ComparableStringLiteral<'a> { fn from(string_literal: &'a ast::StringLiteral) -> Self { Self { - value: string_literal.value.as_str(), + value: &string_literal.value, } } } @@ -1089,10 +1089,7 @@ impl<'a> From<&'a ast::Expr> for ComparableExpr<'a> { kind, value, range: _, - }) => Self::IpyEscapeCommand(ExprIpyEscapeCommand { - kind: *kind, - value: value.as_str(), - }), + }) => Self::IpyEscapeCommand(ExprIpyEscapeCommand { kind: *kind, value }), } } } @@ -1537,10 +1534,7 @@ impl<'a> From<&'a ast::Stmt> for ComparableStmt<'a> { kind, value, range: _, - }) => Self::IpyEscapeCommand(StmtIpyEscapeCommand { - kind: *kind, - value: value.as_str(), - }), + }) => Self::IpyEscapeCommand(StmtIpyEscapeCommand { kind: *kind, value }), ast::Stmt::Expr(ast::StmtExpr { value, range: _ }) => Self::Expr(StmtExpr { value: value.into(), }), diff --git a/crates/ruff_python_ast/src/nodes.rs b/crates/ruff_python_ast/src/nodes.rs index 58d965660717e6..fd06cdff1d6fe6 100644 --- a/crates/ruff_python_ast/src/nodes.rs +++ b/crates/ruff_python_ast/src/nodes.rs @@ -160,7 +160,7 @@ pub enum Stmt { pub struct StmtIpyEscapeCommand { pub range: TextRange, pub kind: IpyEscapeKind, - pub value: String, + pub value: Box, } impl From for Stmt { @@ -671,7 +671,7 @@ impl Expr { pub struct ExprIpyEscapeCommand { pub range: TextRange, pub kind: IpyEscapeKind, - pub value: String, + pub value: Box, } impl From for Expr { @@ -1384,7 +1384,7 @@ impl Default for StringLiteralValueInner { #[derive(Clone, Debug, Default, PartialEq)] pub struct StringLiteral { pub range: TextRange, - pub value: String, + pub value: Box, pub unicode: bool, } @@ -1398,7 +1398,7 @@ impl Deref for StringLiteral { type Target = str; fn deref(&self) -> &Self::Target { - self.value.as_str() + &self.value } } @@ -1426,14 +1426,16 @@ struct ConcatenatedStringLiteral { /// Each string literal that makes up the concatenated string. strings: Vec, /// The concatenated string value. - value: OnceCell, + value: OnceCell>, } impl ConcatenatedStringLiteral { /// Extracts a string slice containing the entire concatenated string. fn to_str(&self) -> &str { - self.value - .get_or_init(|| self.strings.iter().map(StringLiteral::as_str).collect()) + self.value.get_or_init(|| { + let concatenated: String = self.strings.iter().map(StringLiteral::as_str).collect(); + concatenated.into_boxed_str() + }) } } diff --git a/crates/ruff_python_formatter/tests/normalizer.rs b/crates/ruff_python_formatter/tests/normalizer.rs index 2bab8915cc0543..5a7b769f3e054f 100644 --- a/crates/ruff_python_formatter/tests/normalizer.rs +++ b/crates/ruff_python_formatter/tests/normalizer.rs @@ -95,19 +95,22 @@ impl Transformer for Normalizer { &string_literal.value, "\n", ) - .into_owned(); + .into_owned() + .into_boxed_str(); string_literal.value = STRIP_RST_BLOCKS .replace_all( &string_literal.value, "\n", ) - .into_owned(); + .into_owned() + .into_boxed_str(); string_literal.value = STRIP_MARKDOWN_BLOCKS .replace_all( &string_literal.value, "\n", ) - .into_owned(); + .into_owned() + .into_boxed_str(); // Normalize a string by (2) stripping any leading and trailing space from each // line, and (3) removing any blank lines from the start and end of the string. string_literal.value = string_literal @@ -117,6 +120,7 @@ impl Transformer for Normalizer { .collect::>() .join("\n") .trim() - .to_owned(); + .to_owned() + .into_boxed_str(); } } diff --git a/crates/ruff_python_parser/src/function.rs b/crates/ruff_python_parser/src/function.rs index bbfe0961c23816..1700066165e4b4 100644 --- a/crates/ruff_python_parser/src/function.rs +++ b/crates/ruff_python_parser/src/function.rs @@ -40,7 +40,7 @@ pub(crate) fn validate_arguments(arguments: &ast::Parameters) -> Result<(), Lexi let arg_name = arg.name.as_str(); if !all_arg_names.insert(arg_name) { return Err(LexicalError::new( - LexicalErrorType::DuplicateArgumentError(arg_name.to_string()), + LexicalErrorType::DuplicateArgumentError(arg_name.to_string().into_boxed_str()), range.start(), )); } @@ -95,7 +95,9 @@ pub(crate) fn parse_arguments( if let Some(keyword_name) = &name { if !keyword_names.insert(keyword_name.to_string()) { return Err(LexicalError::new( - LexicalErrorType::DuplicateKeywordArgumentError(keyword_name.to_string()), + LexicalErrorType::DuplicateKeywordArgumentError( + keyword_name.to_string().into_boxed_str(), + ), start, )); } @@ -200,22 +202,22 @@ mod tests { function_and_lambda_error! { // Check definitions - test_duplicates_f1: "def f(a, a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string()), - test_duplicates_f2: "def f(a, *, a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string()), - test_duplicates_f3: "def f(a, a=20): pass", LexicalErrorType::DuplicateArgumentError("a".to_string()), - test_duplicates_f4: "def f(a, *a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string()), - test_duplicates_f5: "def f(a, *, **a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string()), - test_duplicates_l1: "lambda a, a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string()), - test_duplicates_l2: "lambda a, *, a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string()), - test_duplicates_l3: "lambda a, a=20: 1", LexicalErrorType::DuplicateArgumentError("a".to_string()), - test_duplicates_l4: "lambda a, *a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string()), - test_duplicates_l5: "lambda a, *, **a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string()), + test_duplicates_f1: "def f(a, a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()), + test_duplicates_f2: "def f(a, *, a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()), + test_duplicates_f3: "def f(a, a=20): pass", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()), + test_duplicates_f4: "def f(a, *a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()), + test_duplicates_f5: "def f(a, *, **a): pass", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()), + test_duplicates_l1: "lambda a, a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()), + test_duplicates_l2: "lambda a, *, a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()), + test_duplicates_l3: "lambda a, a=20: 1", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()), + test_duplicates_l4: "lambda a, *a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()), + test_duplicates_l5: "lambda a, *, **a: 1", LexicalErrorType::DuplicateArgumentError("a".to_string().into_boxed_str()), test_default_arg_error_f: "def f(a, b=20, c): pass", LexicalErrorType::DefaultArgumentError, test_default_arg_error_l: "lambda a, b=20, c: 1", LexicalErrorType::DefaultArgumentError, // Check some calls. test_positional_arg_error_f: "f(b=20, c)", LexicalErrorType::PositionalArgumentError, test_unpacked_arg_error_f: "f(**b, *c)", LexicalErrorType::UnpackedArgumentError, - test_duplicate_kw_f1: "f(a=20, a=30)", LexicalErrorType::DuplicateKeywordArgumentError("a".to_string()), + test_duplicate_kw_f1: "f(a=20, a=30)", LexicalErrorType::DuplicateKeywordArgumentError("a".to_string().into_boxed_str()), } } diff --git a/crates/ruff_python_parser/src/lexer.rs b/crates/ruff_python_parser/src/lexer.rs index 4fd9df80d2ad4e..373e3929b9c75d 100644 --- a/crates/ruff_python_parser/src/lexer.rs +++ b/crates/ruff_python_parser/src/lexer.rs @@ -241,7 +241,7 @@ impl<'source> Lexer<'source> { "yield" => Tok::Yield, _ => { return Ok(Tok::Name { - name: text.to_string(), + name: text.to_string().into_boxed_str(), }) } }; @@ -285,7 +285,7 @@ impl<'source> Lexer<'source> { Ok(int) => int, Err(err) => { return Err(LexicalError::new( - LexicalErrorType::OtherError(format!("{err:?}")), + LexicalErrorType::OtherError(format!("{err:?}").into_boxed_str()), self.token_range().start(), )); } @@ -310,7 +310,7 @@ impl<'source> Lexer<'source> { if self.cursor.eat_char('_') { return Err(LexicalError::new( - LexicalErrorType::OtherError("Invalid Syntax".to_owned()), + LexicalErrorType::OtherError("Invalid Syntax".to_string().into_boxed_str()), self.offset() - TextSize::new(1), )); } @@ -342,7 +342,9 @@ impl<'source> Lexer<'source> { // Improvement: Use `Cow` instead of pushing to value text let value = f64::from_str(number.as_str()).map_err(|_| { LexicalError::new( - LexicalErrorType::OtherError("Invalid decimal literal".to_owned()), + LexicalErrorType::OtherError( + "Invalid decimal literal".to_string().into_boxed_str(), + ), self.token_start(), ) })?; @@ -367,7 +369,9 @@ impl<'source> Lexer<'source> { if start_is_zero && value.as_u8() != Some(0) { // Leading zeros in decimal integer literals are not permitted. return Err(LexicalError::new( - LexicalErrorType::OtherError("Invalid Token".to_owned()), + LexicalErrorType::OtherError( + "Invalid Token".to_string().into_boxed_str(), + ), self.token_range().start(), )); } @@ -375,7 +379,7 @@ impl<'source> Lexer<'source> { } Err(err) => { return Err(LexicalError::new( - LexicalErrorType::OtherError(format!("{err:?}")), + LexicalErrorType::OtherError(format!("{err:?}").into_boxed_str()), self.token_range().start(), )) } @@ -413,7 +417,7 @@ impl<'source> Lexer<'source> { let offset = memchr::memchr2(b'\n', b'\r', bytes).unwrap_or(bytes.len()); self.cursor.skip_bytes(offset); - Tok::Comment(self.token_text().to_string()) + Tok::Comment(self.token_text().to_string().into_boxed_str()) } /// Lex a single IPython escape command. @@ -510,12 +514,15 @@ impl<'source> Lexer<'source> { 2 => IpyEscapeKind::Help2, _ => unreachable!("`question_count` is always 1 or 2"), }; - return Tok::IpyEscapeCommand { kind, value }; + return Tok::IpyEscapeCommand { + kind, + value: value.into_boxed_str(), + }; } '\n' | '\r' | EOF_CHAR => { return Tok::IpyEscapeCommand { kind: escape_kind, - value, + value: value.into_boxed_str(), }; } c => { @@ -675,7 +682,7 @@ impl<'source> Lexer<'source> { normalized }; Ok(Some(Tok::FStringMiddle { - value, + value: value.into_boxed_str(), is_raw: fstring.is_raw_string(), triple_quoted: fstring.is_triple_quoted(), })) @@ -715,7 +722,9 @@ impl<'source> Lexer<'source> { } return Err(LexicalError::new( LexicalErrorType::OtherError( - "EOL while scanning string literal".to_owned(), + "EOL while scanning string literal" + .to_string() + .into_boxed_str(), ), self.offset() - TextSize::new(1), )); @@ -758,7 +767,9 @@ impl<'source> Lexer<'source> { }; let tok = Tok::String { - value: self.source[TextRange::new(value_start, value_end)].to_string(), + value: self.source[TextRange::new(value_start, value_end)] + .to_string() + .into_boxed_str(), kind, triple_quoted, }; @@ -1284,35 +1295,32 @@ impl FusedIterator for Lexer<'_> {} /// /// [lexer]: crate::lexer #[derive(Debug, Clone, PartialEq)] -pub struct LexicalError(Box); +pub struct LexicalError { + /// The type of error that occurred. + error: LexicalErrorType, + /// The location of the error. + location: TextSize, +} impl LexicalError { /// Creates a new `LexicalError` with the given error type and location. pub fn new(error: LexicalErrorType, location: TextSize) -> Self { - Self(Box::new(LexicalErrorInner { error, location })) + Self { error, location } } pub fn error(&self) -> &LexicalErrorType { - &self.0.error + &self.error } pub fn into_error(self) -> LexicalErrorType { - self.0.error + self.error } pub fn location(&self) -> TextSize { - self.0.location + self.location } } -#[derive(Debug, Clone, PartialEq)] -struct LexicalErrorInner { - /// The type of error that occurred. - error: LexicalErrorType, - /// The location of the error. - location: TextSize, -} - impl std::ops::Deref for LexicalError { type Target = LexicalErrorType; @@ -1359,13 +1367,13 @@ pub enum LexicalErrorType { /// A non-default argument follows a default argument. DefaultArgumentError, /// A duplicate argument was found in a function definition. - DuplicateArgumentError(String), + DuplicateArgumentError(Box), /// A positional argument follows a keyword argument. PositionalArgumentError, /// An iterable argument unpacking `*args` follows keyword argument unpacking `**kwargs`. UnpackedArgumentError, /// A keyword argument was repeated. - DuplicateKeywordArgumentError(String), + DuplicateKeywordArgumentError(Box), /// An unrecognized token was encountered. UnrecognizedToken { tok: char }, /// An f-string error containing the [`FStringErrorType`]. @@ -1377,7 +1385,7 @@ pub enum LexicalErrorType { /// Occurs when a syntactically invalid assignment was encountered. AssignmentError, /// An unexpected error occurred. - OtherError(String), + OtherError(Box), } impl std::error::Error for LexicalErrorType {} @@ -2262,25 +2270,21 @@ f"{(lambda x:{x})}" #[test] fn test_fstring_error_location() { assert_debug_snapshot!(lex_error("f'{'"), @r###" - LexicalError( - LexicalErrorInner { - error: FStringError( - UnclosedLbrace, - ), - location: 4, - }, - ) + LexicalError { + error: FStringError( + UnclosedLbrace, + ), + location: 4, + } "###); assert_debug_snapshot!(lex_error("f'{'α"), @r###" - LexicalError( - LexicalErrorInner { - error: FStringError( - UnclosedLbrace, - ), - location: 6, - }, - ) + LexicalError { + error: FStringError( + UnclosedLbrace, + ), + location: 6, + } "###); } } diff --git a/crates/ruff_python_parser/src/python.lalrpop b/crates/ruff_python_parser/src/python.lalrpop index 25c63d1f02ac19..386574b0001b75 100644 --- a/crates/ruff_python_parser/src/python.lalrpop +++ b/crates/ruff_python_parser/src/python.lalrpop @@ -289,7 +289,7 @@ ImportAsAlias: ast::Alias = { DottedName: ast::Identifier = { => ast::Identifier::new(n, (location..end_location).into()), => { - let mut r = n; + let mut r = String::from(n); for x in n2 { r.push('.'); r.push_str(x.1.as_str()); @@ -338,7 +338,7 @@ IpyEscapeCommandStatement: ast::Stmt = { )) } else { Err(LexicalError::new( - LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string()), + LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string().into_boxed_str()), location, ))? } @@ -351,7 +351,7 @@ IpyEscapeCommandExpr: crate::parser::ParenthesizedExpr = { // This should never occur as the lexer won't allow it. if !matches!(c.0, IpyEscapeKind::Magic | IpyEscapeKind::Shell) { return Err(LexicalError::new( - LexicalErrorType::OtherError("IPython escape command expr is only allowed for % and !".to_string()), + LexicalErrorType::OtherError("IPython escape command expr is only allowed for % and !".to_string().into_boxed_str()), location, ))?; } @@ -362,7 +362,7 @@ IpyEscapeCommandExpr: crate::parser::ParenthesizedExpr = { }.into()) } else { Err(LexicalError::new( - LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string()), + LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string().into_boxed_str()), location, ))? } @@ -382,7 +382,7 @@ IpyHelpEndEscapeCommandStatement: ast::Stmt = { ast::Expr::Subscript(ast::ExprSubscript { value, slice, range, .. }) => { let ast::Expr::NumberLiteral(ast::ExprNumberLiteral { value: ast::Number::Int(integer), .. }) = slice.as_ref() else { return Err(LexicalError::new( - LexicalErrorType::OtherError("only integer literals are allowed in Subscript expressions in help end escape command".to_string()), + LexicalErrorType::OtherError("only integer literals are allowed in Subscript expressions in help end escape command".to_string().into_boxed_str()), range.start(), )); }; @@ -398,7 +398,7 @@ IpyHelpEndEscapeCommandStatement: ast::Stmt = { }, _ => { return Err(LexicalError::new( - LexicalErrorType::OtherError("only Name, Subscript and Attribute expressions are allowed in help end escape command".to_string()), + LexicalErrorType::OtherError("only Name, Subscript and Attribute expressions are allowed in help end escape command".to_string().into_boxed_str()), expr.start(), )); } @@ -409,7 +409,7 @@ IpyHelpEndEscapeCommandStatement: ast::Stmt = { if mode != Mode::Ipython { return Err(ParseError::User { error: LexicalError::new( - LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string()), + LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string().into_boxed_str()), location, ), }); @@ -421,7 +421,7 @@ IpyHelpEndEscapeCommandStatement: ast::Stmt = { _ => { return Err(ParseError::User { error: LexicalError::new( - LexicalErrorType::OtherError("maximum of 2 `?` tokens are allowed in help end escape command".to_string()), + LexicalErrorType::OtherError("maximum of 2 `?` tokens are allowed in help end escape command".to_string().into_boxed_str()), location, ), }); @@ -434,7 +434,7 @@ IpyHelpEndEscapeCommandStatement: ast::Stmt = { Ok(ast::Stmt::IpyEscapeCommand( ast::StmtIpyEscapeCommand { kind, - value, + value: value.into_boxed_str(), range: (location..end_location).into() } )) @@ -562,7 +562,7 @@ AsPattern: ast::Pattern = { "as" =>? { if name.as_str() == "_" { Err(LexicalError::new( - LexicalErrorType::OtherError("cannot use '_' as a target".to_string()), + LexicalErrorType::OtherError("cannot use '_' as a target".to_string().into_boxed_str()), location, ))? } else { @@ -1248,7 +1248,7 @@ ParameterListStarArgs "*" >)*> >)?> =>? { if va.is_none() && kwonlyargs.is_empty() && kwarg.is_none() { return Err(LexicalError::new( - LexicalErrorType::OtherError("named arguments must follow bare *".to_string()), + LexicalErrorType::OtherError("named arguments must follow bare *".to_string().into_boxed_str()), location, ))?; } @@ -1677,7 +1677,7 @@ FStringFormatSpec: ast::FStringFormatSpec = { FStringConversion: (TextSize, ast::ConversionFlag) = { "!" =>? { - let conversion = match s.as_str() { + let conversion = match s.as_ref() { "s" => ast::ConversionFlag::Str, "r" => ast::ConversionFlag::Repr, "a" => ast::ConversionFlag::Ascii, @@ -1723,7 +1723,7 @@ Atom: crate::parser::ParenthesizedExpr = { if left.is_none() && right.is_empty() && trailing_comma.is_none() { if mid.expr.is_starred_expr() { return Err(LexicalError::new( - LexicalErrorType::OtherError("cannot use starred expression here".to_string()), + LexicalErrorType::OtherError("cannot use starred expression here".to_string().into_boxed_str()), mid.start(), ))?; } @@ -1752,7 +1752,7 @@ Atom: crate::parser::ParenthesizedExpr = { }.into(), "(" "**" > ")" =>? { Err(LexicalError::new( - LexicalErrorType::OtherError("cannot use double starred expression here".to_string()), + LexicalErrorType::OtherError("cannot use double starred expression here".to_string().into_boxed_str()), location, ).into()) }, @@ -2061,19 +2061,19 @@ extern { float => token::Tok::Float { value: }, complex => token::Tok::Complex { real: , imag: }, string => token::Tok::String { - value: , + value: >, kind: , triple_quoted: }, fstring_middle => token::Tok::FStringMiddle { - value: , + value: >, is_raw: , triple_quoted: }, - name => token::Tok::Name { name: }, + name => token::Tok::Name { name: > }, ipy_escape_command => token::Tok::IpyEscapeCommand { kind: , - value: + value: > }, "\n" => token::Tok::Newline, ";" => token::Tok::Semi, diff --git a/crates/ruff_python_parser/src/python.rs b/crates/ruff_python_parser/src/python.rs index 4c378889765797..abe55991b201a3 100644 --- a/crates/ruff_python_parser/src/python.rs +++ b/crates/ruff_python_parser/src/python.rs @@ -1,5 +1,5 @@ // auto-generated: "lalrpop 0.20.0" -// sha3: 5d684f12592dfd246411a2ca5f6bbed7c1e8ff9dd4129eaba0a98beee6d21cf6 +// sha3: fd05d84d3b654796ff740a7f905ec0ae8915f43f952428717735481947ab55e1 use ruff_text_size::{Ranged, TextLen, TextRange, TextSize}; use ruff_python_ast::{self as ast, Int, IpyEscapeKind}; use crate::{ @@ -50,11 +50,11 @@ mod __parse__Top { Variant0(token::Tok), Variant1((f64, f64)), Variant2(f64), - Variant3((String, bool, bool)), + Variant3((Box, bool, bool)), Variant4(Int), - Variant5((IpyEscapeKind, String)), - Variant6(String), - Variant7((String, StringKind, bool)), + Variant5((IpyEscapeKind, Box)), + Variant6(Box), + Variant7((Box, StringKind, bool)), Variant8(core::option::Option), Variant9(Option>), Variant10(core::option::Option>>), @@ -151,7 +151,7 @@ mod __parse__Top { Variant101(ast::TypeParams), Variant102(core::option::Option), Variant103(ast::UnaryOp), - Variant104(core::option::Option<(String, bool, bool)>), + Variant104(core::option::Option<(Box, bool, bool)>), } const __ACTION: &[i16] = &[ // State 0 @@ -18323,73 +18323,73 @@ mod __parse__Top { fn __symbol_type_mismatch() -> ! { panic!("symbol type mismatch") } - fn __pop_Variant5< + fn __pop_Variant7< >( __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)> - ) -> (TextSize, (IpyEscapeKind, String), TextSize) + ) -> (TextSize, (Box, StringKind, bool), TextSize) { match __symbols.pop() { - Some((__l, __Symbol::Variant5(__v), __r)) => (__l, __v, __r), + Some((__l, __Symbol::Variant7(__v), __r)) => (__l, __v, __r), _ => __symbol_type_mismatch() } } - fn __pop_Variant31< + fn __pop_Variant3< >( __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)> - ) -> (TextSize, (Option<(TextSize, TextSize, Option)>, ast::Expr), TextSize) + ) -> (TextSize, (Box, bool, bool), TextSize) { match __symbols.pop() { - Some((__l, __Symbol::Variant31(__v), __r)) => (__l, __v, __r), + Some((__l, __Symbol::Variant3(__v), __r)) => (__l, __v, __r), _ => __symbol_type_mismatch() } } - fn __pop_Variant13< + fn __pop_Variant5< >( __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)> - ) -> (TextSize, (Option>, Vec, Option>), TextSize) + ) -> (TextSize, (IpyEscapeKind, Box), TextSize) { match __symbols.pop() { - Some((__l, __Symbol::Variant13(__v), __r)) => (__l, __v, __r), + Some((__l, __Symbol::Variant5(__v), __r)) => (__l, __v, __r), _ => __symbol_type_mismatch() } } - fn __pop_Variant59< + fn __pop_Variant31< >( __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)> - ) -> (TextSize, (Option>, crate::parser::ParenthesizedExpr), TextSize) + ) -> (TextSize, (Option<(TextSize, TextSize, Option)>, ast::Expr), TextSize) { match __symbols.pop() { - Some((__l, __Symbol::Variant59(__v), __r)) => (__l, __v, __r), + Some((__l, __Symbol::Variant31(__v), __r)) => (__l, __v, __r), _ => __symbol_type_mismatch() } } - fn __pop_Variant79< + fn __pop_Variant13< >( __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)> - ) -> (TextSize, (Option, Option), TextSize) + ) -> (TextSize, (Option>, Vec, Option>), TextSize) { match __symbols.pop() { - Some((__l, __Symbol::Variant79(__v), __r)) => (__l, __v, __r), + Some((__l, __Symbol::Variant13(__v), __r)) => (__l, __v, __r), _ => __symbol_type_mismatch() } } - fn __pop_Variant7< + fn __pop_Variant59< >( __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)> - ) -> (TextSize, (String, StringKind, bool), TextSize) + ) -> (TextSize, (Option>, crate::parser::ParenthesizedExpr), TextSize) { match __symbols.pop() { - Some((__l, __Symbol::Variant7(__v), __r)) => (__l, __v, __r), + Some((__l, __Symbol::Variant59(__v), __r)) => (__l, __v, __r), _ => __symbol_type_mismatch() } } - fn __pop_Variant3< + fn __pop_Variant79< >( __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)> - ) -> (TextSize, (String, bool, bool), TextSize) + ) -> (TextSize, (Option, Option), TextSize) { match __symbols.pop() { - Some((__l, __Symbol::Variant3(__v), __r)) => (__l, __v, __r), + Some((__l, __Symbol::Variant79(__v), __r)) => (__l, __v, __r), _ => __symbol_type_mismatch() } } @@ -18493,6 +18493,16 @@ mod __parse__Top { _ => __symbol_type_mismatch() } } + fn __pop_Variant6< + >( + __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)> + ) -> (TextSize, Box, TextSize) + { + match __symbols.pop() { + Some((__l, __Symbol::Variant6(__v), __r)) => (__l, __v, __r), + _ => __symbol_type_mismatch() + } + } fn __pop_Variant4< >( __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)> @@ -18523,16 +18533,6 @@ mod __parse__Top { _ => __symbol_type_mismatch() } } - fn __pop_Variant6< - >( - __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)> - ) -> (TextSize, String, TextSize) - { - match __symbols.pop() { - Some((__l, __Symbol::Variant6(__v), __r)) => (__l, __v, __r), - _ => __symbol_type_mismatch() - } - } fn __pop_Variant69< >( __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)> @@ -19113,33 +19113,33 @@ mod __parse__Top { _ => __symbol_type_mismatch() } } - fn __pop_Variant74< + fn __pop_Variant104< >( __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)> - ) -> (TextSize, core::option::Option<(Option<(TextSize, TextSize, Option)>, ast::Expr)>, TextSize) + ) -> (TextSize, core::option::Option<(Box, bool, bool)>, TextSize) { match __symbols.pop() { - Some((__l, __Symbol::Variant74(__v), __r)) => (__l, __v, __r), + Some((__l, __Symbol::Variant104(__v), __r)) => (__l, __v, __r), _ => __symbol_type_mismatch() } } - fn __pop_Variant14< + fn __pop_Variant74< >( __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)> - ) -> (TextSize, core::option::Option<(Option>, Vec, Option>)>, TextSize) + ) -> (TextSize, core::option::Option<(Option<(TextSize, TextSize, Option)>, ast::Expr)>, TextSize) { match __symbols.pop() { - Some((__l, __Symbol::Variant14(__v), __r)) => (__l, __v, __r), + Some((__l, __Symbol::Variant74(__v), __r)) => (__l, __v, __r), _ => __symbol_type_mismatch() } } - fn __pop_Variant104< + fn __pop_Variant14< >( __symbols: &mut alloc::vec::Vec<(TextSize,__Symbol<>,TextSize)> - ) -> (TextSize, core::option::Option<(String, bool, bool)>, TextSize) + ) -> (TextSize, core::option::Option<(Option>, Vec, Option>)>, TextSize) { match __symbols.pop() { - Some((__l, __Symbol::Variant104(__v), __r)) => (__l, __v, __r), + Some((__l, __Symbol::Variant14(__v), __r)) => (__l, __v, __r), _ => __symbol_type_mismatch() } } @@ -33541,7 +33541,7 @@ fn __action69< source_code: &str, mode: Mode, (_, location, _): (TextSize, TextSize, TextSize), - (_, n, _): (TextSize, String, TextSize), + (_, n, _): (TextSize, Box, TextSize), (_, end_location, _): (TextSize, TextSize, TextSize), ) -> ast::Identifier { @@ -33555,13 +33555,13 @@ fn __action70< source_code: &str, mode: Mode, (_, location, _): (TextSize, TextSize, TextSize), - (_, n, _): (TextSize, String, TextSize), + (_, n, _): (TextSize, Box, TextSize), (_, n2, _): (TextSize, alloc::vec::Vec<(token::Tok, ast::Identifier)>, TextSize), (_, end_location, _): (TextSize, TextSize, TextSize), ) -> ast::Identifier { { - let mut r = n; + let mut r = String::from(n); for x in n2 { r.push('.'); r.push_str(x.1.as_str()); @@ -33639,7 +33639,7 @@ fn __action74< source_code: &str, mode: Mode, (_, location, _): (TextSize, TextSize, TextSize), - (_, c, _): (TextSize, (IpyEscapeKind, String), TextSize), + (_, c, _): (TextSize, (IpyEscapeKind, Box), TextSize), (_, end_location, _): (TextSize, TextSize, TextSize), ) -> Result> { @@ -33654,7 +33654,7 @@ fn __action74< )) } else { Err(LexicalError::new( - LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string()), + LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string().into_boxed_str()), location, ))? } @@ -33668,7 +33668,7 @@ fn __action75< source_code: &str, mode: Mode, (_, location, _): (TextSize, TextSize, TextSize), - (_, c, _): (TextSize, (IpyEscapeKind, String), TextSize), + (_, c, _): (TextSize, (IpyEscapeKind, Box), TextSize), (_, end_location, _): (TextSize, TextSize, TextSize), ) -> Result> { @@ -33677,7 +33677,7 @@ fn __action75< // This should never occur as the lexer won't allow it. if !matches!(c.0, IpyEscapeKind::Magic | IpyEscapeKind::Shell) { return Err(LexicalError::new( - LexicalErrorType::OtherError("IPython escape command expr is only allowed for % and !".to_string()), + LexicalErrorType::OtherError("IPython escape command expr is only allowed for % and !".to_string().into_boxed_str()), location, ))?; } @@ -33688,7 +33688,7 @@ fn __action75< }.into()) } else { Err(LexicalError::new( - LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string()), + LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string().into_boxed_str()), location, ))? } @@ -33716,7 +33716,7 @@ fn __action76< ast::Expr::Subscript(ast::ExprSubscript { value, slice, range, .. }) => { let ast::Expr::NumberLiteral(ast::ExprNumberLiteral { value: ast::Number::Int(integer), .. }) = slice.as_ref() else { return Err(LexicalError::new( - LexicalErrorType::OtherError("only integer literals are allowed in Subscript expressions in help end escape command".to_string()), + LexicalErrorType::OtherError("only integer literals are allowed in Subscript expressions in help end escape command".to_string().into_boxed_str()), range.start(), )); }; @@ -33732,7 +33732,7 @@ fn __action76< }, _ => { return Err(LexicalError::new( - LexicalErrorType::OtherError("only Name, Subscript and Attribute expressions are allowed in help end escape command".to_string()), + LexicalErrorType::OtherError("only Name, Subscript and Attribute expressions are allowed in help end escape command".to_string().into_boxed_str()), expr.start(), )); } @@ -33743,7 +33743,7 @@ fn __action76< if mode != Mode::Ipython { return Err(ParseError::User { error: LexicalError::new( - LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string()), + LexicalErrorType::OtherError("IPython escape commands are only allowed in `Mode::Ipython`".to_string().into_boxed_str()), location, ), }); @@ -33755,7 +33755,7 @@ fn __action76< _ => { return Err(ParseError::User { error: LexicalError::new( - LexicalErrorType::OtherError("maximum of 2 `?` tokens are allowed in help end escape command".to_string()), + LexicalErrorType::OtherError("maximum of 2 `?` tokens are allowed in help end escape command".to_string().into_boxed_str()), location, ), }); @@ -33768,7 +33768,7 @@ fn __action76< Ok(ast::Stmt::IpyEscapeCommand( ast::StmtIpyEscapeCommand { kind, - value, + value: value.into_boxed_str(), range: (location..end_location).into() } )) @@ -34127,7 +34127,7 @@ fn __action95< { if name.as_str() == "_" { Err(LexicalError::new( - LexicalErrorType::OtherError("cannot use '_' as a target".to_string()), + LexicalErrorType::OtherError("cannot use '_' as a target".to_string().into_boxed_str()), location, ))? } else { @@ -35910,7 +35910,7 @@ fn __action184< (_, parameters, _): (TextSize, core::option::Option, TextSize), (_, end_location_args, _): (TextSize, TextSize, TextSize), (_, _, _): (TextSize, token::Tok, TextSize), - (_, fstring_middle, _): (TextSize, core::option::Option<(String, bool, bool)>, TextSize), + (_, fstring_middle, _): (TextSize, core::option::Option<(Box, bool, bool)>, TextSize), (_, body, _): (TextSize, crate::parser::ParenthesizedExpr, TextSize), (_, end_location, _): (TextSize, TextSize, TextSize), ) -> Result> @@ -36363,7 +36363,7 @@ fn __action217< source_code: &str, mode: Mode, (_, location, _): (TextSize, TextSize, TextSize), - (_, string, _): (TextSize, (String, StringKind, bool), TextSize), + (_, string, _): (TextSize, (Box, StringKind, bool), TextSize), (_, end_location, _): (TextSize, TextSize, TextSize), ) -> Result> { @@ -36413,7 +36413,7 @@ fn __action220< source_code: &str, mode: Mode, (_, location, _): (TextSize, TextSize, TextSize), - (_, fstring_middle, _): (TextSize, (String, bool, bool), TextSize), + (_, fstring_middle, _): (TextSize, (Box, bool, bool), TextSize), (_, end_location, _): (TextSize, TextSize, TextSize), ) -> Result> { @@ -36514,11 +36514,11 @@ fn __action224< (_, location, _): (TextSize, TextSize, TextSize), (_, _, _): (TextSize, token::Tok, TextSize), (_, name_location, _): (TextSize, TextSize, TextSize), - (_, s, _): (TextSize, String, TextSize), + (_, s, _): (TextSize, Box, TextSize), ) -> Result<(TextSize, ast::ConversionFlag),__lalrpop_util::ParseError> { { - let conversion = match s.as_str() { + let conversion = match s.as_ref() { "s" => ast::ConversionFlag::Str, "r" => ast::ConversionFlag::Repr, "a" => ast::ConversionFlag::Ascii, @@ -36899,7 +36899,7 @@ fn __action249< source_code: &str, mode: Mode, (_, location, _): (TextSize, TextSize, TextSize), - (_, s, _): (TextSize, String, TextSize), + (_, s, _): (TextSize, Box, TextSize), (_, end_location, _): (TextSize, TextSize, TextSize), ) -> ast::Identifier { @@ -37357,8 +37357,8 @@ fn __action281< >( source_code: &str, mode: Mode, - (_, __0, _): (TextSize, (String, bool, bool), TextSize), -) -> core::option::Option<(String, bool, bool)> + (_, __0, _): (TextSize, (Box, bool, bool), TextSize), +) -> core::option::Option<(Box, bool, bool)> { Some(__0) } @@ -37371,7 +37371,7 @@ fn __action282< mode: Mode, __lookbehind: &TextSize, __lookahead: &TextSize, -) -> core::option::Option<(String, bool, bool)> +) -> core::option::Option<(Box, bool, bool)> { None } @@ -39669,7 +39669,7 @@ fn __action445< { if va.is_none() && kwonlyargs.is_empty() && kwarg.is_none() { return Err(LexicalError::new( - LexicalErrorType::OtherError("named arguments must follow bare *".to_string()), + LexicalErrorType::OtherError("named arguments must follow bare *".to_string().into_boxed_str()), location, ))?; } @@ -39794,7 +39794,7 @@ fn __action453< { if va.is_none() && kwonlyargs.is_empty() && kwarg.is_none() { return Err(LexicalError::new( - LexicalErrorType::OtherError("named arguments must follow bare *".to_string()), + LexicalErrorType::OtherError("named arguments must follow bare *".to_string().into_boxed_str()), location, ))?; } @@ -41297,7 +41297,7 @@ fn __action554< if left.is_none() && right.is_empty() && trailing_comma.is_none() { if mid.expr.is_starred_expr() { return Err(LexicalError::new( - LexicalErrorType::OtherError("cannot use starred expression here".to_string()), + LexicalErrorType::OtherError("cannot use starred expression here".to_string().into_boxed_str()), mid.start(), ))?; } @@ -41387,7 +41387,7 @@ fn __action558< { { Err(LexicalError::new( - LexicalErrorType::OtherError("cannot use double starred expression here".to_string()), + LexicalErrorType::OtherError("cannot use double starred expression here".to_string().into_boxed_str()), location, ).into()) } @@ -41995,7 +41995,7 @@ fn __action596< if left.is_none() && right.is_empty() && trailing_comma.is_none() { if mid.expr.is_starred_expr() { return Err(LexicalError::new( - LexicalErrorType::OtherError("cannot use starred expression here".to_string()), + LexicalErrorType::OtherError("cannot use starred expression here".to_string().into_boxed_str()), mid.start(), ))?; } @@ -42085,7 +42085,7 @@ fn __action600< { { Err(LexicalError::new( - LexicalErrorType::OtherError("cannot use double starred expression here".to_string()), + LexicalErrorType::OtherError("cannot use double starred expression here".to_string().into_boxed_str()), location, ).into()) } @@ -48027,7 +48027,7 @@ fn __action789< >( source_code: &str, mode: Mode, - __0: (TextSize, String, TextSize), + __0: (TextSize, Box, TextSize), __1: (TextSize, TextSize, TextSize), ) -> ast::Identifier { @@ -48055,7 +48055,7 @@ fn __action790< >( source_code: &str, mode: Mode, - __0: (TextSize, String, TextSize), + __0: (TextSize, Box, TextSize), __1: (TextSize, alloc::vec::Vec<(token::Tok, ast::Identifier)>, TextSize), __2: (TextSize, TextSize, TextSize), ) -> ast::Identifier @@ -48408,7 +48408,7 @@ fn __action801< source_code: &str, mode: Mode, __0: (TextSize, token::Tok, TextSize), - __1: (TextSize, String, TextSize), + __1: (TextSize, Box, TextSize), ) -> Result<(TextSize, ast::ConversionFlag),__lalrpop_util::ParseError> { let __start0 = __0.0; @@ -48505,7 +48505,7 @@ fn __action804< >( source_code: &str, mode: Mode, - __0: (TextSize, (String, bool, bool), TextSize), + __0: (TextSize, (Box, bool, bool), TextSize), __1: (TextSize, TextSize, TextSize), ) -> Result> { @@ -49209,7 +49209,7 @@ fn __action826< >( source_code: &str, mode: Mode, - __0: (TextSize, String, TextSize), + __0: (TextSize, Box, TextSize), __1: (TextSize, TextSize, TextSize), ) -> ast::Identifier { @@ -49519,7 +49519,7 @@ fn __action836< >( source_code: &str, mode: Mode, - __0: (TextSize, (IpyEscapeKind, String), TextSize), + __0: (TextSize, (IpyEscapeKind, Box), TextSize), __1: (TextSize, TextSize, TextSize), ) -> Result> { @@ -49547,7 +49547,7 @@ fn __action837< >( source_code: &str, mode: Mode, - __0: (TextSize, (IpyEscapeKind, String), TextSize), + __0: (TextSize, (IpyEscapeKind, Box), TextSize), __1: (TextSize, TextSize, TextSize), ) -> Result> { @@ -49609,7 +49609,7 @@ fn __action839< __1: (TextSize, core::option::Option, TextSize), __2: (TextSize, TextSize, TextSize), __3: (TextSize, token::Tok, TextSize), - __4: (TextSize, core::option::Option<(String, bool, bool)>, TextSize), + __4: (TextSize, core::option::Option<(Box, bool, bool)>, TextSize), __5: (TextSize, crate::parser::ParenthesizedExpr, TextSize), __6: (TextSize, TextSize, TextSize), ) -> Result> @@ -52719,7 +52719,7 @@ fn __action937< >( source_code: &str, mode: Mode, - __0: (TextSize, (String, StringKind, bool), TextSize), + __0: (TextSize, (Box, StringKind, bool), TextSize), __1: (TextSize, TextSize, TextSize), ) -> Result> { @@ -64211,7 +64211,7 @@ fn __action1304< >( source_code: &str, mode: Mode, - __0: (TextSize, String, TextSize), + __0: (TextSize, Box, TextSize), ) -> ast::Identifier { let __start0 = __0.2; @@ -64237,7 +64237,7 @@ fn __action1305< >( source_code: &str, mode: Mode, - __0: (TextSize, String, TextSize), + __0: (TextSize, Box, TextSize), __1: (TextSize, alloc::vec::Vec<(token::Tok, ast::Identifier)>, TextSize), ) -> ast::Identifier { @@ -64527,7 +64527,7 @@ fn __action1315< >( source_code: &str, mode: Mode, - __0: (TextSize, (String, bool, bool), TextSize), + __0: (TextSize, (Box, bool, bool), TextSize), ) -> Result> { let __start0 = __0.2; @@ -65035,7 +65035,7 @@ fn __action1333< >( source_code: &str, mode: Mode, - __0: (TextSize, String, TextSize), + __0: (TextSize, Box, TextSize), ) -> ast::Identifier { let __start0 = __0.2; @@ -65347,7 +65347,7 @@ fn __action1344< >( source_code: &str, mode: Mode, - __0: (TextSize, (IpyEscapeKind, String), TextSize), + __0: (TextSize, (IpyEscapeKind, Box), TextSize), ) -> Result> { let __start0 = __0.2; @@ -65373,7 +65373,7 @@ fn __action1345< >( source_code: &str, mode: Mode, - __0: (TextSize, (IpyEscapeKind, String), TextSize), + __0: (TextSize, (IpyEscapeKind, Box), TextSize), ) -> Result> { let __start0 = __0.2; @@ -65430,7 +65430,7 @@ fn __action1347< __0: (TextSize, token::Tok, TextSize), __1: (TextSize, core::option::Option, TextSize), __2: (TextSize, token::Tok, TextSize), - __3: (TextSize, core::option::Option<(String, bool, bool)>, TextSize), + __3: (TextSize, core::option::Option<(Box, bool, bool)>, TextSize), __4: (TextSize, crate::parser::ParenthesizedExpr, TextSize), ) -> Result> { @@ -69997,7 +69997,7 @@ fn __action1494< >( source_code: &str, mode: Mode, - __0: (TextSize, (String, StringKind, bool), TextSize), + __0: (TextSize, (Box, StringKind, bool), TextSize), ) -> Result> { let __start0 = __0.2; @@ -77662,7 +77662,7 @@ fn __action1727< __0: (TextSize, token::Tok, TextSize), __1: (TextSize, ast::Parameters, TextSize), __2: (TextSize, token::Tok, TextSize), - __3: (TextSize, core::option::Option<(String, bool, bool)>, TextSize), + __3: (TextSize, core::option::Option<(Box, bool, bool)>, TextSize), __4: (TextSize, crate::parser::ParenthesizedExpr, TextSize), ) -> Result> { @@ -77693,7 +77693,7 @@ fn __action1728< mode: Mode, __0: (TextSize, token::Tok, TextSize), __1: (TextSize, token::Tok, TextSize), - __2: (TextSize, core::option::Option<(String, bool, bool)>, TextSize), + __2: (TextSize, core::option::Option<(Box, bool, bool)>, TextSize), __3: (TextSize, crate::parser::ParenthesizedExpr, TextSize), ) -> Result> { @@ -79598,7 +79598,7 @@ fn __action1785< __0: (TextSize, token::Tok, TextSize), __1: (TextSize, ast::Parameters, TextSize), __2: (TextSize, token::Tok, TextSize), - __3: (TextSize, (String, bool, bool), TextSize), + __3: (TextSize, (Box, bool, bool), TextSize), __4: (TextSize, crate::parser::ParenthesizedExpr, TextSize), ) -> Result> { @@ -79661,7 +79661,7 @@ fn __action1787< mode: Mode, __0: (TextSize, token::Tok, TextSize), __1: (TextSize, token::Tok, TextSize), - __2: (TextSize, (String, bool, bool), TextSize), + __2: (TextSize, (Box, bool, bool), TextSize), __3: (TextSize, crate::parser::ParenthesizedExpr, TextSize), ) -> Result> { diff --git a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__invalid_leading_zero_big.snap b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__invalid_leading_zero_big.snap index 449450d4f65943..c2906398a51415 100644 --- a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__invalid_leading_zero_big.snap +++ b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__invalid_leading_zero_big.snap @@ -3,12 +3,10 @@ source: crates/ruff_python_parser/src/lexer.rs expression: tokens --- Err( - LexicalError( - LexicalErrorInner { - error: OtherError( - "Invalid Token", - ), - location: 0, - }, - ), + LexicalError { + error: OtherError( + "Invalid Token", + ), + location: 0, + }, ) diff --git a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__invalid_leading_zero_small.snap b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__invalid_leading_zero_small.snap index 449450d4f65943..c2906398a51415 100644 --- a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__invalid_leading_zero_small.snap +++ b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__invalid_leading_zero_small.snap @@ -3,12 +3,10 @@ source: crates/ruff_python_parser/src/lexer.rs expression: tokens --- Err( - LexicalError( - LexicalErrorInner { - error: OtherError( - "Invalid Token", - ), - location: 0, - }, - ), + LexicalError { + error: OtherError( + "Invalid Token", + ), + location: 0, + }, ) diff --git a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__tet_too_low_dedent.snap b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__tet_too_low_dedent.snap index 55d67c5867f392..8a9ba410ae92da 100644 --- a/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__tet_too_low_dedent.snap +++ b/crates/ruff_python_parser/src/snapshots/ruff_python_parser__lexer__tests__tet_too_low_dedent.snap @@ -46,12 +46,10 @@ expression: tokens ), ), Err( - LexicalError( - LexicalErrorInner { - error: IndentationError, - location: 20, - }, - ), + LexicalError { + error: IndentationError, + location: 20, + }, ), Ok( ( diff --git a/crates/ruff_python_parser/src/soft_keywords.rs b/crates/ruff_python_parser/src/soft_keywords.rs index 379ae1c08db380..e4bff73edc28c8 100644 --- a/crates/ruff_python_parser/src/soft_keywords.rs +++ b/crates/ruff_python_parser/src/soft_keywords.rs @@ -203,7 +203,7 @@ fn soft_to_name(tok: &Tok) -> Tok { _ => unreachable!("other tokens never reach here"), }; Tok::Name { - name: name.to_owned(), + name: name.to_string().into_boxed_str(), } } diff --git a/crates/ruff_python_parser/src/string.rs b/crates/ruff_python_parser/src/string.rs index b9d23774e826e0..5b15474cf2dd61 100644 --- a/crates/ruff_python_parser/src/string.rs +++ b/crates/ruff_python_parser/src/string.rs @@ -186,7 +186,9 @@ impl<'a> StringParser<'a> { if self.kind.is_any_bytes() && !first_char.is_ascii() { return Err(LexicalError::new( LexicalErrorType::OtherError( - "bytes can only contain ASCII literal characters".to_owned(), + "bytes can only contain ASCII literal characters" + .to_string() + .into_boxed_str(), ), self.get_pos(), )); @@ -257,7 +259,9 @@ impl<'a> StringParser<'a> { if !ch.is_ascii() { return Err(LexicalError::new( LexicalErrorType::OtherError( - "bytes can only contain ASCII literal characters".to_string(), + "bytes can only contain ASCII literal characters" + .to_string() + .into_boxed_str(), ), self.get_pos(), )); @@ -291,7 +295,7 @@ impl<'a> StringParser<'a> { } } Ok(StringType::Str(ast::StringLiteral { - value, + value: value.into_boxed_str(), unicode: self.kind.is_unicode(), range: self.range, })) @@ -355,7 +359,11 @@ pub(crate) fn concatenated_strings( if has_bytes && byte_literal_count < strings.len() { return Err(LexicalError::new( - LexicalErrorType::OtherError("cannot mix bytes and nonbytes literals".to_owned()), + LexicalErrorType::OtherError( + "cannot mix bytes and nonbytes literals" + .to_string() + .into_boxed_str(), + ), range.start(), )); } @@ -421,7 +429,7 @@ impl From for LexicalError { } /// Represents the different types of errors that can occur during parsing of an f-string. -#[derive(Debug, Clone, PartialEq)] +#[derive(Copy, Debug, Clone, PartialEq)] pub enum FStringErrorType { /// Expected a right brace after an opened left brace. UnclosedLbrace, diff --git a/crates/ruff_python_parser/src/token.rs b/crates/ruff_python_parser/src/token.rs index 9472eb1ed20f09..f177f4eaad9d1d 100644 --- a/crates/ruff_python_parser/src/token.rs +++ b/crates/ruff_python_parser/src/token.rs @@ -16,7 +16,7 @@ pub enum Tok { /// Token value for a name, commonly known as an identifier. Name { /// The name value. - name: String, + name: Box, }, /// Token value for an integer. Int { @@ -38,7 +38,7 @@ pub enum Tok { /// Token value for a string. String { /// The string value. - value: String, + value: Box, /// The kind of string. kind: StringKind, /// Whether the string is triple quoted. @@ -51,7 +51,7 @@ pub enum Tok { /// part of the expression part and isn't an opening or closing brace. FStringMiddle { /// The string value. - value: String, + value: Box, /// Whether the string is raw or not. is_raw: bool, /// Whether the string is triple quoted. @@ -63,12 +63,12 @@ pub enum Tok { /// only when the mode is [`Mode::Ipython`]. IpyEscapeCommand { /// The magic command value. - value: String, + value: Box, /// The kind of magic command. kind: IpyEscapeKind, }, /// Token value for a comment. These are filtered out of the token stream prior to parsing. - Comment(String), + Comment(Box), /// Token value for a newline. Newline, /// Token value for a newline that is not a logical line break. These are filtered out of @@ -919,6 +919,6 @@ mod sizes { use crate::Tok; use static_assertions::assert_eq_size; - assert_eq_size!(Tok, [u8; 32]); + assert_eq_size!(Tok, [u8; 24]); assert_eq_size!(Result, [u8; 32]); }