From 052dee72b8e0bedb2b67b74190d28198bb4e0f26 Mon Sep 17 00:00:00 2001 From: Anders Kaseorg Date: Sun, 11 Dec 2022 02:55:56 -0800 Subject: [PATCH] Parse Python 3.9+ parenthesized context managers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since the upstream grammar for this is not LR(1), we abuse LALRPOP macros and the Into/TryInto traits to build a cover grammar that converts to either tuples or `with` items after additional validation. It’s annoying and ugly, but something like this is basically our only option short of switching to a more powerful parser algorithm. Fixes #4145. Signed-off-by: Anders Kaseorg --- parser/python.lalrpop | 216 +- parser/src/lib.rs | 1 + ...n_parser__with__tests__with_statement.snap | 2094 +++++++++++++++++ parser/src/with.rs | 188 ++ 4 files changed, 2413 insertions(+), 86 deletions(-) create mode 100644 parser/src/snapshots/rustpython_parser__with__tests__with_statement.snap create mode 100644 parser/src/with.rs diff --git a/parser/python.lalrpop b/parser/python.lalrpop index 565130cbf9ffa..1b82e7c8cc92f 100644 --- a/parser/python.lalrpop +++ b/parser/python.lalrpop @@ -10,7 +10,8 @@ use crate::{ lexer, context::set_context, string::parse_strings, - token::StringKind + token::StringKind, + with::{ExprOrWithitems, TupleOrWithitems}, }; use num_bigint::BigInt; @@ -159,10 +160,6 @@ TestOrStarExprList: ast::Expr = { TestList }; -TestOrStarNamedExprList: ast::Expr = { - GenericList -}; - TestOrStarExpr: ast::Expr = { Test, StarExpr, @@ -173,6 +170,12 @@ TestOrStarNamedExpr: ast::Expr = { StarExpr, }; +TestOrStarNamedExprOrWithitem: (ast::Expr, Option>) = { + => (e, None), + => (e, None), + "as" => (e, Some(Box::new(v))), +} + AugAssign: ast::Operator = { "+=" => ast::Operator::Add, "-=" => ast::Operator::Sub, @@ -472,7 +475,7 @@ ExceptClause: ast::Excepthandler = { }; WithStatement: ast::Stmt = { - "with" > ":" => { + "with" ":" => { let type_comment = None; let node = if is_async.is_some() { ast::StmtKind::AsyncWith { items, body, type_comment } @@ -483,6 +486,25 @@ WithStatement: ast::Stmt = { }, }; +// These are only used for their types as macro parameters +ExprGoal: ast::Expr = {}; +ExprOrWithitemsGoal: ExprOrWithitems = {}; + +WithItems: Vec = { + > =>? items.try_into(), + > "as" =>? { + let optional_vars = Some(Box::new(set_context(vars, ast::ExprContext::Store))); + let context_expr = Box::new(first.try_into()?); + Ok(vec![ast::Withitem { context_expr, optional_vars }]) + }, + > "," > =>? { + let optional_vars = n.map(|val| Box::new(set_context(val.1, ast::ExprContext::Store))); + let context_expr = Box::new(first.try_into()?); + items.insert(0, ast::Withitem { context_expr, optional_vars }); + Ok(items) + } +}; + WithItem: ast::Withitem = { => { let optional_vars = n.map(|val| Box::new(set_context(val.1, ast::ExprContext::Store))); @@ -688,7 +710,8 @@ YieldExpr: ast::Expr = { }, }; -Test: ast::Expr = { +Test = TestAs; +TestAs: Goal = { "if" "else" => ast::Expr { location, end_location: Some(end_location), @@ -698,9 +721,9 @@ Test: ast::Expr = { body: Box::new(body), orelse: Box::new(orelse), } - }, - OrTest, - LambdaDef, + }.into(), + OrTestAs, + => e.into(), }; NamedExpressionTest: ast::Expr = { @@ -750,7 +773,8 @@ LambdaDef: ast::Expr = { } } -OrTest: ast::Expr = { +OrTest = OrTestAs; +OrTestAs: Goal = { => { let mut values = vec![e1]; values.extend(e2.into_iter().map(|e| e.1)); @@ -759,12 +783,13 @@ OrTest: ast::Expr = { end_location: Some(end_location), custom: (), node: ast::ExprKind::BoolOp { op: ast::Boolop::Or, values } - } + }.into() }, - AndTest, + AndTestAs, }; -AndTest: ast::Expr = { +AndTest = AndTestAs; +AndTestAs: Goal = { => { let mut values = vec![e1]; values.extend(e2.into_iter().map(|e| e.1)); @@ -773,22 +798,24 @@ AndTest: ast::Expr = { end_location: Some(end_location), custom: (), node: ast::ExprKind::BoolOp { op: ast::Boolop::And, values } - } + }.into() }, - NotTest, + NotTestAs, }; -NotTest: ast::Expr = { +NotTest = NotTestAs; +NotTestAs: Goal = { "not" => ast::Expr { location, end_location: Some(end_location), custom: (), node: ast::ExprKind::UnaryOp { operand: Box::new(e), op: ast::Unaryop::Not } - }, - Comparison, + }.into(), + ComparisonAs, }; -Comparison: ast::Expr = { +Comparison = ComparisonAs; +ComparisonAs: Goal = { => { let (ops, comparators) = comparisons.into_iter().unzip(); ast::Expr { @@ -796,9 +823,9 @@ Comparison: ast::Expr = { end_location: Some(end_location), custom: (), node: ast::ExprKind::Compare { left: Box::new(left), ops, comparators } - } + }.into() }, - Expression, + ExpressionAs, }; CompOp: ast::Cmpop = { @@ -814,44 +841,48 @@ CompOp: ast::Cmpop = { "is" "not" => ast::Cmpop::IsNot, }; -Expression: ast::Expr = { +Expression = ExpressionAs; +ExpressionAs: Goal = { "|" => ast::Expr { location, end_location: Some(end_location), custom: (), node: ast::ExprKind::BinOp { left: Box::new(e1), op: ast::Operator::BitOr, right: Box::new(e2) } - }, - XorExpression, + }.into(), + XorExpressionAs, }; -XorExpression: ast::Expr = { +XorExpression = XorExpressionAs; +XorExpressionAs: Goal = { "^" => ast::Expr { location, end_location: Some(end_location), custom: (), node: ast::ExprKind::BinOp { left: Box::new(e1), op: ast::Operator::BitXor, right: Box::new(e2) } - }, - AndExpression, + }.into(), + AndExpressionAs, }; -AndExpression: ast::Expr = { +AndExpression = AndExpressionAs; +AndExpressionAs: Goal = { "&" => ast::Expr { location, end_location: Some(end_location), custom: (), node: ast::ExprKind::BinOp { left: Box::new(e1), op: ast::Operator::BitAnd, right: Box::new(e2) } - }, - ShiftExpression, + }.into(), + ShiftExpressionAs, }; -ShiftExpression: ast::Expr = { +ShiftExpression = ShiftExpressionAs; +ShiftExpressionAs: Goal = { => ast::Expr { location, end_location: Some(end_location), custom: (), node: ast::ExprKind::BinOp { left: Box::new(e1), op, right: Box::new(e2) } - }, - ArithmeticExpression, + }.into(), + ArithmeticExpressionAs, }; ShiftOp: ast::Operator = { @@ -859,14 +890,15 @@ ShiftOp: ast::Operator = { ">>" => ast::Operator::RShift, }; -ArithmeticExpression: ast::Expr = { +ArithmeticExpression = ArithmeticExpressionAs; +ArithmeticExpressionAs: Goal = { => ast::Expr { location, end_location: Some(end_location), custom: (), node: ast::ExprKind::BinOp { left: Box::new(a), op, right: Box::new(b) } - }, - Term, + }.into(), + TermAs, }; AddOp: ast::Operator = { @@ -874,14 +906,15 @@ AddOp: ast::Operator = { "-" => ast::Operator::Sub, }; -Term: ast::Expr = { +Term = TermAs; +TermAs: Goal = { => ast::Expr { location, end_location: Some(end_location), custom: (), node: ast::ExprKind::BinOp { left: Box::new(a), op, right: Box::new(b) } - }, - Factor, + }.into(), + FactorAs, }; MulOp: ast::Operator = { @@ -892,14 +925,15 @@ MulOp: ast::Operator = { "@" => ast::Operator::MatMult, }; -Factor: ast::Expr = { +Factor = FactorAs; +FactorAs: Goal = { => ast::Expr { location, end_location: Some(end_location), custom: (), node: ast::ExprKind::UnaryOp { operand: Box::new(e), op } - }, - Power, + }.into(), + PowerAs, }; UnaryOp: ast::Unaryop = { @@ -908,48 +942,53 @@ UnaryOp: ast::Unaryop = { "~" => ast::Unaryop::Invert, }; -Power: ast::Expr = { +Power = PowerAs; +PowerAs: Goal = { "**" => ast::Expr { location, end_location: Some(end_location), custom: (), node: ast::ExprKind::BinOp { left: Box::new(e), op: ast::Operator::Pow, right: Box::new(b) } - }, - AtomExpr, + }.into(), + AtomExprAs, }; -AtomExpr: ast::Expr = { - "await" => ast::Expr { - location, - end_location: Some(end_location), - custom: (), - node: ast::ExprKind::Await { value: Box::new(atom) } +AtomExpr = AtomExprAs; +AtomExprAs: Goal = { + "await" => { + ast::Expr { + location, + end_location: Some(end_location), + custom: (), + node: ast::ExprKind::Await { value: Box::new(atom) } + }.into() }, - AtomExpr2, + AtomExpr2As, } -AtomExpr2: ast::Expr = { - Atom, +AtomExpr2 = AtomExpr2As; +AtomExpr2As: Goal = { + AtomAs, "(" ")" => { ast::Expr { location, end_location: Some(end_location), custom: (), node: ast::ExprKind::Call { func: Box::new(f), args: a.args, keywords: a.keywords } - } + }.into() }, "[" "]" => ast::Expr { location, end_location: Some(end_location), custom: (), node: ast::ExprKind::Subscript { value: Box::new(e), slice: Box::new(s), ctx: ast::ExprContext::Load } - }, + }.into(), "." => ast::Expr { location, end_location: Some(end_location), custom: (), node: ast::ExprKind::Attribute { value: Box::new(e), attr, ctx: ast::ExprContext::Load } - }, + }.into(), }; SubscriptList: ast::Expr = { @@ -991,20 +1030,21 @@ SliceOp: Option = { ":" => e, } -Atom: ast::Expr = { - =>? parse_strings(s).map_err(|e| e.into()), +Atom = AtomAs; +AtomAs: Goal = { + =>? Ok(parse_strings(s)?.into()), => ast::Expr { location, end_location: Some(end_location), custom: (), node: ast::ExprKind::Constant { value, kind: None } - }, + }.into(), => ast::Expr { location, end_location: Some(end_location), custom: (), node: ast::ExprKind::Name { id: name, ctx: ast::ExprContext::Load } - }, + }.into(), "[" "]" => { let elts = e.unwrap_or_default(); ast::Expr { @@ -1012,7 +1052,7 @@ Atom: ast::Expr = { end_location: Some(end_location), custom: (), node: ast::ExprKind::List { elts, ctx: ast::ExprContext::Load } - } + }.into() }, "[" "]" => { ast::Expr { @@ -1020,34 +1060,38 @@ Atom: ast::Expr = { end_location: Some(end_location), custom: (), node: ast::ExprKind::ListComp { elt: Box::new(elt), generators } - } - }, - "(" ")" =>? { - match elt.node { - ast::ExprKind::Starred { .. } => { - Err(LexicalError{ - error: LexicalErrorType::OtherError("cannot use starred expression here".to_string()), - location: elt.location, - }.into()) - } - _ => { - Ok(elt) + }.into() + }, + "(" > ")" =>? { + if items.len() == 1 && items[0].1.is_none() && trailing_comma.is_none() { + match items[0].0.node { + ast::ExprKind::Starred { .. } => { + Err(LexicalError{ + error: LexicalErrorType::OtherError("cannot use starred expression here".to_string()), + location: items[0].0.location, + }.into()) + } + _ => { + Ok(items.into_iter().next().unwrap().0.into()) + } } + } else { + TupleOrWithitems { location, end_location, items }.try_into() } }, "(" ")" => ast::Expr::new( location, end_location, ast::ExprKind::Tuple { elts: Vec::new(), ctx: ast::ExprContext::Load } - ), - "(" ")" => e, + ).into(), + "(" ")" => e.into(), "(" ")" => { ast::Expr { location, end_location: Some(end_location), custom: (), node: ast::ExprKind::GeneratorExp { elt: Box::new(elt), generators } - } + }.into() }, "(" "**" ")" =>? { Err(LexicalError{ @@ -1099,7 +1143,7 @@ Atom: ast::Expr = { end_location: Some(end_location), custom: (), node: ast::ExprKind::Dict { keys, values } - } + }.into() }, "{" "}" => { ast::Expr { @@ -1111,26 +1155,26 @@ Atom: ast::Expr = { value: Box::new(e1.1), generators, } - } + }.into() }, "{" "}" => ast::Expr { location, end_location: Some(end_location), custom: (), node: ast::ExprKind::Set { elts } - }, + }.into(), "{" "}" => { ast::Expr { location, end_location: Some(end_location), custom: (), node: ast::ExprKind::SetComp { elt: Box::new(elt), generators } - } + }.into() }, - "True" => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: true.into(), kind: None }), - "False" => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: false.into(), kind: None }), - "None" => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: ast::Constant::None, kind: None }), - "..." => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: ast::Constant::Ellipsis, kind: None }), + "True" => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: true.into(), kind: None }).into(), + "False" => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: false.into(), kind: None }).into(), + "None" => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: ast::Constant::None, kind: None }).into(), + "..." => ast::Expr::new(location, end_location, ast::ExprKind::Constant { value: ast::Constant::Ellipsis, kind: None }).into(), }; ListLiteralValues: Vec = { diff --git a/parser/src/lib.rs b/parser/src/lib.rs index 0bd363a82ed2b..66e565370bf5e 100644 --- a/parser/src/lib.rs +++ b/parser/src/lib.rs @@ -33,3 +33,4 @@ mod python; mod context; mod string; pub mod token; +mod with; diff --git a/parser/src/snapshots/rustpython_parser__with__tests__with_statement.snap b/parser/src/snapshots/rustpython_parser__with__tests__with_statement.snap new file mode 100644 index 0000000000000..e82ee2d0f3e71 --- /dev/null +++ b/parser/src/snapshots/rustpython_parser__with__tests__with_statement.snap @@ -0,0 +1,2094 @@ +--- +source: compiler/parser/src/with.rs +expression: "parse_program(source, \"\").unwrap()" +--- +[ + Located { + location: Location { + row: 1, + column: 0, + }, + end_location: Some( + Location { + row: 2, + column: 0, + }, + ), + custom: (), + node: With { + items: [ + Withitem { + context_expr: Located { + location: Location { + row: 1, + column: 5, + }, + end_location: Some( + Location { + row: 1, + column: 6, + }, + ), + custom: (), + node: Constant { + value: Int( + 0, + ), + kind: None, + }, + }, + optional_vars: None, + }, + ], + body: [ + Located { + location: Location { + row: 1, + column: 8, + }, + end_location: Some( + Location { + row: 1, + column: 12, + }, + ), + custom: (), + node: Pass, + }, + ], + type_comment: None, + }, + }, + Located { + location: Location { + row: 2, + column: 0, + }, + end_location: Some( + Location { + row: 3, + column: 0, + }, + ), + custom: (), + node: With { + items: [ + Withitem { + context_expr: Located { + location: Location { + row: 2, + column: 5, + }, + end_location: Some( + Location { + row: 2, + column: 6, + }, + ), + custom: (), + node: Constant { + value: Int( + 0, + ), + kind: None, + }, + }, + optional_vars: Some( + Located { + location: Location { + row: 2, + column: 10, + }, + end_location: Some( + Location { + row: 2, + column: 11, + }, + ), + custom: (), + node: Name { + id: "x", + ctx: Store, + }, + }, + ), + }, + ], + body: [ + Located { + location: Location { + row: 2, + column: 13, + }, + end_location: Some( + Location { + row: 2, + column: 17, + }, + ), + custom: (), + node: Pass, + }, + ], + type_comment: None, + }, + }, + Located { + location: Location { + row: 3, + column: 0, + }, + end_location: Some( + Location { + row: 4, + column: 0, + }, + ), + custom: (), + node: With { + items: [ + Withitem { + context_expr: Located { + location: Location { + row: 3, + column: 5, + }, + end_location: Some( + Location { + row: 3, + column: 6, + }, + ), + custom: (), + node: Constant { + value: Int( + 0, + ), + kind: None, + }, + }, + optional_vars: None, + }, + Withitem { + context_expr: Located { + location: Location { + row: 3, + column: 8, + }, + end_location: Some( + Location { + row: 3, + column: 9, + }, + ), + custom: (), + node: Constant { + value: Int( + 1, + ), + kind: None, + }, + }, + optional_vars: None, + }, + ], + body: [ + Located { + location: Location { + row: 3, + column: 11, + }, + end_location: Some( + Location { + row: 3, + column: 15, + }, + ), + custom: (), + node: Pass, + }, + ], + type_comment: None, + }, + }, + Located { + location: Location { + row: 4, + column: 0, + }, + end_location: Some( + Location { + row: 5, + column: 0, + }, + ), + custom: (), + node: With { + items: [ + Withitem { + context_expr: Located { + location: Location { + row: 4, + column: 5, + }, + end_location: Some( + Location { + row: 4, + column: 6, + }, + ), + custom: (), + node: Constant { + value: Int( + 0, + ), + kind: None, + }, + }, + optional_vars: Some( + Located { + location: Location { + row: 4, + column: 10, + }, + end_location: Some( + Location { + row: 4, + column: 11, + }, + ), + custom: (), + node: Name { + id: "x", + ctx: Store, + }, + }, + ), + }, + Withitem { + context_expr: Located { + location: Location { + row: 4, + column: 13, + }, + end_location: Some( + Location { + row: 4, + column: 14, + }, + ), + custom: (), + node: Constant { + value: Int( + 1, + ), + kind: None, + }, + }, + optional_vars: Some( + Located { + location: Location { + row: 4, + column: 18, + }, + end_location: Some( + Location { + row: 4, + column: 19, + }, + ), + custom: (), + node: Name { + id: "y", + ctx: Store, + }, + }, + ), + }, + ], + body: [ + Located { + location: Location { + row: 4, + column: 21, + }, + end_location: Some( + Location { + row: 4, + column: 25, + }, + ), + custom: (), + node: Pass, + }, + ], + type_comment: None, + }, + }, + Located { + location: Location { + row: 5, + column: 0, + }, + end_location: Some( + Location { + row: 6, + column: 0, + }, + ), + custom: (), + node: With { + items: [ + Withitem { + context_expr: Located { + location: Location { + row: 5, + column: 7, + }, + end_location: Some( + Location { + row: 5, + column: 18, + }, + ), + custom: (), + node: IfExp { + test: Located { + location: Location { + row: 5, + column: 10, + }, + end_location: Some( + Location { + row: 5, + column: 11, + }, + ), + custom: (), + node: Constant { + value: Int( + 1, + ), + kind: None, + }, + }, + body: Located { + location: Location { + row: 5, + column: 5, + }, + end_location: Some( + Location { + row: 5, + column: 6, + }, + ), + custom: (), + node: Constant { + value: Int( + 0, + ), + kind: None, + }, + }, + orelse: Located { + location: Location { + row: 5, + column: 17, + }, + end_location: Some( + Location { + row: 5, + column: 18, + }, + ), + custom: (), + node: Constant { + value: Int( + 2, + ), + kind: None, + }, + }, + }, + }, + optional_vars: None, + }, + ], + body: [ + Located { + location: Location { + row: 5, + column: 20, + }, + end_location: Some( + Location { + row: 5, + column: 24, + }, + ), + custom: (), + node: Pass, + }, + ], + type_comment: None, + }, + }, + Located { + location: Location { + row: 6, + column: 0, + }, + end_location: Some( + Location { + row: 7, + column: 0, + }, + ), + custom: (), + node: With { + items: [ + Withitem { + context_expr: Located { + location: Location { + row: 6, + column: 7, + }, + end_location: Some( + Location { + row: 6, + column: 18, + }, + ), + custom: (), + node: IfExp { + test: Located { + location: Location { + row: 6, + column: 10, + }, + end_location: Some( + Location { + row: 6, + column: 11, + }, + ), + custom: (), + node: Constant { + value: Int( + 1, + ), + kind: None, + }, + }, + body: Located { + location: Location { + row: 6, + column: 5, + }, + end_location: Some( + Location { + row: 6, + column: 6, + }, + ), + custom: (), + node: Constant { + value: Int( + 0, + ), + kind: None, + }, + }, + orelse: Located { + location: Location { + row: 6, + column: 17, + }, + end_location: Some( + Location { + row: 6, + column: 18, + }, + ), + custom: (), + node: Constant { + value: Int( + 2, + ), + kind: None, + }, + }, + }, + }, + optional_vars: Some( + Located { + location: Location { + row: 6, + column: 22, + }, + end_location: Some( + Location { + row: 6, + column: 23, + }, + ), + custom: (), + node: Name { + id: "x", + ctx: Store, + }, + }, + ), + }, + ], + body: [ + Located { + location: Location { + row: 6, + column: 25, + }, + end_location: Some( + Location { + row: 6, + column: 29, + }, + ), + custom: (), + node: Pass, + }, + ], + type_comment: None, + }, + }, + Located { + location: Location { + row: 7, + column: 0, + }, + end_location: Some( + Location { + row: 8, + column: 0, + }, + ), + custom: (), + node: With { + items: [ + Withitem { + context_expr: Located { + location: Location { + row: 7, + column: 5, + }, + end_location: Some( + Location { + row: 7, + column: 7, + }, + ), + custom: (), + node: Tuple { + elts: [], + ctx: Load, + }, + }, + optional_vars: None, + }, + ], + body: [ + Located { + location: Location { + row: 7, + column: 9, + }, + end_location: Some( + Location { + row: 7, + column: 13, + }, + ), + custom: (), + node: Pass, + }, + ], + type_comment: None, + }, + }, + Located { + location: Location { + row: 8, + column: 0, + }, + end_location: Some( + Location { + row: 9, + column: 0, + }, + ), + custom: (), + node: With { + items: [ + Withitem { + context_expr: Located { + location: Location { + row: 8, + column: 5, + }, + end_location: Some( + Location { + row: 8, + column: 7, + }, + ), + custom: (), + node: Tuple { + elts: [], + ctx: Load, + }, + }, + optional_vars: Some( + Located { + location: Location { + row: 8, + column: 11, + }, + end_location: Some( + Location { + row: 8, + column: 12, + }, + ), + custom: (), + node: Name { + id: "x", + ctx: Store, + }, + }, + ), + }, + ], + body: [ + Located { + location: Location { + row: 8, + column: 14, + }, + end_location: Some( + Location { + row: 8, + column: 18, + }, + ), + custom: (), + node: Pass, + }, + ], + type_comment: None, + }, + }, + Located { + location: Location { + row: 9, + column: 0, + }, + end_location: Some( + Location { + row: 10, + column: 0, + }, + ), + custom: (), + node: With { + items: [ + Withitem { + context_expr: Located { + location: Location { + row: 9, + column: 6, + }, + end_location: Some( + Location { + row: 9, + column: 7, + }, + ), + custom: (), + node: Constant { + value: Int( + 0, + ), + kind: None, + }, + }, + optional_vars: None, + }, + ], + body: [ + Located { + location: Location { + row: 9, + column: 10, + }, + end_location: Some( + Location { + row: 9, + column: 14, + }, + ), + custom: (), + node: Pass, + }, + ], + type_comment: None, + }, + }, + Located { + location: Location { + row: 10, + column: 0, + }, + end_location: Some( + Location { + row: 11, + column: 0, + }, + ), + custom: (), + node: With { + items: [ + Withitem { + context_expr: Located { + location: Location { + row: 10, + column: 6, + }, + end_location: Some( + Location { + row: 10, + column: 7, + }, + ), + custom: (), + node: Constant { + value: Int( + 0, + ), + kind: None, + }, + }, + optional_vars: Some( + Located { + location: Location { + row: 10, + column: 12, + }, + end_location: Some( + Location { + row: 10, + column: 13, + }, + ), + custom: (), + node: Name { + id: "x", + ctx: Store, + }, + }, + ), + }, + ], + body: [ + Located { + location: Location { + row: 10, + column: 15, + }, + end_location: Some( + Location { + row: 10, + column: 19, + }, + ), + custom: (), + node: Pass, + }, + ], + type_comment: None, + }, + }, + Located { + location: Location { + row: 11, + column: 0, + }, + end_location: Some( + Location { + row: 12, + column: 0, + }, + ), + custom: (), + node: With { + items: [ + Withitem { + context_expr: Located { + location: Location { + row: 11, + column: 6, + }, + end_location: Some( + Location { + row: 11, + column: 7, + }, + ), + custom: (), + node: Constant { + value: Int( + 0, + ), + kind: None, + }, + }, + optional_vars: None, + }, + ], + body: [ + Located { + location: Location { + row: 11, + column: 11, + }, + end_location: Some( + Location { + row: 11, + column: 15, + }, + ), + custom: (), + node: Pass, + }, + ], + type_comment: None, + }, + }, + Located { + location: Location { + row: 12, + column: 0, + }, + end_location: Some( + Location { + row: 13, + column: 0, + }, + ), + custom: (), + node: With { + items: [ + Withitem { + context_expr: Located { + location: Location { + row: 12, + column: 6, + }, + end_location: Some( + Location { + row: 12, + column: 8, + }, + ), + custom: (), + node: Tuple { + elts: [ + Located { + location: Location { + row: 12, + column: 6, + }, + end_location: Some( + Location { + row: 12, + column: 7, + }, + ), + custom: (), + node: Constant { + value: Int( + 0, + ), + kind: None, + }, + }, + ], + ctx: Load, + }, + }, + optional_vars: Some( + Located { + location: Location { + row: 12, + column: 13, + }, + end_location: Some( + Location { + row: 12, + column: 14, + }, + ), + custom: (), + node: Name { + id: "x", + ctx: Store, + }, + }, + ), + }, + ], + body: [ + Located { + location: Location { + row: 12, + column: 16, + }, + end_location: Some( + Location { + row: 12, + column: 20, + }, + ), + custom: (), + node: Pass, + }, + ], + type_comment: None, + }, + }, + Located { + location: Location { + row: 13, + column: 0, + }, + end_location: Some( + Location { + row: 14, + column: 0, + }, + ), + custom: (), + node: With { + items: [ + Withitem { + context_expr: Located { + location: Location { + row: 13, + column: 6, + }, + end_location: Some( + Location { + row: 13, + column: 7, + }, + ), + custom: (), + node: Constant { + value: Int( + 0, + ), + kind: None, + }, + }, + optional_vars: None, + }, + Withitem { + context_expr: Located { + location: Location { + row: 13, + column: 9, + }, + end_location: Some( + Location { + row: 13, + column: 10, + }, + ), + custom: (), + node: Constant { + value: Int( + 1, + ), + kind: None, + }, + }, + optional_vars: None, + }, + ], + body: [ + Located { + location: Location { + row: 13, + column: 13, + }, + end_location: Some( + Location { + row: 13, + column: 17, + }, + ), + custom: (), + node: Pass, + }, + ], + type_comment: None, + }, + }, + Located { + location: Location { + row: 14, + column: 0, + }, + end_location: Some( + Location { + row: 15, + column: 0, + }, + ), + custom: (), + node: With { + items: [ + Withitem { + context_expr: Located { + location: Location { + row: 14, + column: 6, + }, + end_location: Some( + Location { + row: 14, + column: 10, + }, + ), + custom: (), + node: Tuple { + elts: [ + Located { + location: Location { + row: 14, + column: 6, + }, + end_location: Some( + Location { + row: 14, + column: 7, + }, + ), + custom: (), + node: Constant { + value: Int( + 0, + ), + kind: None, + }, + }, + Located { + location: Location { + row: 14, + column: 9, + }, + end_location: Some( + Location { + row: 14, + column: 10, + }, + ), + custom: (), + node: Constant { + value: Int( + 1, + ), + kind: None, + }, + }, + ], + ctx: Load, + }, + }, + optional_vars: Some( + Located { + location: Location { + row: 14, + column: 15, + }, + end_location: Some( + Location { + row: 14, + column: 16, + }, + ), + custom: (), + node: Name { + id: "x", + ctx: Store, + }, + }, + ), + }, + ], + body: [ + Located { + location: Location { + row: 14, + column: 18, + }, + end_location: Some( + Location { + row: 14, + column: 22, + }, + ), + custom: (), + node: Pass, + }, + ], + type_comment: None, + }, + }, + Located { + location: Location { + row: 15, + column: 0, + }, + end_location: Some( + Location { + row: 16, + column: 0, + }, + ), + custom: (), + node: With { + items: [ + Withitem { + context_expr: Located { + location: Location { + row: 15, + column: 6, + }, + end_location: Some( + Location { + row: 15, + column: 9, + }, + ), + custom: (), + node: Tuple { + elts: [ + Located { + location: Location { + row: 15, + column: 6, + }, + end_location: Some( + Location { + row: 15, + column: 8, + }, + ), + custom: (), + node: Starred { + value: Located { + location: Location { + row: 15, + column: 7, + }, + end_location: Some( + Location { + row: 15, + column: 8, + }, + ), + custom: (), + node: Name { + id: "a", + ctx: Load, + }, + }, + ctx: Load, + }, + }, + ], + ctx: Load, + }, + }, + optional_vars: None, + }, + ], + body: [ + Located { + location: Location { + row: 15, + column: 12, + }, + end_location: Some( + Location { + row: 15, + column: 16, + }, + ), + custom: (), + node: Pass, + }, + ], + type_comment: None, + }, + }, + Located { + location: Location { + row: 16, + column: 0, + }, + end_location: Some( + Location { + row: 17, + column: 0, + }, + ), + custom: (), + node: With { + items: [ + Withitem { + context_expr: Located { + location: Location { + row: 16, + column: 6, + }, + end_location: Some( + Location { + row: 16, + column: 9, + }, + ), + custom: (), + node: Tuple { + elts: [ + Located { + location: Location { + row: 16, + column: 6, + }, + end_location: Some( + Location { + row: 16, + column: 8, + }, + ), + custom: (), + node: Starred { + value: Located { + location: Location { + row: 16, + column: 7, + }, + end_location: Some( + Location { + row: 16, + column: 8, + }, + ), + custom: (), + node: Name { + id: "a", + ctx: Load, + }, + }, + ctx: Load, + }, + }, + ], + ctx: Load, + }, + }, + optional_vars: Some( + Located { + location: Location { + row: 16, + column: 14, + }, + end_location: Some( + Location { + row: 16, + column: 15, + }, + ), + custom: (), + node: Name { + id: "x", + ctx: Store, + }, + }, + ), + }, + ], + body: [ + Located { + location: Location { + row: 16, + column: 17, + }, + end_location: Some( + Location { + row: 16, + column: 21, + }, + ), + custom: (), + node: Pass, + }, + ], + type_comment: None, + }, + }, + Located { + location: Location { + row: 17, + column: 0, + }, + end_location: Some( + Location { + row: 18, + column: 0, + }, + ), + custom: (), + node: With { + items: [ + Withitem { + context_expr: Located { + location: Location { + row: 17, + column: 6, + }, + end_location: Some( + Location { + row: 17, + column: 11, + }, + ), + custom: (), + node: Tuple { + elts: [ + Located { + location: Location { + row: 17, + column: 6, + }, + end_location: Some( + Location { + row: 17, + column: 7, + }, + ), + custom: (), + node: Constant { + value: Int( + 0, + ), + kind: None, + }, + }, + Located { + location: Location { + row: 17, + column: 9, + }, + end_location: Some( + Location { + row: 17, + column: 11, + }, + ), + custom: (), + node: Starred { + value: Located { + location: Location { + row: 17, + column: 10, + }, + end_location: Some( + Location { + row: 17, + column: 11, + }, + ), + custom: (), + node: Name { + id: "a", + ctx: Load, + }, + }, + ctx: Load, + }, + }, + ], + ctx: Load, + }, + }, + optional_vars: None, + }, + ], + body: [ + Located { + location: Location { + row: 17, + column: 14, + }, + end_location: Some( + Location { + row: 17, + column: 18, + }, + ), + custom: (), + node: Pass, + }, + ], + type_comment: None, + }, + }, + Located { + location: Location { + row: 18, + column: 0, + }, + end_location: Some( + Location { + row: 19, + column: 0, + }, + ), + custom: (), + node: With { + items: [ + Withitem { + context_expr: Located { + location: Location { + row: 18, + column: 6, + }, + end_location: Some( + Location { + row: 18, + column: 11, + }, + ), + custom: (), + node: Tuple { + elts: [ + Located { + location: Location { + row: 18, + column: 6, + }, + end_location: Some( + Location { + row: 18, + column: 7, + }, + ), + custom: (), + node: Constant { + value: Int( + 0, + ), + kind: None, + }, + }, + Located { + location: Location { + row: 18, + column: 9, + }, + end_location: Some( + Location { + row: 18, + column: 11, + }, + ), + custom: (), + node: Starred { + value: Located { + location: Location { + row: 18, + column: 10, + }, + end_location: Some( + Location { + row: 18, + column: 11, + }, + ), + custom: (), + node: Name { + id: "a", + ctx: Load, + }, + }, + ctx: Load, + }, + }, + ], + ctx: Load, + }, + }, + optional_vars: Some( + Located { + location: Location { + row: 18, + column: 16, + }, + end_location: Some( + Location { + row: 18, + column: 17, + }, + ), + custom: (), + node: Name { + id: "x", + ctx: Store, + }, + }, + ), + }, + ], + body: [ + Located { + location: Location { + row: 18, + column: 19, + }, + end_location: Some( + Location { + row: 18, + column: 23, + }, + ), + custom: (), + node: Pass, + }, + ], + type_comment: None, + }, + }, + Located { + location: Location { + row: 19, + column: 0, + }, + end_location: Some( + Location { + row: 20, + column: 0, + }, + ), + custom: (), + node: With { + items: [ + Withitem { + context_expr: Located { + location: Location { + row: 19, + column: 6, + }, + end_location: Some( + Location { + row: 19, + column: 12, + }, + ), + custom: (), + node: NamedExpr { + target: Located { + location: Location { + row: 19, + column: 6, + }, + end_location: Some( + Location { + row: 19, + column: 12, + }, + ), + custom: (), + node: Name { + id: "a", + ctx: Store, + }, + }, + value: Located { + location: Location { + row: 19, + column: 11, + }, + end_location: Some( + Location { + row: 19, + column: 12, + }, + ), + custom: (), + node: Constant { + value: Int( + 0, + ), + kind: None, + }, + }, + }, + }, + optional_vars: None, + }, + ], + body: [ + Located { + location: Location { + row: 19, + column: 15, + }, + end_location: Some( + Location { + row: 19, + column: 19, + }, + ), + custom: (), + node: Pass, + }, + ], + type_comment: None, + }, + }, + Located { + location: Location { + row: 20, + column: 0, + }, + end_location: Some( + Location { + row: 21, + column: 0, + }, + ), + custom: (), + node: With { + items: [ + Withitem { + context_expr: Located { + location: Location { + row: 20, + column: 6, + }, + end_location: Some( + Location { + row: 20, + column: 12, + }, + ), + custom: (), + node: NamedExpr { + target: Located { + location: Location { + row: 20, + column: 6, + }, + end_location: Some( + Location { + row: 20, + column: 12, + }, + ), + custom: (), + node: Name { + id: "a", + ctx: Store, + }, + }, + value: Located { + location: Location { + row: 20, + column: 11, + }, + end_location: Some( + Location { + row: 20, + column: 12, + }, + ), + custom: (), + node: Constant { + value: Int( + 0, + ), + kind: None, + }, + }, + }, + }, + optional_vars: Some( + Located { + location: Location { + row: 20, + column: 17, + }, + end_location: Some( + Location { + row: 20, + column: 18, + }, + ), + custom: (), + node: Name { + id: "x", + ctx: Store, + }, + }, + ), + }, + ], + body: [ + Located { + location: Location { + row: 20, + column: 20, + }, + end_location: Some( + Location { + row: 20, + column: 24, + }, + ), + custom: (), + node: Pass, + }, + ], + type_comment: None, + }, + }, + Located { + location: Location { + row: 21, + column: 0, + }, + end_location: Some( + Location { + row: 22, + column: 0, + }, + ), + custom: (), + node: With { + items: [ + Withitem { + context_expr: Located { + location: Location { + row: 21, + column: 6, + }, + end_location: Some( + Location { + row: 21, + column: 12, + }, + ), + custom: (), + node: NamedExpr { + target: Located { + location: Location { + row: 21, + column: 6, + }, + end_location: Some( + Location { + row: 21, + column: 12, + }, + ), + custom: (), + node: Name { + id: "a", + ctx: Store, + }, + }, + value: Located { + location: Location { + row: 21, + column: 11, + }, + end_location: Some( + Location { + row: 21, + column: 12, + }, + ), + custom: (), + node: Constant { + value: Int( + 0, + ), + kind: None, + }, + }, + }, + }, + optional_vars: None, + }, + Withitem { + context_expr: Located { + location: Location { + row: 21, + column: 14, + }, + end_location: Some( + Location { + row: 21, + column: 20, + }, + ), + custom: (), + node: NamedExpr { + target: Located { + location: Location { + row: 21, + column: 14, + }, + end_location: Some( + Location { + row: 21, + column: 20, + }, + ), + custom: (), + node: Name { + id: "b", + ctx: Store, + }, + }, + value: Located { + location: Location { + row: 21, + column: 19, + }, + end_location: Some( + Location { + row: 21, + column: 20, + }, + ), + custom: (), + node: Constant { + value: Int( + 1, + ), + kind: None, + }, + }, + }, + }, + optional_vars: None, + }, + ], + body: [ + Located { + location: Location { + row: 21, + column: 23, + }, + end_location: Some( + Location { + row: 21, + column: 27, + }, + ), + custom: (), + node: Pass, + }, + ], + type_comment: None, + }, + }, + Located { + location: Location { + row: 22, + column: 0, + }, + end_location: Some( + Location { + row: 23, + column: 0, + }, + ), + custom: (), + node: With { + items: [ + Withitem { + context_expr: Located { + location: Location { + row: 22, + column: 6, + }, + end_location: Some( + Location { + row: 22, + column: 20, + }, + ), + custom: (), + node: Tuple { + elts: [ + Located { + location: Location { + row: 22, + column: 6, + }, + end_location: Some( + Location { + row: 22, + column: 12, + }, + ), + custom: (), + node: NamedExpr { + target: Located { + location: Location { + row: 22, + column: 6, + }, + end_location: Some( + Location { + row: 22, + column: 12, + }, + ), + custom: (), + node: Name { + id: "a", + ctx: Store, + }, + }, + value: Located { + location: Location { + row: 22, + column: 11, + }, + end_location: Some( + Location { + row: 22, + column: 12, + }, + ), + custom: (), + node: Constant { + value: Int( + 0, + ), + kind: None, + }, + }, + }, + }, + Located { + location: Location { + row: 22, + column: 14, + }, + end_location: Some( + Location { + row: 22, + column: 20, + }, + ), + custom: (), + node: NamedExpr { + target: Located { + location: Location { + row: 22, + column: 14, + }, + end_location: Some( + Location { + row: 22, + column: 20, + }, + ), + custom: (), + node: Name { + id: "b", + ctx: Store, + }, + }, + value: Located { + location: Location { + row: 22, + column: 19, + }, + end_location: Some( + Location { + row: 22, + column: 20, + }, + ), + custom: (), + node: Constant { + value: Int( + 1, + ), + kind: None, + }, + }, + }, + }, + ], + ctx: Load, + }, + }, + optional_vars: Some( + Located { + location: Location { + row: 22, + column: 25, + }, + end_location: Some( + Location { + row: 22, + column: 26, + }, + ), + custom: (), + node: Name { + id: "x", + ctx: Store, + }, + }, + ), + }, + ], + body: [ + Located { + location: Location { + row: 22, + column: 28, + }, + end_location: Some( + Location { + row: 22, + column: 32, + }, + ), + custom: (), + node: Pass, + }, + ], + type_comment: None, + }, + }, +] diff --git a/parser/src/with.rs b/parser/src/with.rs new file mode 100644 index 0000000000000..afc482360176c --- /dev/null +++ b/parser/src/with.rs @@ -0,0 +1,188 @@ +//! Intermediate types for `with` statement cover grammar. +//! +//! When we start parsing a `with` statement, we don't initially know +//! whether we're looking at a tuple or a Python 3.9+ parenthesized +//! collection of contexts: +//! +//! ```python +//! with (a, b, c) as t: # tuple +//! with (a, b, c): # withitems +//! ``` +//! +//! Since LALRPOP requires us to commit to an output type before we +//! have enough information to decide, we build a cover grammar that's +//! convertible either way. This module contains the necessary +//! intermediate data types. + +use crate::ast::{self, Location}; +use crate::error::{LexicalError, LexicalErrorType}; +use crate::token::Tok; +use lalrpop_util::ParseError as LalrpopError; + +/// Represents a parenthesized collection that we might later convert +/// to a tuple or to `with` items. +/// +/// It can be converted to either `Expr` or `ExprOrWithitems` with +/// `.try_into()`. The `Expr` conversion will fail if any `as` +/// variables are present. The `ExprOrWithitems` conversion cannot +/// fail (but we need it to have the same interface so we can use +/// LALRPOP macros to declare the cover grammar without much code +/// duplication). +pub struct TupleOrWithitems { + pub location: Location, + pub end_location: Location, + pub items: Vec<(ast::Expr, Option>)>, +} + +impl TryFrom for ast::Expr { + type Error = LalrpopError; + fn try_from(tuple_or_withitems: TupleOrWithitems) -> Result { + Ok(ast::Expr { + location: tuple_or_withitems.location, + end_location: Some(tuple_or_withitems.end_location), + custom: (), + node: ast::ExprKind::Tuple { + elts: tuple_or_withitems + .items + .into_iter() + .map(|(expr, optional_vars)| { + if let Some(vars) = optional_vars { + Err(LexicalError { + error: LexicalErrorType::OtherError( + "cannot use 'as' here".to_string(), + ), + location: vars.location, + })? + } + Ok(expr) + }) + .collect::, Self::Error>>()?, + ctx: ast::ExprContext::Load, + }, + }) + } +} + +impl TryFrom for ExprOrWithitems { + type Error = LalrpopError; + fn try_from(items: TupleOrWithitems) -> Result { + Ok(ExprOrWithitems::TupleOrWithitems(items)) + } +} + +/// Represents either a non-tuple expression, or a parenthesized +/// collection that we might later convert to a tuple or to `with` +/// items. +/// +/// It can be constructed from an `Expr` with `.into()`. (The same +/// interface can be used to convert an `Expr` into itself, which is +/// also important for our LALRPOP macro setup.) +/// +/// It can be converted to either `Expr` or `Vec` with +/// `.try_into()`. The `Expr` conversion will fail if any `as` +/// clauses are present. The `Vec` conversion will fail if +/// both `as` clauses and starred expressions are present. +pub enum ExprOrWithitems { + Expr(ast::Expr), + TupleOrWithitems(TupleOrWithitems), +} + +impl From for ExprOrWithitems { + fn from(expr: ast::Expr) -> ExprOrWithitems { + ExprOrWithitems::Expr(expr) + } +} + +impl TryFrom for ast::Expr { + type Error = LalrpopError; + fn try_from(expr_or_withitems: ExprOrWithitems) -> Result { + match expr_or_withitems { + ExprOrWithitems::Expr(expr) => Ok(expr), + ExprOrWithitems::TupleOrWithitems(items) => items.try_into(), + } + } +} + +impl TryFrom for Vec { + type Error = LalrpopError; + fn try_from(expr_or_withitems: ExprOrWithitems) -> Result, Self::Error> { + match expr_or_withitems { + ExprOrWithitems::TupleOrWithitems(tuple_or_withitems) + if !tuple_or_withitems.items.iter().any(|(context_expr, _)| { + matches!(context_expr.node, ast::ExprKind::Starred { .. }) + }) => + { + Ok(tuple_or_withitems + .items + .into_iter() + .map(|(context_expr, optional_vars)| ast::Withitem { + context_expr: Box::new(context_expr), + optional_vars, + }) + .collect()) + } + _ => Ok(vec![ast::Withitem { + context_expr: Box::new(expr_or_withitems.try_into()?), + optional_vars: None, + }]), + } + } +} + +#[cfg(test)] +mod tests { + use crate::parser::parse_program; + + #[test] + fn test_with_statement() { + let source = "\ +with 0: pass +with 0 as x: pass +with 0, 1: pass +with 0 as x, 1 as y: pass +with 0 if 1 else 2: pass +with 0 if 1 else 2 as x: pass +with (): pass +with () as x: pass +with (0): pass +with (0) as x: pass +with (0,): pass +with (0,) as x: pass +with (0, 1): pass +with (0, 1) as x: pass +with (*a,): pass +with (*a,) as x: pass +with (0, *a): pass +with (0, *a) as x: pass +with (a := 0): pass +with (a := 0) as x: pass +with (a := 0, b := 1): pass +with (a := 0, b := 1) as x: pass +"; + insta::assert_debug_snapshot!(parse_program(source, "").unwrap()); + } + + #[test] + fn test_with_statement_invalid() { + for source in [ + "with 0,: pass", + "with 0 as x,: pass", + "with 0 as *x: pass", + "with *a: pass", + "with *a as x: pass", + "with (*a): pass", + "with (*a) as x: pass", + "with *a, 0 as x: pass", + "with (*a, 0 as x): pass", + "with 0 as x, *a: pass", + "with (0 as x, *a): pass", + "with (0 as x) as y: pass", + "with (0 as x), 1: pass", + "with ((0 as x)): pass", + "with a := 0 as x: pass", + "with (a := 0 as x): pass", + ] { + assert!(parse_program(source, "").is_err()); + } + } +}