From 48f7714819525ec5b361ea480eaecea61620a56b Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Tue, 3 Dec 2024 20:09:29 +1100 Subject: [PATCH 1/6] Rename `Parser::expected_tokens` as `Parser::expected_token_types`. Because the `Token` type is similar to but different to the `TokenType` type, and the difference is important, so we want to avoid confusion. --- compiler/rustc_builtin_macros/src/format.rs | 2 +- .../rustc_parse/src/parser/diagnostics.rs | 19 ++++++++-------- compiler/rustc_parse/src/parser/expr.rs | 2 +- compiler/rustc_parse/src/parser/item.rs | 2 +- compiler/rustc_parse/src/parser/mod.rs | 22 +++++++++---------- compiler/rustc_parse/src/parser/path.rs | 2 +- compiler/rustc_parse/src/parser/ty.rs | 2 +- 7 files changed, 26 insertions(+), 25 deletions(-) diff --git a/compiler/rustc_builtin_macros/src/format.rs b/compiler/rustc_builtin_macros/src/format.rs index 73d762d21e51..df3750049b71 100644 --- a/compiler/rustc_builtin_macros/src/format.rs +++ b/compiler/rustc_builtin_macros/src/format.rs @@ -95,7 +95,7 @@ fn parse_args<'a>(ecx: &ExtCtxt<'a>, sp: Span, tts: TokenStream) -> PResult<'a, while p.token != token::Eof { if !p.eat(&token::Comma) { if first { - p.clear_expected_tokens(); + p.clear_expected_token_types(); } match p.expect(&token::Comma) { diff --git a/compiler/rustc_parse/src/parser/diagnostics.rs b/compiler/rustc_parse/src/parser/diagnostics.rs index 8417701ac0cd..938c1e2abb80 100644 --- a/compiler/rustc_parse/src/parser/diagnostics.rs +++ b/compiler/rustc_parse/src/parser/diagnostics.rs @@ -483,9 +483,10 @@ impl<'a> Parser<'a> { }) } - self.expected_tokens.extend(edible.iter().chain(inedible).cloned().map(TokenType::Token)); + self.expected_token_types + .extend(edible.iter().chain(inedible).cloned().map(TokenType::Token)); let mut expected = self - .expected_tokens + .expected_token_types .iter() .filter(|token| { // Filter out suggestions that suggest the same token which was found and deemed incorrect. @@ -785,17 +786,17 @@ impl<'a> Parser<'a> { let Some((curr_ident, _)) = self.token.ident() else { return; }; - let expected_tokens: &[TokenType] = + let expected_token_types: &[TokenType] = expected.len().checked_sub(10).map_or(&expected, |index| &expected[index..]); - let expected_keywords: Vec = expected_tokens + let expected_keywords: Vec = expected_token_types .iter() .filter_map(|token| if let TokenType::Keyword(kw) = token { Some(*kw) } else { None }) .collect(); - // When there are a few keywords in the last ten elements of `self.expected_tokens` and the current - // token is an identifier, it's probably a misspelled keyword. - // This handles code like `async Move {}`, misspelled `if` in match guard, misspelled `else` in `if`-`else` - // and mispelled `where` in a where clause. + // When there are a few keywords in the last ten elements of `self.expected_token_types` + // and the current token is an identifier, it's probably a misspelled keyword. This handles + // code like `async Move {}`, misspelled `if` in match guard, misspelled `else` in + // `if`-`else` and mispelled `where` in a where clause. if !expected_keywords.is_empty() && !curr_ident.is_used_keyword() && let Some(misspelled_kw) = find_similar_kw(curr_ident, &expected_keywords) @@ -3016,7 +3017,7 @@ impl<'a> Parser<'a> { /// Check for exclusive ranges written as `..<` pub(crate) fn maybe_err_dotdotlt_syntax(&self, maybe_lt: Token, mut err: Diag<'a>) -> Diag<'a> { if maybe_lt == token::Lt - && (self.expected_tokens.contains(&TokenType::Token(token::Gt)) + && (self.expected_token_types.contains(&TokenType::Token(token::Gt)) || matches!(self.token.kind, token::Literal(..))) { err.span_suggestion( diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs index 2f34dcb93083..1e84b2a0cf8e 100644 --- a/compiler/rustc_parse/src/parser/expr.rs +++ b/compiler/rustc_parse/src/parser/expr.rs @@ -153,7 +153,7 @@ impl<'a> Parser<'a> { return Ok((lhs, parsed_something)); } - self.expected_tokens.push(TokenType::Operator); + self.expected_token_types.push(TokenType::Operator); while let Some(op) = self.check_assoc_op() { let lhs_span = self.interpolated_or_expr_span(&lhs); let cur_op_span = self.token.span; diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs index e6a8eda42e8d..58fc90f3939d 100644 --- a/compiler/rustc_parse/src/parser/item.rs +++ b/compiler/rustc_parse/src/parser/item.rs @@ -2630,7 +2630,7 @@ impl<'a> Parser<'a> { if !self.eat_keyword_case(kw::Fn, case) { // It is possible for `expect_one_of` to recover given the contents of - // `self.expected_tokens`, therefore, do not use `self.unexpected()` which doesn't + // `self.expected_token_types`, therefore, do not use `self.unexpected()` which doesn't // account for this. match self.expect_one_of(&[], &[]) { Ok(Recovered::Yes(_)) => {} diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index 4bc8f5913b2b..d41c6c2cd1c9 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -141,7 +141,7 @@ pub struct Parser<'a> { pub prev_token: Token, pub capture_cfg: bool, restrictions: Restrictions, - expected_tokens: Vec, + expected_token_types: Vec, token_cursor: TokenCursor, // The number of calls to `bump`, i.e. the position in the token stream. num_bump_calls: u32, @@ -490,7 +490,7 @@ impl<'a> Parser<'a> { prev_token: Token::dummy(), capture_cfg: false, restrictions: Restrictions::empty(), - expected_tokens: Vec::new(), + expected_token_types: Vec::new(), token_cursor: TokenCursor { curr: TokenTreeCursor::new(stream), stack: Vec::new() }, num_bump_calls: 0, break_last_token: 0, @@ -554,7 +554,7 @@ impl<'a> Parser<'a> { /// Expects and consumes the token `t`. Signals an error if the next token is not `t`. pub fn expect(&mut self, t: &TokenKind) -> PResult<'a, Recovered> { - if self.expected_tokens.is_empty() { + if self.expected_token_types.is_empty() { if self.token == *t { self.bump(); Ok(Recovered::No) @@ -619,13 +619,13 @@ impl<'a> Parser<'a> { /// Checks if the next token is `tok`, and returns `true` if so. /// - /// This method will automatically add `tok` to `expected_tokens` if `tok` is not + /// This method will automatically add `tok` to `expected_token_types` if `tok` is not /// encountered. #[inline] fn check(&mut self, tok: &TokenKind) -> bool { let is_present = self.token == *tok; if !is_present { - self.expected_tokens.push(TokenType::Token(tok.clone())); + self.expected_token_types.push(TokenType::Token(tok.clone())); } is_present } @@ -666,7 +666,7 @@ impl<'a> Parser<'a> { #[inline] #[must_use] fn check_keyword(&mut self, kw: Symbol) -> bool { - self.expected_tokens.push(TokenType::Keyword(kw)); + self.expected_token_types.push(TokenType::Keyword(kw)); self.token.is_keyword(kw) } @@ -755,7 +755,7 @@ impl<'a> Parser<'a> { if ok { true } else { - self.expected_tokens.push(typ); + self.expected_token_types.push(typ); false } } @@ -832,7 +832,7 @@ impl<'a> Parser<'a> { true } _ => { - self.expected_tokens.push(TokenType::Token(expected)); + self.expected_token_types.push(TokenType::Token(expected)); false } } @@ -1180,7 +1180,7 @@ impl<'a> Parser<'a> { self.token_spacing = next_spacing; // Diagnostics. - self.expected_tokens.clear(); + self.expected_token_types.clear(); } /// Advance the parser by one token. @@ -1670,8 +1670,8 @@ impl<'a> Parser<'a> { DebugParser { parser: self, lookahead } } - pub fn clear_expected_tokens(&mut self) { - self.expected_tokens.clear(); + pub fn clear_expected_token_types(&mut self) { + self.expected_token_types.clear(); } pub fn approx_token_stream_pos(&self) -> u32 { diff --git a/compiler/rustc_parse/src/parser/path.rs b/compiler/rustc_parse/src/parser/path.rs index f2f0c6dfad56..3505ec88d044 100644 --- a/compiler/rustc_parse/src/parser/path.rs +++ b/compiler/rustc_parse/src/parser/path.rs @@ -300,7 +300,7 @@ impl<'a> Parser<'a> { ) }; let check_args_start = |this: &mut Self| { - this.expected_tokens.extend_from_slice(&[ + this.expected_token_types.extend_from_slice(&[ TokenType::Token(token::Lt), TokenType::Token(token::OpenDelim(Delimiter::Parenthesis)), ]); diff --git a/compiler/rustc_parse/src/parser/ty.rs b/compiler/rustc_parse/src/parser/ty.rs index 6e720db4edf7..e4f67d97aa50 100644 --- a/compiler/rustc_parse/src/parser/ty.rs +++ b/compiler/rustc_parse/src/parser/ty.rs @@ -1280,7 +1280,7 @@ impl<'a> Parser<'a> { } pub(super) fn check_lifetime(&mut self) -> bool { - self.expected_tokens.push(TokenType::Lifetime); + self.expected_token_types.push(TokenType::Lifetime); self.token.is_lifetime() } From fb5ba8a6d49d1ed46cfdaa12ac6ee7002eb7d8fd Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 4 Dec 2024 15:36:49 +1100 Subject: [PATCH 2/6] Tweak some parser `check`/`eat` methods. The most significant is `check_keyword`: it now only pushes to `expected_token_types` if the keyword check fails, which matches how all the other `check` methods work. The remainder are just tweaks to make these methods more consistent with each other. --- compiler/rustc_parse/src/parser/mod.rs | 45 ++++++++++++-------------- 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index d41c6c2cd1c9..0ee3e2bfb0d3 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -666,19 +666,20 @@ impl<'a> Parser<'a> { #[inline] #[must_use] fn check_keyword(&mut self, kw: Symbol) -> bool { - self.expected_token_types.push(TokenType::Keyword(kw)); - self.token.is_keyword(kw) + let is_keyword = self.token.is_keyword(kw); + if !is_keyword { + self.expected_token_types.push(TokenType::Keyword(kw)); + } + is_keyword } #[inline] #[must_use] fn check_keyword_case(&mut self, kw: Symbol, case: Case) -> bool { if self.check_keyword(kw) { - return true; - } - + true // Do an ASCII case-insensitive match, because all keywords are ASCII. - if case == Case::Insensitive + } else if case == Case::Insensitive && let Some((ident, IdentIsRaw::No)) = self.token.ident() && ident.as_str().eq_ignore_ascii_case(kw.as_str()) { @@ -694,12 +695,11 @@ impl<'a> Parser<'a> { #[inline] #[must_use] pub fn eat_keyword(&mut self, kw: Symbol) -> bool { - if self.check_keyword(kw) { + let is_keyword = self.check_keyword(kw); + if is_keyword { self.bump(); - true - } else { - false } + is_keyword } /// Eats a keyword, optionally ignoring the case. @@ -709,19 +709,17 @@ impl<'a> Parser<'a> { #[must_use] fn eat_keyword_case(&mut self, kw: Symbol, case: Case) -> bool { if self.eat_keyword(kw) { - return true; - } - - if case == Case::Insensitive + true + } else if case == Case::Insensitive && let Some((ident, IdentIsRaw::No)) = self.token.ident() && ident.as_str().to_lowercase() == kw.as_str().to_lowercase() { self.dcx().emit_err(errors::KwBadCase { span: ident.span, kw: kw.as_str() }); self.bump(); - return true; + true + } else { + false } - - false } /// If the next token is the given keyword, eats it and returns `true`. @@ -730,12 +728,11 @@ impl<'a> Parser<'a> { #[inline] #[must_use] pub fn eat_keyword_noexpect(&mut self, kw: Symbol) -> bool { - if self.token.is_keyword(kw) { + let is_keyword = self.token.is_keyword(kw); + if is_keyword { self.bump(); - true - } else { - false } + is_keyword } /// If the given word is not a keyword, signals an error. @@ -752,12 +749,10 @@ impl<'a> Parser<'a> { #[inline] fn check_or_expected(&mut self, ok: bool, typ: TokenType) -> bool { - if ok { - true - } else { + if !ok { self.expected_token_types.push(typ); - false } + ok } fn check_ident(&mut self) -> bool { From d5370d981f58ebadf575f075a6f0d8c35bc704e8 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 4 Dec 2024 15:50:46 +1100 Subject: [PATCH 3/6] Remove `bra`/`ket` naming. This is a naming convention used in a handful of spots in the parser for delimiters. It confused me when I first saw it a long time ago, and I've never liked it. A web search says "Bra-ket notation" exists in linear algebra but the terminology has zero prior use in a programming context, as far as I can tell. This commit changes it to `open`/`close`, which is consistent with the rest of the compiler. --- .../rustc_parse/src/parser/diagnostics.rs | 8 ++-- compiler/rustc_parse/src/parser/mod.rs | 40 +++++++++---------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/compiler/rustc_parse/src/parser/diagnostics.rs b/compiler/rustc_parse/src/parser/diagnostics.rs index 938c1e2abb80..1a8f8f6069aa 100644 --- a/compiler/rustc_parse/src/parser/diagnostics.rs +++ b/compiler/rustc_parse/src/parser/diagnostics.rs @@ -1125,11 +1125,11 @@ impl<'a> Parser<'a> { Ok(self.mk_expr_err(lo.to(self.token.span), guar)) } - /// Eats and discards tokens until one of `kets` is encountered. Respects token trees, + /// Eats and discards tokens until one of `closes` is encountered. Respects token trees, /// passes through any errors encountered. Used for error recovery. - pub(super) fn eat_to_tokens(&mut self, kets: &[&TokenKind]) { - if let Err(err) = - self.parse_seq_to_before_tokens(kets, &[], SeqSep::none(), |p| Ok(p.parse_token_tree())) + pub(super) fn eat_to_tokens(&mut self, closes: &[&TokenKind]) { + if let Err(err) = self + .parse_seq_to_before_tokens(closes, &[], SeqSep::none(), |p| Ok(p.parse_token_tree())) { err.cancel(); } diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs index 0ee3e2bfb0d3..3e82a9cf1bb3 100644 --- a/compiler/rustc_parse/src/parser/mod.rs +++ b/compiler/rustc_parse/src/parser/mod.rs @@ -882,14 +882,14 @@ impl<'a> Parser<'a> { } } - /// Checks if the next token is contained within `kets`, and returns `true` if so. + /// Checks if the next token is contained within `closes`, and returns `true` if so. fn expect_any_with_type( &mut self, - kets_expected: &[&TokenKind], - kets_not_expected: &[&TokenKind], + closes_expected: &[&TokenKind], + closes_not_expected: &[&TokenKind], ) -> bool { - kets_expected.iter().any(|k| self.check(k)) - || kets_not_expected.iter().any(|k| self.check_noexpect(k)) + closes_expected.iter().any(|k| self.check(k)) + || closes_not_expected.iter().any(|k| self.check_noexpect(k)) } /// Parses a sequence until the specified delimiters. The function @@ -897,8 +897,8 @@ impl<'a> Parser<'a> { /// closing bracket. fn parse_seq_to_before_tokens( &mut self, - kets_expected: &[&TokenKind], - kets_not_expected: &[&TokenKind], + closes_expected: &[&TokenKind], + closes_not_expected: &[&TokenKind], sep: SeqSep, mut f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>, ) -> PResult<'a, (ThinVec, Trailing, Recovered)> { @@ -907,7 +907,7 @@ impl<'a> Parser<'a> { let mut trailing = Trailing::No; let mut v = ThinVec::new(); - while !self.expect_any_with_type(kets_expected, kets_not_expected) { + while !self.expect_any_with_type(closes_expected, closes_not_expected) { if let token::CloseDelim(..) | token::Eof = self.token.kind { break; } @@ -1006,7 +1006,7 @@ impl<'a> Parser<'a> { // we will try to recover in `maybe_recover_struct_lit_bad_delims` return Err(expect_err); } else if let [token::CloseDelim(Delimiter::Parenthesis)] = - kets_expected + closes_expected { return Err(expect_err); } else { @@ -1020,7 +1020,7 @@ impl<'a> Parser<'a> { } } if sep.trailing_sep_allowed - && self.expect_any_with_type(kets_expected, kets_not_expected) + && self.expect_any_with_type(closes_expected, closes_not_expected) { trailing = Trailing::Yes; break; @@ -1096,11 +1096,11 @@ impl<'a> Parser<'a> { /// closing bracket. fn parse_seq_to_before_end( &mut self, - ket: &TokenKind, + close: &TokenKind, sep: SeqSep, f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>, ) -> PResult<'a, (ThinVec, Trailing, Recovered)> { - self.parse_seq_to_before_tokens(&[ket], &[], sep, f) + self.parse_seq_to_before_tokens(&[close], &[], sep, f) } /// Parses a sequence, including only the closing delimiter. The function @@ -1108,15 +1108,15 @@ impl<'a> Parser<'a> { /// closing bracket. fn parse_seq_to_end( &mut self, - ket: &TokenKind, + close: &TokenKind, sep: SeqSep, f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>, ) -> PResult<'a, (ThinVec, Trailing)> { - let (val, trailing, recovered) = self.parse_seq_to_before_end(ket, sep, f)?; - if matches!(recovered, Recovered::No) && !self.eat(ket) { + let (val, trailing, recovered) = self.parse_seq_to_before_end(close, sep, f)?; + if matches!(recovered, Recovered::No) && !self.eat(close) { self.dcx().span_delayed_bug( self.token.span, - "recovered but `parse_seq_to_before_end` did not give us the ket token", + "recovered but `parse_seq_to_before_end` did not give us the close token", ); } Ok((val, trailing)) @@ -1127,13 +1127,13 @@ impl<'a> Parser<'a> { /// closing bracket. fn parse_unspanned_seq( &mut self, - bra: &TokenKind, - ket: &TokenKind, + open: &TokenKind, + close: &TokenKind, sep: SeqSep, f: impl FnMut(&mut Parser<'a>) -> PResult<'a, T>, ) -> PResult<'a, (ThinVec, Trailing)> { - self.expect(bra)?; - self.parse_seq_to_end(ket, sep, f) + self.expect(open)?; + self.parse_seq_to_end(close, sep, f) } /// Parses a comma-separated sequence, including both delimiters. From b9bf0b4b10148aa914243a527d9010aba9b7b827 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 4 Dec 2024 15:55:06 +1100 Subject: [PATCH 4/6] Speed up `Parser::expected_token_types`. The parser pushes a `TokenType` to `Parser::expected_token_types` on every call to the various `check`/`eat` methods, and clears it on every call to `bump`. Some of those `TokenType` values are full tokens that require cloning and dropping. This is a *lot* of work for something that is only used in error messages and it accounts for a significant fraction of parsing execution time. This commit overhauls `TokenType` so that `Parser::expected_token_types` can be implemented as a bitset. This requires changing `TokenType` to a C-style parameterless enum, and adding `TokenTypeSet` which uses a `u128` for the bits. (The new `TokenType` has 105 variants.) The new types `ExpTokenPair` and `ExpKeywordPair` are now arguments to the `check`/`eat` methods. This is for maximum speed. The elements in the pairs are always statically known; e.g. a `token::BinOp(token::Star)` is always paired with a `TokenType::Star`. So we now compute `TokenType`s in advance and pass them in to `check`/`eat` rather than the current approach of constructing them on insertion into `expected_token_types`. Values of these pair types can be produced by the new `exp!` macro, which is used at every `check`/`eat` call site. The macro is for convenience, allowing any pair to be generated from a single identifier. The ident/keyword filtering in `expected_one_of_not_found` is no longer necessary. It was there to account for some sloppiness in `TokenKind`/`TokenType` comparisons. The existing `TokenType` is moved to a new file `token_type.rs`, and all its new infrastructure is added to that file. There is more boilerplate code than I would like, but I can't see how to make it shorter. --- compiler/rustc_builtin_macros/src/asm.rs | 112 ++-- compiler/rustc_builtin_macros/src/assert.rs | 3 +- compiler/rustc_builtin_macros/src/cfg.rs | 5 +- compiler/rustc_builtin_macros/src/format.rs | 7 +- .../rustc_builtin_macros/src/pattern_type.rs | 5 +- compiler/rustc_builtin_macros/src/util.rs | 6 +- compiler/rustc_expand/src/module.rs | 6 +- .../rustc_expand/src/proc_macro_server.rs | 4 +- compiler/rustc_parse/src/parser/attr.rs | 41 +- .../rustc_parse/src/parser/diagnostics.rs | 139 ++-- compiler/rustc_parse/src/parser/expr.rs | 308 +++++---- compiler/rustc_parse/src/parser/generics.rs | 33 +- compiler/rustc_parse/src/parser/item.rs | 370 ++++++----- compiler/rustc_parse/src/parser/mod.rs | 226 +++---- compiler/rustc_parse/src/parser/pat.rs | 77 +-- compiler/rustc_parse/src/parser/path.rs | 34 +- compiler/rustc_parse/src/parser/stmt.rs | 27 +- compiler/rustc_parse/src/parser/token_type.rs | 620 ++++++++++++++++++ compiler/rustc_parse/src/parser/ty.rs | 92 +-- src/tools/rustfmt/src/parse/macros/cfg_if.rs | 11 +- .../rustfmt/src/parse/macros/lazy_static.rs | 17 +- src/tools/rustfmt/src/parse/parser.rs | 5 +- 22 files changed, 1356 insertions(+), 792 deletions(-) create mode 100644 compiler/rustc_parse/src/parser/token_type.rs diff --git a/compiler/rustc_builtin_macros/src/asm.rs b/compiler/rustc_builtin_macros/src/asm.rs index cce70fb2ea4b..6ae697d4030d 100644 --- a/compiler/rustc_builtin_macros/src/asm.rs +++ b/compiler/rustc_builtin_macros/src/asm.rs @@ -1,16 +1,16 @@ use ast::token::IdentIsRaw; use lint::BuiltinLintDiag; -use rustc_ast::AsmMacro; use rustc_ast::ptr::P; -use rustc_ast::token::{self, Delimiter}; use rustc_ast::tokenstream::TokenStream; +use rustc_ast::{AsmMacro, token}; use rustc_data_structures::fx::{FxHashMap, FxIndexMap}; use rustc_errors::PResult; use rustc_expand::base::*; use rustc_index::bit_set::GrowableBitSet; -use rustc_parse::parser::Parser; +use rustc_parse::exp; +use rustc_parse::parser::{ExpKeywordPair, Parser}; use rustc_session::lint; -use rustc_span::{ErrorGuaranteed, Ident, InnerSpan, Span, Symbol, kw, sym}; +use rustc_span::{ErrorGuaranteed, Ident, InnerSpan, Span, Symbol, kw}; use rustc_target::asm::InlineAsmArch; use smallvec::smallvec; use {rustc_ast as ast, rustc_parse_format as parse}; @@ -38,16 +38,16 @@ pub struct AsmArgs { /// - `Err(_)` if the current token matches the keyword, but was not expected fn eat_operand_keyword<'a>( p: &mut Parser<'a>, - symbol: Symbol, + exp: ExpKeywordPair, asm_macro: AsmMacro, ) -> PResult<'a, bool> { if matches!(asm_macro, AsmMacro::Asm) { - Ok(p.eat_keyword(symbol)) + Ok(p.eat_keyword(exp)) } else { let span = p.token.span; - if p.eat_keyword_noexpect(symbol) { + if p.eat_keyword_noexpect(exp.kw) { // in gets printed as `r#in` otherwise - let symbol = if symbol == kw::In { "in" } else { symbol.as_str() }; + let symbol = if exp.kw == kw::In { "in" } else { exp.kw.as_str() }; Err(p.dcx().create_err(errors::AsmUnsupportedOperand { span, symbol, @@ -95,13 +95,13 @@ pub fn parse_asm_args<'a>( let mut allow_templates = true; while p.token != token::Eof { - if !p.eat(&token::Comma) { + if !p.eat(exp!(Comma)) { if allow_templates { // After a template string, we always expect *only* a comma... return Err(dcx.create_err(errors::AsmExpectedComma { span: p.token.span })); } else { // ...after that delegate to `expect` to also include the other expected tokens. - return Err(p.expect(&token::Comma).err().unwrap()); + return Err(p.expect(exp!(Comma)).err().unwrap()); } } if p.token == token::Eof { @@ -109,14 +109,14 @@ pub fn parse_asm_args<'a>( } // accept trailing commas // Parse clobber_abi - if p.eat_keyword(sym::clobber_abi) { + if p.eat_keyword(exp!(ClobberAbi)) { parse_clobber_abi(p, &mut args)?; allow_templates = false; continue; } // Parse options - if p.eat_keyword(sym::options) { + if p.eat_keyword(exp!(Options)) { parse_options(p, &mut args, asm_macro)?; allow_templates = false; continue; @@ -128,7 +128,7 @@ pub fn parse_asm_args<'a>( let name = if p.token.is_ident() && p.look_ahead(1, |t| *t == token::Eq) { let (ident, _) = p.token.ident().unwrap(); p.bump(); - p.expect(&token::Eq)?; + p.expect(exp!(Eq))?; allow_templates = false; Some(ident.name) } else { @@ -136,57 +136,57 @@ pub fn parse_asm_args<'a>( }; let mut explicit_reg = false; - let op = if eat_operand_keyword(p, kw::In, asm_macro)? { + let op = if eat_operand_keyword(p, exp!(In), asm_macro)? { let reg = parse_reg(p, &mut explicit_reg)?; - if p.eat_keyword(kw::Underscore) { + if p.eat_keyword(exp!(Underscore)) { let err = dcx.create_err(errors::AsmUnderscoreInput { span: p.token.span }); return Err(err); } let expr = p.parse_expr()?; ast::InlineAsmOperand::In { reg, expr } - } else if eat_operand_keyword(p, sym::out, asm_macro)? { + } else if eat_operand_keyword(p, exp!(Out), asm_macro)? { let reg = parse_reg(p, &mut explicit_reg)?; - let expr = if p.eat_keyword(kw::Underscore) { None } else { Some(p.parse_expr()?) }; + let expr = if p.eat_keyword(exp!(Underscore)) { None } else { Some(p.parse_expr()?) }; ast::InlineAsmOperand::Out { reg, expr, late: false } - } else if eat_operand_keyword(p, sym::lateout, asm_macro)? { + } else if eat_operand_keyword(p, exp!(Lateout), asm_macro)? { let reg = parse_reg(p, &mut explicit_reg)?; - let expr = if p.eat_keyword(kw::Underscore) { None } else { Some(p.parse_expr()?) }; + let expr = if p.eat_keyword(exp!(Underscore)) { None } else { Some(p.parse_expr()?) }; ast::InlineAsmOperand::Out { reg, expr, late: true } - } else if eat_operand_keyword(p, sym::inout, asm_macro)? { + } else if eat_operand_keyword(p, exp!(Inout), asm_macro)? { let reg = parse_reg(p, &mut explicit_reg)?; - if p.eat_keyword(kw::Underscore) { + if p.eat_keyword(exp!(Underscore)) { let err = dcx.create_err(errors::AsmUnderscoreInput { span: p.token.span }); return Err(err); } let expr = p.parse_expr()?; - if p.eat(&token::FatArrow) { + if p.eat(exp!(FatArrow)) { let out_expr = - if p.eat_keyword(kw::Underscore) { None } else { Some(p.parse_expr()?) }; + if p.eat_keyword(exp!(Underscore)) { None } else { Some(p.parse_expr()?) }; ast::InlineAsmOperand::SplitInOut { reg, in_expr: expr, out_expr, late: false } } else { ast::InlineAsmOperand::InOut { reg, expr, late: false } } - } else if eat_operand_keyword(p, sym::inlateout, asm_macro)? { + } else if eat_operand_keyword(p, exp!(Inlateout), asm_macro)? { let reg = parse_reg(p, &mut explicit_reg)?; - if p.eat_keyword(kw::Underscore) { + if p.eat_keyword(exp!(Underscore)) { let err = dcx.create_err(errors::AsmUnderscoreInput { span: p.token.span }); return Err(err); } let expr = p.parse_expr()?; - if p.eat(&token::FatArrow) { + if p.eat(exp!(FatArrow)) { let out_expr = - if p.eat_keyword(kw::Underscore) { None } else { Some(p.parse_expr()?) }; + if p.eat_keyword(exp!(Underscore)) { None } else { Some(p.parse_expr()?) }; ast::InlineAsmOperand::SplitInOut { reg, in_expr: expr, out_expr, late: true } } else { ast::InlineAsmOperand::InOut { reg, expr, late: true } } - } else if eat_operand_keyword(p, sym::label, asm_macro)? { + } else if eat_operand_keyword(p, exp!(Label), asm_macro)? { let block = p.parse_block()?; ast::InlineAsmOperand::Label { block } - } else if p.eat_keyword(kw::Const) { + } else if p.eat_keyword(exp!(Const)) { let anon_const = p.parse_expr_anon_const()?; ast::InlineAsmOperand::Const { anon_const } - } else if p.eat_keyword(sym::sym) { + } else if p.eat_keyword(exp!(Sym)) { let expr = p.parse_expr()?; let ast::ExprKind::Path(qself, path) = &expr.kind else { let err = dcx.create_err(errors::AsmSymNoPath { span: expr.span }); @@ -389,31 +389,31 @@ fn parse_options<'a>( ) -> PResult<'a, ()> { let span_start = p.prev_token.span; - p.expect(&token::OpenDelim(Delimiter::Parenthesis))?; - - while !p.eat(&token::CloseDelim(Delimiter::Parenthesis)) { - const OPTIONS: [(Symbol, ast::InlineAsmOptions); ast::InlineAsmOptions::COUNT] = [ - (sym::pure, ast::InlineAsmOptions::PURE), - (sym::nomem, ast::InlineAsmOptions::NOMEM), - (sym::readonly, ast::InlineAsmOptions::READONLY), - (sym::preserves_flags, ast::InlineAsmOptions::PRESERVES_FLAGS), - (sym::noreturn, ast::InlineAsmOptions::NORETURN), - (sym::nostack, ast::InlineAsmOptions::NOSTACK), - (sym::may_unwind, ast::InlineAsmOptions::MAY_UNWIND), - (sym::att_syntax, ast::InlineAsmOptions::ATT_SYNTAX), - (kw::Raw, ast::InlineAsmOptions::RAW), + p.expect(exp!(OpenParen))?; + + while !p.eat(exp!(CloseParen)) { + const OPTIONS: [(ExpKeywordPair, ast::InlineAsmOptions); ast::InlineAsmOptions::COUNT] = [ + (exp!(Pure), ast::InlineAsmOptions::PURE), + (exp!(Nomem), ast::InlineAsmOptions::NOMEM), + (exp!(Readonly), ast::InlineAsmOptions::READONLY), + (exp!(PreservesFlags), ast::InlineAsmOptions::PRESERVES_FLAGS), + (exp!(Noreturn), ast::InlineAsmOptions::NORETURN), + (exp!(Nostack), ast::InlineAsmOptions::NOSTACK), + (exp!(MayUnwind), ast::InlineAsmOptions::MAY_UNWIND), + (exp!(AttSyntax), ast::InlineAsmOptions::ATT_SYNTAX), + (exp!(Raw), ast::InlineAsmOptions::RAW), ]; 'blk: { - for (symbol, option) in OPTIONS { + for (exp, option) in OPTIONS { let kw_matched = if asm_macro.is_supported_option(option) { - p.eat_keyword(symbol) + p.eat_keyword(exp) } else { - p.eat_keyword_noexpect(symbol) + p.eat_keyword_noexpect(exp.kw) }; if kw_matched { - try_set_option(p, args, asm_macro, symbol, option); + try_set_option(p, args, asm_macro, exp.kw, option); break 'blk; } } @@ -422,10 +422,10 @@ fn parse_options<'a>( } // Allow trailing commas - if p.eat(&token::CloseDelim(Delimiter::Parenthesis)) { + if p.eat(exp!(CloseParen)) { break; } - p.expect(&token::Comma)?; + p.expect(exp!(Comma))?; } let new_span = span_start.to(p.prev_token.span); @@ -437,14 +437,14 @@ fn parse_options<'a>( fn parse_clobber_abi<'a>(p: &mut Parser<'a>, args: &mut AsmArgs) -> PResult<'a, ()> { let span_start = p.prev_token.span; - p.expect(&token::OpenDelim(Delimiter::Parenthesis))?; + p.expect(exp!(OpenParen))?; - if p.eat(&token::CloseDelim(Delimiter::Parenthesis)) { + if p.eat(exp!(CloseParen)) { return Err(p.dcx().create_err(errors::NonABI { span: p.token.span })); } let mut new_abis = Vec::new(); - while !p.eat(&token::CloseDelim(Delimiter::Parenthesis)) { + while !p.eat(exp!(CloseParen)) { match p.parse_str_lit() { Ok(str_lit) => { new_abis.push((str_lit.symbol_unescaped, str_lit.span)); @@ -456,10 +456,10 @@ fn parse_clobber_abi<'a>(p: &mut Parser<'a>, args: &mut AsmArgs) -> PResult<'a, }; // Allow trailing commas - if p.eat(&token::CloseDelim(Delimiter::Parenthesis)) { + if p.eat(exp!(CloseParen)) { break; } - p.expect(&token::Comma)?; + p.expect(exp!(Comma))?; } let full_span = span_start.to(p.prev_token.span); @@ -482,7 +482,7 @@ fn parse_reg<'a>( p: &mut Parser<'a>, explicit_reg: &mut bool, ) -> PResult<'a, ast::InlineAsmRegOrRegClass> { - p.expect(&token::OpenDelim(Delimiter::Parenthesis))?; + p.expect(exp!(OpenParen))?; let result = match p.token.uninterpolate().kind { token::Ident(name, IdentIsRaw::No) => ast::InlineAsmRegOrRegClass::RegClass(name), token::Literal(token::Lit { kind: token::LitKind::Str, symbol, suffix: _ }) => { @@ -496,7 +496,7 @@ fn parse_reg<'a>( } }; p.bump(); - p.expect(&token::CloseDelim(Delimiter::Parenthesis))?; + p.expect(exp!(CloseParen))?; Ok(result) } diff --git a/compiler/rustc_builtin_macros/src/assert.rs b/compiler/rustc_builtin_macros/src/assert.rs index 95b31c7e47a2..c659b1cff59b 100644 --- a/compiler/rustc_builtin_macros/src/assert.rs +++ b/compiler/rustc_builtin_macros/src/assert.rs @@ -7,6 +7,7 @@ use rustc_ast::{DelimArgs, Expr, ExprKind, MacCall, Path, PathSegment, UnOp, tok use rustc_ast_pretty::pprust; use rustc_errors::PResult; use rustc_expand::base::{DummyResult, ExpandResult, ExtCtxt, MacEager, MacroExpanderResult}; +use rustc_parse::exp; use rustc_parse::parser::Parser; use rustc_span::{DUMMY_SP, Ident, Span, Symbol, sym}; use thin_vec::thin_vec; @@ -143,7 +144,7 @@ fn parse_assert<'a>(cx: &ExtCtxt<'a>, sp: Span, stream: TokenStream) -> PResult< cx.dcx().emit_err(errors::AssertMissingComma { span: parser.token.span, comma }); parse_custom_message(&mut parser) - } else if parser.eat(&token::Comma) { + } else if parser.eat(exp!(Comma)) { parse_custom_message(&mut parser) } else { None diff --git a/compiler/rustc_builtin_macros/src/cfg.rs b/compiler/rustc_builtin_macros/src/cfg.rs index 6e90f1682e30..85b8ef79c050 100644 --- a/compiler/rustc_builtin_macros/src/cfg.rs +++ b/compiler/rustc_builtin_macros/src/cfg.rs @@ -6,6 +6,7 @@ use rustc_ast::token; use rustc_ast::tokenstream::TokenStream; use rustc_errors::PResult; use rustc_expand::base::{DummyResult, ExpandResult, ExtCtxt, MacEager, MacroExpanderResult}; +use rustc_parse::exp; use rustc_span::Span; use {rustc_ast as ast, rustc_attr_parsing as attr}; @@ -48,9 +49,9 @@ fn parse_cfg<'a>( let cfg = p.parse_meta_item_inner()?; - let _ = p.eat(&token::Comma); + let _ = p.eat(exp!(Comma)); - if !p.eat(&token::Eof) { + if !p.eat(exp!(Eof)) { return Err(cx.dcx().create_err(errors::OneCfgPattern { span })); } diff --git a/compiler/rustc_builtin_macros/src/format.rs b/compiler/rustc_builtin_macros/src/format.rs index df3750049b71..528eb7725f5c 100644 --- a/compiler/rustc_builtin_macros/src/format.rs +++ b/compiler/rustc_builtin_macros/src/format.rs @@ -12,6 +12,7 @@ use rustc_errors::{Applicability, Diag, MultiSpan, PResult, SingleLabelManySpans use rustc_expand::base::*; use rustc_lint_defs::builtin::NAMED_ARGUMENTS_USED_POSITIONALLY; use rustc_lint_defs::{BufferedEarlyLint, BuiltinLintDiag, LintId}; +use rustc_parse::exp; use rustc_parse_format as parse; use rustc_span::{BytePos, ErrorGuaranteed, Ident, InnerSpan, Span, Symbol}; @@ -93,12 +94,12 @@ fn parse_args<'a>(ecx: &ExtCtxt<'a>, sp: Span, tts: TokenStream) -> PResult<'a, let mut first = true; while p.token != token::Eof { - if !p.eat(&token::Comma) { + if !p.eat(exp!(Comma)) { if first { p.clear_expected_token_types(); } - match p.expect(&token::Comma) { + match p.expect(exp!(Comma)) { Err(err) => { match token::TokenKind::Comma.similar_tokens() { Some(tks) if tks.contains(&p.token.kind) => { @@ -122,7 +123,7 @@ fn parse_args<'a>(ecx: &ExtCtxt<'a>, sp: Span, tts: TokenStream) -> PResult<'a, match p.token.ident() { Some((ident, _)) if p.look_ahead(1, |t| *t == token::Eq) => { p.bump(); - p.expect(&token::Eq)?; + p.expect(exp!(Eq))?; let expr = p.parse_expr()?; if let Some((_, prev)) = args.by_name(ident.name) { ecx.dcx().emit_err(errors::FormatDuplicateArg { diff --git a/compiler/rustc_builtin_macros/src/pattern_type.rs b/compiler/rustc_builtin_macros/src/pattern_type.rs index 90b5f097b32b..a600a9f316a7 100644 --- a/compiler/rustc_builtin_macros/src/pattern_type.rs +++ b/compiler/rustc_builtin_macros/src/pattern_type.rs @@ -3,7 +3,8 @@ use rustc_ast::tokenstream::TokenStream; use rustc_ast::{Pat, Ty, ast}; use rustc_errors::PResult; use rustc_expand::base::{self, DummyResult, ExpandResult, ExtCtxt, MacroExpanderResult}; -use rustc_span::{Span, sym}; +use rustc_parse::exp; +use rustc_span::Span; pub(crate) fn expand<'cx>( cx: &'cx mut ExtCtxt<'_>, @@ -24,7 +25,7 @@ fn parse_pat_ty<'a>(cx: &mut ExtCtxt<'a>, stream: TokenStream) -> PResult<'a, (P let mut parser = cx.new_parser_from_tts(stream); let ty = parser.parse_ty()?; - parser.expect_keyword(sym::is)?; + parser.expect_keyword(exp!(Is))?; let pat = parser.parse_pat_no_top_alt(None, None)?; Ok((ty, pat)) diff --git a/compiler/rustc_builtin_macros/src/util.rs b/compiler/rustc_builtin_macros/src/util.rs index 2a28dfaf3c43..be12d21a8000 100644 --- a/compiler/rustc_builtin_macros/src/util.rs +++ b/compiler/rustc_builtin_macros/src/util.rs @@ -7,7 +7,7 @@ use rustc_expand::expand::AstFragment; use rustc_feature::AttributeTemplate; use rustc_lint_defs::BuiltinLintDiag; use rustc_lint_defs::builtin::DUPLICATE_MACRO_ATTRIBUTES; -use rustc_parse::{parser, validate_attr}; +use rustc_parse::{exp, parser, validate_attr}; use rustc_session::errors::report_lit_error; use rustc_span::{BytePos, Span, Symbol}; @@ -204,7 +204,7 @@ pub(crate) fn get_single_expr_from_tts( Ok(ret) => ret, Err(guar) => return ExpandResult::Ready(Err(guar)), }; - let _ = p.eat(&token::Comma); + let _ = p.eat(exp!(Comma)); if p.token != token::Eof { cx.dcx().emit_err(errors::OnlyOneArgument { span, name }); @@ -237,7 +237,7 @@ pub(crate) fn get_exprs_from_tts( let expr = cx.expander().fully_expand_fragment(AstFragment::Expr(expr)).make_expr(); es.push(expr); - if p.eat(&token::Comma) { + if p.eat(exp!(Comma)) { continue; } if p.token != token::Eof { diff --git a/compiler/rustc_expand/src/module.rs b/compiler/rustc_expand/src/module.rs index a001b1d3dc8a..9c35b26772b6 100644 --- a/compiler/rustc_expand/src/module.rs +++ b/compiler/rustc_expand/src/module.rs @@ -2,9 +2,9 @@ use std::iter::once; use std::path::{self, Path, PathBuf}; use rustc_ast::ptr::P; -use rustc_ast::{AttrVec, Attribute, Inline, Item, ModSpans, token}; +use rustc_ast::{AttrVec, Attribute, Inline, Item, ModSpans}; use rustc_errors::{Diag, ErrorGuaranteed}; -use rustc_parse::{new_parser_from_file, unwrap_or_emit_fatal, validate_attr}; +use rustc_parse::{exp, new_parser_from_file, unwrap_or_emit_fatal, validate_attr}; use rustc_session::Session; use rustc_session::parse::ParseSess; use rustc_span::{Ident, Span, sym}; @@ -70,7 +70,7 @@ pub(crate) fn parse_external_mod( let mut parser = unwrap_or_emit_fatal(new_parser_from_file(&sess.psess, &mp.file_path, Some(span))); let (inner_attrs, items, inner_span) = - parser.parse_mod(&token::Eof).map_err(|err| ModError::ParserError(err))?; + parser.parse_mod(exp!(Eof)).map_err(|err| ModError::ParserError(err))?; attrs.extend(inner_attrs); (items, inner_span, mp.file_path) }; diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index 8577aa110af8..7eb09a64e96a 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -15,7 +15,7 @@ use rustc_data_structures::sync::Lrc; use rustc_errors::{Diag, ErrorGuaranteed, MultiSpan, PResult}; use rustc_parse::lexer::nfc_normalize; use rustc_parse::parser::Parser; -use rustc_parse::{new_parser_from_source_str, source_str_to_stream, unwrap_or_emit_fatal}; +use rustc_parse::{exp, new_parser_from_source_str, source_str_to_stream, unwrap_or_emit_fatal}; use rustc_session::parse::ParseSess; use rustc_span::def_id::CrateNum; use rustc_span::{BytePos, FileName, Pos, SourceFile, Span, Symbol, sym}; @@ -473,7 +473,7 @@ impl server::FreeFunctions for Rustc<'_, '_> { unwrap_or_emit_fatal(new_parser_from_source_str(self.psess(), name, s.to_owned())); let first_span = parser.token.span.data(); - let minus_present = parser.eat(&token::BinOp(token::Minus)); + let minus_present = parser.eat(exp!(Minus)); let lit_span = parser.token.span.data(); let token::Literal(mut lit) = parser.token.kind else { diff --git a/compiler/rustc_parse/src/parser/attr.rs b/compiler/rustc_parse/src/parser/attr.rs index 9da4ab5a7882..2691e6f56d68 100644 --- a/compiler/rustc_parse/src/parser/attr.rs +++ b/compiler/rustc_parse/src/parser/attr.rs @@ -1,8 +1,7 @@ -use rustc_ast::token::{self, Delimiter}; -use rustc_ast::{self as ast, Attribute, attr}; +use rustc_ast::{self as ast, Attribute, attr, token}; use rustc_errors::codes::*; use rustc_errors::{Diag, PResult}; -use rustc_span::{BytePos, Span, kw}; +use rustc_span::{BytePos, Span}; use thin_vec::ThinVec; use tracing::debug; @@ -10,7 +9,7 @@ use super::{ AttrWrapper, Capturing, FnParseMode, ForceCollect, Parser, ParserRange, PathStyle, Trailing, UsePreAttrPos, }; -use crate::{errors, fluent_generated as fluent, maybe_whole}; +use crate::{errors, exp, fluent_generated as fluent, maybe_whole}; // Public for rustfmt usage #[derive(Debug)] @@ -45,7 +44,7 @@ impl<'a> Parser<'a> { let mut just_parsed_doc_comment = false; let start_pos = self.num_bump_calls; loop { - let attr = if self.check(&token::Pound) { + let attr = if self.check(exp!(Pound)) { let prev_outer_attr_sp = outer_attrs.last().map(|attr: &Attribute| attr.span); let inner_error_reason = if just_parsed_doc_comment { @@ -126,14 +125,14 @@ impl<'a> Parser<'a> { let lo = self.token.span; // Attributes can't have attributes of their own [Editor's note: not with that attitude] self.collect_tokens_no_attrs(|this| { - assert!(this.eat(&token::Pound), "parse_attribute called in non-attribute position"); + assert!(this.eat(exp!(Pound)), "parse_attribute called in non-attribute position"); let style = - if this.eat(&token::Not) { ast::AttrStyle::Inner } else { ast::AttrStyle::Outer }; + if this.eat(exp!(Not)) { ast::AttrStyle::Inner } else { ast::AttrStyle::Outer }; - this.expect(&token::OpenDelim(Delimiter::Bracket))?; + this.expect(exp!(OpenBracket))?; let item = this.parse_attr_item(ForceCollect::No)?; - this.expect(&token::CloseDelim(Delimiter::Bracket))?; + this.expect(exp!(CloseBracket))?; let attr_sp = lo.to(this.prev_token.span); // Emit error if inner attribute is encountered and forbidden. @@ -274,10 +273,10 @@ impl<'a> Parser<'a> { // Attr items don't have attributes. self.collect_tokens(None, AttrWrapper::empty(), force_collect, |this, _empty_attrs| { - let is_unsafe = this.eat_keyword(kw::Unsafe); + let is_unsafe = this.eat_keyword(exp!(Unsafe)); let unsafety = if is_unsafe { let unsafe_span = this.prev_token.span; - this.expect(&token::OpenDelim(Delimiter::Parenthesis))?; + this.expect(exp!(OpenParen))?; ast::Safety::Unsafe(unsafe_span) } else { ast::Safety::Default @@ -286,7 +285,7 @@ impl<'a> Parser<'a> { let path = this.parse_path(PathStyle::Mod)?; let args = this.parse_attr_args()?; if is_unsafe { - this.expect(&token::CloseDelim(Delimiter::Parenthesis))?; + this.expect(exp!(CloseParen))?; } Ok(( ast::AttrItem { unsafety, path, args, tokens: None }, @@ -306,7 +305,7 @@ impl<'a> Parser<'a> { loop { let start_pos = self.num_bump_calls; // Only try to parse if it is an inner attribute (has `!`). - let attr = if self.check(&token::Pound) && self.look_ahead(1, |t| t == &token::Not) { + let attr = if self.check(exp!(Pound)) && self.look_ahead(1, |t| t == &token::Not) { Some(self.parse_attribute(InnerAttrPolicy::Permitted)?) } else if let token::DocComment(comment_kind, attr_style, data) = self.token.kind { if attr_style == ast::AttrStyle::Inner { @@ -358,7 +357,7 @@ impl<'a> Parser<'a> { &mut self, ) -> PResult<'a, (ast::MetaItemInner, Vec<(ast::AttrItem, Span)>)> { let cfg_predicate = self.parse_meta_item_inner()?; - self.expect(&token::Comma)?; + self.expect(exp!(Comma))?; // Presumably, the majority of the time there will only be one attr. let mut expanded_attrs = Vec::with_capacity(1); @@ -366,7 +365,7 @@ impl<'a> Parser<'a> { let lo = self.token.span; let item = self.parse_attr_item(ForceCollect::Yes)?; expanded_attrs.push((item, lo.to(self.prev_token.span))); - if !self.eat(&token::Comma) { + if !self.eat(exp!(Comma)) { break; } } @@ -380,7 +379,7 @@ impl<'a> Parser<'a> { let mut nmis = ThinVec::with_capacity(1); while self.token != token::Eof { nmis.push(self.parse_meta_item_inner()?); - if !self.eat(&token::Comma) { + if !self.eat(exp!(Comma)) { break; } } @@ -413,13 +412,13 @@ impl<'a> Parser<'a> { let lo = self.token.span; let is_unsafe = if unsafe_allowed == AllowLeadingUnsafe::Yes { - self.eat_keyword(kw::Unsafe) + self.eat_keyword(exp!(Unsafe)) } else { false }; let unsafety = if is_unsafe { let unsafe_span = self.prev_token.span; - self.expect(&token::OpenDelim(Delimiter::Parenthesis))?; + self.expect(exp!(OpenParen))?; ast::Safety::Unsafe(unsafe_span) } else { @@ -429,7 +428,7 @@ impl<'a> Parser<'a> { let path = self.parse_path(PathStyle::Mod)?; let kind = self.parse_meta_item_kind()?; if is_unsafe { - self.expect(&token::CloseDelim(Delimiter::Parenthesis))?; + self.expect(exp!(CloseParen))?; } let span = lo.to(self.prev_token.span); @@ -437,9 +436,9 @@ impl<'a> Parser<'a> { } pub(crate) fn parse_meta_item_kind(&mut self) -> PResult<'a, ast::MetaItemKind> { - Ok(if self.eat(&token::Eq) { + Ok(if self.eat(exp!(Eq)) { ast::MetaItemKind::NameValue(self.parse_unsuffixed_meta_item_lit()?) - } else if self.check(&token::OpenDelim(Delimiter::Parenthesis)) { + } else if self.check(exp!(OpenParen)) { let (list, _) = self.parse_paren_comma_seq(|p| p.parse_meta_item_inner())?; ast::MetaItemKind::List(list) } else { diff --git a/compiler/rustc_parse/src/parser/diagnostics.rs b/compiler/rustc_parse/src/parser/diagnostics.rs index 1a8f8f6069aa..7e9b9219e7ac 100644 --- a/compiler/rustc_parse/src/parser/diagnostics.rs +++ b/compiler/rustc_parse/src/parser/diagnostics.rs @@ -29,7 +29,8 @@ use tracing::{debug, trace}; use super::pat::Expected; use super::{ - BlockMode, CommaRecoveryMode, Parser, PathStyle, Restrictions, SemiColonMode, SeqSep, TokenType, + BlockMode, CommaRecoveryMode, ExpTokenPair, Parser, PathStyle, Restrictions, SemiColonMode, + SeqSep, TokenType, }; use crate::errors::{ AddParen, AmbiguousPlus, AsyncMoveBlockIn2015, AttributeOnParamType, AwaitSuggestion, @@ -47,7 +48,7 @@ use crate::errors::{ UnexpectedConstParamDeclarationSugg, UnmatchedAngleBrackets, UseEqInstead, WrapType, }; use crate::parser::attr::InnerAttrPolicy; -use crate::{fluent_generated as fluent, parser}; +use crate::{exp, fluent_generated as fluent}; /// Creates a placeholder argument. pub(super) fn dummy_arg(ident: Ident, guar: ErrorGuaranteed) -> Param { @@ -462,8 +463,8 @@ impl<'a> Parser<'a> { pub(super) fn expected_one_of_not_found( &mut self, - edible: &[TokenKind], - inedible: &[TokenKind], + edible: &[ExpTokenPair<'_>], + inedible: &[ExpTokenPair<'_>], ) -> PResult<'a, ErrorGuaranteed> { debug!("expected_one_of_not_found(edible: {:?}, inedible: {:?})", edible, inedible); fn tokens_to_string(tokens: &[TokenType]) -> String { @@ -483,50 +484,17 @@ impl<'a> Parser<'a> { }) } - self.expected_token_types - .extend(edible.iter().chain(inedible).cloned().map(TokenType::Token)); - let mut expected = self - .expected_token_types - .iter() - .filter(|token| { - // Filter out suggestions that suggest the same token which was found and deemed incorrect. - fn is_ident_eq_keyword(found: &TokenKind, expected: &TokenType) -> bool { - if let TokenKind::Ident(current_sym, _) = found - && let TokenType::Keyword(suggested_sym) = expected - { - return current_sym == suggested_sym; - } - false - } - - if **token != parser::TokenType::Token(self.token.kind.clone()) { - let eq = is_ident_eq_keyword(&self.token.kind, &token); - // If the suggestion is a keyword and the found token is an ident, - // the content of which are equal to the suggestion's content, - // we can remove that suggestion (see the `return false` below). - - // If this isn't the case however, and the suggestion is a token the - // content of which is the same as the found token's, we remove it as well. - if !eq { - if let TokenType::Token(kind) = token { - if self.token == *kind { - return false; - } - } - return true; - } - } - false - }) - .cloned() - .collect::>(); + for exp in edible.iter().chain(inedible.iter()) { + self.expected_token_types.insert(exp.token_type); + } + let mut expected: Vec<_> = self.expected_token_types.iter().collect(); expected.sort_by_cached_key(|x| x.to_string()); expected.dedup(); let sm = self.psess.source_map(); // Special-case "expected `;`" errors. - if expected.contains(&TokenType::Token(token::Semi)) { + if expected.contains(&TokenType::Semi) { // If the user is trying to write a ternary expression, recover it and // return an Err to prevent a cascade of irrelevant diagnostics. if self.prev_token == token::Question @@ -578,7 +546,7 @@ impl<'a> Parser<'a> { || (sm.is_multiline( self.prev_token.span.shrink_to_hi().until(self.token.span.shrink_to_lo()), ) && t == &token::Pound) - }) && !expected.contains(&TokenType::Token(token::Comma)) + }) && !expected.contains(&TokenType::Comma) { // Missing semicolon typo. This is triggered if the next token could either start a // new statement or is a block close. For example: @@ -598,7 +566,7 @@ impl<'a> Parser<'a> { if self.token == TokenKind::EqEq && self.prev_token.is_ident() - && expected.iter().any(|tok| matches!(tok, TokenType::Token(TokenKind::Eq))) + && expected.contains(&TokenType::Eq) { // Likely typo: `=` → `==` in let expr or enum item return Err(self.dcx().create_err(UseEqInstead { span: self.token.span })); @@ -637,15 +605,8 @@ impl<'a> Parser<'a> { // Look for usages of '=>' where '>=' was probably intended if self.token == token::FatArrow - && expected - .iter() - .any(|tok| matches!(tok, TokenType::Operator | TokenType::Token(TokenKind::Le))) - && !expected.iter().any(|tok| { - matches!( - tok, - TokenType::Token(TokenKind::FatArrow) | TokenType::Token(TokenKind::Comma) - ) - }) + && expected.iter().any(|tok| matches!(tok, TokenType::Operator | TokenType::Le)) + && !expected.iter().any(|tok| matches!(tok, TokenType::FatArrow | TokenType::Comma)) { err.span_suggestion( self.token.span, @@ -742,7 +703,7 @@ impl<'a> Parser<'a> { }; if self.check_too_many_raw_str_terminators(&mut err) { - if expected.contains(&TokenType::Token(token::Semi)) && self.eat(&token::Semi) { + if expected.contains(&TokenType::Semi) && self.eat(exp!(Semi)) { let guar = err.emit(); return Ok(guar); } else { @@ -788,10 +749,8 @@ impl<'a> Parser<'a> { }; let expected_token_types: &[TokenType] = expected.len().checked_sub(10).map_or(&expected, |index| &expected[index..]); - let expected_keywords: Vec = expected_token_types - .iter() - .filter_map(|token| if let TokenType::Keyword(kw) = token { Some(*kw) } else { None }) - .collect(); + let expected_keywords: Vec = + expected_token_types.iter().filter_map(|token| token.is_keyword()).collect(); // When there are a few keywords in the last ten elements of `self.expected_token_types` // and the current token is an identifier, it's probably a misspelled keyword. This handles @@ -1053,7 +1012,7 @@ impl<'a> Parser<'a> { (Err(snapshot_err), Err(err)) => { // We don't know what went wrong, emit the normal error. snapshot_err.cancel(); - self.consume_block(Delimiter::Brace, ConsumeClosingDelim::Yes); + self.consume_block(exp!(OpenBrace), exp!(CloseBrace), ConsumeClosingDelim::Yes); Err(err) } (Ok(_), Ok(mut tail)) => { @@ -1090,7 +1049,7 @@ impl<'a> Parser<'a> { Applicability::MaybeIncorrect, ); let guar = err.emit(); - self.eat_to_tokens(&[&token::CloseDelim(Delimiter::Brace)]); + self.eat_to_tokens(&[exp!(CloseBrace)]); guar } token::OpenDelim(Delimiter::Parenthesis) @@ -1098,7 +1057,7 @@ impl<'a> Parser<'a> { { // We are within a function call or tuple, we can emit the error // and recover. - self.eat_to_tokens(&[&token::CloseDelim(Delimiter::Parenthesis), &token::Comma]); + self.eat_to_tokens(&[exp!(CloseParen), exp!(Comma)]); err.multipart_suggestion_verbose( "you might have meant to open the body of the closure", @@ -1127,7 +1086,7 @@ impl<'a> Parser<'a> { /// Eats and discards tokens until one of `closes` is encountered. Respects token trees, /// passes through any errors encountered. Used for error recovery. - pub(super) fn eat_to_tokens(&mut self, closes: &[&TokenKind]) { + pub(super) fn eat_to_tokens(&mut self, closes: &[ExpTokenPair<'_>]) { if let Err(err) = self .parse_seq_to_before_tokens(closes, &[], SeqSep::none(), |p| Ok(p.parse_token_tree())) { @@ -1148,7 +1107,7 @@ impl<'a> Parser<'a> { pub(super) fn check_trailing_angle_brackets( &mut self, segment: &PathSegment, - end: &[&TokenKind], + end: &[ExpTokenPair<'_>], ) -> Option { if !self.may_recover() { return None; @@ -1231,7 +1190,7 @@ impl<'a> Parser<'a> { // second case. if self.look_ahead(position, |t| { trace!("check_trailing_angle_brackets: t={:?}", t); - end.contains(&&t.kind) + end.iter().any(|exp| exp.tok == &t.kind) }) { // Eat from where we started until the end token so that parsing can continue // as if we didn't have those extra angle brackets. @@ -1299,11 +1258,11 @@ impl<'a> Parser<'a> { ) -> PResult<'a, ErrorGuaranteed> { if let ExprKind::Binary(binop, _, _) = &expr.kind && let ast::BinOpKind::Lt = binop.node - && self.eat(&token::Comma) + && self.eat(exp!(Comma)) { let x = self.parse_seq_to_before_end( - &token::Gt, - SeqSep::trailing_allowed(token::Comma), + exp!(Gt), + SeqSep::trailing_allowed(exp!(Comma)), |p| match p.parse_generic_arg(None)? { Some(arg) => Ok(arg), // If we didn't eat a generic arg, then we should error. @@ -1312,7 +1271,7 @@ impl<'a> Parser<'a> { ); match x { Ok((_, _, Recovered::No)) => { - if self.eat(&token::Gt) { + if self.eat(exp!(Gt)) { // We made sense of it. Improve the error message. e.span_suggestion_verbose( binop.span.shrink_to_lo(), @@ -1875,7 +1834,7 @@ impl<'a> Parser<'a> { ty_span: Span, ty: P, ) -> PResult<'a, P> { - self.expect(&token::PathSep)?; + self.expect(exp!(PathSep))?; let mut path = ast::Path { segments: ThinVec::new(), span: DUMMY_SP, tokens: None }; self.parse_path_segments(&mut path.segments, T::PATH_STYLE, None)?; @@ -1957,10 +1916,10 @@ impl<'a> Parser<'a> { } pub(super) fn expect_semi(&mut self) -> PResult<'a, ()> { - if self.eat(&token::Semi) || self.recover_colon_as_semi() { + if self.eat(exp!(Semi)) || self.recover_colon_as_semi() { return Ok(()); } - self.expect(&token::Semi).map(drop) // Error unconditionally + self.expect(exp!(Semi)).map(drop) // Error unconditionally } pub(super) fn recover_colon_as_semi(&mut self) -> bool { @@ -2005,15 +1964,15 @@ impl<'a> Parser<'a> { } fn recover_await_macro(&mut self) -> PResult<'a, (Span, P, bool)> { - self.expect(&token::Not)?; - self.expect(&token::OpenDelim(Delimiter::Parenthesis))?; + self.expect(exp!(Not))?; + self.expect(exp!(OpenParen))?; let expr = self.parse_expr()?; - self.expect(&token::CloseDelim(Delimiter::Parenthesis))?; + self.expect(exp!(CloseParen))?; Ok((self.prev_token.span, expr, false)) } fn recover_await_prefix(&mut self, await_sp: Span) -> PResult<'a, (Span, P, bool)> { - let is_question = self.eat(&token::Question); // Handle `await? `. + let is_question = self.eat(exp!(Question)); // Handle `await? `. let expr = if self.token == token::OpenDelim(Delimiter::Brace) { // Handle `await { }`. // This needs to be handled separately from the next arm to avoid @@ -2075,7 +2034,7 @@ impl<'a> Parser<'a> { let try_span = lo.to(self.token.span); //we take the try!( span self.bump(); //remove ( let is_empty = self.token == token::CloseDelim(Delimiter::Parenthesis); //check if the block is empty - self.consume_block(Delimiter::Parenthesis, ConsumeClosingDelim::No); //eat the block + self.consume_block(exp!(OpenParen), exp!(CloseParen), ConsumeClosingDelim::No); //eat the block let hi = self.token.span; self.bump(); //remove ) let mut err = self.dcx().struct_span_err(lo.to(hi), "use of deprecated `try` macro"); @@ -2131,13 +2090,14 @@ impl<'a> Parser<'a> { pub(super) fn recover_seq_parse_error( &mut self, - delim: Delimiter, + open: ExpTokenPair<'_>, + close: ExpTokenPair<'_>, lo: Span, err: Diag<'a>, ) -> P { let guar = err.emit(); // Recover from parse error, callers expect the closing delim to be consumed. - self.consume_block(delim, ConsumeClosingDelim::Yes); + self.consume_block(open, close, ConsumeClosingDelim::Yes); self.mk_expr(lo.to(self.prev_token.span), ExprKind::Err(guar)) } @@ -2226,7 +2186,7 @@ impl<'a> Parser<'a> { } pub(super) fn check_for_for_in_in_typo(&mut self, in_span: Span) { - if self.eat_keyword(kw::In) { + if self.eat_keyword(exp!(In)) { // a common typo: `for _ in in bar {}` self.dcx().emit_err(InInTypo { span: self.prev_token.span, @@ -2367,7 +2327,7 @@ impl<'a> Parser<'a> { pub(super) fn recover_arg_parse(&mut self) -> PResult<'a, (P, P)> { let pat = self.parse_pat_no_top_alt(Some(Expected::ArgumentName), None)?; - self.expect(&token::Colon)?; + self.expect(exp!(Colon))?; let ty = self.parse_ty()?; self.dcx().emit_err(PatternMethodParamWithoutBody { span: pat.span }); @@ -2385,12 +2345,17 @@ impl<'a> Parser<'a> { Ok(param) } - pub(super) fn consume_block(&mut self, delim: Delimiter, consume_close: ConsumeClosingDelim) { + pub(super) fn consume_block( + &mut self, + open: ExpTokenPair<'_>, + close: ExpTokenPair<'_>, + consume_close: ConsumeClosingDelim, + ) { let mut brace_depth = 0; loop { - if self.eat(&token::OpenDelim(delim)) { + if self.eat(open) { brace_depth += 1; - } else if self.check(&token::CloseDelim(delim)) { + } else if self.check(close) { if brace_depth == 0 { if let ConsumeClosingDelim::Yes = consume_close { // Some of the callers of this method expect to be able to parse the @@ -2546,7 +2511,7 @@ impl<'a> Parser<'a> { match self.recover_const_arg(arg.span(), err) { Ok(arg) => { args.push(AngleBracketedArg::Arg(arg)); - if self.eat(&token::Comma) { + if self.eat(exp!(Comma)) { return Ok(true); // Continue } } @@ -3017,7 +2982,7 @@ impl<'a> Parser<'a> { /// Check for exclusive ranges written as `..<` pub(crate) fn maybe_err_dotdotlt_syntax(&self, maybe_lt: Token, mut err: Diag<'a>) -> Diag<'a> { if maybe_lt == token::Lt - && (self.expected_token_types.contains(&TokenType::Token(token::Gt)) + && (self.expected_token_types.contains(TokenType::Gt) || matches!(self.token.kind, token::Literal(..))) { err.span_suggestion( @@ -3147,9 +3112,9 @@ impl<'a> Parser<'a> { /// Parse and throw away a parenthesized comma separated /// sequence of patterns until `)` is reached. fn skip_pat_list(&mut self) -> PResult<'a, ()> { - while !self.check(&token::CloseDelim(Delimiter::Parenthesis)) { + while !self.check(exp!(CloseParen)) { self.parse_pat_no_top_alt(None, None)?; - if !self.eat(&token::Comma) { + if !self.eat(exp!(Comma)) { return Ok(()); } } diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs index 1e84b2a0cf8e..2f4adf2af9e9 100644 --- a/compiler/rustc_parse/src/parser/expr.rs +++ b/compiler/rustc_parse/src/parser/expr.rs @@ -35,10 +35,10 @@ use super::diagnostics::SnapshotParser; use super::pat::{CommaRecoveryMode, Expected, RecoverColon, RecoverComma}; use super::ty::{AllowPlus, RecoverQPath, RecoverReturnSign}; use super::{ - AttrWrapper, BlockMode, ClosureSpans, ForceCollect, Parser, PathStyle, Restrictions, - SemiColonMode, SeqSep, TokenType, Trailing, UsePreAttrPos, + AttrWrapper, BlockMode, ClosureSpans, ExpTokenPair, ForceCollect, Parser, PathStyle, + Restrictions, SemiColonMode, SeqSep, TokenType, Trailing, UsePreAttrPos, }; -use crate::{errors, maybe_recover_from_interpolated_ty_qpath}; +use crate::{errors, exp, maybe_recover_from_interpolated_ty_qpath}; #[derive(Debug)] pub(super) enum DestructuredFloat { @@ -153,7 +153,7 @@ impl<'a> Parser<'a> { return Ok((lhs, parsed_something)); } - self.expected_token_types.push(TokenType::Operator); + self.expected_token_types.insert(TokenType::Operator); while let Some(op) = self.check_assoc_op() { let lhs_span = self.interpolated_or_expr_span(&lhs); let cur_op_span = self.token.span; @@ -873,9 +873,9 @@ impl<'a> Parser<'a> { /// Parse `mut?` or `raw [ const | mut ]`. fn parse_borrow_modifiers(&mut self) -> (ast::BorrowKind, ast::Mutability) { - if self.check_keyword(kw::Raw) && self.look_ahead(1, Token::is_mutability) { + if self.check_keyword(exp!(Raw)) && self.look_ahead(1, Token::is_mutability) { // `raw [ const | mut ]`. - let found_raw = self.eat_keyword(kw::Raw); + let found_raw = self.eat_keyword(exp!(Raw)); assert!(found_raw); let mutability = self.parse_const_or_mut().unwrap(); (ast::BorrowKind::Raw, mutability) @@ -908,7 +908,7 @@ impl<'a> Parser<'a> { // a `return` which could be suggested otherwise. self.eat_noexpect(&token::Question) } else { - self.eat(&token::Question) + self.eat(exp!(Question)) }; if has_question { // `expr?` @@ -926,7 +926,7 @@ impl<'a> Parser<'a> { self.dcx().emit_err(errors::ExprRArrowCall { span }); true } else { - self.eat(&token::Dot) + self.eat(exp!(Dot)) }; if has_dot { // expr.f @@ -1251,7 +1251,7 @@ impl<'a> Parser<'a> { .map(|args| self.mk_expr(lo.to(self.prev_token.span), self.mk_call(fun, args))); match self.maybe_recover_struct_lit_bad_delims(lo, open_paren, seq, snapshot) { Ok(expr) => expr, - Err(err) => self.recover_seq_parse_error(Delimiter::Parenthesis, lo, err), + Err(err) => self.recover_seq_parse_error(exp!(OpenParen), exp!(CloseParen), lo, err), } } @@ -1268,10 +1268,8 @@ impl<'a> Parser<'a> { match (self.may_recover(), seq, snapshot) { (true, Err(err), Some((mut snapshot, ExprKind::Path(None, path)))) => { snapshot.bump(); // `(` - match snapshot.parse_struct_fields(path.clone(), false, Delimiter::Parenthesis) { - Ok((fields, ..)) - if snapshot.eat(&token::CloseDelim(Delimiter::Parenthesis)) => - { + match snapshot.parse_struct_fields(path.clone(), false, exp!(CloseParen)) { + Ok((fields, ..)) if snapshot.eat(exp!(CloseParen)) => { // We are certain we have `Enum::Foo(a: 3, b: 4)`, suggest // `Enum::Foo { a: 3, b: 4 }` or `Enum::Foo(3, 4)`. self.restore_snapshot(snapshot); @@ -1328,7 +1326,7 @@ impl<'a> Parser<'a> { self.bump(); // `[` let index = self.parse_expr()?; self.suggest_missing_semicolon_before_array(prev_span, open_delim_span)?; - self.expect(&token::CloseDelim(Delimiter::Bracket))?; + self.expect(exp!(CloseBracket))?; Ok(self.mk_expr( lo.to(self.prev_token.span), self.mk_index(base, index, open_delim_span.to(self.prev_token.span)), @@ -1337,12 +1335,12 @@ impl<'a> Parser<'a> { /// Assuming we have just parsed `.`, continue parsing into an expression. fn parse_dot_suffix(&mut self, self_arg: P, lo: Span) -> PResult<'a, P> { - if self.token.uninterpolated_span().at_least_rust_2018() && self.eat_keyword(kw::Await) { + if self.token.uninterpolated_span().at_least_rust_2018() && self.eat_keyword(exp!(Await)) { return Ok(self.mk_await_expr(self_arg, lo)); } // Post-fix match - if self.eat_keyword(kw::Match) { + if self.eat_keyword(exp!(Match)) { let match_span = self.prev_token.span; self.psess.gated_spans.gate(sym::postfix_match, match_span); return self.parse_match_block(lo, match_span, self_arg, MatchKind::Postfix); @@ -1350,10 +1348,10 @@ impl<'a> Parser<'a> { let fn_span_lo = self.token.span; let mut seg = self.parse_path_segment(PathStyle::Expr, None)?; - self.check_trailing_angle_brackets(&seg, &[&token::OpenDelim(Delimiter::Parenthesis)]); + self.check_trailing_angle_brackets(&seg, &[exp!(OpenParen)]); self.check_turbofish_missing_angle_brackets(&mut seg); - if self.check(&token::OpenDelim(Delimiter::Parenthesis)) { + if self.check(exp!(OpenParen)) { // Method call `expr.f()` let args = self.parse_expr_paren_seq()?; let fn_span = fn_span_lo.to(self.prev_token.span); @@ -1415,18 +1413,18 @@ impl<'a> Parser<'a> { let restrictions = self.restrictions; self.with_res(restrictions - Restrictions::ALLOW_LET, |this| { - // Note: when adding new syntax here, don't forget to adjust `TokenKind::can_begin_expr()`. + // Note: adding new syntax here? Don't forget to adjust `TokenKind::can_begin_expr()`. let lo = this.token.span; if let token::Literal(_) = this.token.kind { // This match arm is a special-case of the `_` match arm below and // could be removed without changing functionality, but it's faster // to have it here, especially for programs with large constants. this.parse_expr_lit() - } else if this.check(&token::OpenDelim(Delimiter::Parenthesis)) { + } else if this.check(exp!(OpenParen)) { this.parse_expr_tuple_parens(restrictions) - } else if this.check(&token::OpenDelim(Delimiter::Brace)) { + } else if this.check(exp!(OpenBrace)) { this.parse_expr_block(None, lo, BlockCheckMode::Default) - } else if this.check(&token::BinOp(token::Or)) || this.check(&token::OrOr) { + } else if this.check(exp!(Or)) || this.check(exp!(OrOr)) { this.parse_expr_closure().map_err(|mut err| { // If the input is something like `if a { 1 } else { 2 } | if a { 3 } else { 4 }` // then suggest parens around the lhs. @@ -1435,41 +1433,41 @@ impl<'a> Parser<'a> { } err }) - } else if this.check(&token::OpenDelim(Delimiter::Bracket)) { - this.parse_expr_array_or_repeat(Delimiter::Bracket) + } else if this.check(exp!(OpenBracket)) { + this.parse_expr_array_or_repeat(exp!(CloseBracket)) } else if this.is_builtin() { this.parse_expr_builtin() } else if this.check_path() { this.parse_expr_path_start() - } else if this.check_keyword(kw::Move) - || this.check_keyword(kw::Static) + } else if this.check_keyword(exp!(Move)) + || this.check_keyword(exp!(Static)) || this.check_const_closure() { this.parse_expr_closure() - } else if this.eat_keyword(kw::If) { + } else if this.eat_keyword(exp!(If)) { this.parse_expr_if() - } else if this.check_keyword(kw::For) { + } else if this.check_keyword(exp!(For)) { if this.choose_generics_over_qpath(1) { this.parse_expr_closure() } else { - assert!(this.eat_keyword(kw::For)); + assert!(this.eat_keyword(exp!(For))); this.parse_expr_for(None, lo) } - } else if this.eat_keyword(kw::While) { + } else if this.eat_keyword(exp!(While)) { this.parse_expr_while(None, lo) } else if let Some(label) = this.eat_label() { this.parse_expr_labeled(label, true) - } else if this.eat_keyword(kw::Loop) { + } else if this.eat_keyword(exp!(Loop)) { this.parse_expr_loop(None, lo).map_err(|mut err| { err.span_label(lo, "while parsing this `loop` expression"); err }) - } else if this.eat_keyword(kw::Match) { + } else if this.eat_keyword(exp!(Match)) { this.parse_expr_match().map_err(|mut err| { err.span_label(lo, "while parsing this `match` expression"); err }) - } else if this.eat_keyword(kw::Unsafe) { + } else if this.eat_keyword(exp!(Unsafe)) { this.parse_expr_block(None, lo, BlockCheckMode::Unsafe(ast::UserProvided)).map_err( |mut err| { err.span_label(lo, "while parsing this `unsafe` expression"); @@ -1481,23 +1479,23 @@ impl<'a> Parser<'a> { } else if this.may_recover() && this.is_do_catch_block() { this.recover_do_catch() } else if this.is_try_block() { - this.expect_keyword(kw::Try)?; + this.expect_keyword(exp!(Try))?; this.parse_try_block(lo) - } else if this.eat_keyword(kw::Return) { + } else if this.eat_keyword(exp!(Return)) { this.parse_expr_return() - } else if this.eat_keyword(kw::Continue) { + } else if this.eat_keyword(exp!(Continue)) { this.parse_expr_continue(lo) - } else if this.eat_keyword(kw::Break) { + } else if this.eat_keyword(exp!(Break)) { this.parse_expr_break() - } else if this.eat_keyword(kw::Yield) { + } else if this.eat_keyword(exp!(Yield)) { this.parse_expr_yield() } else if this.is_do_yeet() { this.parse_expr_yeet() - } else if this.eat_keyword(kw::Become) { + } else if this.eat_keyword(exp!(Become)) { this.parse_expr_become() - } else if this.check_keyword(kw::Let) { + } else if this.check_keyword(exp!(Let)) { this.parse_expr_let(restrictions) - } else if this.eat_keyword(kw::Underscore) { + } else if this.eat_keyword(exp!(Underscore)) { Ok(this.mk_expr(this.prev_token.span, ExprKind::Underscore)) } else if this.token.uninterpolated_span().at_least_rust_2018() { // `Span::at_least_rust_2018()` is somewhat expensive; don't get it repeatedly. @@ -1505,11 +1503,11 @@ impl<'a> Parser<'a> { // check for `gen {}` and `gen move {}` // or `async gen {}` and `async gen move {}` && (this.is_gen_block(kw::Gen, 0) - || (this.check_keyword(kw::Async) && this.is_gen_block(kw::Gen, 1))) + || (this.check_keyword(exp!(Async)) && this.is_gen_block(kw::Gen, 1))) { // FIXME: (async) gen closures aren't yet parsed. this.parse_gen_block() - } else if this.check_keyword(kw::Async) { + } else if this.check_keyword(exp!(Async)) { // FIXME(gen_blocks): Parse `gen async` and suggest swap if this.is_gen_block(kw::Async, 0) { // Check for `async {` and `async move {`, @@ -1541,15 +1539,20 @@ impl<'a> Parser<'a> { fn parse_expr_tuple_parens(&mut self, restrictions: Restrictions) -> PResult<'a, P> { let lo = self.token.span; - self.expect(&token::OpenDelim(Delimiter::Parenthesis))?; + self.expect(exp!(OpenParen))?; let (es, trailing_comma) = match self.parse_seq_to_end( - &token::CloseDelim(Delimiter::Parenthesis), - SeqSep::trailing_allowed(token::Comma), + exp!(CloseParen), + SeqSep::trailing_allowed(exp!(Comma)), |p| p.parse_expr_catch_underscore(restrictions.intersection(Restrictions::ALLOW_LET)), ) { Ok(x) => x, Err(err) => { - return Ok(self.recover_seq_parse_error(Delimiter::Parenthesis, lo, err)); + return Ok(self.recover_seq_parse_error( + exp!(OpenParen), + exp!(CloseParen), + lo, + err, + )); } }; let kind = if es.len() == 1 && matches!(trailing_comma, Trailing::No) { @@ -1563,25 +1566,24 @@ impl<'a> Parser<'a> { self.maybe_recover_from_bad_qpath(expr) } - fn parse_expr_array_or_repeat(&mut self, close_delim: Delimiter) -> PResult<'a, P> { + fn parse_expr_array_or_repeat(&mut self, close: ExpTokenPair<'_>) -> PResult<'a, P> { let lo = self.token.span; self.bump(); // `[` or other open delim - let close = &token::CloseDelim(close_delim); let kind = if self.eat(close) { // Empty vector ExprKind::Array(ThinVec::new()) } else { // Non-empty vector let first_expr = self.parse_expr()?; - if self.eat(&token::Semi) { + if self.eat(exp!(Semi)) { // Repeating array syntax: `[ 0; 512 ]` let count = self.parse_expr_anon_const()?; self.expect(close)?; ExprKind::Repeat(first_expr, count) - } else if self.eat(&token::Comma) { + } else if self.eat(exp!(Comma)) { // Vector with two or more elements. - let sep = SeqSep::trailing_allowed(token::Comma); + let sep = SeqSep::trailing_allowed(exp!(Comma)); let (mut exprs, _) = self.parse_seq_to_end(close, sep, |p| p.parse_expr())?; exprs.insert(0, first_expr); ExprKind::Array(exprs) @@ -1615,7 +1617,7 @@ impl<'a> Parser<'a> { }; // `!`, as an operator, is prefix, so we know this isn't that. - let (span, kind) = if self.eat(&token::Not) { + let (span, kind) = if self.eat(exp!(Not)) { // MACRO INVOCATION expression if qself.is_some() { self.dcx().emit_err(errors::MacroInvocationWithQualifiedPath(path.span)); @@ -1623,7 +1625,7 @@ impl<'a> Parser<'a> { let lo = path.span; let mac = P(MacCall { path, args: self.parse_delim_args()? }); (lo.to(self.prev_token.span), ExprKind::MacCall(mac)) - } else if self.check(&token::OpenDelim(Delimiter::Brace)) + } else if self.check(exp!(OpenBrace)) && let Some(expr) = self.maybe_parse_struct_expr(&qself, &path) { if qself.is_some() { @@ -1646,13 +1648,13 @@ impl<'a> Parser<'a> { ) -> PResult<'a, P> { let lo = label_.ident.span; let label = Some(label_); - let ate_colon = self.eat(&token::Colon); + let ate_colon = self.eat(exp!(Colon)); let tok_sp = self.token.span; - let expr = if self.eat_keyword(kw::While) { + let expr = if self.eat_keyword(exp!(While)) { self.parse_expr_while(label, lo) - } else if self.eat_keyword(kw::For) { + } else if self.eat_keyword(exp!(For)) { self.parse_expr_for(label, lo) - } else if self.eat_keyword(kw::Loop) { + } else if self.eat_keyword(exp!(Loop)) { self.parse_expr_loop(label, lo) } else if self.check_noexpect(&token::OpenDelim(Delimiter::Brace)) || self.token.is_whole_block() @@ -1958,7 +1960,7 @@ impl<'a> Parser<'a> { self.psess.gated_spans.gate(sym::builtin_syntax, ident.span); self.bump(); - self.expect(&TokenKind::OpenDelim(Delimiter::Parenthesis))?; + self.expect(exp!(OpenParen))?; let ret = if let Some(res) = parse(self, lo, ident)? { Ok(res) } else { @@ -1968,7 +1970,7 @@ impl<'a> Parser<'a> { }); return Err(err); }; - self.expect(&TokenKind::CloseDelim(Delimiter::Parenthesis))?; + self.expect(exp!(CloseParen))?; ret } @@ -1976,14 +1978,12 @@ impl<'a> Parser<'a> { /// Built-in macro for `offset_of!` expressions. pub(crate) fn parse_expr_offset_of(&mut self, lo: Span) -> PResult<'a, P> { let container = self.parse_ty()?; - self.expect(&TokenKind::Comma)?; + self.expect(exp!(Comma))?; let fields = self.parse_floating_field_access()?; let trailing_comma = self.eat_noexpect(&TokenKind::Comma); - if let Err(mut e) = - self.expect_one_of(&[], &[TokenKind::CloseDelim(Delimiter::Parenthesis)]) - { + if let Err(mut e) = self.expect_one_of(&[], &[exp!(CloseParen)]) { if trailing_comma { e.note("unexpected third argument to offset_of"); } else { @@ -2006,7 +2006,7 @@ impl<'a> Parser<'a> { /// Built-in macro for type ascription expressions. pub(crate) fn parse_expr_type_ascribe(&mut self, lo: Span) -> PResult<'a, P> { let expr = self.parse_expr()?; - self.expect(&token::Comma)?; + self.expect(exp!(Comma))?; let ty = self.parse_ty()?; let span = lo.to(self.token.span); Ok(self.mk_expr(span, ExprKind::Type(expr, ty))) @@ -2018,7 +2018,7 @@ impl<'a> Parser<'a> { kind: UnsafeBinderCastKind, ) -> PResult<'a, P> { let expr = self.parse_expr()?; - let ty = if self.eat(&TokenKind::Comma) { Some(self.parse_ty()?) } else { None }; + let ty = if self.eat(exp!(Comma)) { Some(self.parse_ty()?) } else { None }; let span = lo.to(self.token.span); Ok(self.mk_expr(span, ExprKind::UnsafeBinderCast(kind, expr, ty))) } @@ -2214,7 +2214,7 @@ impl<'a> Parser<'a> { } let lo = self.token.span; - let minus_present = self.eat(&token::BinOp(token::Minus)); + let minus_present = self.eat(exp!(Minus)); let (token_lit, span) = self.parse_token_lit()?; let expr = self.mk_expr(span, ExprKind::Lit(token_lit)); @@ -2236,7 +2236,7 @@ impl<'a> Parser<'a> { /// expression. fn maybe_suggest_brackets_instead_of_braces(&mut self, lo: Span) -> Option> { let mut snapshot = self.create_snapshot_for_diagnostic(); - match snapshot.parse_expr_array_or_repeat(Delimiter::Brace) { + match snapshot.parse_expr_array_or_repeat(exp!(CloseBrace)) { Ok(arr) => { let guar = self.dcx().emit_err(errors::ArrayBracketsInsteadOfSpaces { span: arr.span, @@ -2272,8 +2272,8 @@ impl<'a> Parser<'a> { let mut snapshot = self.create_snapshot_for_diagnostic(); snapshot.bump(); match snapshot.parse_seq_to_before_end( - &token::CloseDelim(Delimiter::Bracket), - SeqSep::trailing_allowed(token::Comma), + exp!(CloseBracket), + SeqSep::trailing_allowed(exp!(Comma)), |p| p.parse_expr(), ) { Ok(_) @@ -2337,7 +2337,7 @@ impl<'a> Parser<'a> { let lo = self.token.span; let before = self.prev_token.clone(); - let binder = if self.check_keyword(kw::For) { + let binder = if self.check_keyword(exp!(For)) { let lo = self.token.span; let (lifetime_defs, _) = self.parse_late_bound_lifetime_defs()?; let span = lo.to(self.prev_token.span); @@ -2352,7 +2352,7 @@ impl<'a> Parser<'a> { let constness = self.parse_closure_constness(); let movability = - if self.eat_keyword(kw::Static) { Movability::Static } else { Movability::Movable }; + if self.eat_keyword(exp!(Static)) { Movability::Static } else { Movability::Movable }; let coroutine_kind = if self.token.uninterpolated_span().at_least_rust_2018() { self.parse_coroutine_kind(Case::Sensitive) @@ -2433,10 +2433,10 @@ impl<'a> Parser<'a> { /// Parses an optional `move` prefix to a closure-like construct. fn parse_capture_clause(&mut self) -> PResult<'a, CaptureBy> { - if self.eat_keyword(kw::Move) { + if self.eat_keyword(exp!(Move)) { let move_kw_span = self.prev_token.span; // Check for `move async` and recover - if self.check_keyword(kw::Async) { + if self.check_keyword(exp!(Async)) { let move_async_span = self.token.span.with_lo(self.prev_token.span.data().lo); Err(self .dcx() @@ -2453,15 +2453,15 @@ impl<'a> Parser<'a> { fn parse_fn_block_decl(&mut self) -> PResult<'a, (P, Span)> { let arg_start = self.token.span.lo(); - let inputs = if self.eat(&token::OrOr) { + let inputs = if self.eat(exp!(OrOr)) { ThinVec::new() } else { - self.expect(&token::BinOp(token::Or))?; + self.expect(exp!(Or))?; let args = self .parse_seq_to_before_tokens( - &[&token::BinOp(token::Or)], + &[exp!(Or)], &[&token::OrOr], - SeqSep::trailing_allowed(token::Comma), + SeqSep::trailing_allowed(exp!(Comma)), |p| p.parse_fn_block_param(), )? .0; @@ -2481,7 +2481,7 @@ impl<'a> Parser<'a> { let attrs = self.parse_outer_attributes()?; self.collect_tokens(None, attrs, ForceCollect::No, |this, attrs| { let pat = this.parse_pat_no_top_alt(Some(Expected::ParameterName), None)?; - let ty = if this.eat(&token::Colon) { + let ty = if this.eat(exp!(Colon)) { this.parse_ty()? } else { this.mk_ty(pat.span, TyKind::Infer) @@ -2566,7 +2566,7 @@ impl<'a> Parser<'a> { } else { let attrs = self.parse_outer_attributes()?; // For recovery. let maybe_fatarrow = self.token.clone(); - let block = if self.check(&token::OpenDelim(Delimiter::Brace)) { + let block = if self.check(exp!(OpenBrace)) { self.parse_block()? } else if let Some(block) = recover_block_from_condition(self) { block @@ -2609,7 +2609,7 @@ impl<'a> Parser<'a> { self.error_on_if_block_attrs(lo, false, block.span, attrs); block }; - let els = if self.eat_keyword(kw::Else) { Some(self.parse_expr_else()?) } else { None }; + let els = if self.eat_keyword(exp!(Else)) { Some(self.parse_expr_else()?) } else { None }; Ok(self.mk_expr(lo.to(self.prev_token.span), ExprKind::If(cond, thn, els))) } @@ -2662,7 +2662,7 @@ impl<'a> Parser<'a> { }); self.bump(); } else { - self.expect(&token::Eq)?; + self.expect(exp!(Eq))?; } let attrs = self.parse_outer_attributes()?; let (expr, _) = @@ -2675,9 +2675,9 @@ impl<'a> Parser<'a> { fn parse_expr_else(&mut self) -> PResult<'a, P> { let else_span = self.prev_token.span; // `else` let attrs = self.parse_outer_attributes()?; // For recovery. - let expr = if self.eat_keyword(kw::If) { + let expr = if self.eat_keyword(exp!(If)) { ensure_sufficient_stack(|| self.parse_expr_if())? - } else if self.check(&TokenKind::OpenDelim(Delimiter::Brace)) { + } else if self.check(exp!(OpenBrace)) { self.parse_simple_block()? } else { let snapshot = self.create_snapshot_for_diagnostic(); @@ -2719,7 +2719,7 @@ impl<'a> Parser<'a> { // while true {} // } // ^ - if self.check(&TokenKind::OpenDelim(Delimiter::Brace)) + if self.check(exp!(OpenBrace)) && (classify::expr_requires_semi_to_be_stmt(&cond) || matches!(cond.kind, ExprKind::MacCall(..))) => @@ -2805,7 +2805,7 @@ impl<'a> Parser<'a> { begin_paren, ) { (Ok(pat), _) => pat, // Happy path. - (Err(err), Some((start_span, left))) if self.eat_keyword(kw::In) => { + (Err(err), Some((start_span, left))) if self.eat_keyword(exp!(In)) => { // We know for sure we have seen `for ($SOMETHING in`. In the happy path this would // happen right before the return of this method. let attrs = self.parse_outer_attributes()?; @@ -2839,7 +2839,7 @@ impl<'a> Parser<'a> { } (Err(err), _) => return Err(err), // Some other error, bubble up. }; - if !self.eat_keyword(kw::In) { + if !self.eat_keyword(exp!(In)) { self.error_missing_in_for_loop(); } self.check_for_for_in_in_typo(self.prev_token.span); @@ -2851,7 +2851,7 @@ impl<'a> Parser<'a> { /// Parses `for await? in ` (`for` token already eaten). fn parse_expr_for(&mut self, opt_label: Option