From 2479f6cb48989f85c89671cef5b7d553d6151068 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Thu, 4 Jul 2024 10:16:51 +1000 Subject: [PATCH] Remove `TokenStream::flattened` and `InvisibleOrigin::FlattenToken`. They are no longer needed. This does slightly worsen the error message for a single test, but that test contains code that is so badly broken that I'm not worried about it. --- compiler/rustc_ast/src/token.rs | 8 +--- compiler/rustc_ast/src/tokenstream.rs | 44 +------------------ compiler/rustc_ast_lowering/src/lib.rs | 2 +- compiler/rustc_builtin_macros/src/cfg_eval.rs | 10 ++--- compiler/rustc_expand/src/config.rs | 6 --- tests/ui/macros/syntax-error-recovery.rs | 4 +- tests/ui/macros/syntax-error-recovery.stderr | 6 +-- 7 files changed, 11 insertions(+), 69 deletions(-) diff --git a/compiler/rustc_ast/src/token.rs b/compiler/rustc_ast/src/token.rs index ad49683e85991..fa264edd0c32e 100644 --- a/compiler/rustc_ast/src/token.rs +++ b/compiler/rustc_ast/src/token.rs @@ -47,10 +47,6 @@ pub enum InvisibleOrigin { // Converted from `proc_macro::Delimiter` in // `proc_macro::Delimiter::to_internal`, i.e. returned by a proc macro. ProcMacro, - - // Converted from `TokenKind::NtLifetime` in `TokenStream::flatten_token`. - // Treated similarly to `ProcMacro`. - FlattenToken, } impl PartialEq for InvisibleOrigin { @@ -142,9 +138,7 @@ impl Delimiter { match self { Delimiter::Parenthesis | Delimiter::Bracket | Delimiter::Brace => false, Delimiter::Invisible(InvisibleOrigin::MetaVar(_)) => false, - Delimiter::Invisible(InvisibleOrigin::FlattenToken | InvisibleOrigin::ProcMacro) => { - true - } + Delimiter::Invisible(InvisibleOrigin::ProcMacro) => true, } } diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index 887475a1a6710..1f3b271fc6ed7 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -24,7 +24,7 @@ use rustc_span::{DUMMY_SP, Span, SpanDecoder, SpanEncoder, Symbol, sym}; use crate::ast::AttrStyle; use crate::ast_traits::{HasAttrs, HasTokens}; -use crate::token::{self, Delimiter, InvisibleOrigin, Token, TokenKind}; +use crate::token::{self, Delimiter, Token, TokenKind}; use crate::{AttrVec, Attribute}; /// Part of a `TokenStream`. @@ -470,48 +470,6 @@ impl TokenStream { TokenStream::new(tts) } - fn flatten_token(token: &Token, spacing: Spacing) -> TokenTree { - match token.kind { - token::NtIdent(ident, is_raw) => { - TokenTree::Token(Token::new(token::Ident(ident.name, is_raw), ident.span), spacing) - } - token::NtLifetime(ident, is_raw) => TokenTree::Delimited( - DelimSpan::from_single(token.span), - DelimSpacing::new(Spacing::JointHidden, spacing), - Delimiter::Invisible(InvisibleOrigin::FlattenToken), - TokenStream::token_alone(token::Lifetime(ident.name, is_raw), ident.span), - ), - _ => TokenTree::Token(*token, spacing), - } - } - - fn flatten_token_tree(tree: &TokenTree) -> TokenTree { - match tree { - TokenTree::Token(token, spacing) => TokenStream::flatten_token(token, *spacing), - TokenTree::Delimited(span, spacing, delim, tts) => { - TokenTree::Delimited(*span, *spacing, *delim, tts.flattened()) - } - } - } - - #[must_use] - pub fn flattened(&self) -> TokenStream { - fn can_skip(stream: &TokenStream) -> bool { - stream.trees().all(|tree| match tree { - TokenTree::Token(token, _) => { - !matches!(token.kind, token::NtIdent(..) | token::NtLifetime(..)) - } - TokenTree::Delimited(.., inner) => can_skip(inner), - }) - } - - if can_skip(self) { - return self.clone(); - } - - self.trees().map(|tree| TokenStream::flatten_token_tree(tree)).collect() - } - // If `vec` is not empty, try to glue `tt` onto its last token. The return // value indicates if gluing took place. fn try_glue_to_last(vec: &mut Vec, tt: &TokenTree) -> bool { diff --git a/compiler/rustc_ast_lowering/src/lib.rs b/compiler/rustc_ast_lowering/src/lib.rs index 0b2969a49ba88..3dd0328f431d8 100644 --- a/compiler/rustc_ast_lowering/src/lib.rs +++ b/compiler/rustc_ast_lowering/src/lib.rs @@ -934,7 +934,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { } fn lower_delim_args(&self, args: &DelimArgs) -> DelimArgs { - DelimArgs { dspan: args.dspan, delim: args.delim, tokens: args.tokens.flattened() } + DelimArgs { dspan: args.dspan, delim: args.delim, tokens: args.tokens.clone() } } /// Lower an associated item constraint. diff --git a/compiler/rustc_builtin_macros/src/cfg_eval.rs b/compiler/rustc_builtin_macros/src/cfg_eval.rs index b337de1c9e307..9431710924d10 100644 --- a/compiler/rustc_builtin_macros/src/cfg_eval.rs +++ b/compiler/rustc_builtin_macros/src/cfg_eval.rs @@ -93,11 +93,7 @@ impl CfgEval<'_> { // the location of `#[cfg]` and `#[cfg_attr]` in the token stream. The tokenization // process is lossless, so this process is invisible to proc-macros. - // 'Flatten' all nonterminals (i.e. `TokenKind::Nt{Ident,Lifetime}`) - // to `None`-delimited groups containing the corresponding tokens. This - // is normally delayed until the proc-macro server actually needs to - // provide tokens to a proc-macro. We do this earlier, so that we can - // handle cases like: + // Interesting cases: // // ```rust // #[cfg_eval] #[cfg] $item @@ -105,8 +101,8 @@ impl CfgEval<'_> { // // where `$item` is `#[cfg_attr] struct Foo {}`. We want to make // sure to evaluate *all* `#[cfg]` and `#[cfg_attr]` attributes - the simplest - // way to do this is to do a single parse of a stream without any nonterminals. - let orig_tokens = annotatable.to_tokens().flattened(); + // way to do this is to do a single parse of the token stream. + let orig_tokens = annotatable.to_tokens(); // Re-parse the tokens, setting the `capture_cfg` flag to save extra information // to the captured `AttrTokenStream` (specifically, we capture diff --git a/compiler/rustc_expand/src/config.rs b/compiler/rustc_expand/src/config.rs index 2847ea1232a2f..681abb9ac511f 100644 --- a/compiler/rustc_expand/src/config.rs +++ b/compiler/rustc_expand/src/config.rs @@ -225,12 +225,6 @@ impl<'a> StripUnconfigured<'a> { inner = self.configure_tokens(&inner); Some(AttrTokenTree::Delimited(sp, spacing, delim, inner)) } - AttrTokenTree::Token( - Token { kind: TokenKind::NtIdent(..) | TokenKind::NtLifetime(..), .. }, - _, - ) => { - panic!("Nonterminal should have been flattened: {:?}", tree); - } AttrTokenTree::Token( Token { kind: TokenKind::OpenDelim(_) | TokenKind::CloseDelim(_), .. }, _, diff --git a/tests/ui/macros/syntax-error-recovery.rs b/tests/ui/macros/syntax-error-recovery.rs index 6cf9d54e82636..e1681ea32a2ee 100644 --- a/tests/ui/macros/syntax-error-recovery.rs +++ b/tests/ui/macros/syntax-error-recovery.rs @@ -5,14 +5,14 @@ macro_rules! values { $( #[$attr] $token $($inner)? = $value, + //~^ ERROR expected one of `!` or `::`, found `` )* } }; } -//~^^^^^ ERROR expected one of `(`, `,`, `=`, `{`, or `}`, found `ty` metavariable +//~^^^^^^ ERROR expected one of `(`, `,`, `=`, `{`, or `}`, found `ty` metavariable //~| ERROR macro expansion ignores `ty` metavariable and any tokens following values!(STRING(1) as (String) => cfg(test),); -//~^ ERROR expected one of `!` or `::`, found `` fn main() {} diff --git a/tests/ui/macros/syntax-error-recovery.stderr b/tests/ui/macros/syntax-error-recovery.stderr index 61758fb9d7dc6..a2059aa1aa802 100644 --- a/tests/ui/macros/syntax-error-recovery.stderr +++ b/tests/ui/macros/syntax-error-recovery.stderr @@ -22,10 +22,10 @@ LL | values!(STRING(1) as (String) => cfg(test),); = note: the usage of `values!` is likely invalid in item context error: expected one of `!` or `::`, found `` - --> $DIR/syntax-error-recovery.rs:15:9 + --> $DIR/syntax-error-recovery.rs:7:17 | -LL | values!(STRING(1) as (String) => cfg(test),); - | ^^^^^^ expected one of `!` or `::` +LL | $token $($inner)? = $value, + | ^^^^^^ expected one of `!` or `::` error: aborting due to 3 previous errors