Rollup merge of rust-lang#66935 - petrochenkov:attrtok2, r=Centril

syntax: Unify macro and attribute arguments in AST

The unified form (`ast::MacArgs`) represents parsed arguments instead of an unstructured token stream that was previously used for attributes.
It also tracks some spans and delimiter kinds better for fn-like macros and macro definitions.

I've been talking about implementing this with @nnethercote in rust-lang#65750 (comment).
The parsed representation is closer to `MetaItem` and requires less token juggling during conversions, so it potentially may be faster.

r? @Centril

src/librustc/hir/lowering.rs

@@ -1003,7 +1003,7 @@ impl<'a> LoweringContext<'a> {
             AttrKind::Normal(ref item) => {
                 AttrKind::Normal(AttrItem {
                     path: item.path.clone(),
-                    tokens: self.lower_token_stream(item.tokens.clone()),
+                    args: self.lower_mac_args(&item.args),
                 })
             }
             AttrKind::DocComment(comment) => AttrKind::DocComment(comment)
@@ -1017,6 +1017,16 @@ impl<'a> LoweringContext<'a> {
         }
     }
 
+    fn lower_mac_args(&mut self, args: &MacArgs) -> MacArgs {
+        match *args {
+            MacArgs::Empty => MacArgs::Empty,
+            MacArgs::Delimited(dspan, delim, ref tokens) =>
+                MacArgs::Delimited(dspan, delim, self.lower_token_stream(tokens.clone())),
+            MacArgs::Eq(eq_span, ref tokens) =>
+                MacArgs::Eq(eq_span, self.lower_token_stream(tokens.clone())),
+        }
+    }
+
     fn lower_token_stream(&mut self, tokens: TokenStream) -> TokenStream {
         tokens
             .into_trees()

src/librustc/hir/lowering/item.rs

@@ -233,7 +233,7 @@ impl LoweringContext<'_> {
 
         if let ItemKind::MacroDef(ref def) = i.kind {
             if !def.legacy || attr::contains_name(&i.attrs, sym::macro_export) {
-                let body = self.lower_token_stream(def.stream());
+                let body = self.lower_token_stream(def.body.inner_tokens());
                 let hir_id = self.lower_node_id(i.id);
                 self.exported_macros.push(hir::MacroDef {
                     name: ident.name,

src/librustc_lint/builtin.rs

@@ -1450,10 +1450,10 @@ impl KeywordIdents {
 
 impl EarlyLintPass for KeywordIdents {
     fn check_mac_def(&mut self, cx: &EarlyContext<'_>, mac_def: &ast::MacroDef, _id: ast::NodeId) {
-        self.check_tokens(cx, mac_def.stream());
+        self.check_tokens(cx, mac_def.body.inner_tokens());
     }
     fn check_mac(&mut self, cx: &EarlyContext<'_>, mac: &ast::Mac) {
-        self.check_tokens(cx, mac.tts.clone().into());
+        self.check_tokens(cx, mac.args.inner_tokens());
     }
     fn check_ident(&mut self, cx: &EarlyContext<'_>, ident: ast::Ident) {
         self.check_ident_token(cx, UnderMacro(false), ident);

src/librustc_metadata/rmeta/decoder/cstore_impl.rs

@@ -32,6 +32,8 @@ use syntax::source_map;
 use syntax::source_map::Spanned;
 use syntax::symbol::Symbol;
 use syntax::expand::allocator::AllocatorKind;
+use syntax::ptr::P;
+use syntax::tokenstream::DelimSpan;
 use syntax_pos::{Span, FileName};
 
 macro_rules! provide {
@@ -427,6 +429,7 @@ impl CStore {
 
         let source_file = sess.parse_sess.source_map().new_source_file(source_name, def.body);
         let local_span = Span::with_root_ctxt(source_file.start_pos, source_file.end_pos);
+        let dspan = DelimSpan::from_single(local_span);
         let (body, mut errors) = source_file_to_stream(&sess.parse_sess, source_file, None);
         emit_unclosed_delims(&mut errors, &sess.parse_sess);
 
@@ -448,7 +451,7 @@ impl CStore {
             span: local_span,
             attrs: attrs.iter().cloned().collect(),
             kind: ast::ItemKind::MacroDef(ast::MacroDef {
-                tokens: body.into(),
+                body: P(ast::MacArgs::Delimited(dspan, ast::MacDelimiter::Brace, body)),
                 legacy: def.legacy,
             }),
             vis: source_map::respan(local_span.shrink_to_lo(), ast::VisibilityKind::Inherited),

src/librustc_parse/config.rs

@@ -101,7 +101,7 @@ impl<'a> StripUnconfigured<'a> {
         if !attr.has_name(sym::cfg_attr) {
             return vec![attr];
         }
-        if attr.get_normal_item().tokens.is_empty() {
+        if let ast::MacArgs::Empty = attr.get_normal_item().args {
             self.sess.span_diagnostic
                 .struct_span_err(
                     attr.span,

src/librustc_parse/lib.rs

@@ -277,7 +277,9 @@ pub fn parse_in_attr<'a, T>(
 ) -> PResult<'a, T> {
     let mut parser = Parser::new(
         sess,
-        attr.get_normal_item().tokens.clone(),
+        // FIXME(#66940, Centril | petrochenkov): refactor this function so it doesn't
+        // require reconstructing and immediately re-parsing delimiters.
+        attr.get_normal_item().args.outer_tokens(),
         None,
         false,
         false,
@@ -409,7 +411,7 @@ fn prepend_attrs(
             brackets.push(stream);
         }
 
-        brackets.push(item.tokens.clone());
+        brackets.push(item.args.outer_tokens());
 
         // The span we list here for `#` and for `[ ... ]` are both wrong in
         // that it encompasses more than each token, but it hopefully is "good

src/librustc_parse/parser/attr.rs

@@ -2,8 +2,7 @@ use super::{SeqSep, Parser, TokenType, PathStyle};
 use syntax::attr;
 use syntax::ast;
 use syntax::util::comments;
-use syntax::token::{self, Nonterminal, DelimToken};
-use syntax::tokenstream::{TokenStream, TokenTree};
+use syntax::token::{self, Nonterminal};
 use syntax_pos::{Span, Symbol};
 use errors::PResult;
 
@@ -181,31 +180,8 @@ impl<'a> Parser<'a> {
             item
         } else {
             let path = self.parse_path(PathStyle::Mod)?;
-            let tokens = if self.check(&token::OpenDelim(DelimToken::Paren)) ||
-               self.check(&token::OpenDelim(DelimToken::Bracket)) ||
-               self.check(&token::OpenDelim(DelimToken::Brace)) {
-                   self.parse_token_tree().into()
-            } else if self.eat(&token::Eq) {
-                let eq = TokenTree::token(token::Eq, self.prev_span);
-                let mut is_interpolated_expr = false;
-                if let token::Interpolated(nt) = &self.token.kind {
-                    if let token::NtExpr(..) = **nt {
-                        is_interpolated_expr = true;
-                    }
-                }
-                let token_tree = if is_interpolated_expr {
-                    // We need to accept arbitrary interpolated expressions to continue
-                    // supporting things like `doc = $expr` that work on stable.
-                    // Non-literal interpolated expressions are rejected after expansion.
-                    self.parse_token_tree()
-                } else {
-                    self.parse_unsuffixed_lit()?.token_tree()
-                };
-                TokenStream::new(vec![eq.into(), token_tree.into()])
-            } else {
-                TokenStream::default()
-            };
-            ast::AttrItem { path, tokens }
+            let args = self.parse_attr_args()?;
+            ast::AttrItem { path, args }
         })
     }
 
@@ -244,7 +220,7 @@ impl<'a> Parser<'a> {
         Ok(attrs)
     }
 
-    fn parse_unsuffixed_lit(&mut self) -> PResult<'a, ast::Lit> {
+    pub(super) fn parse_unsuffixed_lit(&mut self) -> PResult<'a, ast::Lit> {
         let lit = self.parse_lit()?;
         debug!("checking if {:?} is unusuffixed", lit);
 

src/librustc_parse/parser/expr.rs

@@ -922,13 +922,11 @@ impl<'a> Parser<'a> {
                     // `!`, as an operator, is prefix, so we know this isn't that.
                     if self.eat(&token::Not) {
                         // MACRO INVOCATION expression
-                        let (delim, tts) = self.expect_delimited_token_tree()?;
+                        let args = self.parse_mac_args()?;
                         hi = self.prev_span;
                         ex = ExprKind::Mac(Mac {
                             path,
-                            tts,
-                            delim,
-                            span: lo.to(hi),
+                            args,
                             prior_type_ascription: self.last_type_ascription,
                         });
                     } else if self.check(&token::OpenDelim(token::Brace)) {

src/librustc_parse/parser/item.rs

@@ -8,12 +8,12 @@ use syntax::ast::{ItemKind, ImplItem, ImplItemKind, TraitItem, TraitItemKind, Us
 use syntax::ast::{PathSegment, IsAuto, Constness, IsAsync, Unsafety, Defaultness, Extern, StrLit};
 use syntax::ast::{Visibility, VisibilityKind, Mutability, FnHeader, ForeignItem, ForeignItemKind};
 use syntax::ast::{Ty, TyKind, Generics, TraitRef, EnumDef, Variant, VariantData, StructField};
-use syntax::ast::{Mac, MacDelimiter, Block, BindingMode, FnDecl, FnSig, SelfKind, Param};
+use syntax::ast::{Mac, MacArgs, MacDelimiter, Block, BindingMode, FnDecl, FnSig, SelfKind, Param};
 use syntax::print::pprust;
 use syntax::ptr::P;
 use syntax::ThinVec;
 use syntax::token;
-use syntax::tokenstream::{TokenTree, TokenStream};
+use syntax::tokenstream::{DelimSpan, TokenTree, TokenStream};
 use syntax::source_map::{self, respan, Span};
 use syntax::struct_span_err;
 use syntax_pos::BytePos;
@@ -432,22 +432,18 @@ impl<'a> Parser<'a> {
             let prev_span = self.prev_span;
             self.complain_if_pub_macro(&visibility.node, prev_span);
 
-            let mac_lo = self.token.span;
-
             // Item macro
             let path = self.parse_path(PathStyle::Mod)?;
             self.expect(&token::Not)?;
-            let (delim, tts) = self.expect_delimited_token_tree()?;
-            if delim != MacDelimiter::Brace && !self.eat(&token::Semi) {
+            let args = self.parse_mac_args()?;
+            if args.need_semicolon() && !self.eat(&token::Semi) {
                 self.report_invalid_macro_expansion_item();
             }
 
             let hi = self.prev_span;
             let mac = Mac {
                 path,
-                tts,
-                delim,
-                span: mac_lo.to(hi),
+                args,
                 prior_type_ascription: self.last_type_ascription,
             };
             let item =
@@ -500,7 +496,6 @@ impl<'a> Parser<'a> {
         if self.token.is_path_start() &&
                 !(self.is_async_fn() && self.token.span.rust_2015()) {
             let prev_span = self.prev_span;
-            let lo = self.token.span;
             let path = self.parse_path(PathStyle::Mod)?;
 
             if path.segments.len() == 1 {
@@ -518,16 +513,14 @@ impl<'a> Parser<'a> {
             *at_end = true;
 
             // eat a matched-delimiter token tree:
-            let (delim, tts) = self.expect_delimited_token_tree()?;
-            if delim != MacDelimiter::Brace {
+            let args = self.parse_mac_args()?;
+            if args.need_semicolon() {
                 self.expect_semi()?;
             }
 
             Ok(Some(Mac {
                 path,
-                tts,
-                delim,
-                span: lo.to(self.prev_span),
+                args,
                 prior_type_ascription: self.last_type_ascription,
             }))
         } else {
@@ -1624,33 +1617,31 @@ impl<'a> Parser<'a> {
         vis: &Visibility,
         lo: Span
     ) -> PResult<'a, Option<P<Item>>> {
-        let token_lo = self.token.span;
         let (ident, def) = if self.eat_keyword(kw::Macro) {
             let ident = self.parse_ident()?;
-            let tokens = if self.check(&token::OpenDelim(token::Brace)) {
-                match self.parse_token_tree() {
-                    TokenTree::Delimited(_, _, tts) => tts,
-                    _ => unreachable!(),
-                }
+            let body = if self.check(&token::OpenDelim(token::Brace)) {
+                self.parse_mac_args()?
             } else if self.check(&token::OpenDelim(token::Paren)) {
-                let args = self.parse_token_tree();
+                let params = self.parse_token_tree();
+                let pspan = params.span();
                 let body = if self.check(&token::OpenDelim(token::Brace)) {
                     self.parse_token_tree()
                 } else {
-                    self.unexpected()?;
-                    unreachable!()
+                    return self.unexpected();
                 };
-                TokenStream::new(vec![
-                    args.into(),
-                    TokenTree::token(token::FatArrow, token_lo.to(self.prev_span)).into(),
+                let bspan = body.span();
+                let tokens = TokenStream::new(vec![
+                    params.into(),
+                    TokenTree::token(token::FatArrow, pspan.between(bspan)).into(),
                     body.into(),
-                ])
+                ]);
+                let dspan = DelimSpan::from_pair(pspan.shrink_to_lo(), bspan.shrink_to_hi());
+                P(MacArgs::Delimited(dspan, MacDelimiter::Brace, tokens))
             } else {
-                self.unexpected()?;
-                unreachable!()
+                return self.unexpected();
             };
 
-            (ident, ast::MacroDef { tokens: tokens.into(), legacy: false })
+            (ident, ast::MacroDef { body, legacy: false })
         } else if self.check_keyword(sym::macro_rules) &&
                   self.look_ahead(1, |t| *t == token::Not) &&
                   self.look_ahead(2, |t| t.is_ident()) {
@@ -1660,12 +1651,12 @@ impl<'a> Parser<'a> {
             self.bump();
 
             let ident = self.parse_ident()?;
-            let (delim, tokens) = self.expect_delimited_token_tree()?;
-            if delim != MacDelimiter::Brace && !self.eat(&token::Semi) {
+            let body = self.parse_mac_args()?;
+            if body.need_semicolon() && !self.eat(&token::Semi) {
                 self.report_invalid_macro_expansion_item();
             }
 
-            (ident, ast::MacroDef { tokens, legacy: true })
+            (ident, ast::MacroDef { body, legacy: true })
         } else {
             return Ok(None);
         };

src/librustc_parse/parser/mod.rs

@@ -16,7 +16,7 @@ use crate::lexer::UnmatchedBrace;
 
 use syntax::ast::{
     self, DUMMY_NODE_ID, AttrStyle, Attribute, CrateSugar, Extern, Ident, StrLit,
-    IsAsync, MacDelimiter, Mutability, Visibility, VisibilityKind, Unsafety,
+    IsAsync, MacArgs, MacDelimiter, Mutability, Visibility, VisibilityKind, Unsafety,
 };
 
 use syntax::print::pprust;
@@ -1010,27 +1010,49 @@ impl<'a> Parser<'a> {
         }
     }
 
-    fn expect_delimited_token_tree(&mut self) -> PResult<'a, (MacDelimiter, TokenStream)> {
-        let delim = match self.token.kind {
-            token::OpenDelim(delim) => delim,
-            _ => {
-                let msg = "expected open delimiter";
-                let mut err = self.fatal(msg);
-                err.span_label(self.token.span, msg);
-                return Err(err)
+    fn parse_mac_args(&mut self) -> PResult<'a, P<MacArgs>> {
+        self.parse_mac_args_common(true).map(P)
+    }
+
+    fn parse_attr_args(&mut self) -> PResult<'a, MacArgs> {
+        self.parse_mac_args_common(false)
+    }
+
+    fn parse_mac_args_common(&mut self, delimited_only: bool) -> PResult<'a, MacArgs> {
+        Ok(if self.check(&token::OpenDelim(DelimToken::Paren)) ||
+                       self.check(&token::OpenDelim(DelimToken::Bracket)) ||
+                       self.check(&token::OpenDelim(DelimToken::Brace)) {
+            match self.parse_token_tree() {
+                TokenTree::Delimited(dspan, delim, tokens) =>
+                    // We've confirmed above that there is a delimiter so unwrapping is OK.
+                    MacArgs::Delimited(dspan, MacDelimiter::from_token(delim).unwrap(), tokens),
+                _ => unreachable!(),
             }
-        };
-        let tts = match self.parse_token_tree() {
-            TokenTree::Delimited(_, _, tts) => tts,
-            _ => unreachable!(),
-        };
-        let delim = match delim {
-            token::Paren => MacDelimiter::Parenthesis,
-            token::Bracket => MacDelimiter::Bracket,
-            token::Brace => MacDelimiter::Brace,
-            token::NoDelim => self.bug("unexpected no delimiter"),
-        };
-        Ok((delim, tts.into()))
+        } else if !delimited_only {
+            if self.eat(&token::Eq) {
+                let eq_span = self.prev_span;
+                let mut is_interpolated_expr = false;
+                if let token::Interpolated(nt) = &self.token.kind {
+                    if let token::NtExpr(..) = **nt {
+                        is_interpolated_expr = true;
+                    }
+                }
+                let token_tree = if is_interpolated_expr {
+                    // We need to accept arbitrary interpolated expressions to continue
+                    // supporting things like `doc = $expr` that work on stable.
+                    // Non-literal interpolated expressions are rejected after expansion.
+                    self.parse_token_tree()
+                } else {
+                    self.parse_unsuffixed_lit()?.token_tree()
+                };
+
+                MacArgs::Eq(eq_span, token_tree.into())
+            } else {
+                MacArgs::Empty
+            }
+        } else {
+            return self.unexpected();
+        })
     }
 
     fn parse_or_use_outer_attributes(