Auto merge of #75800 - Aaron1011:feature/full-nt-tokens, r=petrochenkov

Attach tokens to all AST types used in `Nonterminal`

We perform token capturing when we have outer attributes (for nonterminals that support attributes - e.g. `Stmt`), or when we parse a `Nonterminal` for a `macro_rules!` argument. The full list of `Nonterminals` affected by this PR is:

* `NtBlock`
* `NtStmt`
* `NtTy`
* `NtMeta`
* `NtPath`
* `NtVis`
* `NtLiteral`

Of these nonterminals, only `NtStmt` and `NtLiteral` (which is actually just an `Expr`), support outer attributes - the rest only ever have token capturing perform when they match a `macro_rules!` argument.

This makes progress towards solving #43081 - we now collect tokens for everything that might need them. However, we still need to handle `#[cfg]`, inner attributes, and misc pretty-printing issues (e.g. #75734)

I've separated the changes into (mostly) independent commits, which could be split into individual PRs for each `Nonterminal` variant. The purpose of having them all in one PR is to do a single Crater run for all of them.

Most of the changes in this PR are trivial (adding `tokens: None` everywhere we construct the various AST structs). The significant changes are:

* `ast::Visibility` is changed from `type Visibility = Spanned<VisibilityKind>` to a `struct Visibility { kind, span, tokens }`.
* `maybe_collect_tokens` is made generic, and used for both `ast::Expr` and `ast::Stmt`.
* Some of the statement-parsing functions are refactored so that we can capture the trailing semicolon.
* `Nonterminal` and `Expr` both grew by 8 bytes, as some of the structs which are stored inline (rather than behind a `P`) now have an `Option<TokenStream>` field. Hopefully the performance impact of doing this is negligible.

compiler/rustc_ast/src/ast.rs

@@ -96,6 +96,7 @@ pub struct Path {
     /// The segments in the path: the things separated by `::`.
     /// Global paths begin with `kw::PathRoot`.
     pub segments: Vec<PathSegment>,
+    pub tokens: Option<TokenStream>,
 }
 
 impl PartialEq<Symbol> for Path {
@@ -117,7 +118,7 @@ impl Path {
     // Convert a span and an identifier to the corresponding
     // one-segment path.
     pub fn from_ident(ident: Ident) -> Path {
-        Path { segments: vec![PathSegment::from_ident(ident)], span: ident.span }
+        Path { segments: vec![PathSegment::from_ident(ident)], span: ident.span, tokens: None }
     }
 
     pub fn is_global(&self) -> bool {
@@ -540,6 +541,7 @@ pub struct Block {
     /// Distinguishes between `unsafe { ... }` and `{ ... }`.
     pub rules: BlockCheckMode,
     pub span: Span,
+    pub tokens: Option<TokenStream>,
 }
 
 /// A match pattern.
@@ -586,7 +588,7 @@ impl Pat {
             _ => return None,
         };
 
-        Some(P(Ty { kind, id: self.id, span: self.span }))
+        Some(P(Ty { kind, id: self.id, span: self.span, tokens: None }))
     }
 
     /// Walk top-down and call `it` in each place where a pattern occurs
@@ -916,6 +918,7 @@ pub struct Stmt {
     pub id: NodeId,
     pub kind: StmtKind,
     pub span: Span,
+    pub tokens: Option<TokenStream>,
 }
 
 impl Stmt {
@@ -1068,7 +1071,7 @@ pub struct Expr {
 
 // `Expr` is used a lot. Make sure it doesn't unintentionally get bigger.
 #[cfg(target_arch = "x86_64")]
-rustc_data_structures::static_assert_size!(Expr, 104);
+rustc_data_structures::static_assert_size!(Expr, 112);
 
 impl Expr {
     /// Returns `true` if this expression would be valid somewhere that expects a value;
@@ -1168,7 +1171,7 @@ impl Expr {
             _ => return None,
         };
 
-        Some(P(Ty { kind, id: self.id, span: self.span }))
+        Some(P(Ty { kind, id: self.id, span: self.span, tokens: None }))
     }
 
     pub fn precedence(&self) -> ExprPrecedence {
@@ -1866,6 +1869,7 @@ pub struct Ty {
     pub id: NodeId,
     pub kind: TyKind,
     pub span: Span,
+    pub tokens: Option<TokenStream>,
 }
 
 #[derive(Clone, Encodable, Decodable, Debug)]
@@ -2144,7 +2148,7 @@ impl Param {
     /// Builds a `Param` object from `ExplicitSelf`.
     pub fn from_self(attrs: AttrVec, eself: ExplicitSelf, eself_ident: Ident) -> Param {
         let span = eself.span.to(eself_ident.span);
-        let infer_ty = P(Ty { id: DUMMY_NODE_ID, kind: TyKind::ImplicitSelf, span });
+        let infer_ty = P(Ty { id: DUMMY_NODE_ID, kind: TyKind::ImplicitSelf, span, tokens: None });
         let param = |mutbl, ty| Param {
             attrs,
             pat: P(Pat {
@@ -2167,6 +2171,7 @@ impl Param {
                     id: DUMMY_NODE_ID,
                     kind: TyKind::Rptr(lt, MutTy { ty: infer_ty, mutbl }),
                     span,
+                    tokens: None,
                 }),
             ),
         }
@@ -2416,6 +2421,7 @@ impl<D: Decoder> rustc_serialize::Decodable<D> for AttrId {
 pub struct AttrItem {
     pub path: Path,
     pub args: MacArgs,
+    pub tokens: Option<TokenStream>,
 }
 
 /// A list of attributes.
@@ -2485,7 +2491,12 @@ pub enum CrateSugar {
     JustCrate,
 }
 
-pub type Visibility = Spanned<VisibilityKind>;
+#[derive(Clone, Encodable, Decodable, Debug)]
+pub struct Visibility {
+    pub kind: VisibilityKind,
+    pub span: Span,
+    pub tokens: Option<TokenStream>,
+}
 
 #[derive(Clone, Encodable, Decodable, Debug)]
 pub enum VisibilityKind {

compiler/rustc_ast/src/attr/mod.rs

@@ -330,7 +330,7 @@ crate fn mk_attr_id() -> AttrId {
 }
 
 pub fn mk_attr(style: AttrStyle, path: Path, args: MacArgs, span: Span) -> Attribute {
-    mk_attr_from_item(style, AttrItem { path, args }, span)
+    mk_attr_from_item(style, AttrItem { path, args, tokens: None }, span)
 }
 
 pub fn mk_attr_from_item(style: AttrStyle, item: AttrItem, span: Span) -> Attribute {
@@ -415,7 +415,7 @@ impl MetaItem {
                     }
                 }
                 let span = span.with_hi(segments.last().unwrap().ident.span.hi());
-                Path { span, segments }
+                Path { span, segments, tokens: None }
             }
             Some(TokenTree::Token(Token { kind: token::Interpolated(nt), .. })) => match *nt {
                 token::Nonterminal::NtMeta(ref item) => return item.meta(item.path.span),

compiler/rustc_ast/src/mut_visit.rs

@@ -14,7 +14,7 @@ use crate::tokenstream::*;
 
 use rustc_data_structures::map_in_place::MapInPlace;
 use rustc_data_structures::sync::Lrc;
-use rustc_span::source_map::{respan, Spanned};
+use rustc_span::source_map::Spanned;
 use rustc_span::symbol::Ident;
 use rustc_span::Span;
 
@@ -451,7 +451,7 @@ pub fn noop_visit_ty_constraint<T: MutVisitor>(
 }
 
 pub fn noop_visit_ty<T: MutVisitor>(ty: &mut P<Ty>, vis: &mut T) {
-    let Ty { id, kind, span } = ty.deref_mut();
+    let Ty { id, kind, span, tokens: _ } = ty.deref_mut();
     vis.visit_id(id);
     match kind {
         TyKind::Infer | TyKind::ImplicitSelf | TyKind::Err | TyKind::Never | TyKind::CVarArgs => {}
@@ -513,7 +513,7 @@ pub fn noop_visit_ident<T: MutVisitor>(Ident { name: _, span }: &mut Ident, vis:
     vis.visit_span(span);
 }
 
-pub fn noop_visit_path<T: MutVisitor>(Path { segments, span }: &mut Path, vis: &mut T) {
+pub fn noop_visit_path<T: MutVisitor>(Path { segments, span, tokens: _ }: &mut Path, vis: &mut T) {
     vis.visit_span(span);
     for PathSegment { ident, id, args } in segments {
         vis.visit_ident(ident);
@@ -579,7 +579,7 @@ pub fn noop_visit_local<T: MutVisitor>(local: &mut P<Local>, vis: &mut T) {
 pub fn noop_visit_attribute<T: MutVisitor>(attr: &mut Attribute, vis: &mut T) {
     let Attribute { kind, id: _, style: _, span } = attr;
     match kind {
-        AttrKind::Normal(AttrItem { path, args }) => {
+        AttrKind::Normal(AttrItem { path, args, tokens: _ }) => {
             vis.visit_path(path);
             visit_mac_args(args, vis);
         }
@@ -709,7 +709,7 @@ pub fn noop_visit_interpolated<T: MutVisitor>(nt: &mut token::Nonterminal, vis:
         token::NtLifetime(ident) => vis.visit_ident(ident),
         token::NtLiteral(expr) => vis.visit_expr(expr),
         token::NtMeta(item) => {
-            let AttrItem { path, args } = item.deref_mut();
+            let AttrItem { path, args, tokens: _ } = item.deref_mut();
             vis.visit_path(path);
             visit_mac_args(args, vis);
         }
@@ -871,7 +871,7 @@ pub fn noop_visit_mt<T: MutVisitor>(MutTy { ty, mutbl: _ }: &mut MutTy, vis: &mu
 }
 
 pub fn noop_visit_block<T: MutVisitor>(block: &mut P<Block>, vis: &mut T) {
-    let Block { id, stmts, rules: _, span } = block.deref_mut();
+    let Block { id, stmts, rules: _, span, tokens: _ } = block.deref_mut();
     vis.visit_id(id);
     stmts.flat_map_in_place(|stmt| vis.flat_map_stmt(stmt));
     vis.visit_span(span);
@@ -978,11 +978,13 @@ pub fn noop_visit_mod<T: MutVisitor>(module: &mut Mod, vis: &mut T) {
 
 pub fn noop_visit_crate<T: MutVisitor>(krate: &mut Crate, vis: &mut T) {
     visit_clobber(krate, |Crate { module, attrs, span, proc_macros }| {
+        let item_vis =
+            Visibility { kind: VisibilityKind::Public, span: span.shrink_to_lo(), tokens: None };
         let item = P(Item {
             ident: Ident::invalid(),
             attrs,
             id: DUMMY_NODE_ID,
-            vis: respan(span.shrink_to_lo(), VisibilityKind::Public),
+            vis: item_vis,
             span,
             kind: ItemKind::Mod(module),
             tokens: None,
@@ -1284,12 +1286,15 @@ pub fn noop_filter_map_expr<T: MutVisitor>(mut e: P<Expr>, vis: &mut T) -> Optio
 }
 
 pub fn noop_flat_map_stmt<T: MutVisitor>(
-    Stmt { kind, mut span, mut id }: Stmt,
+    Stmt { kind, mut span, mut id, tokens }: Stmt,
     vis: &mut T,
 ) -> SmallVec<[Stmt; 1]> {
     vis.visit_id(&mut id);
     vis.visit_span(&mut span);
-    noop_flat_map_stmt_kind(kind, vis).into_iter().map(|kind| Stmt { id, kind, span }).collect()
+    noop_flat_map_stmt_kind(kind, vis)
+        .into_iter()
+        .map(|kind| Stmt { id, kind, span, tokens: tokens.clone() })
+        .collect()
 }
 
 pub fn noop_flat_map_stmt_kind<T: MutVisitor>(
@@ -1314,13 +1319,13 @@ pub fn noop_flat_map_stmt_kind<T: MutVisitor>(
     }
 }
 
-pub fn noop_visit_vis<T: MutVisitor>(Spanned { node, span }: &mut Visibility, vis: &mut T) {
-    match node {
+pub fn noop_visit_vis<T: MutVisitor>(visibility: &mut Visibility, vis: &mut T) {
+    match &mut visibility.kind {
         VisibilityKind::Public | VisibilityKind::Crate(_) | VisibilityKind::Inherited => {}
         VisibilityKind::Restricted { path, id } => {
             vis.visit_path(path);
             vis.visit_id(id);
         }
     }
-    vis.visit_span(span);
+    vis.visit_span(&mut visibility.span);
 }

compiler/rustc_ast/src/token.rs

@@ -700,7 +700,7 @@ pub enum Nonterminal {
 
 // `Nonterminal` is used a lot. Make sure it doesn't unintentionally get bigger.
 #[cfg(target_arch = "x86_64")]
-rustc_data_structures::static_assert_size!(Nonterminal, 40);
+rustc_data_structures::static_assert_size!(Nonterminal, 48);
 
 #[derive(Debug, Copy, Clone, PartialEq, Encodable, Decodable)]
 pub enum NonterminalKind {

compiler/rustc_ast/src/visit.rs

@@ -879,7 +879,7 @@ pub fn walk_arm<'a, V: Visitor<'a>>(visitor: &mut V, arm: &'a Arm) {
 }
 
 pub fn walk_vis<'a, V: Visitor<'a>>(visitor: &mut V, vis: &'a Visibility) {
-    if let VisibilityKind::Restricted { ref path, id } = vis.node {
+    if let VisibilityKind::Restricted { ref path, id } = vis.kind {
         visitor.visit_path(path, id);
     }
 }

compiler/rustc_ast_lowering/src/item.rs

@@ -251,7 +251,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
             ItemKind::ExternCrate(orig_name) => hir::ItemKind::ExternCrate(orig_name),
             ItemKind::Use(ref use_tree) => {
                 // Start with an empty prefix.
-                let prefix = Path { segments: vec![], span: use_tree.span };
+                let prefix = Path { segments: vec![], span: use_tree.span, tokens: None };
 
                 self.lower_use_tree(use_tree, &prefix, id, vis, ident, attrs)
             }
@@ -488,7 +488,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 *ident = tree.ident();
 
                 // First, apply the prefix to the path.
-                let mut path = Path { segments, span: path.span };
+                let mut path = Path { segments, span: path.span, tokens: None };
 
                 // Correctly resolve `self` imports.
                 if path.segments.len() > 1
@@ -540,8 +540,11 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 hir::ItemKind::Use(path, hir::UseKind::Single)
             }
             UseTreeKind::Glob => {
-                let path =
-                    self.lower_path(id, &Path { segments, span: path.span }, ParamMode::Explicit);
+                let path = self.lower_path(
+                    id,
+                    &Path { segments, span: path.span, tokens: None },
+                    ParamMode::Explicit,
+                );
                 hir::ItemKind::Use(path, hir::UseKind::Glob)
             }
             UseTreeKind::Nested(ref trees) => {
@@ -569,7 +572,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
                 // for that we return the `{}` import (called the
                 // `ListStem`).
 
-                let prefix = Path { segments, span: prefix.span.to(path.span) };
+                let prefix = Path { segments, span: prefix.span.to(path.span), tokens: None };
 
                 // Add all the nested `PathListItem`s to the HIR.
                 for &(ref use_tree, id) in trees {
@@ -927,7 +930,7 @@ impl<'hir> LoweringContext<'_, 'hir> {
         v: &Visibility,
         explicit_owner: Option<NodeId>,
     ) -> hir::Visibility<'hir> {
-        let node = match v.node {
+        let node = match v.kind {
             VisibilityKind::Public => hir::VisibilityKind::Public,
             VisibilityKind::Crate(sugar) => hir::VisibilityKind::Crate(sugar),
             VisibilityKind::Restricted { ref path, id } => {

compiler/rustc_ast_lowering/src/lib.rs

@@ -967,6 +967,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             AttrKind::Normal(ref item) => AttrKind::Normal(AttrItem {
                 path: item.path.clone(),
                 args: self.lower_mac_args(&item.args),
+                tokens: None,
             }),
             AttrKind::DocComment(comment_kind, data) => AttrKind::DocComment(comment_kind, data),
         };
@@ -1106,6 +1107,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                                 id: node_id,
                                 kind: TyKind::ImplTrait(impl_trait_node_id, bounds.clone()),
                                 span: constraint.span,
+                                tokens: None,
                             },
                             itctx,
                         );

compiler/rustc_ast_passes/src/ast_validation.rs

@@ -198,13 +198,13 @@ impl<'a> AstValidator<'a> {
     }
 
     fn invalid_visibility(&self, vis: &Visibility, note: Option<&str>) {
-        if let VisibilityKind::Inherited = vis.node {
+        if let VisibilityKind::Inherited = vis.kind {
             return;
         }
 
         let mut err =
             struct_span_err!(self.session, vis.span, E0449, "unnecessary visibility qualifier");
-        if vis.node.is_pub() {
+        if vis.kind.is_pub() {
             err.span_label(vis.span, "`pub` not permitted here because it's implied");
         }
         if let Some(note) = note {

compiler/rustc_ast_passes/src/feature_gate.rs

@@ -594,7 +594,7 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> {
     }
 
     fn visit_vis(&mut self, vis: &'a ast::Visibility) {
-        if let ast::VisibilityKind::Crate(ast::CrateSugar::JustCrate) = vis.node {
+        if let ast::VisibilityKind::Crate(ast::CrateSugar::JustCrate) = vis.kind {
             gate_feature_post!(
                 &self,
                 crate_visibility_modifier,

compiler/rustc_ast_pretty/src/pprust.rs

@@ -1359,7 +1359,7 @@ impl<'a> State<'a> {
     }
 
     crate fn print_visibility(&mut self, vis: &ast::Visibility) {
-        match vis.node {
+        match vis.kind {
             ast::VisibilityKind::Public => self.word_nbsp("pub"),
             ast::VisibilityKind::Crate(sugar) => match sugar {
                 ast::CrateSugar::PubCrate => self.word_nbsp("pub(crate)"),

compiler/rustc_ast_pretty/src/pprust/tests.rs

@@ -1,7 +1,6 @@
 use super::*;
 
 use rustc_ast as ast;
-use rustc_span::source_map::respan;
 use rustc_span::symbol::Ident;
 use rustc_span::with_default_session_globals;
 
@@ -45,7 +44,11 @@ fn test_variant_to_string() {
 
         let var = ast::Variant {
             ident,
-            vis: respan(rustc_span::DUMMY_SP, ast::VisibilityKind::Inherited),
+            vis: ast::Visibility {
+                span: rustc_span::DUMMY_SP,
+                kind: ast::VisibilityKind::Inherited,
+                tokens: None,
+            },
             attrs: Vec::new(),
             id: ast::DUMMY_NODE_ID,
             data: ast::VariantData::Unit(ast::DUMMY_NODE_ID),

compiler/rustc_builtin_macros/src/cmdline_attrs.rs

@@ -15,7 +15,7 @@ pub fn inject(mut krate: ast::Crate, parse_sess: &ParseSess, attrs: &[String]) -
         );
 
         let start_span = parser.token.span;
-        let AttrItem { path, args } = match parser.parse_attr_item() {
+        let AttrItem { path, args, tokens: _ } = match parser.parse_attr_item() {
             Ok(ai) => ai,
             Err(mut err) => {
                 err.emit();

compiler/rustc_builtin_macros/src/concat_idents.rs

@@ -61,6 +61,7 @@ pub fn expand_concat_idents<'cx>(
                 id: ast::DUMMY_NODE_ID,
                 kind: ast::TyKind::Path(None, ast::Path::from_ident(self.ident)),
                 span: self.ident.span,
+                tokens: None,
             }))
         }
     }

compiler/rustc_builtin_macros/src/deriving/debug.rs

@@ -133,5 +133,5 @@ fn stmt_let_underscore(cx: &mut ExtCtxt<'_>, sp: Span, expr: P<ast::Expr>) -> as
         span: sp,
         attrs: ast::AttrVec::new(),
     });
-    ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Local(local), span: sp }
+    ast::Stmt { id: ast::DUMMY_NODE_ID, kind: ast::StmtKind::Local(local), span: sp, tokens: None }
 }