Auto merge of rust-lang#75800 - Aaron1011:feature/full-nt-tokens, r=p…

…etrochenkov

Attach tokens to all AST types used in `Nonterminal`

We perform token capturing when we have outer attributes (for nonterminals that support attributes - e.g. `Stmt`), or when we parse a `Nonterminal` for a `macro_rules!` argument. The full list of `Nonterminals` affected by this PR is:

* `NtBlock`
* `NtStmt`
* `NtTy`
* `NtMeta`
* `NtPath`
* `NtVis`
* `NtLiteral`

Of these nonterminals, only `NtStmt` and `NtLiteral` (which is actually just an `Expr`), support outer attributes - the rest only ever have token capturing perform when they match a `macro_rules!` argument.

This makes progress towards solving rust-lang#43081 - we now collect tokens for everything that might need them. However, we still need to handle `#[cfg]`, inner attributes, and misc pretty-printing issues (e.g. rust-lang#75734)

I've separated the changes into (mostly) independent commits, which could be split into individual PRs for each `Nonterminal` variant. The purpose of having them all in one PR is to do a single Crater run for all of them.

Most of the changes in this PR are trivial (adding `tokens: None` everywhere we construct the various AST structs). The significant changes are:

* `ast::Visibility` is changed from `type Visibility = Spanned<VisibilityKind>` to a `struct Visibility { kind, span, tokens }`.
* `maybe_collect_tokens` is made generic, and used for both `ast::Expr` and `ast::Stmt`.
* Some of the statement-parsing functions are refactored so that we can capture the trailing semicolon.
* `Nonterminal` and `Expr` both grew by 8 bytes, as some of the structs which are stored inline (rather than behind a `P`) now have an `Option<TokenStream>` field. Hopefully the performance impact of doing this is negligible.

clippy_lints/src/enum_variants.rs

@@ -285,7 +285,7 @@ impl EarlyLintPass for EnumVariantNames {
                             );
                         }
                     }
-                    if item.vis.node.is_pub() {
+                    if item.vis.kind.is_pub() {
                         let matching = partial_match(mod_camel, &item_camel);
                         let rmatching = partial_rmatch(mod_camel, &item_camel);
                         let nchars = mod_camel.chars().count();
@@ -316,7 +316,7 @@ impl EarlyLintPass for EnumVariantNames {
             }
         }
         if let ItemKind::Enum(ref def, _) = item.kind {
-            let lint = match item.vis.node {
+            let lint = match item.vis.kind {
                 VisibilityKind::Public => PUB_ENUM_VARIANT_NAMES,
                 _ => ENUM_VARIANT_NAMES,
             };

clippy_lints/src/manual_non_exhaustive.rs

@@ -122,7 +122,7 @@ fn check_manual_non_exhaustive_enum(cx: &EarlyContext<'_>, item: &Item, variants
 
 fn check_manual_non_exhaustive_struct(cx: &EarlyContext<'_>, item: &Item, data: &VariantData) {
     fn is_private(field: &StructField) -> bool {
-        matches!(field.vis.node, VisibilityKind::Inherited)
+        matches!(field.vis.kind, VisibilityKind::Inherited)
     }
 
     fn is_non_exhaustive_marker(field: &StructField) -> bool {
@@ -141,7 +141,7 @@ fn check_manual_non_exhaustive_struct(cx: &EarlyContext<'_>, item: &Item, data:
 
     let fields = data.fields();
     let private_fields = fields.iter().filter(|f| is_private(f)).count();
-    let public_fields = fields.iter().filter(|f| f.vis.node.is_pub()).count();
+    let public_fields = fields.iter().filter(|f| f.vis.kind.is_pub()).count();
 
     if_chain! {
         if private_fields == 1 && public_fields >= 1 && public_fields == fields.len() - 1;

clippy_lints/src/single_component_path_imports.rs

@@ -41,7 +41,7 @@ impl EarlyLintPass for SingleComponentPathImports {
         if_chain! {
             if !in_macro(item.span);
             if cx.sess.opts.edition == Edition::Edition2018;
-            if !item.vis.node.is_pub();
+            if !item.vis.kind.is_pub();
             if let ItemKind::Use(use_tree) = &item.kind;
             if let segments = &use_tree.prefix.segments;
             if segments.len() == 1;

clippy_lints/src/utils/ast_utils.rs

@@ -394,7 +394,7 @@ pub fn eq_defaultness(l: Defaultness, r: Defaultness) -> bool {
 
 pub fn eq_vis(l: &Visibility, r: &Visibility) -> bool {
     use VisibilityKind::*;
-    match (&l.node, &r.node) {
+    match (&l.kind, &r.kind) {
         (Public, Public) | (Inherited, Inherited) | (Crate(_), Crate(_)) => true,
         (Restricted { path: l, .. }, Restricted { path: r, .. }) => eq_path(l, r),
         _ => false,