Rollup merge of rust-lang#63867 - petrochenkov:dhelpers, r=matthewjasper

resolve: Block expansion of a derive container until all its derives are resolved

So, it turns out there's one more reason to block expansion of a `#[derive]` container until all the derives inside it are resolved, beside `Copy` (rust-lang#63248).

The set of derive helper attributes registered by derives in the container also has to be known before the derives themselves are expanded, otherwise it may be too late (see rust-lang#63468 (comment) and the `#[stable_hasher]`-related test failures in rust-lang#63468).

So, we stop our attempts to unblock the container earlier, as soon as the `Copy` status is known, and just block until all its derives are resolved.
After all the derives are resolved we immediately go and process their helper attributes in the item, without delaying it until expansion of the individual derives.

Unblocks rust-lang#63468
r? @matthewjasper (as a reviewer of rust-lang#63248)
cc @c410-f3r

src/librustc_metadata/decoder.rs

@@ -527,9 +527,7 @@ impl<'a, 'tcx> CrateMetadata {
                     attributes.iter().cloned().map(Symbol::intern).collect::<Vec<_>>();
                 (
                     trait_name,
-                    SyntaxExtensionKind::Derive(Box::new(ProcMacroDerive {
-                        client, attrs: helper_attrs.clone()
-                    })),
+                    SyntaxExtensionKind::Derive(Box::new(ProcMacroDerive { client })),
                     helper_attrs,
                 )
             }

src/librustc_resolve/macros.rs

@@ -13,7 +13,7 @@ use rustc::{ty, lint, span_bug};
 use syntax::ast::{self, NodeId, Ident};
 use syntax::attr::StabilityLevel;
 use syntax::edition::Edition;
-use syntax::ext::base::{self, Indeterminate, SpecialDerives};
+use syntax::ext::base::{self, InvocationRes, Indeterminate, SpecialDerives};
 use syntax::ext::base::{MacroKind, SyntaxExtension};
 use syntax::ext::expand::{AstFragment, Invocation, InvocationKind};
 use syntax::ext::hygiene::{self, ExpnId, ExpnData, ExpnKind};
@@ -142,7 +142,7 @@ impl<'a> base::Resolver for Resolver<'a> {
 
     fn resolve_macro_invocation(
         &mut self, invoc: &Invocation, eager_expansion_root: ExpnId, force: bool
-    ) -> Result<Option<Lrc<SyntaxExtension>>, Indeterminate> {
+    ) -> Result<InvocationRes, Indeterminate> {
         let invoc_id = invoc.expansion_data.id;
         let parent_scope = match self.invocation_parent_scopes.get(&invoc_id) {
             Some(parent_scope) => *parent_scope,
@@ -165,25 +165,24 @@ impl<'a> base::Resolver for Resolver<'a> {
             InvocationKind::Derive { ref path, .. } =>
                 (path, MacroKind::Derive, &[][..], false),
             InvocationKind::DeriveContainer { ref derives, .. } => {
-                // Block expansion of derives in the container until we know whether one of them
-                // is a built-in `Copy`. Skip the resolution if there's only one derive - either
-                // it's not a `Copy` and we don't need to do anything, or it's a `Copy` and it
-                // will automatically knows about itself.
-                let mut result = Ok(None);
-                if derives.len() > 1 {
-                    for path in derives {
-                        match self.resolve_macro_path(path, Some(MacroKind::Derive),
-                                                      &parent_scope, true, force) {
-                            Ok((Some(ref ext), _)) if ext.is_derive_copy => {
-                                self.add_derives(invoc_id, SpecialDerives::COPY);
-                                return Ok(None);
-                            }
-                            Err(Determinacy::Undetermined) => result = Err(Indeterminate),
-                            _ => {}
-                        }
-                    }
+                // Block expansion of the container until we resolve all derives in it.
+                // This is required for two reasons:
+                // - Derive helper attributes are in scope for the item to which the `#[derive]`
+                //   is applied, so they have to be produced by the container's expansion rather
+                //   than by individual derives.
+                // - Derives in the container need to know whether one of them is a built-in `Copy`.
+                // FIXME: Try to avoid repeated resolutions for derives here and in expansion.
+                let mut exts = Vec::new();
+                for path in derives {
+                    exts.push(match self.resolve_macro_path(
+                        path, Some(MacroKind::Derive), &parent_scope, true, force
+                    ) {
+                        Ok((Some(ext), _)) => ext,
+                        Ok(_) | Err(Determinacy::Determined) => self.dummy_ext(MacroKind::Derive),
+                        Err(Determinacy::Undetermined) => return Err(Indeterminate),
+                    })
                 }
-                return result;
+                return Ok(InvocationRes::DeriveContainer(exts));
             }
         };
 
@@ -203,7 +202,7 @@ impl<'a> base::Resolver for Resolver<'a> {
             self.definitions.add_parent_module_of_macro_def(invoc_id, normal_module_def_id);
         }
 
-        Ok(Some(ext))
+        Ok(InvocationRes::Single(ext))
     }
 
     fn check_unused_macros(&self) {

src/libsyntax/ext/base.rs

@@ -11,6 +11,7 @@ use crate::ptr::P;
 use crate::symbol::{kw, sym, Ident, Symbol};
 use crate::{ThinVec, MACRO_ARGUMENTS};
 use crate::tokenstream::{self, TokenStream, TokenTree};
+use crate::visit::Visitor;
 
 use errors::{DiagnosticBuilder, DiagnosticId};
 use smallvec::{smallvec, SmallVec};
@@ -72,6 +73,17 @@ impl Annotatable {
         }
     }
 
+    pub fn visit_with<'a, V: Visitor<'a>>(&'a self, visitor: &mut V) {
+        match self {
+            Annotatable::Item(item) => visitor.visit_item(item),
+            Annotatable::TraitItem(trait_item) => visitor.visit_trait_item(trait_item),
+            Annotatable::ImplItem(impl_item) => visitor.visit_impl_item(impl_item),
+            Annotatable::ForeignItem(foreign_item) => visitor.visit_foreign_item(foreign_item),
+            Annotatable::Stmt(stmt) => visitor.visit_stmt(stmt),
+            Annotatable::Expr(expr) => visitor.visit_expr(expr),
+        }
+    }
+
     pub fn expect_item(self) -> P<ast::Item> {
         match self {
             Annotatable::Item(i) => i,
@@ -700,6 +712,12 @@ impl SyntaxExtension {
 
 pub type NamedSyntaxExtension = (Name, SyntaxExtension);
 
+/// Result of resolving a macro invocation.
+pub enum InvocationRes {
+    Single(Lrc<SyntaxExtension>),
+    DeriveContainer(Vec<Lrc<SyntaxExtension>>),
+}
+
 /// Error type that denotes indeterminacy.
 pub struct Indeterminate;
 
@@ -727,7 +745,7 @@ pub trait Resolver {
 
     fn resolve_macro_invocation(
         &mut self, invoc: &Invocation, eager_expansion_root: ExpnId, force: bool
-    ) -> Result<Option<Lrc<SyntaxExtension>>, Indeterminate>;
+    ) -> Result<InvocationRes, Indeterminate>;
 
     fn check_unused_macros(&self);
 

src/libsyntax/ext/expand.rs

@@ -4,7 +4,7 @@ use crate::attr::{self, HasAttrs};
 use crate::source_map::respan;
 use crate::config::StripUnconfigured;
 use crate::ext::base::*;
-use crate::ext::proc_macro::collect_derives;
+use crate::ext::proc_macro::{collect_derives, MarkAttrs};
 use crate::ext::hygiene::{ExpnId, SyntaxContext, ExpnData, ExpnKind};
 use crate::ext::tt::macro_rules::annotate_err_with_kind;
 use crate::ext::placeholders::{placeholder, PlaceholderExpander};
@@ -307,10 +307,10 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
 
             let eager_expansion_root =
                 if self.monotonic { invoc.expansion_data.id } else { orig_expansion_data.id };
-            let ext = match self.cx.resolver.resolve_macro_invocation(
+            let res = match self.cx.resolver.resolve_macro_invocation(
                 &invoc, eager_expansion_root, force
             ) {
-                Ok(ext) => ext,
+                Ok(res) => res,
                 Err(Indeterminate) => {
                     undetermined_invocations.push(invoc);
                     continue
@@ -322,54 +322,72 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
             self.cx.current_expansion = invoc.expansion_data.clone();
 
             // FIXME(jseyfried): Refactor out the following logic
-            let (expanded_fragment, new_invocations) = if let Some(ext) = ext {
-                let fragment = self.expand_invoc(invoc, &ext.kind);
-                self.collect_invocations(fragment, &[])
-            } else if let InvocationKind::DeriveContainer { derives: traits, item } = invoc.kind {
-                if !item.derive_allowed() {
-                    let attr = attr::find_by_name(item.attrs(), sym::derive)
-                        .expect("`derive` attribute should exist");
-                    let span = attr.span;
-                    let mut err = self.cx.mut_span_err(span,
-                                                        "`derive` may only be applied to \
-                                                        structs, enums and unions");
-                    if let ast::AttrStyle::Inner = attr.style {
-                        let trait_list = traits.iter()
-                            .map(|t| t.to_string()).collect::<Vec<_>>();
-                        let suggestion = format!("#[derive({})]", trait_list.join(", "));
-                        err.span_suggestion(
-                            span, "try an outer attribute", suggestion,
-                            // We don't 𝑘𝑛𝑜𝑤 that the following item is an ADT
-                            Applicability::MaybeIncorrect
-                        );
-                    }
-                    err.emit();
+            let (expanded_fragment, new_invocations) = match res {
+                InvocationRes::Single(ext) => {
+                    let fragment = self.expand_invoc(invoc, &ext.kind);
+                    self.collect_invocations(fragment, &[])
                 }
+                InvocationRes::DeriveContainer(exts) => {
+                    let (derives, item) = match invoc.kind {
+                        InvocationKind::DeriveContainer { derives, item } => (derives, item),
+                        _ => unreachable!(),
+                    };
+                    if !item.derive_allowed() {
+                        let attr = attr::find_by_name(item.attrs(), sym::derive)
+                            .expect("`derive` attribute should exist");
+                        let span = attr.span;
+                        let mut err = self.cx.mut_span_err(span,
+                            "`derive` may only be applied to structs, enums and unions");
+                        if let ast::AttrStyle::Inner = attr.style {
+                            let trait_list = derives.iter()
+                                .map(|t| t.to_string()).collect::<Vec<_>>();
+                            let suggestion = format!("#[derive({})]", trait_list.join(", "));
+                            err.span_suggestion(
+                                span, "try an outer attribute", suggestion,
+                                // We don't 𝑘𝑛𝑜𝑤 that the following item is an ADT
+                                Applicability::MaybeIncorrect
+                            );
+                        }
+                        err.emit();
+                    }
 
-                let mut item = self.fully_configure(item);
-                item.visit_attrs(|attrs| attrs.retain(|a| a.path != sym::derive));
-                let derive_placeholders =
-                    all_derive_placeholders.entry(invoc.expansion_data.id).or_default();
-
-                derive_placeholders.reserve(traits.len());
-                invocations.reserve(traits.len());
-                for path in traits {
-                    let expn_id = ExpnId::fresh(None);
-                    derive_placeholders.push(NodeId::placeholder_from_expn_id(expn_id));
-                    invocations.push(Invocation {
-                        kind: InvocationKind::Derive { path, item: item.clone() },
-                        fragment_kind: invoc.fragment_kind,
-                        expansion_data: ExpansionData {
-                            id: expn_id,
-                            ..invoc.expansion_data.clone()
-                        },
-                    });
+                    let mut item = self.fully_configure(item);
+                    item.visit_attrs(|attrs| attrs.retain(|a| a.path != sym::derive));
+                    let mut helper_attrs = Vec::new();
+                    let mut has_copy = false;
+                    for ext in exts {
+                        helper_attrs.extend(&ext.helper_attrs);
+                        has_copy |= ext.is_derive_copy;
+                    }
+                    // Mark derive helpers inside this item as known and used.
+                    // FIXME: This is a hack, derive helpers should be integrated with regular name
+                    // resolution instead. For example, helpers introduced by a derive container
+                    // can be in scope for all code produced by that container's expansion.
+                    item.visit_with(&mut MarkAttrs(&helper_attrs));
+                    if has_copy {
+                        self.cx.resolver.add_derives(invoc.expansion_data.id, SpecialDerives::COPY);
+                    }
+
+                    let derive_placeholders =
+                        all_derive_placeholders.entry(invoc.expansion_data.id).or_default();
+                    derive_placeholders.reserve(derives.len());
+                    invocations.reserve(derives.len());
+                    for path in derives {
+                        let expn_id = ExpnId::fresh(None);
+                        derive_placeholders.push(NodeId::placeholder_from_expn_id(expn_id));
+                        invocations.push(Invocation {
+                            kind: InvocationKind::Derive { path, item: item.clone() },
+                            fragment_kind: invoc.fragment_kind,
+                            expansion_data: ExpansionData {
+                                id: expn_id,
+                                ..invoc.expansion_data.clone()
+                            },
+                        });
+                    }
+                    let fragment = invoc.fragment_kind
+                        .expect_from_annotatables(::std::iter::once(item));
+                    self.collect_invocations(fragment, derive_placeholders)
                 }
-                let fragment = invoc.fragment_kind
-                    .expect_from_annotatables(::std::iter::once(item));
-                self.collect_invocations(fragment, derive_placeholders)
-            } else {
-                unreachable!()
             };
 
             if expanded_fragments.len() < depth {

src/libsyntax/ext/proc_macro.rs

@@ -78,7 +78,6 @@ pub struct ProcMacroDerive {
     pub client: proc_macro::bridge::client::Client<
         fn(proc_macro::TokenStream) -> proc_macro::TokenStream,
     >,
-    pub attrs: Vec<ast::Name>,
 }
 
 impl MultiItemModifier for ProcMacroDerive {
@@ -111,9 +110,6 @@ impl MultiItemModifier for ProcMacroDerive {
             }
         }
 
-        // Mark attributes as known, and used.
-        MarkAttrs(&self.attrs).visit_item(&item);
-
         let token = token::Interpolated(Lrc::new(token::NtItem(item)));
         let input = tokenstream::TokenTree::token(token, DUMMY_SP).into();
 
@@ -164,7 +160,7 @@ impl MultiItemModifier for ProcMacroDerive {
     }
 }
 
-struct MarkAttrs<'a>(&'a [ast::Name]);
+crate struct MarkAttrs<'a>(crate &'a [ast::Name]);
 
 impl<'a> Visitor<'a> for MarkAttrs<'a> {
     fn visit_attribute(&mut self, attr: &Attribute) {

src/test/ui/derives/deriving-bounds.stderr

@@ -1,15 +1,3 @@
-error: cannot find derive macro `Send` in this scope
-  --> $DIR/deriving-bounds.rs:1:10
-   |
-LL | #[derive(Send)]
-   |          ^^^^
-   |
-note: unsafe traits like `Send` should be implemented explicitly
-  --> $DIR/deriving-bounds.rs:1:10
-   |
-LL | #[derive(Send)]
-   |          ^^^^
-
 error: cannot find derive macro `Sync` in this scope
   --> $DIR/deriving-bounds.rs:5:10
    |
@@ -22,5 +10,17 @@ note: unsafe traits like `Sync` should be implemented explicitly
 LL | #[derive(Sync)]
    |          ^^^^
 
+error: cannot find derive macro `Send` in this scope
+  --> $DIR/deriving-bounds.rs:1:10
+   |
+LL | #[derive(Send)]
+   |          ^^^^
+   |
+note: unsafe traits like `Send` should be implemented explicitly
+  --> $DIR/deriving-bounds.rs:1:10
+   |
+LL | #[derive(Send)]
+   |          ^^^^
+
 error: aborting due to 2 previous errors
 

src/test/ui/feature-gate/issue-43106-gating-of-derive-2.stderr

@@ -1,5 +1,5 @@
 error: cannot find derive macro `x3300` in this scope
-  --> $DIR/issue-43106-gating-of-derive-2.rs:4:14
+  --> $DIR/issue-43106-gating-of-derive-2.rs:12:14
    |
 LL |     #[derive(x3300)]
    |              ^^^^^
@@ -11,7 +11,7 @@ LL |     #[derive(x3300)]
    |              ^^^^^
 
 error: cannot find derive macro `x3300` in this scope
-  --> $DIR/issue-43106-gating-of-derive-2.rs:12:14
+  --> $DIR/issue-43106-gating-of-derive-2.rs:4:14
    |
 LL |     #[derive(x3300)]
    |              ^^^^^

src/test/ui/feature-gate/issue-43106-gating-of-derive.rs

@@ -1,9 +1,6 @@
 // `#![derive]` raises errors when it occurs at contexts other than ADT
 // definitions.
 
-#![derive(Debug)]
-//~^ ERROR `derive` may only be applied to structs, enums and unions
-
 #[derive(Debug)]
 //~^ ERROR `derive` may only be applied to structs, enums and unions
 mod derive {

src/test/ui/feature-gate/issue-43106-gating-of-derive.stderr

@@ -1,38 +1,32 @@
 error: `derive` may only be applied to structs, enums and unions
   --> $DIR/issue-43106-gating-of-derive.rs:4:1
    |
-LL | #![derive(Debug)]
-   | ^^^^^^^^^^^^^^^^^ help: try an outer attribute: `#[derive(Debug)]`
-
-error: `derive` may only be applied to structs, enums and unions
-  --> $DIR/issue-43106-gating-of-derive.rs:7:1
-   |
 LL | #[derive(Debug)]
    | ^^^^^^^^^^^^^^^^
 
 error: `derive` may only be applied to structs, enums and unions
-  --> $DIR/issue-43106-gating-of-derive.rs:10:17
+  --> $DIR/issue-43106-gating-of-derive.rs:7:17
    |
 LL |     mod inner { #![derive(Debug)] }
    |                 ^^^^^^^^^^^^^^^^^ help: try an outer attribute: `#[derive(Debug)]`
 
 error: `derive` may only be applied to structs, enums and unions
-  --> $DIR/issue-43106-gating-of-derive.rs:13:5
+  --> $DIR/issue-43106-gating-of-derive.rs:10:5
    |
 LL |     #[derive(Debug)]
    |     ^^^^^^^^^^^^^^^^
 
 error: `derive` may only be applied to structs, enums and unions
-  --> $DIR/issue-43106-gating-of-derive.rs:26:5
+  --> $DIR/issue-43106-gating-of-derive.rs:23:5
    |
 LL |     #[derive(Debug)]
    |     ^^^^^^^^^^^^^^^^
 
 error: `derive` may only be applied to structs, enums and unions
-  --> $DIR/issue-43106-gating-of-derive.rs:30:5
+  --> $DIR/issue-43106-gating-of-derive.rs:27:5
    |
 LL |     #[derive(Debug)]
    |     ^^^^^^^^^^^^^^^^
 
-error: aborting due to 6 previous errors
+error: aborting due to 5 previous errors