expand: Refactor module loading

compiler/rustc_builtin_macros/src/source_util.rs

@@ -4,7 +4,7 @@ use rustc_ast::token;
 use rustc_ast::tokenstream::TokenStream;
 use rustc_ast_pretty::pprust;
 use rustc_expand::base::{self, *};
-use rustc_expand::module::DirectoryOwnership;
+use rustc_expand::module::DirOwnership;
 use rustc_parse::parser::{ForceCollect, Parser};
 use rustc_parse::{self, new_parser_from_file};
 use rustc_session::lint::builtin::INCOMPLETE_INCLUDE;
@@ -101,7 +101,7 @@ pub fn expand_include<'cx>(
         None => return DummyResult::any(sp),
     };
     // The file will be added to the code map by the parser
-    let mut file = match cx.resolve_path(file, sp) {
+    let file = match cx.resolve_path(file, sp) {
         Ok(f) => f,
         Err(mut err) => {
             err.emit();
@@ -114,10 +114,9 @@ pub fn expand_include<'cx>(
     // then the path of `bar.rs` should be relative to the directory of `file`.
     // See https://github.com/rust-lang/rust/pull/69838/files#r395217057 for a discussion.
     // `MacroExpander::fully_expand_fragment` later restores, so "stack discipline" is maintained.
-    file.pop();
-    cx.current_expansion.directory_ownership = DirectoryOwnership::Owned { relative: None };
-    let mod_path = cx.current_expansion.module.mod_path.clone();
-    cx.current_expansion.module = Rc::new(ModuleData { mod_path, directory: file });
+    let dir_path = file.parent().unwrap_or(&file).to_owned();
+    cx.current_expansion.module = Rc::new(cx.current_expansion.module.with_dir_path(dir_path));
+    cx.current_expansion.dir_ownership = DirOwnership::Owned { relative: None };
 
     struct ExpandResult<'a> {
         p: Parser<'a>,

compiler/rustc_expand/src/base.rs

@@ -1,11 +1,11 @@
 use crate::expand::{self, AstFragment, Invocation};
-use crate::module::DirectoryOwnership;
+use crate::module::DirOwnership;
 
 use rustc_ast::ptr::P;
 use rustc_ast::token::{self, Nonterminal};
 use rustc_ast::tokenstream::{CanSynthesizeMissingTokens, LazyTokenStream, TokenStream};
 use rustc_ast::visit::{AssocCtxt, Visitor};
-use rustc_ast::{self as ast, AstLike, Attribute, NodeId, PatKind};
+use rustc_ast::{self as ast, AstLike, Attribute, Item, NodeId, PatKind};
 use rustc_attr::{self as attr, Deprecation, Stability};
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::sync::{self, Lrc};
@@ -894,21 +894,40 @@ pub trait ResolverExpand {
     fn cfg_accessible(&mut self, expn_id: ExpnId, path: &ast::Path) -> Result<bool, Indeterminate>;
 }
 
-#[derive(Clone)]
+#[derive(Clone, Default)]
 pub struct ModuleData {
+    /// Path to the module starting from the crate name, like `my_crate::foo::bar`.
     pub mod_path: Vec<Ident>,
-    pub directory: PathBuf,
+    /// Stack of paths to files loaded by out-of-line module items,
+    /// used to detect and report recursive module inclusions.
+    pub file_path_stack: Vec<PathBuf>,
+    /// Directory to search child module files in,
+    /// often (but not necessarily) the parent of the top file path on the `file_path_stack`.
+    pub dir_path: PathBuf,
+}
+
+impl ModuleData {
+    pub fn with_dir_path(&self, dir_path: PathBuf) -> ModuleData {
+        ModuleData {
+            mod_path: self.mod_path.clone(),
+            file_path_stack: self.file_path_stack.clone(),
+            dir_path,
+        }
+    }
 }
 
 #[derive(Clone)]
 pub struct ExpansionData {
     pub id: ExpnId,
     pub depth: usize,
     pub module: Rc<ModuleData>,
-    pub directory_ownership: DirectoryOwnership,
+    pub dir_ownership: DirOwnership,
     pub prior_type_ascription: Option<(Span, bool)>,
 }
 
+type OnExternModLoaded<'a> =
+    Option<&'a dyn Fn(Ident, Vec<Attribute>, Vec<P<Item>>, Span) -> (Vec<Attribute>, Vec<P<Item>>)>;
+
 /// One of these is made during expansion and incrementally updated as we go;
 /// when a macro expansion occurs, the resulting nodes have the `backtrace()
 /// -> expn_data` of their expansion context stored into their span.
@@ -926,15 +945,15 @@ pub struct ExtCtxt<'a> {
     /// Called directly after having parsed an external `mod foo;` in expansion.
     ///
     /// `Ident` is the module name.
-    pub(super) extern_mod_loaded: Option<&'a dyn Fn(&ast::Crate, Ident)>,
+    pub(super) extern_mod_loaded: OnExternModLoaded<'a>,
 }
 
 impl<'a> ExtCtxt<'a> {
     pub fn new(
         sess: &'a Session,
         ecfg: expand::ExpansionConfig<'a>,
         resolver: &'a mut dyn ResolverExpand,
-        extern_mod_loaded: Option<&'a dyn Fn(&ast::Crate, Ident)>,
+        extern_mod_loaded: OnExternModLoaded<'a>,
     ) -> ExtCtxt<'a> {
         ExtCtxt {
             sess,
@@ -946,8 +965,8 @@ impl<'a> ExtCtxt<'a> {
             current_expansion: ExpansionData {
                 id: ExpnId::root(),
                 depth: 0,
-                module: Rc::new(ModuleData { mod_path: Vec::new(), directory: PathBuf::new() }),
-                directory_ownership: DirectoryOwnership::Owned { relative: None },
+                module: Default::default(),
+                dir_ownership: DirOwnership::Owned { relative: None },
                 prior_type_ascription: None,
             },
             force_mode: false,

compiler/rustc_expand/src/expand.rs

@@ -3,7 +3,7 @@ use crate::config::StripUnconfigured;
 use crate::configure;
 use crate::hygiene::SyntaxContext;
 use crate::mbe::macro_rules::annotate_err_with_kind;
-use crate::module::{parse_external_mod, push_directory, Directory, DirectoryOwnership};
+use crate::module::{mod_dir_path, parse_external_mod, DirOwnership, ParsedExternalMod};
 use crate::placeholders::{placeholder, PlaceholderExpander};
 
 use rustc_ast as ast;
@@ -355,16 +355,17 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
     // FIXME: Avoid visiting the crate as a `Mod` item,
     // make crate a first class expansion target instead.
     pub fn expand_crate(&mut self, mut krate: ast::Crate) -> ast::Crate {
-        let mut module = ModuleData {
-            mod_path: vec![Ident::from_str(&self.cx.ecfg.crate_name)],
-            directory: match self.cx.source_map().span_to_unmapped_path(krate.span) {
-                FileName::Real(name) => name.into_local_path(),
-                other => PathBuf::from(other.to_string()),
-            },
+        let file_path = match self.cx.source_map().span_to_unmapped_path(krate.span) {
+            FileName::Real(name) => name.into_local_path(),
+            other => PathBuf::from(other.to_string()),
         };
-        module.directory.pop();
-        self.cx.root_path = module.directory.clone();
-        self.cx.current_expansion.module = Rc::new(module);
+        let dir_path = file_path.parent().unwrap_or(&file_path).to_owned();
+        self.cx.root_path = dir_path.clone();
+        self.cx.current_expansion.module = Rc::new(ModuleData {
+            mod_path: vec![Ident::from_str(&self.cx.ecfg.crate_name)],
+            file_path_stack: vec![file_path],
+            dir_path,
+        });
 
         let krate_item = AstFragment::Items(smallvec![P(ast::Item {
             attrs: krate.attrs,
@@ -1245,10 +1246,12 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> {
     }
 
     fn visit_block(&mut self, block: &mut P<Block>) {
-        let old_directory_ownership = self.cx.current_expansion.directory_ownership;
-        self.cx.current_expansion.directory_ownership = DirectoryOwnership::UnownedViaBlock;
+        let orig_dir_ownership = mem::replace(
+            &mut self.cx.current_expansion.dir_ownership,
+            DirOwnership::UnownedViaBlock,
+        );
         noop_visit_block(block, self);
-        self.cx.current_expansion.directory_ownership = old_directory_ownership;
+        self.cx.current_expansion.dir_ownership = orig_dir_ownership;
     }
 
     fn flat_map_item(&mut self, item: P<ast::Item>) -> SmallVec<[P<ast::Item>; 1]> {
@@ -1276,63 +1279,76 @@ impl<'a, 'b> MutVisitor for InvocationCollector<'a, 'b> {
                 })
             }
             ast::ItemKind::Mod(_, ref mut mod_kind) if ident != Ident::invalid() => {
-                let sess = &self.cx.sess.parse_sess;
-                let orig_ownership = self.cx.current_expansion.directory_ownership;
-                let mut module = (*self.cx.current_expansion.module).clone();
-
-                let pushed = &mut false; // Record `parse_external_mod` pushing so we can pop.
-                let dir = Directory { ownership: orig_ownership, path: module.directory };
-                let Directory { ownership, path } = match mod_kind {
-                    ModKind::Loaded(_, Inline::Yes, _) => {
+                let (file_path, dir_path, dir_ownership) = match mod_kind {
+                    ModKind::Loaded(_, inline, _) => {
                         // Inline `mod foo { ... }`, but we still need to push directories.
+                        assert!(
+                            *inline == Inline::Yes,
+                            "`mod` item is loaded from a file for the second time"
+                        );
+                        let (dir_path, dir_ownership) = mod_dir_path(
+                            &self.cx.sess,
+                            ident,
+                            &attrs,
+                            &self.cx.current_expansion.module,
+                            self.cx.current_expansion.dir_ownership,
+                        );
                         item.attrs = attrs;
-                        push_directory(&self.cx.sess, ident, &item.attrs, dir)
-                    }
-                    ModKind::Loaded(_, Inline::No, _) => {
-                        panic!("`mod` item is loaded from a file for the second time")
+                        (None, dir_path, dir_ownership)
                     }
                     ModKind::Unloaded => {
                         // We have an outline `mod foo;` so we need to parse the file.
-                        let (items, inner_span, dir) =
-                            parse_external_mod(&self.cx.sess, ident, span, dir, &mut attrs, pushed);
+                        let old_attrs_len = attrs.len();
+                        let ParsedExternalMod {
+                            mut items,
+                            inner_span,
+                            file_path,
+                            dir_path,
+                            dir_ownership,
+                        } = parse_external_mod(
+                            &self.cx.sess,
+                            ident,
+                            span,
+                            &self.cx.current_expansion.module,
+                            self.cx.current_expansion.dir_ownership,
+                            &mut attrs,
+                        );
 
-                        let krate =
-                            ast::Crate { attrs, items, span: inner_span, proc_macros: vec![] };
                         if let Some(extern_mod_loaded) = self.cx.extern_mod_loaded {
-                            extern_mod_loaded(&krate, ident);
+                            (attrs, items) = extern_mod_loaded(ident, attrs, items, inner_span);
                         }
 
-                        *mod_kind = ModKind::Loaded(krate.items, Inline::No, inner_span);
-                        item.attrs = krate.attrs;
-                        // File can have inline attributes, e.g., `#![cfg(...)]` & co. => Reconfigure.
-                        item = match self.configure(item) {
-                            Some(node) => node,
-                            None => {
-                                if *pushed {
-                                    sess.included_mod_stack.borrow_mut().pop();
-                                }
-                                return Default::default();
-                            }
-                        };
-                        dir
+                        *mod_kind = ModKind::Loaded(items, Inline::No, inner_span);
+                        item.attrs = attrs;
+                        if item.attrs.len() > old_attrs_len {
+                            // If we loaded an out-of-line module and added some inner attributes,
+                            // then we need to re-configure it.
+                            // FIXME: Attributes also need to be recollected
+                            // for resolution and expansion.
+                            item = configure!(self, item);
+                        }
+                        (Some(file_path), dir_path, dir_ownership)
                     }
                 };
 
                 // Set the module info before we flat map.
-                self.cx.current_expansion.directory_ownership = ownership;
-                module.directory = path;
+                let mut module = self.cx.current_expansion.module.with_dir_path(dir_path);
                 module.mod_path.push(ident);
+                if let Some(file_path) = file_path {
+                    module.file_path_stack.push(file_path);
+                }
+
                 let orig_module =
                     mem::replace(&mut self.cx.current_expansion.module, Rc::new(module));
+                let orig_dir_ownership =
+                    mem::replace(&mut self.cx.current_expansion.dir_ownership, dir_ownership);
 
                 let result = noop_flat_map_item(item, self);
 
                 // Restore the module info.
+                self.cx.current_expansion.dir_ownership = orig_dir_ownership;
                 self.cx.current_expansion.module = orig_module;
-                self.cx.current_expansion.directory_ownership = orig_ownership;
-                if *pushed {
-                    sess.included_mod_stack.borrow_mut().pop();
-                }
+
                 result
             }
             _ => {

compiler/rustc_expand/src/lib.rs

@@ -1,5 +1,7 @@
+#![feature(bool_to_option)]
 #![feature(crate_visibility_modifier)]
 #![feature(decl_macro)]
+#![feature(destructuring_assignment)]
 #![feature(or_patterns)]
 #![feature(proc_macro_diagnostic)]
 #![feature(proc_macro_internals)]