Auto merge of #34032 - jseyfried:load_macros_in_expansion, r=nrc

Support `#[macro_use]` on macro-expanded crates

This PR loads macros from `#[macro_use]` crates during expansion so that
 - macro-expanded `#[macro_use]` crates work (fixes #33936, fixes #28071), and
 - macros imported from crates have the same scope as macros imported from modules.

This is a [breaking-change]. For example, this will break:
```rust
macro_rules! m {
    () => { #[macro_use(foo)] extern crate core; } //~ ERROR imported macro not found
}
m!();
```
Also, this will break:
```rust
macro_rules! try { () => {} }
// #[macro_use] mod bar { macro_rules! try { ... } } //< ... just like this would ...
fn main() { try!(); } //< ... making this an error
```

r? @nrc

3 files changed, 53 insertions, 29 deletions

diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs
index 303187aeba8..95624a43373 100644
--- a/src/libsyntax/ext/base.rs
+++ b/src/libsyntax/ext/base.rs
@@ -536,6 +536,17 @@ fn initial_syntax_expander_table<'feat>(ecfg: &expand::ExpansionConfig<'feat>)
     syntax_expanders
 }
 
+pub trait MacroLoader {
+    fn load_crate(&mut self, extern_crate: &ast::Item, allows_macros: bool) -> Vec<ast::MacroDef>;
+}
+
+pub struct DummyMacroLoader;
+impl MacroLoader for DummyMacroLoader {
+    fn load_crate(&mut self, _: &ast::Item, _: bool) -> Vec<ast::MacroDef> {
+        Vec::new()
+    }
+}
+
 /// One of these is made during expansion and incrementally updated as we go;
 /// when a macro expansion occurs, the resulting nodes have the backtrace()
 /// -> expn_info of their expansion context stored into their span.
@@ -546,6 +557,7 @@ pub struct ExtCtxt<'a> {
     pub ecfg: expand::ExpansionConfig<'a>,
     pub crate_root: Option<&'static str>,
     pub feature_gated_cfgs: &'a mut Vec<GatedCfgAttr>,
+    pub loader: &'a mut MacroLoader,
 
     pub mod_path: Vec<ast::Ident> ,
     pub exported_macros: Vec<ast::MacroDef>,
@@ -561,7 +573,9 @@ pub struct ExtCtxt<'a> {
 impl<'a> ExtCtxt<'a> {
     pub fn new(parse_sess: &'a parse::ParseSess, cfg: ast::CrateConfig,
                ecfg: expand::ExpansionConfig<'a>,
-               feature_gated_cfgs: &'a mut Vec<GatedCfgAttr>) -> ExtCtxt<'a> {
+               feature_gated_cfgs: &'a mut Vec<GatedCfgAttr>,
+               loader: &'a mut MacroLoader)
+               -> ExtCtxt<'a> {
         let env = initial_syntax_expander_table(&ecfg);
         ExtCtxt {
             parse_sess: parse_sess,
@@ -572,6 +586,7 @@ impl<'a> ExtCtxt<'a> {
             crate_root: None,
             feature_gated_cfgs: feature_gated_cfgs,
             exported_macros: Vec::new(),
+            loader: loader,
             syntax_env: env,
             recursion_count: 0,
 
@@ -925,4 +940,10 @@ impl SyntaxEnv {
         let last_chain_index = self.chain.len() - 1;
         &mut self.chain[last_chain_index].info
     }
+
+    pub fn is_crate_root(&mut self) -> bool {
+        // The first frame is pushed in `SyntaxEnv::new()` and the second frame is
+        // pushed when folding the crate root pseudo-module (c.f. noop_fold_crate).
+        self.chain.len() == 2
+    }
 }
diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index 7fee27c5dd4..15d192b59b8 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -726,13 +726,11 @@ fn expand_annotatable(a: Annotatable,
     let new_items: SmallVector<Annotatable> = match a {
         Annotatable::Item(it) => match it.node {
             ast::ItemKind::Mac(..) => {
-                let new_items: SmallVector<P<ast::Item>> = it.and_then(|it| match it.node {
+                it.and_then(|it| match it.node {
                     ItemKind::Mac(mac) =>
                         expand_mac_invoc(mac, Some(it.ident), it.attrs, it.span, fld),
                     _ => unreachable!(),
-                });
-
-                new_items.into_iter().map(|i| Annotatable::Item(i)).collect()
+                })
             }
             ast::ItemKind::Mod(_) | ast::ItemKind::ForeignMod(_) => {
                 let valid_ident =
@@ -748,10 +746,19 @@ fn expand_annotatable(a: Annotatable,
                 if valid_ident {
                     fld.cx.mod_pop();
                 }
-                result.into_iter().map(|i| Annotatable::Item(i)).collect()
+                result
             },
-            _ => noop_fold_item(it, fld).into_iter().map(|i| Annotatable::Item(i)).collect(),
-        },
+            ast::ItemKind::ExternCrate(_) => {
+                // We need to error on `#[macro_use] extern crate` when it isn't at the
+                // crate root, because `$crate` won't work properly.
+                let allows_macros = fld.cx.syntax_env.is_crate_root();
+                for def in fld.cx.loader.load_crate(&it, allows_macros) {
+                    fld.cx.insert_macro(def);
+                }
+                SmallVector::one(it)
+            },
+            _ => noop_fold_item(it, fld),
+        }.into_iter().map(|i| Annotatable::Item(i)).collect(),
 
         Annotatable::TraitItem(it) => match it.node {
             ast::TraitItemKind::Method(_, Some(_)) => {
@@ -1137,8 +1144,6 @@ impl<'feat> ExpansionConfig<'feat> {
 }
 
 pub fn expand_crate(mut cx: ExtCtxt,
-                    // these are the macros being imported to this crate:
-                    imported_macros: Vec<ast::MacroDef>,
                     user_exts: Vec<NamedSyntaxExtension>,
                     c: Crate) -> (Crate, HashSet<Name>) {
     if std_inject::no_core(&c) {
@@ -1151,10 +1156,6 @@ pub fn expand_crate(mut cx: ExtCtxt,
     let ret = {
         let mut expander = MacroExpander::new(&mut cx);
 
-        for def in imported_macros {
-            expander.cx.insert_macro(def);
-        }
-
         for (name, extension) in user_exts {
             expander.cx.syntax_env.insert(name, extension);
         }
@@ -1220,7 +1221,7 @@ mod tests {
     use ast;
     use ast::Name;
     use codemap;
-    use ext::base::ExtCtxt;
+    use ext::base::{ExtCtxt, DummyMacroLoader};
     use ext::mtwt;
     use fold::Folder;
     use parse;
@@ -1291,9 +1292,9 @@ mod tests {
             src,
             Vec::new(), &sess).unwrap();
         // should fail:
-        let mut gated_cfgs = vec![];
-        let ecx = ExtCtxt::new(&sess, vec![], test_ecfg(), &mut gated_cfgs);
-        expand_crate(ecx, vec![], vec![], crate_ast);
+        let (mut gated_cfgs, mut loader) = (vec![], DummyMacroLoader);
+        let ecx = ExtCtxt::new(&sess, vec![], test_ecfg(), &mut gated_cfgs, &mut loader);
+        expand_crate(ecx, vec![], crate_ast);
     }
 
     // make sure that macros can't escape modules
@@ -1306,9 +1307,9 @@ mod tests {
             "<test>".to_string(),
             src,
             Vec::new(), &sess).unwrap();
-        let mut gated_cfgs = vec![];
-        let ecx = ExtCtxt::new(&sess, vec![], test_ecfg(), &mut gated_cfgs);
-        expand_crate(ecx, vec![], vec![], crate_ast);
+        let (mut gated_cfgs, mut loader) = (vec![], DummyMacroLoader);
+        let ecx = ExtCtxt::new(&sess, vec![], test_ecfg(), &mut gated_cfgs, &mut loader);
+        expand_crate(ecx, vec![], crate_ast);
     }
 
     // macro_use modules should allow macros to escape
@@ -1320,18 +1321,18 @@ mod tests {
             "<test>".to_string(),
             src,
             Vec::new(), &sess).unwrap();
-        let mut gated_cfgs = vec![];
-        let ecx = ExtCtxt::new(&sess, vec![], test_ecfg(), &mut gated_cfgs);
-        expand_crate(ecx, vec![], vec![], crate_ast);
+        let (mut gated_cfgs, mut loader) = (vec![], DummyMacroLoader);
+        let ecx = ExtCtxt::new(&sess, vec![], test_ecfg(), &mut gated_cfgs, &mut loader);
+        expand_crate(ecx, vec![], crate_ast);
     }
 
     fn expand_crate_str(crate_str: String) -> ast::Crate {
         let ps = parse::ParseSess::new();
         let crate_ast = panictry!(string_to_parser(&ps, crate_str).parse_crate_mod());
         // the cfg argument actually does matter, here...
-        let mut gated_cfgs = vec![];
-        let ecx = ExtCtxt::new(&ps, vec![], test_ecfg(), &mut gated_cfgs);
-        expand_crate(ecx, vec![], vec![], crate_ast).0
+        let (mut gated_cfgs, mut loader) = (vec![], DummyMacroLoader);
+        let ecx = ExtCtxt::new(&ps, vec![], test_ecfg(), &mut gated_cfgs, &mut loader);
+        expand_crate(ecx, vec![], crate_ast).0
     }
 
     // find the pat_ident paths in a crate
diff --git a/src/libsyntax/test.rs b/src/libsyntax/test.rs
index 6fbbed2ee98..2ac4aac65de 100644
--- a/src/libsyntax/test.rs
+++ b/src/libsyntax/test.rs
@@ -25,7 +25,7 @@ use codemap;
 use errors;
 use config;
 use entry::{self, EntryPointType};
-use ext::base::ExtCtxt;
+use ext::base::{ExtCtxt, DummyMacroLoader};
 use ext::build::AstBuilder;
 use ext::expand::ExpansionConfig;
 use fold::Folder;
@@ -271,12 +271,14 @@ fn generate_test_harness(sess: &ParseSess,
     let krate = cleaner.fold_crate(krate);
 
     let mut feature_gated_cfgs = vec![];
+    let mut loader = DummyMacroLoader;
     let mut cx: TestCtxt = TestCtxt {
         sess: sess,
         span_diagnostic: sd,
         ext_cx: ExtCtxt::new(sess, vec![],
                              ExpansionConfig::default("test".to_string()),
-                             &mut feature_gated_cfgs),
+                             &mut feature_gated_cfgs,
+                             &mut loader),
         path: Vec::new(),
         testfns: Vec::new(),
         reexport_test_harness_main: reexport_test_harness_main,