Rollup merge of #36438 - jseyfried:node_ids_in_expansion, r=nrc

Assign node ids during macro expansion

After this PR,
 - The `ExtCtxt` can access `resolve`'s `Resolver` through the trait object `ext::base::Resolver`.
  - The `Resolver` trait object can load macros and replaces today's `MacroLoader` trait object.
  - The macro expander uses the `Resolver` trait object to resolve macro invocations.
 - The macro expander assigns node ids and builds the `Resolver`'s `macros_at_scope` map.
   - This is groundwork for merging import resolution and expansion.
 - Performance of expansion together with node id assignment improves by ~5%.

**EDIT:** Since Github is reordering the commits, here is `git log`:
 - b54e1e399741579612f13e2df98a25ea9447989d: Differentiate between monotonic and non-monotonic expansion and only assign node ids during monotonic expansion.
 - 78c00398780db6f59ebf43e765fa9368dad436d2: Expand generated test harnesses and macro registries.
 - f3c2dca3539e6edc745f9c91898cb97d281865c1: Remove scope placeholders from the crate root.
 - c86c8d41a26b2037e80c9fd028a59313a78b3a66: Perform node id assignment and `macros_at_scope` construction during the `InvocationCollector` and `PlaceholderExpander` folds.
 - 72a636975fc5d0bb4af45af7bdd97987cc722a6a: Move macro resolution into `librustc_resolve`.
 - 20b43b23230ce063ccf99a4841d85790ad311bdf: Rewrite the unit tests in `ext/expand.rs` as a `compile-fail` test.
 - a9821e1658240bb2c056f260a4b6bc9789301fae: Refactor `ExtCtxt` to use a `Resolver` instead of a `MacroLoader`.
 - 60440b226d2f70bdae803443ff7ad2e2af2c9b10: Refactor `noop_fold_stmt_kind` out of `noop_fold_stmt`.
 - 50f94f6c95c944f08c4af264f48260e42efefd47: Avoid needless reexpansions.

r? @nrc

4 files changed, 81 insertions, 58 deletions

diff --git a/src/libsyntax_ext/deriving/mod.rs b/src/libsyntax_ext/deriving/mod.rs
index fcbce363890..6162beb80ec 100644
--- a/src/libsyntax_ext/deriving/mod.rs
+++ b/src/libsyntax_ext/deriving/mod.rs
@@ -11,8 +11,7 @@
 //! The compiler code necessary to implement the `#[derive]` extensions.
 
 use syntax::ast::{self, MetaItem};
-use syntax::ext::base::{Annotatable, ExtCtxt, SyntaxEnv};
-use syntax::ext::base::MultiModifier;
+use syntax::ext::base::{Annotatable, ExtCtxt};
 use syntax::ext::build::AstBuilder;
 use syntax::feature_gate;
 use syntax::codemap;
@@ -89,7 +88,7 @@ fn allow_unstable(cx: &mut ExtCtxt, span: Span, attr_name: &str) -> Span {
     }
 }
 
-fn expand_derive(cx: &mut ExtCtxt,
+pub fn expand_derive(cx: &mut ExtCtxt,
                  span: Span,
                  mitem: &MetaItem,
                  annotatable: Annotatable)
@@ -243,10 +242,6 @@ fn expand_derive(cx: &mut ExtCtxt,
 
 macro_rules! derive_traits {
     ($( $name:expr => $func:path, )+) => {
-        pub fn register_all(env: &mut SyntaxEnv) {
-            env.insert(intern("derive"), MultiModifier(Box::new(expand_derive)));
-        }
-
         pub fn is_builtin_trait(name: &str) -> bool {
             match name {
                 $( $name )|+ => true,
diff --git a/src/libsyntax_ext/format.rs b/src/libsyntax_ext/format.rs
index 06b16095d19..892ebcfa761 100644
--- a/src/libsyntax_ext/format.rs
+++ b/src/libsyntax_ext/format.rs
@@ -17,7 +17,6 @@ use syntax::ast;
 use syntax::ext::base::*;
 use syntax::ext::base;
 use syntax::ext::build::AstBuilder;
-use syntax::fold::Folder;
 use syntax::parse::token::{self, keywords};
 use syntax::ptr::P;
 use syntax_pos::{Span, DUMMY_SP};
@@ -702,10 +701,12 @@ pub fn expand_preparsed_format_args(ecx: &mut ExtCtxt,
     let arg_types: Vec<_> = (0..args.len()).map(|_| Vec::new()).collect();
     let arg_unique_types: Vec<_> = (0..args.len()).map(|_| Vec::new()).collect();
     let macsp = ecx.call_site();
-    // Expand the format literal so that efmt.span will have a backtrace. This
-    // is essential for locating a bug when the format literal is generated in
-    // a macro. (e.g. println!("{}"), which uses concat!($fmt, "\n")).
-    let efmt = ecx.expander().fold_expr(efmt);
+    let msg = "format argument must be a string literal.";
+    let fmt = match expr_to_spanned_string(ecx, efmt, msg) {
+        Some(fmt) => fmt,
+        None => return DummyResult::raw_expr(sp),
+    };
+
     let mut cx = Context {
         ecx: ecx,
         args: args,
@@ -723,14 +724,10 @@ pub fn expand_preparsed_format_args(ecx: &mut ExtCtxt,
         str_pieces: Vec::new(),
         all_pieces_simple: true,
         macsp: macsp,
-        fmtsp: efmt.span,
-    };
-    let fmt = match expr_to_string(cx.ecx, efmt, "format argument must be a string literal.") {
-        Some((fmt, _)) => fmt,
-        None => return DummyResult::raw_expr(sp),
+        fmtsp: fmt.span,
     };
 
-    let mut parser = parse::Parser::new(&fmt);
+    let mut parser = parse::Parser::new(&fmt.node.0);
     let mut pieces = vec![];
 
     loop {
diff --git a/src/libsyntax_ext/lib.rs b/src/libsyntax_ext/lib.rs
index 2065d92fd6e..3a6212e5445 100644
--- a/src/libsyntax_ext/lib.rs
+++ b/src/libsyntax_ext/lib.rs
@@ -34,11 +34,6 @@ extern crate syntax_pos;
 extern crate rustc_macro;
 extern crate rustc_errors as errors;
 
-use syntax::ext::base::{MacroExpanderFn, NormalTT};
-use syntax::ext::base::{SyntaxEnv, SyntaxExtension};
-use syntax::parse::token::intern;
-
-
 mod asm;
 mod cfg;
 mod concat;
@@ -53,28 +48,67 @@ pub mod rustc_macro_registrar;
 // for custom_derive
 pub mod deriving;
 
-pub fn register_builtins(env: &mut SyntaxEnv) {
-    // utility function to simplify creating NormalTT syntax extensions
-    fn builtin_normal_expander(f: MacroExpanderFn) -> SyntaxExtension {
-        NormalTT(Box::new(f), None, false)
+use std::rc::Rc;
+use syntax::ast;
+use syntax::ext::base::{MacroExpanderFn, MacroRulesTT, NormalTT, MultiModifier};
+use syntax::ext::hygiene::Mark;
+use syntax::parse::token::intern;
+
+pub fn register_builtins(resolver: &mut syntax::ext::base::Resolver, enable_quotes: bool) {
+    let mut register = |name, ext| {
+        resolver.add_macro(Mark::root(), ast::Ident::with_empty_ctxt(intern(name)), Rc::new(ext));
+    };
+
+    register("macro_rules", MacroRulesTT);
+
+    macro_rules! register {
+        ($( $name:ident: $f:expr, )*) => { $(
+            register(stringify!($name), NormalTT(Box::new($f as MacroExpanderFn), None, false));
+        )* }
     }
 
-    env.insert(intern("asm"), builtin_normal_expander(asm::expand_asm));
-    env.insert(intern("cfg"), builtin_normal_expander(cfg::expand_cfg));
-    env.insert(intern("concat"),
-               builtin_normal_expander(concat::expand_syntax_ext));
-    env.insert(intern("concat_idents"),
-               builtin_normal_expander(concat_idents::expand_syntax_ext));
-    env.insert(intern("env"), builtin_normal_expander(env::expand_env));
-    env.insert(intern("option_env"),
-               builtin_normal_expander(env::expand_option_env));
-    env.insert(intern("format_args"),
-               // format_args uses `unstable` things internally.
-               NormalTT(Box::new(format::expand_format_args), None, true));
-    env.insert(intern("log_syntax"),
-               builtin_normal_expander(log_syntax::expand_syntax_ext));
-    env.insert(intern("trace_macros"),
-               builtin_normal_expander(trace_macros::expand_trace_macros));
-
-    deriving::register_all(env);
+    if enable_quotes {
+        use syntax::ext::quote::*;
+        register! {
+            quote_tokens: expand_quote_tokens,
+            quote_expr: expand_quote_expr,
+            quote_ty: expand_quote_ty,
+            quote_item: expand_quote_item,
+            quote_pat: expand_quote_pat,
+            quote_arm: expand_quote_arm,
+            quote_stmt: expand_quote_stmt,
+            quote_matcher: expand_quote_matcher,
+            quote_attr: expand_quote_attr,
+            quote_arg: expand_quote_arg,
+            quote_block: expand_quote_block,
+            quote_meta_item: expand_quote_meta_item,
+            quote_path: expand_quote_path,
+        }
+    }
+
+    use syntax::ext::source_util::*;
+    register! {
+        line: expand_line,
+        column: expand_column,
+        file: expand_file,
+        stringify: expand_stringify,
+        include: expand_include,
+        include_str: expand_include_str,
+        include_bytes: expand_include_bytes,
+        module_path: expand_mod,
+
+        asm: asm::expand_asm,
+        cfg: cfg::expand_cfg,
+        concat: concat::expand_syntax_ext,
+        concat_idents: concat_idents::expand_syntax_ext,
+        env: env::expand_env,
+        option_env: env::expand_option_env,
+        log_syntax: log_syntax::expand_syntax_ext,
+        trace_macros: trace_macros::expand_trace_macros,
+    }
+
+    // format_args uses `unstable` things internally.
+    register("format_args", NormalTT(Box::new(format::expand_format_args), None, true));
+
+    register("derive", MultiModifier(Box::new(deriving::expand_derive)));
 }
diff --git a/src/libsyntax_ext/rustc_macro_registrar.rs b/src/libsyntax_ext/rustc_macro_registrar.rs
index 7693e2416f4..ce3e53cdf97 100644
--- a/src/libsyntax_ext/rustc_macro_registrar.rs
+++ b/src/libsyntax_ext/rustc_macro_registrar.rs
@@ -13,12 +13,13 @@ use std::mem;
 use errors;
 use syntax::ast::{self, Ident, NodeId};
 use syntax::codemap::{ExpnInfo, NameAndSpan, MacroAttribute};
-use syntax::ext::base::{ExtCtxt, DummyMacroLoader};
+use syntax::ext::base::ExtCtxt;
 use syntax::ext::build::AstBuilder;
 use syntax::ext::expand::ExpansionConfig;
 use syntax::parse::ParseSess;
 use syntax::parse::token::{self, InternedString};
 use syntax::feature_gate::Features;
+use syntax::fold::Folder;
 use syntax::ptr::P;
 use syntax_pos::{Span, DUMMY_SP};
 use syntax::visit::{self, Visitor};
@@ -39,16 +40,14 @@ struct CollectCustomDerives<'a> {
 }
 
 pub fn modify(sess: &ParseSess,
+              resolver: &mut ::syntax::ext::base::Resolver,
               mut krate: ast::Crate,
               is_rustc_macro_crate: bool,
               num_crate_types: usize,
               handler: &errors::Handler,
               features: &Features) -> ast::Crate {
-    let mut loader = DummyMacroLoader;
-    let mut cx = ExtCtxt::new(sess,
-                              Vec::new(),
-                              ExpansionConfig::default("rustc_macro".to_string()),
-                              &mut loader);
+    let ecfg = ExpansionConfig::default("rustc_macro".to_string());
+    let mut cx = ExtCtxt::new(sess, Vec::new(), ecfg, resolver);
 
     let mut collect = CollectCustomDerives {
         derives: Vec::new(),
@@ -268,13 +267,11 @@ fn mk_registrar(cx: &mut ExtCtxt,
         i.vis = ast::Visibility::Public;
         i
     });
-    let module = cx.item_mod(span,
-                             span,
-                             ast::Ident::with_empty_ctxt(token::gensym("registrar")),
-                             Vec::new(),
-                             vec![krate, func]);
-    module.map(|mut i| {
+    let ident = ast::Ident::with_empty_ctxt(token::gensym("registrar"));
+    let module = cx.item_mod(span, span, ident, Vec::new(), vec![krate, func]).map(|mut i| {
         i.vis = ast::Visibility::Public;
         i
-    })
+    });
+
+    cx.monotonic_expander().fold_item(module).pop().unwrap()
 }