Reformat metadata for exported macros

Instead of copy-pasting the whole macro_rules! item from the original .rs file,
we serialize a separate name, attributes list, and body, the latter as
pretty-printed TTs.  The compilation of macro_rules! macros is decoupled
somewhat from the expansion of macros in item position.

This filters out comments, and facilitates selective imports.

6 files changed, 57 insertions, 74 deletions

diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs
index 55aa73b4faa..36fc8a691dc 100644
--- a/src/libsyntax/ast.rs
+++ b/src/libsyntax/ast.rs
@@ -476,7 +476,7 @@ pub struct Crate {
     pub attrs: Vec<Attribute>,
     pub config: CrateConfig,
     pub span: Span,
-    pub exported_macros: Vec<P<Item>>
+    pub exported_macros: Vec<MacroDef>,
 }
 
 pub type MetaItem = Spanned<MetaItem_>;
@@ -1698,6 +1698,19 @@ pub enum InlinedItem {
     IIForeign(P<ForeignItem>),
 }
 
+/// A macro definition, in this crate or imported from another.
+///
+/// Not parsed directly, but created on macro import or `macro_rules!` expansion.
+#[derive(Clone, PartialEq, Eq, RustcEncodable, RustcDecodable, Hash, Show)]
+pub struct MacroDef {
+    pub ident: Ident,
+    pub attrs: Vec<Attribute>,
+    pub id: NodeId,
+    pub span: Span,
+    pub imported_from: Option<Ident>,
+    pub body: Vec<TokenTree>,
+}
+
 #[cfg(test)]
 mod test {
     use serialize::json;
diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs
index 4b38e6c0cb2..9a06745967f 100644
--- a/src/libsyntax/ext/base.rs
+++ b/src/libsyntax/ext/base.rs
@@ -28,19 +28,6 @@ use fold::Folder;
 use std::collections::HashMap;
 use std::rc::Rc;
 
-// new-style macro! tt code:
-//
-//    MacResult, NormalTT, IdentTT
-//
-// also note that ast::Mac used to have a bunch of extraneous cases and
-// is now probably a redundant AST node, can be merged with
-// ast::MacInvocTT.
-
-pub struct MacroDef {
-    pub name: String,
-    pub ext: SyntaxExtension
-}
-
 pub trait ItemDecorator {
     fn expand(&self,
               ecx: &mut ExtCtxt,
@@ -140,13 +127,6 @@ impl<F> IdentMacroExpander for F
 /// methods are spliced into the AST at the callsite of the macro (or
 /// just into the compiler's internal macro table, for `make_def`).
 pub trait MacResult {
-    /// Attempt to define a new macro.
-    // this should go away; the idea that a macro might expand into
-    // either a macro definition or an expression, depending on what
-    // the context wants, is kind of silly.
-    fn make_def(&mut self) -> Option<MacroDef> {
-        None
-    }
     /// Create an expression.
     fn make_expr(self: Box<Self>) -> Option<P<ast::Expr>> {
         None
@@ -469,7 +449,7 @@ pub struct ExtCtxt<'a> {
 
     pub mod_path: Vec<ast::Ident> ,
     pub trace_mac: bool,
-    pub exported_macros: Vec<P<ast::Item>>,
+    pub exported_macros: Vec<ast::MacroDef>,
 
     pub syntax_env: SyntaxEnv,
     pub recursion_count: uint,
diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index 13cbc83f730..d56df2d7fb4 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -432,7 +432,7 @@ pub fn expand_item(it: P<ast::Item>, fld: &mut MacroExpander)
     }
 
     let mut new_items = match it.node {
-        ast::ItemMac(..) => expand_item_mac(it, None, fld),
+        ast::ItemMac(..) => expand_item_mac(it, fld),
         ast::ItemMod(_) | ast::ItemForeignMod(_) => {
             let valid_ident =
                 it.ident.name != parse::token::special_idents::invalid.name;
@@ -549,7 +549,6 @@ fn contains_macro_use(fld: &mut MacroExpander, attrs: &[ast::Attribute]) -> bool
 // Support for item-position macro invocations, exactly the same
 // logic as for expression-position macro invocations.
 pub fn expand_item_mac(it: P<ast::Item>,
-                       imported_from: Option<ast::Ident>,
                        fld: &mut MacroExpander) -> SmallVector<P<ast::Item>> {
     let (extname, path_span, tts) = match it.node {
         ItemMac(codemap::Spanned {
@@ -630,18 +629,20 @@ pub fn expand_item_mac(it: P<ast::Item>,
                         }
                     });
                     // DON'T mark before expansion.
-                    let MacroDef { name, ext }
-                        = macro_rules::add_new_extension(fld.cx, it.span, it.ident,
-                                                         imported_from, tts);
-
-                    fld.cx.syntax_env.insert(intern(name.as_slice()), ext);
-
-                    if match imported_from {
-                        None => attr::contains_name(it.attrs.as_slice(), "macro_export"),
-                        Some(_) => fld.cx.ecfg.reexported_macros.iter()
-                                       .any(|e| e.as_slice() == name.as_slice()),
-                    } {
-                        fld.cx.exported_macros.push(it);
+
+                    let def = ast::MacroDef {
+                        ident: it.ident,
+                        attrs: it.attrs.clone(),
+                        id: ast::DUMMY_NODE_ID,
+                        span: it.span,
+                        imported_from: None,
+                        body: tts,
+                    };
+                    let ext = macro_rules::compile(fld.cx, &def);
+                    fld.cx.syntax_env.insert(def.ident.name, ext);
+
+                    if attr::contains_name(def.attrs.as_slice(), "macro_export") {
+                        fld.cx.exported_macros.push(def);
                     }
 
                     // macro_rules! has a side effect but expands to nothing.
@@ -680,9 +681,6 @@ pub fn expand_item_mac(it: P<ast::Item>,
 }
 
 /// Expand a stmt
-//
-// I don't understand why this returns a vector... it looks like we're
-// half done adding machinery to allow macros to expand into multiple statements.
 fn expand_stmt(s: Stmt, fld: &mut MacroExpander) -> SmallVector<P<Stmt>> {
     let (mac, style) = match s.node {
         StmtMac(mac, style) => (mac, style),
@@ -1195,30 +1193,23 @@ impl ExpansionConfig {
     }
 }
 
-pub struct ExportedMacros {
-    pub crate_name: Ident,
-    pub macros: Vec<String>,
-}
-
 pub fn expand_crate(parse_sess: &parse::ParseSess,
                     cfg: ExpansionConfig,
                     // these are the macros being imported to this crate:
-                    imported_macros: Vec<ExportedMacros>,
+                    imported_macros: Vec<ast::MacroDef>,
                     user_exts: Vec<NamedSyntaxExtension>,
                     c: Crate) -> Crate {
     let mut cx = ExtCtxt::new(parse_sess, c.config.clone(), cfg);
     let mut expander = MacroExpander::new(&mut cx);
 
-    for ExportedMacros { crate_name, macros } in imported_macros.into_iter() {
-        let name = format!("<{} macros>", token::get_ident(crate_name));
+    for def in imported_macros.iter() {
+        let ext = macro_rules::compile(expander.cx, def);
+        expander.cx.syntax_env.insert(def.ident.name, ext);
+
+        if expander.cx.ecfg.reexported_macros.iter()
+            .any(|e| e[] == token::get_ident(def.ident).get()) {
 
-        for source in macros.into_iter() {
-            let item = parse::parse_item_from_source_str(name.clone(),
-                                                         source,
-                                                         expander.cx.cfg(),
-                                                         expander.cx.parse_sess())
-                    .expect("expected a serialized item");
-            expand_item_mac(item, Some(crate_name), &mut expander);
+            expander.cx.exported_macros.push(def.clone());
         }
     }
 
diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs
index a278604b167..cf0d2c6474b 100644
--- a/src/libsyntax/ext/tt/macro_rules.rs
+++ b/src/libsyntax/ext/tt/macro_rules.rs
@@ -11,7 +11,7 @@
 use ast::{Ident, TtDelimited, TtSequence, TtToken};
 use ast;
 use codemap::{Span, DUMMY_SP};
-use ext::base::{ExtCtxt, MacResult, MacroDef};
+use ext::base::{ExtCtxt, MacResult, SyntaxExtension};
 use ext::base::{NormalTT, TTMacroExpander};
 use ext::tt::macro_parser::{Success, Error, Failure};
 use ext::tt::macro_parser::{NamedMatch, MatchedSeq, MatchedNonterminal};
@@ -208,15 +208,9 @@ fn generic_extension<'cx>(cx: &'cx ExtCtxt,
 //
 // Holy self-referential!
 
-/// This procedure performs the expansion of the
-/// macro_rules! macro. It parses the RHS and adds
-/// an extension to the current context.
-pub fn add_new_extension<'cx>(cx: &'cx mut ExtCtxt,
-                              sp: Span,
-                              name: Ident,
-                              imported_from: Option<Ident>,
-                              arg: Vec<ast::TokenTree> )
-                              -> MacroDef {
+/// Converts a `macro_rules!` invocation into a syntax extension.
+pub fn compile<'cx>(cx: &'cx mut ExtCtxt,
+                    def: &ast::MacroDef) -> SyntaxExtension {
 
     let lhs_nm =  gensym_ident("lhs");
     let rhs_nm =  gensym_ident("rhs");
@@ -254,7 +248,7 @@ pub fn add_new_extension<'cx>(cx: &'cx mut ExtCtxt,
     let arg_reader = new_tt_reader(&cx.parse_sess().span_diagnostic,
                                    None,
                                    None,
-                                   arg.clone());
+                                   def.body.clone());
     let argument_map = parse_or_else(cx.parse_sess(),
                                      cx.cfg(),
                                      arg_reader,
@@ -263,23 +257,20 @@ pub fn add_new_extension<'cx>(cx: &'cx mut ExtCtxt,
     // Extract the arguments:
     let lhses = match *argument_map[lhs_nm] {
         MatchedSeq(ref s, _) => /* FIXME (#2543) */ (*s).clone(),
-        _ => cx.span_bug(sp, "wrong-structured lhs")
+        _ => cx.span_bug(def.span, "wrong-structured lhs")
     };
 
     let rhses = match *argument_map[rhs_nm] {
         MatchedSeq(ref s, _) => /* FIXME (#2543) */ (*s).clone(),
-        _ => cx.span_bug(sp, "wrong-structured rhs")
+        _ => cx.span_bug(def.span, "wrong-structured rhs")
     };
 
     let exp = box MacroRulesMacroExpander {
-        name: name,
-        imported_from: imported_from,
+        name: def.ident,
+        imported_from: def.imported_from,
         lhses: lhses,
         rhses: rhses,
     };
 
-    MacroDef {
-        name: token::get_ident(name).to_string(),
-        ext: NormalTT(exp, Some(sp))
-    }
+    NormalTT(exp, Some(def.span))
 }
diff --git a/src/libsyntax/fold.rs b/src/libsyntax/fold.rs
index 35b2e5dbc53..6b5d333ef8f 100644
--- a/src/libsyntax/fold.rs
+++ b/src/libsyntax/fold.rs
@@ -1115,7 +1115,7 @@ pub fn noop_fold_mod<T: Folder>(Mod {inner, view_items, items}: Mod, folder: &mu
     }
 }
 
-pub fn noop_fold_crate<T: Folder>(Crate {module, attrs, config, exported_macros, span}: Crate,
+pub fn noop_fold_crate<T: Folder>(Crate {module, attrs, config, mut exported_macros, span}: Crate,
                                   folder: &mut T) -> Crate {
     let config = folder.fold_meta_items(config);
 
@@ -1146,6 +1146,10 @@ pub fn noop_fold_crate<T: Folder>(Crate {module, attrs, config, exported_macros,
         }, Vec::new(), span)
     };
 
+    for def in exported_macros.iter_mut() {
+        def.id = folder.new_id(def.id);
+    }
+
     Crate {
         module: module,
         attrs: attrs,
diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs
index ee7edceaf69..b0969a573e6 100644
--- a/src/libsyntax/parse/mod.rs
+++ b/src/libsyntax/parse/mod.rs
@@ -169,6 +169,8 @@ pub fn parse_stmt_from_source_str(name: String,
 
 // Note: keep in sync with `with_hygiene::parse_tts_from_source_str`
 // until #16472 is resolved.
+//
+// Warning: This parses with quote_depth > 0, which is not the default.
 pub fn parse_tts_from_source_str(name: String,
                                  source: String,
                                  cfg: ast::CrateConfig,
@@ -310,6 +312,8 @@ pub mod with_hygiene {
 
     // Note: keep this in sync with `super::parse_tts_from_source_str` until
     // #16472 is resolved.
+    //
+    // Warning: This parses with quote_depth > 0, which is not the default.
     pub fn parse_tts_from_source_str(name: String,
                                      source: String,
                                      cfg: ast::CrateConfig,