Auto merge of #40220 - jseyfried:ast_macro_def, r=nrc

syntax: add `ast::ItemKind::MacroDef`, simplify hygiene info

This PR
 - adds a new variant `MacroDef` to `ast::ItemKind` for `macro_rules!` and eventually `macro` items,
 - [breaking-change] forbids macro defs without a name (`macro_rules! { () => {} }` compiles today),
 - removes `ast::MacroDef`, and
 - no longer uses `Mark` and `Invocation` to identify and characterize macro definitions.
   - We used to apply (at least) two `Mark`s to an expanded identifier's `SyntaxContext` -- the definition mark(s) and the expansion mark(s). We now only apply the latter.

r? @nrc

5 files changed, 28 insertions, 111 deletions

diff --git a/src/libsyntax/ext/base.rs b/src/libsyntax/ext/base.rs
index e242cf2777f..dc7e7673eb0 100644
--- a/src/libsyntax/ext/base.rs
+++ b/src/libsyntax/ext/base.rs
@@ -15,7 +15,7 @@ use attr::HasAttrs;
 use codemap::{self, CodeMap, ExpnInfo, Spanned, respan};
 use syntax_pos::{Span, ExpnId, NO_EXPANSION};
 use errors::{DiagnosticBuilder, FatalError};
-use ext::expand::{self, Expansion};
+use ext::expand::{self, Expansion, Invocation};
 use ext::hygiene::Mark;
 use fold::{self, Folder};
 use parse::{self, parser, DirectoryOwnership};
@@ -552,14 +552,15 @@ pub trait Resolver {
     fn is_whitelisted_legacy_custom_derive(&self, name: Name) -> bool;
 
     fn visit_expansion(&mut self, mark: Mark, expansion: &Expansion, derives: &[Mark]);
-    fn add_ext(&mut self, ident: ast::Ident, ext: Rc<SyntaxExtension>);
-    fn add_expansions_at_stmt(&mut self, id: ast::NodeId, macros: Vec<Mark>);
+    fn add_builtin(&mut self, ident: ast::Ident, ext: Rc<SyntaxExtension>);
 
     fn resolve_imports(&mut self);
     // Resolves attribute and derive legacy macros from `#![plugin(..)]`.
     fn find_legacy_attr_invoc(&mut self, attrs: &mut Vec<Attribute>) -> Option<Attribute>;
-    fn resolve_macro(&mut self, scope: Mark, path: &ast::Path, kind: MacroKind,
-                     force: bool) -> Result<Rc<SyntaxExtension>, Determinacy>;
+    fn resolve_invoc(&mut self, invoc: &mut Invocation, scope: Mark, force: bool)
+                     -> Result<Option<Rc<SyntaxExtension>>, Determinacy>;
+    fn resolve_macro(&mut self, scope: Mark, path: &ast::Path, kind: MacroKind, force: bool)
+                     -> Result<Rc<SyntaxExtension>, Determinacy>;
 }
 
 #[derive(Copy, Clone, Debug)]
@@ -577,11 +578,14 @@ impl Resolver for DummyResolver {
     fn is_whitelisted_legacy_custom_derive(&self, _name: Name) -> bool { false }
 
     fn visit_expansion(&mut self, _invoc: Mark, _expansion: &Expansion, _derives: &[Mark]) {}
-    fn add_ext(&mut self, _ident: ast::Ident, _ext: Rc<SyntaxExtension>) {}
-    fn add_expansions_at_stmt(&mut self, _id: ast::NodeId, _macros: Vec<Mark>) {}
+    fn add_builtin(&mut self, _ident: ast::Ident, _ext: Rc<SyntaxExtension>) {}
 
     fn resolve_imports(&mut self) {}
     fn find_legacy_attr_invoc(&mut self, _attrs: &mut Vec<Attribute>) -> Option<Attribute> { None }
+    fn resolve_invoc(&mut self, _invoc: &mut Invocation, _scope: Mark, _force: bool)
+                     -> Result<Option<Rc<SyntaxExtension>>, Determinacy> {
+        Err(Determinacy::Determined)
+    }
     fn resolve_macro(&mut self, _scope: Mark, _path: &ast::Path, _kind: MacroKind,
                      _force: bool) -> Result<Rc<SyntaxExtension>, Determinacy> {
         Err(Determinacy::Determined)
diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index f7dcd00e409..10168f010a0 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -154,7 +154,7 @@ impl ExpansionKind {
 pub struct Invocation {
     pub kind: InvocationKind,
     expansion_kind: ExpansionKind,
-    expansion_data: ExpansionData,
+    pub expansion_data: ExpansionData,
 }
 
 pub enum InvocationKind {
@@ -251,7 +251,7 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
 
             let scope =
                 if self.monotonic { invoc.expansion_data.mark } else { orig_expansion_data.mark };
-            let ext = match self.resolve_invoc(&mut invoc, scope, force) {
+            let ext = match self.cx.resolver.resolve_invoc(&mut invoc, scope, force) {
                 Ok(ext) => Some(ext),
                 Err(Determinacy::Determined) => None,
                 Err(Determinacy::Undetermined) => {
@@ -364,64 +364,6 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
         result
     }
 
-    fn resolve_invoc(&mut self, invoc: &mut Invocation, scope: Mark, force: bool)
-                     -> Result<Option<Rc<SyntaxExtension>>, Determinacy> {
-        let (attr, traits, item) = match invoc.kind {
-            InvocationKind::Bang { ref mac, .. } => {
-                return self.cx.resolver.resolve_macro(scope, &mac.node.path,
-                                                      MacroKind::Bang, force).map(Some);
-            }
-            InvocationKind::Attr { attr: None, .. } => return Ok(None),
-            InvocationKind::Derive { name, span, .. } => {
-                let path = ast::Path::from_ident(span, Ident::with_empty_ctxt(name));
-                return self.cx.resolver.resolve_macro(scope, &path,
-                                                      MacroKind::Derive, force).map(Some)
-            }
-            InvocationKind::Attr { ref mut attr, ref traits, ref mut item } => (attr, traits, item),
-        };
-
-        let (attr_name, path) = {
-            let attr = attr.as_ref().unwrap();
-            (attr.name(), ast::Path::from_ident(attr.span, Ident::with_empty_ctxt(attr.name())))
-        };
-
-        let mut determined = true;
-        match self.cx.resolver.resolve_macro(scope, &path, MacroKind::Attr, force) {
-            Ok(ext) => return Ok(Some(ext)),
-            Err(Determinacy::Undetermined) => determined = false,
-            Err(Determinacy::Determined) if force => return Err(Determinacy::Determined),
-            _ => {}
-        }
-
-        for &(name, span) in traits {
-            let path = ast::Path::from_ident(span, Ident::with_empty_ctxt(name));
-            match self.cx.resolver.resolve_macro(scope, &path, MacroKind::Derive, force) {
-                Ok(ext) => if let SyntaxExtension::ProcMacroDerive(_, ref inert_attrs) = *ext {
-                    if inert_attrs.contains(&attr_name) {
-                        // FIXME(jseyfried) Avoid `mem::replace` here.
-                        let dummy_item = placeholder(ExpansionKind::Items, ast::DUMMY_NODE_ID)
-                            .make_items().pop().unwrap();
-                        *item = mem::replace(item, Annotatable::Item(dummy_item))
-                            .map_attrs(|mut attrs| {
-                                let inert_attr = attr.take().unwrap();
-                                attr::mark_known(&inert_attr);
-                                if self.cx.ecfg.proc_macro_enabled() {
-                                    *attr = find_attr_invoc(&mut attrs);
-                                }
-                                attrs.push(inert_attr);
-                                attrs
-                            });
-                    }
-                    return Err(Determinacy::Undetermined);
-                },
-                Err(Determinacy::Undetermined) => determined = false,
-                Err(Determinacy::Determined) => {}
-            }
-        }
-
-        Err(if determined { Determinacy::Determined } else { Determinacy::Undetermined })
-    }
-
     fn expand_invoc(&mut self, invoc: Invocation, ext: Rc<SyntaxExtension>) -> Expansion {
         match invoc.kind {
             InvocationKind::Bang { .. } => self.expand_bang_invoc(invoc, ext),
@@ -502,7 +444,8 @@ impl<'a, 'b> MacroExpander<'a, 'b> {
 
         let extname = path.segments.last().unwrap().identifier.name;
         let ident = ident.unwrap_or(keywords::Invalid.ident());
-        let marked_tts = mark_tts(mac.node.stream(), mark);
+        let marked_tts =
+            noop_fold_tts(mac.node.stream(), &mut Marker { mark: mark, expn_id: None });
         let opt_expanded = match *ext {
             NormalTT(ref expandfun, exp_span, allow_internal_unstable) => {
                 if ident.name != keywords::Invalid.name() {
@@ -814,7 +757,7 @@ impl<'a, 'b> InvocationCollector<'a, 'b> {
     }
 }
 
-fn find_attr_invoc(attrs: &mut Vec<ast::Attribute>) -> Option<ast::Attribute> {
+pub fn find_attr_invoc(attrs: &mut Vec<ast::Attribute>) -> Option<ast::Attribute> {
     for i in 0 .. attrs.len() {
         if !attr::is_known(&attrs[i]) && !is_builtin_attr(&attrs[i]) {
              return Some(attrs.remove(i));
@@ -960,17 +903,7 @@ impl<'a, 'b> Folder for InvocationCollector<'a, 'b> {
         match item.node {
             ast::ItemKind::Mac(..) => {
                 self.check_attributes(&item.attrs);
-                let is_macro_def = if let ItemKind::Mac(ref mac) = item.node {
-                    mac.node.path.segments[0].identifier.name == "macro_rules"
-                } else {
-                    unreachable!()
-                };
-
-                item.and_then(|mut item| match item.node {
-                    ItemKind::Mac(_) if is_macro_def => {
-                        item.id = Mark::fresh().as_placeholder_id();
-                        SmallVector::one(P(item))
-                    }
+                item.and_then(|item| match item.node {
                     ItemKind::Mac(mac) => {
                         self.collect(ExpansionKind::Items, InvocationKind::Bang {
                             mac: mac,
@@ -1090,7 +1023,10 @@ impl<'a, 'b> Folder for InvocationCollector<'a, 'b> {
     }
 
     fn fold_item_kind(&mut self, item: ast::ItemKind) -> ast::ItemKind {
-        noop_fold_item_kind(self.cfg.configure_item_kind(item), self)
+        match item {
+            ast::ItemKind::MacroDef(..) => item,
+            _ => noop_fold_item_kind(self.cfg.configure_item_kind(item), self),
+        }
     }
 
     fn new_id(&mut self, id: ast::NodeId) -> ast::NodeId {
@@ -1171,8 +1107,3 @@ impl Folder for Marker {
         span
     }
 }
-
-// apply a given mark to the given token trees. Used prior to expansion of a macro.
-pub fn mark_tts(tts: TokenStream, m: Mark) -> TokenStream {
-    noop_fold_tts(tts, &mut Marker{mark:m, expn_id: None})
-}
diff --git a/src/libsyntax/ext/hygiene.rs b/src/libsyntax/ext/hygiene.rs
index 2af5c2ea999..57f5ab73d37 100644
--- a/src/libsyntax/ext/hygiene.rs
+++ b/src/libsyntax/ext/hygiene.rs
@@ -31,7 +31,7 @@ pub struct SyntaxContextData {
 }
 
 /// A mark is a unique id associated with a macro expansion.
-#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, Default)]
+#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, Default, RustcEncodable, RustcDecodable)]
 pub struct Mark(u32);
 
 impl Mark {
@@ -118,13 +118,6 @@ impl SyntaxContext {
             })
         })
     }
-
-    /// If `ident` is macro expanded, return the source ident from the macro definition
-    /// and the mark of the expansion that created the macro definition.
-    pub fn source(self) -> (Self /* source context */, Mark /* source macro */) {
-         let macro_def_ctxt = self.data().prev_ctxt.data();
-         (macro_def_ctxt.prev_ctxt, macro_def_ctxt.outer_mark)
-    }
 }
 
 impl fmt::Debug for SyntaxContext {
diff --git a/src/libsyntax/ext/placeholders.rs b/src/libsyntax/ext/placeholders.rs
index e2fb1946e90..f60b1d17a5e 100644
--- a/src/libsyntax/ext/placeholders.rs
+++ b/src/libsyntax/ext/placeholders.rs
@@ -21,7 +21,6 @@ use util::move_map::MoveMap;
 use util::small_vector::SmallVector;
 
 use std::collections::HashMap;
-use std::mem;
 
 pub fn placeholder(kind: ExpansionKind, id: ast::NodeId) -> Expansion {
     fn mac_placeholder() -> ast::Mac {
@@ -174,20 +173,11 @@ impl<'a, 'b> Folder for PlaceholderExpander<'a, 'b> {
 
     fn fold_block(&mut self, block: P<ast::Block>) -> P<ast::Block> {
         noop_fold_block(block, self).map(|mut block| {
-            let mut macros = Vec::new();
             let mut remaining_stmts = block.stmts.len();
 
             block.stmts = block.stmts.move_flat_map(|mut stmt| {
                 remaining_stmts -= 1;
 
-                // `macro_rules!` macro definition
-                if let ast::StmtKind::Item(ref item) = stmt.node {
-                    if let ast::ItemKind::Mac(_) = item.node {
-                        macros.push(Mark::from_placeholder_id(item.id));
-                        return None;
-                    }
-                }
-
                 match stmt.node {
                     // Avoid wasting a node id on a trailing expression statement,
                     // which shares a HIR node with the expression itself.
@@ -201,11 +191,6 @@ impl<'a, 'b> Folder for PlaceholderExpander<'a, 'b> {
                     _ => {}
                 }
 
-                if self.monotonic && !macros.is_empty() {
-                    let macros = mem::replace(&mut macros, Vec::new());
-                    self.cx.resolver.add_expansions_at_stmt(stmt.id, macros);
-                }
-
                 Some(stmt)
             });
 
diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs
index 1d386c1a3ac..7aa1230f9ae 100644
--- a/src/libsyntax/ext/tt/macro_rules.rs
+++ b/src/libsyntax/ext/tt/macro_rules.rs
@@ -153,7 +153,7 @@ fn generic_extension<'cx>(cx: &'cx ExtCtxt,
 // Holy self-referential!
 
 /// Converts a `macro_rules!` invocation into a syntax extension.
-pub fn compile(sess: &ParseSess, def: &ast::MacroDef) -> SyntaxExtension {
+pub fn compile(sess: &ParseSess, def: &ast::Item) -> SyntaxExtension {
     let lhs_nm = ast::Ident::with_empty_ctxt(Symbol::gensym("lhs"));
     let rhs_nm = ast::Ident::with_empty_ctxt(Symbol::gensym("rhs"));
 
@@ -183,7 +183,11 @@ pub fn compile(sess: &ParseSess, def: &ast::MacroDef) -> SyntaxExtension {
     ];
 
     // Parse the macro_rules! invocation
-    let argument_map = match parse(sess, def.body.clone().into(), &argument_gram, None) {
+    let body = match def.node {
+        ast::ItemKind::MacroDef(ref body) => body.clone().into(),
+        _ => unreachable!(),
+    };
+    let argument_map = match parse(sess, body, &argument_gram, None) {
         Success(m) => m,
         Failure(sp, tok) => {
             let s = parse_failure_msg(tok);