Auto merge of #40847 - jseyfried:decl_macro, r=nrc

Initial implementation of declarative macros 2.0

Implement declarative macros 2.0 (rust-lang/rfcs#1584) behind `#![feature(decl_macro)]`.
Differences from `macro_rules!` include:
 - new syntax: `macro m(..) { .. }` instead of `macro_rules! m { (..) => { .. } }`
 - declarative macros are items:
```rust
// crate A:
pub mod foo {
    m!(); // use before definition; declaration order is irrelevant
    pub macro m() {} // `pub`, `pub(super)`, etc. work
}
fn main() {
    foo::m!(); // named like other items
    { use foo::m as n; n!(); } // imported like other items
}
pub use foo::m; // re-exported like other items

// crate B:
extern crate A; // no need for `#[macro_use]`
A::foo::m!(); A::m!();
```
 - Racket-like hygiene for items, imports, methods, fields, type parameters, privacy, etc.
   - Intuitively, names in a macro definition are resolved in the macro definition's scope, not the scope in which the macro is used.
   - This [explaination](http://beautifulracket.com/explainer/hygiene.html) of hygiene for Racket applies here (except for the "Breaking Hygiene" section). I wrote a similar [explanation](https://github.com/jseyfried/rfcs/blob/hygiene/text/0000-hygiene.md) for Rust.
   - Generally speaking, if `fn f() { <body> }` resolves, `pub macro m() { <body> } ... m!()` also resolves, even if `m!()` is in a separate crate.
   - `::foo::bar` in a `macro` behaves like `$crate::foo::bar` in a `macro_rules!`, except it can access everything visible from the `macro` (thus more permissive).
   - See [`src/test/{run-pass, compile-fail}/hygiene`](https://github.com/rust-lang/rust/pull/40847/commits/afe7d89858fd72b983e24727d6f4058293153c19) for examples. Small example:
```rust
mod foo {
    fn f() { println!("hello world"); }
    pub macro m() { f(); }
}
fn main() { foo::m!(); }
```

Limitations:
 - This does not address planned changes to matchers (`expr`,`ty`, etc.), c.f. #26361.
 - Lints (including stability and deprecation) and `unsafe` are not hygienic.
   - adding hygiene here will be mostly or entirely backwards compatible
 - Nested macro definitions (a `macro` inside another `macro`) don't always work correctly when invoked from external crates.
   - pending improvements in how we encode macro definitions in crate metadata
 - There is no way to "escape" hygiene without using a procedural macro.

r? @nrc

1 files changed, 48 insertions, 20 deletions

diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs
index c28f678cb51..8d7c8c5248b 100644
--- a/src/libsyntax/parse/parser.rs
+++ b/src/libsyntax/parse/parser.rs
@@ -1958,7 +1958,7 @@ impl<'a> Parser<'a> {
             token::Lifetime(ident) => {
                 let ident_span = self.span;
                 self.bump();
-                Lifetime { name: ident.name, span: ident_span, id: ast::DUMMY_NODE_ID }
+                Lifetime { ident: ident, span: ident_span, id: ast::DUMMY_NODE_ID }
             }
             _ => self.span_bug(self.span, "not a lifetime")
         }
@@ -3758,31 +3758,59 @@ impl<'a> Parser<'a> {
     fn eat_macro_def(&mut self, attrs: &[Attribute], vis: &Visibility)
                      -> PResult<'a, Option<P<Item>>> {
         let lo = self.span;
-        match self.token {
-            token::Ident(ident) if ident.name == "macro_rules" => {
-                if self.look_ahead(1, |t| *t == token::Not) {
-                    let prev_span = self.prev_span;
-                    self.complain_if_pub_macro(vis, prev_span);
-                    self.bump();
-                    self.bump();
+        let (ident, def) = match self.token {
+            token::Ident(ident) if ident.name == keywords::Macro.name() => {
+                self.bump();
+                let ident = self.parse_ident()?;
+                let tokens = if self.check(&token::OpenDelim(token::Brace)) {
+                    match self.parse_token_tree() {
+                        TokenTree::Delimited(_, ref delimited) => delimited.stream(),
+                        _ => unreachable!(),
+                    }
+                } else if self.check(&token::OpenDelim(token::Paren)) {
+                    let args = self.parse_token_tree();
+                    let body = if self.check(&token::OpenDelim(token::Brace)) {
+                        self.parse_token_tree()
+                    } else {
+                        self.unexpected()?;
+                        unreachable!()
+                    };
+                    TokenStream::concat(vec![
+                        args.into(),
+                        TokenTree::Token(lo.to(self.prev_span), token::FatArrow).into(),
+                        body.into(),
+                    ])
+                } else {
+                    self.unexpected()?;
+                    unreachable!()
+                };
+
+                (ident, ast::MacroDef { tokens: tokens.into(), legacy: false })
+            }
+            token::Ident(ident) if ident.name == "macro_rules" &&
+                                   self.look_ahead(1, |t| *t == token::Not) => {
+                let prev_span = self.prev_span;
+                self.complain_if_pub_macro(vis, prev_span);
+                self.bump();
+                self.bump();
+
+                let ident = self.parse_ident()?;
+                let (delim, tokens) = self.expect_delimited_token_tree()?;
+                if delim != token::Brace {
+                    if !self.eat(&token::Semi) {
+                        let msg = "macros that expand to items must either \
+                                   be surrounded with braces or followed by a semicolon";
+                        self.span_err(self.prev_span, msg);
+                    }
                 }
+
+                (ident, ast::MacroDef { tokens: tokens, legacy: true })
             }
             _ => return Ok(None),
         };
 
-        let id = self.parse_ident()?;
-        let (delim, tts) = self.expect_delimited_token_tree()?;
-        if delim != token::Brace {
-            if !self.eat(&token::Semi) {
-                let msg = "macros that expand to items must either be surrounded with braces \
-                           or followed by a semicolon";
-                self.span_err(self.prev_span, msg);
-            }
-        }
-
         let span = lo.to(self.prev_span);
-        let kind = ItemKind::MacroDef(tts);
-        Ok(Some(self.mk_item(span, id, kind, Visibility::Inherited, attrs.to_owned())))
+        Ok(Some(self.mk_item(span, ident, ItemKind::MacroDef(def), vis.clone(), attrs.to_vec())))
     }
 
     fn parse_stmt_without_recovery(&mut self,