Auto merge of #37213 - jseyfried:refactor_crate_var, r=nrc

macros: improve `$crate`

This PR refactors the implementation of `$crate` so that
 - `$crate` is only allowed at the start of a path (like `super`),
 - we can make `$crate` work with inter-crate re-exports (groundwork for macro modularization), and
 - we can support importing macros from an extern crate that is not declared at the crate root (also groundwork for macro modularization).

This is a [breaking-change]. For example, the following would break:
```rust
fn foo() {}
macro_rules! m { () => {
    $crate foo $crate () $crate $crate;
    //^ Today, `$crate` is allowed just about anywhere in unexported macros.
} }
fn main() {
    m!();
}
```
r? @nrc

8 files changed, 13 insertions, 69 deletions

diff --git a/src/libsyntax/ext/expand.rs b/src/libsyntax/ext/expand.rs
index 6aeb46fd522..e84a9208029 100644
--- a/src/libsyntax/ext/expand.rs
+++ b/src/libsyntax/ext/expand.rs
@@ -939,6 +939,6 @@ impl Folder for Marker {
 }
 
 // apply a given mark to the given token trees. Used prior to expansion of a macro.
-fn mark_tts(tts: &[TokenTree], m: Mark) -> Vec<TokenTree> {
+pub fn mark_tts(tts: &[TokenTree], m: Mark) -> Vec<TokenTree> {
     noop_fold_tts(tts, &mut Marker{mark:m, expn_id: None})
 }
diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs
index a74d335d604..5496d27c087 100644
--- a/src/libsyntax/ext/tt/macro_rules.rs
+++ b/src/libsyntax/ext/tt/macro_rules.rs
@@ -58,7 +58,6 @@ impl<'a> ParserAnyMacro<'a> {
 
 struct MacroRulesMacroExpander {
     name: ast::Ident,
-    imported_from: Option<ast::Ident>,
     lhses: Vec<TokenTree>,
     rhses: Vec<TokenTree>,
     valid: bool,
@@ -76,7 +75,6 @@ impl TTMacroExpander for MacroRulesMacroExpander {
         generic_extension(cx,
                           sp,
                           self.name,
-                          self.imported_from,
                           arg,
                           &self.lhses,
                           &self.rhses)
@@ -87,7 +85,6 @@ impl TTMacroExpander for MacroRulesMacroExpander {
 fn generic_extension<'cx>(cx: &'cx ExtCtxt,
                           sp: Span,
                           name: ast::Ident,
-                          imported_from: Option<ast::Ident>,
                           arg: &[TokenTree],
                           lhses: &[TokenTree],
                           rhses: &[TokenTree])
@@ -116,10 +113,8 @@ fn generic_extension<'cx>(cx: &'cx ExtCtxt,
                     _ => cx.span_bug(sp, "malformed macro rhs"),
                 };
                 // rhs has holes ( `$id` and `$(...)` that need filled)
-                let trncbr = new_tt_reader(&cx.parse_sess.span_diagnostic,
-                                           Some(named_matches),
-                                           imported_from,
-                                           rhs);
+                let trncbr =
+                    new_tt_reader(&cx.parse_sess.span_diagnostic, Some(named_matches), rhs);
                 let mut p = Parser::new(cx.parse_sess(), cx.cfg().clone(), Box::new(trncbr));
                 p.directory = cx.current_expansion.module.directory.clone();
                 p.restrictions = match cx.current_expansion.no_noninline_mod {
@@ -223,7 +218,7 @@ pub fn compile(sess: &ParseSess, def: &ast::MacroDef) -> SyntaxExtension {
     ];
 
     // Parse the macro_rules! invocation (`none` is for no interpolations):
-    let arg_reader = new_tt_reader(&sess.span_diagnostic, None, None, def.body.clone());
+    let arg_reader = new_tt_reader(&sess.span_diagnostic, None, def.body.clone());
 
     let argument_map = match parse(sess, &Vec::new(), arg_reader, &argument_gram) {
         Success(m) => m,
@@ -269,7 +264,6 @@ pub fn compile(sess: &ParseSess, def: &ast::MacroDef) -> SyntaxExtension {
 
     let exp: Box<_> = Box::new(MacroRulesMacroExpander {
         name: def.ident,
-        imported_from: def.imported_from,
         lhses: lhses,
         rhses: rhses,
         valid: valid,
diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs
index 205c709d6cb..38a926b6e87 100644
--- a/src/libsyntax/ext/tt/transcribe.rs
+++ b/src/libsyntax/ext/tt/transcribe.rs
@@ -14,7 +14,7 @@ use syntax_pos::{Span, DUMMY_SP};
 use errors::{Handler, DiagnosticBuilder};
 use ext::tt::macro_parser::{NamedMatch, MatchedSeq, MatchedNonterminal};
 use parse::token::{DocComment, MatchNt, SubstNt};
-use parse::token::{Token, Interpolated, NtIdent, NtTT, SpecialMacroVar};
+use parse::token::{Token, Interpolated, NtIdent, NtTT};
 use parse::token;
 use parse::lexer::TokenAndSpan;
 use tokenstream::{self, TokenTree};
@@ -39,10 +39,7 @@ pub struct TtReader<'a> {
     stack: Vec<TtFrame>,
     /* for MBE-style macro transcription */
     interpolations: HashMap<Ident, Rc<NamedMatch>>,
-    imported_from: Option<Ident>,
 
-    // Some => return imported_from as the next token
-    crate_name_next: Option<Span>,
     repeat_idx: Vec<usize>,
     repeat_len: Vec<usize>,
     /* cached: */
@@ -59,10 +56,9 @@ pub struct TtReader<'a> {
 /// (and should) be None.
 pub fn new_tt_reader(sp_diag: &Handler,
                      interp: Option<HashMap<Ident, Rc<NamedMatch>>>,
-                     imported_from: Option<Ident>,
                      src: Vec<tokenstream::TokenTree>)
                      -> TtReader {
-    new_tt_reader_with_doc_flag(sp_diag, interp, imported_from, src, false)
+    new_tt_reader_with_doc_flag(sp_diag, interp, src, false)
 }
 
 /// The extra `desugar_doc_comments` flag enables reading doc comments
@@ -73,7 +69,6 @@ pub fn new_tt_reader(sp_diag: &Handler,
 /// (and should) be None.
 pub fn new_tt_reader_with_doc_flag(sp_diag: &Handler,
                                    interp: Option<HashMap<Ident, Rc<NamedMatch>>>,
-                                   imported_from: Option<Ident>,
                                    src: Vec<tokenstream::TokenTree>,
                                    desugar_doc_comments: bool)
                                    -> TtReader {
@@ -93,8 +88,6 @@ pub fn new_tt_reader_with_doc_flag(sp_diag: &Handler,
             None => HashMap::new(),
             Some(x) => x,
         },
-        imported_from: imported_from,
-        crate_name_next: None,
         repeat_idx: Vec::new(),
         repeat_len: Vec::new(),
         desugar_doc_comments: desugar_doc_comments,
@@ -189,14 +182,6 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan {
         sp: r.cur_span.clone(),
     };
     loop {
-        match r.crate_name_next.take() {
-            None => (),
-            Some(sp) => {
-                r.cur_span = sp;
-                r.cur_tok = token::Ident(r.imported_from.unwrap());
-                return ret_val;
-            },
-        }
         let should_pop = match r.stack.last() {
             None => {
                 assert_eq!(ret_val.tok, token::Eof);
@@ -346,18 +331,6 @@ pub fn tt_next_token(r: &mut TtReader) -> TokenAndSpan {
                    sep: None
                 });
             }
-            TokenTree::Token(sp, token::SpecialVarNt(SpecialMacroVar::CrateMacroVar)) => {
-                r.stack.last_mut().unwrap().idx += 1;
-
-                if r.imported_from.is_some() {
-                    r.cur_span = sp;
-                    r.cur_tok = token::ModSep;
-                    r.crate_name_next = Some(sp);
-                    return ret_val;
-                }
-
-                // otherwise emit nothing and proceed to the next token
-            }
             TokenTree::Token(sp, tok) => {
                 r.cur_span = sp;
                 r.cur_tok = tok;
diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs
index 1e286c143de..1a84a750463 100644
--- a/src/libsyntax/parse/mod.rs
+++ b/src/libsyntax/parse/mod.rs
@@ -276,7 +276,7 @@ pub fn filemap_to_tts(sess: &ParseSess, filemap: Rc<FileMap>)
 pub fn tts_to_parser<'a>(sess: &'a ParseSess,
                          tts: Vec<tokenstream::TokenTree>,
                          cfg: ast::CrateConfig) -> Parser<'a> {
-    let trdr = lexer::new_tt_reader(&sess.span_diagnostic, None, None, tts);
+    let trdr = lexer::new_tt_reader(&sess.span_diagnostic, None, tts);
     let mut p = Parser::new(sess, cfg, Box::new(trdr));
     p.check_unknown_macro_variable();
     p
diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs
index 61268d457ce..eac78f5e6c6 100644
--- a/src/libsyntax/parse/parser.rs
+++ b/src/libsyntax/parse/parser.rs
@@ -48,8 +48,7 @@ use parse::classify;
 use parse::common::SeqSep;
 use parse::lexer::{Reader, TokenAndSpan};
 use parse::obsolete::ObsoleteSyntax;
-use parse::token::{self, intern, MatchNt, SubstNt, SpecialVarNt, InternedString};
-use parse::token::{keywords, SpecialMacroVar};
+use parse::token::{self, intern, keywords, MatchNt, SubstNt, InternedString};
 use parse::{new_sub_parser_from_file, ParseSess};
 use util::parser::{AssocOp, Fixity};
 use print::pprust;
@@ -2653,8 +2652,12 @@ impl<'a> Parser<'a> {
                                           num_captures: name_num
                                       })));
                 } else if self.token.is_keyword(keywords::Crate) {
+                    let ident = match self.token {
+                        token::Ident(id) => ast::Ident { name: token::intern("$crate"), ..id },
+                        _ => unreachable!(),
+                    };
                     self.bump();
-                    return Ok(TokenTree::Token(sp, SpecialVarNt(SpecialMacroVar::CrateMacroVar)));
+                    return Ok(TokenTree::Token(sp, token::Ident(ident)));
                 } else {
                     sp = mk_sp(sp.lo, self.span.hi);
                     self.parse_ident().unwrap_or_else(|mut e| {
diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs
index 3d4dd9ec064..26b5b99c8cc 100644
--- a/src/libsyntax/parse/token.rs
+++ b/src/libsyntax/parse/token.rs
@@ -53,21 +53,6 @@ pub enum DelimToken {
 }
 
 #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug, Copy)]
-pub enum SpecialMacroVar {
-    /// `$crate` will be filled in with the name of the crate a macro was
-    /// imported from, if any.
-    CrateMacroVar,
-}
-
-impl SpecialMacroVar {
-    pub fn as_str(self) -> &'static str {
-        match self {
-            SpecialMacroVar::CrateMacroVar => "crate",
-        }
-    }
-}
-
-#[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug, Copy)]
 pub enum Lit {
     Byte(ast::Name),
     Char(ast::Name),
@@ -148,8 +133,6 @@ pub enum Token {
     // In right-hand-sides of MBE macros:
     /// A syntactic variable that will be filled in by macro expansion.
     SubstNt(ast::Ident),
-    /// A macro variable with special meaning.
-    SpecialVarNt(SpecialMacroVar),
 
     // Junk. These carry no data because we don't really care about the data
     // they *would* carry, and don't really want to allocate a new ident for
diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs
index 89834da2821..c6f63d77242 100644
--- a/src/libsyntax/print/pprust.rs
+++ b/src/libsyntax/print/pprust.rs
@@ -285,8 +285,6 @@ pub fn token_to_string(tok: &Token) -> String {
         token::Comment              => "/* */".to_string(),
         token::Shebang(s)           => format!("/* shebang: {}*/", s),
 
-        token::SpecialVarNt(var)    => format!("${}", var.as_str()),
-
         token::Interpolated(ref nt) => match *nt {
             token::NtExpr(ref e)        => expr_to_string(&e),
             token::NtMeta(ref e)        => meta_item_to_string(&e),
diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs
index b35b4617ea1..f22f920a7fa 100644
--- a/src/libsyntax/tokenstream.rs
+++ b/src/libsyntax/tokenstream.rs
@@ -134,7 +134,6 @@ impl TokenTree {
                     AttrStyle::Inner => 3,
                 }
             }
-            TokenTree::Token(_, token::SpecialVarNt(..)) => 2,
             TokenTree::Token(_, token::MatchNt(..)) => 3,
             TokenTree::Token(_, token::Interpolated(Nonterminal::NtTT(..))) => 1,
             TokenTree::Delimited(_, ref delimed) => delimed.tts.len() + 2,
@@ -188,11 +187,6 @@ impl TokenTree {
                 }
                 delimed.tts[index - 1].clone()
             }
-            (&TokenTree::Token(sp, token::SpecialVarNt(var)), _) => {
-                let v = [TokenTree::Token(sp, token::Dollar),
-                         TokenTree::Token(sp, token::Ident(token::str_to_ident(var.as_str())))];
-                v[index].clone()
-            }
             (&TokenTree::Token(sp, token::MatchNt(name, kind)), _) => {
                 let v = [TokenTree::Token(sp, token::SubstNt(name)),
                          TokenTree::Token(sp, token::Colon),
@@ -224,7 +218,6 @@ impl TokenTree {
         // `None` is because we're not interpolating
         let arg_rdr = lexer::new_tt_reader_with_doc_flag(&cx.parse_sess().span_diagnostic,
                                                          None,
-                                                         None,
                                                          tts.iter().cloned().collect(),
                                                          true);
         macro_parser::parse(cx.parse_sess(), cx.cfg(), arg_rdr, mtch)