Initial implementation of RFC 2151, Raw Identifiers

4 files changed, 15 insertions, 14 deletions

diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs
index 667653b5f7f..714b7ca981d 100644
--- a/src/libsyntax/ext/tt/macro_parser.rs
+++ b/src/libsyntax/ext/tt/macro_parser.rs
@@ -365,7 +365,7 @@ pub fn parse_failure_msg(tok: Token) -> String {
 /// Perform a token equality check, ignoring syntax context (that is, an unhygienic comparison)
 fn token_name_eq(t1: &Token, t2: &Token) -> bool {
     if let (Some(id1), Some(id2)) = (t1.ident(), t2.ident()) {
-        id1.name == id2.name
+        id1.name == id2.name && t1.is_raw_ident() == t2.is_raw_ident()
     } else if let (&token::Lifetime(id1), &token::Lifetime(id2)) = (t1, t2) {
         id1.name == id2.name
     } else {
@@ -711,9 +711,10 @@ pub fn parse(
 
 /// The token is an identifier, but not `_`.
 /// We prohibit passing `_` to macros expecting `ident` for now.
-fn get_macro_ident(token: &Token) -> Option<Ident> {
+fn get_macro_ident(token: &Token) -> Option<(Ident, bool)> {
     match *token {
-        token::Ident(ident) if ident.name != keywords::Underscore.name() => Some(ident),
+        token::Ident(ident, is_raw) if ident.name != keywords::Underscore.name() =>
+            Some((ident, is_raw)),
         _ => None,
     }
 }
@@ -737,7 +738,7 @@ fn may_begin_with(name: &str, token: &Token) -> bool {
         "ident" => get_macro_ident(token).is_some(),
         "vis" => match *token {
             // The follow-set of :vis + "priv" keyword + interpolated
-            Token::Comma | Token::Ident(_) | Token::Interpolated(_) => true,
+            Token::Comma | Token::Ident(..) | Token::Interpolated(_) => true,
             _ => token.can_begin_type(),
         },
         "block" => match *token {
@@ -746,7 +747,7 @@ fn may_begin_with(name: &str, token: &Token) -> bool {
                 token::NtItem(_)
                 | token::NtPat(_)
                 | token::NtTy(_)
-                | token::NtIdent(_)
+                | token::NtIdent(..)
                 | token::NtMeta(_)
                 | token::NtPath(_)
                 | token::NtVis(_) => false, // none of these may start with '{'.
@@ -755,7 +756,7 @@ fn may_begin_with(name: &str, token: &Token) -> bool {
             _ => false,
         },
         "path" | "meta" => match *token {
-            Token::ModSep | Token::Ident(_) => true,
+            Token::ModSep | Token::Ident(..) => true,
             Token::Interpolated(ref nt) => match nt.0 {
                 token::NtPath(_) | token::NtMeta(_) => true,
                 _ => may_be_ident(&nt.0),
@@ -763,7 +764,7 @@ fn may_begin_with(name: &str, token: &Token) -> bool {
             _ => false,
         },
         "pat" => match *token {
-            Token::Ident(_) |               // box, ref, mut, and other identifiers (can stricten)
+            Token::Ident(..) |               // box, ref, mut, and other identifiers (can stricten)
             Token::OpenDelim(token::Paren) |    // tuple pattern
             Token::OpenDelim(token::Bracket) |  // slice pattern
             Token::BinOp(token::And) |          // reference
@@ -823,9 +824,9 @@ fn parse_nt<'a>(p: &mut Parser<'a>, sp: Span, name: &str) -> Nonterminal {
         "expr" => token::NtExpr(panictry!(p.parse_expr())),
         "ty" => token::NtTy(panictry!(p.parse_ty())),
         // this could be handled like a token, since it is one
-        "ident" => if let Some(ident) = get_macro_ident(&p.token) {
+        "ident" => if let Some((ident, is_raw)) = get_macro_ident(&p.token) {
             p.bump();
-            token::NtIdent(respan(p.prev_span, ident))
+            token::NtIdent(respan(p.prev_span, ident), is_raw)
         } else {
             let token_str = pprust::token_to_string(&p.token);
             p.fatal(&format!("expected ident, found {}", &token_str)).emit();
diff --git a/src/libsyntax/ext/tt/macro_rules.rs b/src/libsyntax/ext/tt/macro_rules.rs
index a4b2c3990f5..10e5926eb9e 100644
--- a/src/libsyntax/ext/tt/macro_rules.rs
+++ b/src/libsyntax/ext/tt/macro_rules.rs
@@ -831,7 +831,7 @@ fn is_in_follow(tok: &quoted::TokenTree, frag: &str) -> Result<bool, (String, &'
             "pat" => match *tok {
                 TokenTree::Token(_, ref tok) => match *tok {
                     FatArrow | Comma | Eq | BinOp(token::Or) => Ok(true),
-                    Ident(i) if i.name == "if" || i.name == "in" => Ok(true),
+                    Ident(i, false) if i.name == "if" || i.name == "in" => Ok(true),
                     _ => Ok(false)
                 },
                 _ => Ok(false),
@@ -840,7 +840,7 @@ fn is_in_follow(tok: &quoted::TokenTree, frag: &str) -> Result<bool, (String, &'
                 TokenTree::Token(_, ref tok) => match *tok {
                     OpenDelim(token::DelimToken::Brace) | OpenDelim(token::DelimToken::Bracket) |
                     Comma | FatArrow | Colon | Eq | Gt | Semi | BinOp(token::Or) => Ok(true),
-                    Ident(i) if i.name == "as" || i.name == "where" => Ok(true),
+                    Ident(i, false) if i.name == "as" || i.name == "where" => Ok(true),
                     _ => Ok(false)
                 },
                 TokenTree::MetaVarDecl(_, _, frag) if frag.name == "block" => Ok(true),
@@ -860,7 +860,7 @@ fn is_in_follow(tok: &quoted::TokenTree, frag: &str) -> Result<bool, (String, &'
                 match *tok {
                     TokenTree::Token(_, ref tok) => match *tok {
                         Comma => Ok(true),
-                        Ident(i) if i.name != "priv" => Ok(true),
+                        Ident(i, is_raw) if is_raw || i.name != "priv" => Ok(true),
                         ref tok => Ok(tok.can_begin_type())
                     },
                     TokenTree::MetaVarDecl(_, _, frag) if frag.name == "ident"
diff --git a/src/libsyntax/ext/tt/quoted.rs b/src/libsyntax/ext/tt/quoted.rs
index 122bb9ba024..11eb9b940ae 100644
--- a/src/libsyntax/ext/tt/quoted.rs
+++ b/src/libsyntax/ext/tt/quoted.rs
@@ -296,7 +296,7 @@ where
                         name: keywords::DollarCrate.name(),
                         ..ident
                     };
-                    TokenTree::Token(span, token::Ident(ident))
+                    TokenTree::Token(span, token::Ident(ident, false))
                 } else {
                     TokenTree::MetaVar(span, ident)
                 }
diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs
index 7883c4bbc16..0f5855a3225 100644
--- a/src/libsyntax/ext/tt/transcribe.rs
+++ b/src/libsyntax/ext/tt/transcribe.rs
@@ -169,7 +169,7 @@ pub fn transcribe(cx: &ExtCtxt,
                         Ident { ctxt: ident.ctxt.apply_mark(cx.current_expansion.mark), ..ident };
                     sp = sp.with_ctxt(sp.ctxt().apply_mark(cx.current_expansion.mark));
                     result.push(TokenTree::Token(sp, token::Dollar).into());
-                    result.push(TokenTree::Token(sp, token::Ident(ident)).into());
+                    result.push(TokenTree::Token(sp, token::Ident(ident, false)).into());
                 }
             }
             quoted::TokenTree::Delimited(mut span, delimited) => {