From fad1648e0f8299a8b108f85c2b1055eb37bdab9e Mon Sep 17 00:00:00 2001
From: Lymia Aluysia <lymia@lymiahugs.com>
Date: Fri, 9 Mar 2018 23:56:40 -0600
Subject: Initial implementation of RFC 2151, Raw Identifiers

---
 src/libsyntax/parse/lexer/mod.rs |  69 ++++++++++++++++---------
 src/libsyntax/parse/mod.rs       |  20 ++++----
 src/libsyntax/parse/parser.rs    |  29 ++++++-----
 src/libsyntax/parse/token.rs     | 108 ++++++++++++++++++++++++++++-----------
 4 files changed, 153 insertions(+), 73 deletions(-)

(limited to 'src/libsyntax/parse')

diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs
index 0e20eb49d39..0596fb44abe 100644
--- a/src/libsyntax/parse/lexer/mod.rs
+++ b/src/libsyntax/parse/lexer/mod.rs
@@ -14,7 +14,7 @@ use codemap::{CodeMap, FilePathMapping};
 use errors::{FatalError, DiagnosticBuilder};
 use parse::{token, ParseSess};
 use str::char_at;
-use symbol::Symbol;
+use symbol::{Symbol, keywords};
 use std_unicode::property::Pattern_White_Space;
 
 use std::borrow::Cow;
@@ -1115,26 +1115,49 @@ impl<'a> StringReader<'a> {
     /// token, and updates the interner
     fn next_token_inner(&mut self) -> Result<token::Token, ()> {
         let c = self.ch;
-        if ident_start(c) &&
-           match (c.unwrap(), self.nextch(), self.nextnextch()) {
-            // Note: r as in r" or r#" is part of a raw string literal,
-            // b as in b' is part of a byte literal.
-            // They are not identifiers, and are handled further down.
-            ('r', Some('"'), _) |
-            ('r', Some('#'), _) |
-            ('b', Some('"'), _) |
-            ('b', Some('\''), _) |
-            ('b', Some('r'), Some('"')) |
-            ('b', Some('r'), Some('#')) => false,
-            _ => true,
-        } {
-            let start = self.pos;
-            while ident_continue(self.ch) {
-                self.bump();
-            }
 
-            // FIXME: perform NFKC normalization here. (Issue #2253)
-            return Ok(self.with_str_from(start, |string| token::Ident(self.mk_ident(string))));
+        if ident_start(c) {
+            let (is_ident_start, is_raw_ident) =
+                match (c.unwrap(), self.nextch(), self.nextnextch()) {
+                    // r# followed by an identifier starter is a raw identifier.
+                    // This is an exception to the r# case below.
+                    ('r', Some('#'), x) if ident_start(x) => (true, true),
+                    // r as in r" or r#" is part of a raw string literal.
+                    // b as in b' is part of a byte literal.
+                    // They are not identifiers, and are handled further down.
+                    ('r', Some('"'), _) |
+                    ('r', Some('#'), _) |
+                    ('b', Some('"'), _) |
+                    ('b', Some('\''), _) |
+                    ('b', Some('r'), Some('"')) |
+                    ('b', Some('r'), Some('#')) => (false, false),
+                    _ => (true, false),
+                };
+            if is_ident_start {
+                let raw_start = self.pos;
+                if is_raw_ident {
+                    // Consume the 'r#' characters.
+                    self.bump();
+                    self.bump();
+                }
+
+                let start = self.pos;
+                while ident_continue(self.ch) {
+                    self.bump();
+                }
+
+                return Ok(self.with_str_from(start, |string| {
+                    // FIXME: perform NFKC normalization here. (Issue #2253)
+                    let ident = self.mk_ident(string);
+                    if is_raw_ident && (token::is_path_segment_keyword(ident) ||
+                                        ident.name == keywords::Underscore.name()) {
+                        self.fatal_span_(raw_start, self.pos,
+                            &format!("`r#{}` is not currently supported.", ident.name)
+                        ).raise();
+                    }
+                    token::Ident(ident, is_raw_ident)
+                }));
+            }
         }
 
         if is_dec_digit(c) {
@@ -1801,7 +1824,7 @@ mod tests {
             assert_eq!(string_reader.next_token().tok, token::Whitespace);
             let tok1 = string_reader.next_token();
             let tok2 = TokenAndSpan {
-                tok: token::Ident(id),
+                tok: token::Ident(id, false),
                 sp: Span::new(BytePos(21), BytePos(23), NO_EXPANSION),
             };
             assert_eq!(tok1, tok2);
@@ -1811,7 +1834,7 @@ mod tests {
             // read another token:
             let tok3 = string_reader.next_token();
             let tok4 = TokenAndSpan {
-                tok: token::Ident(Ident::from_str("main")),
+                tok: mk_ident("main"),
                 sp: Span::new(BytePos(24), BytePos(28), NO_EXPANSION),
             };
             assert_eq!(tok3, tok4);
@@ -1830,7 +1853,7 @@ mod tests {
 
     // make the identifier by looking up the string in the interner
     fn mk_ident(id: &str) -> token::Token {
-        token::Ident(Ident::from_str(id))
+        token::Token::from_ast_ident(Ident::from_str(id))
     }
 
     #[test]
diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs
index f7e5d40b524..4acfdab53c0 100644
--- a/src/libsyntax/parse/mod.rs
+++ b/src/libsyntax/parse/mod.rs
@@ -741,9 +741,9 @@ mod tests {
             match (tts.len(), tts.get(0), tts.get(1), tts.get(2), tts.get(3)) {
                 (
                     4,
-                    Some(&TokenTree::Token(_, token::Ident(name_macro_rules))),
+                    Some(&TokenTree::Token(_, token::Ident(name_macro_rules, false))),
                     Some(&TokenTree::Token(_, token::Not)),
-                    Some(&TokenTree::Token(_, token::Ident(name_zip))),
+                    Some(&TokenTree::Token(_, token::Ident(name_zip, false))),
                     Some(&TokenTree::Delimited(_, ref macro_delimed)),
                 )
                 if name_macro_rules.name == "macro_rules"
@@ -762,7 +762,7 @@ mod tests {
                                 (
                                     2,
                                     Some(&TokenTree::Token(_, token::Dollar)),
-                                    Some(&TokenTree::Token(_, token::Ident(ident))),
+                                    Some(&TokenTree::Token(_, token::Ident(ident, false))),
                                 )
                                 if first_delimed.delim == token::Paren && ident.name == "a" => {},
                                 _ => panic!("value 3: {:?}", *first_delimed),
@@ -772,7 +772,7 @@ mod tests {
                                 (
                                     2,
                                     Some(&TokenTree::Token(_, token::Dollar)),
-                                    Some(&TokenTree::Token(_, token::Ident(ident))),
+                                    Some(&TokenTree::Token(_, token::Ident(ident, false))),
                                 )
                                 if second_delimed.delim == token::Paren
                                 && ident.name == "a" => {},
@@ -793,17 +793,18 @@ mod tests {
             let tts = string_to_stream("fn a (b : i32) { b; }".to_string());
 
             let expected = TokenStream::concat(vec![
-                TokenTree::Token(sp(0, 2), token::Ident(Ident::from_str("fn"))).into(),
-                TokenTree::Token(sp(3, 4), token::Ident(Ident::from_str("a"))).into(),
+                TokenTree::Token(sp(0, 2), token::Ident(Ident::from_str("fn"), false)).into(),
+                TokenTree::Token(sp(3, 4), token::Ident(Ident::from_str("a"), false)).into(),
                 TokenTree::Delimited(
                     sp(5, 14),
                     tokenstream::Delimited {
                         delim: token::DelimToken::Paren,
                         tts: TokenStream::concat(vec![
-                            TokenTree::Token(sp(6, 7), token::Ident(Ident::from_str("b"))).into(),
+                            TokenTree::Token(sp(6, 7),
+                                             token::Ident(Ident::from_str("b"), false)).into(),
                             TokenTree::Token(sp(8, 9), token::Colon).into(),
                             TokenTree::Token(sp(10, 13),
-                                             token::Ident(Ident::from_str("i32"))).into(),
+                                             token::Ident(Ident::from_str("i32"), false)).into(),
                         ]).into(),
                     }).into(),
                 TokenTree::Delimited(
@@ -811,7 +812,8 @@ mod tests {
                     tokenstream::Delimited {
                         delim: token::DelimToken::Brace,
                         tts: TokenStream::concat(vec![
-                            TokenTree::Token(sp(17, 18), token::Ident(Ident::from_str("b"))).into(),
+                            TokenTree::Token(sp(17, 18),
+                                             token::Ident(Ident::from_str("b"), false)).into(),
                             TokenTree::Token(sp(18, 19), token::Semi).into(),
                         ]).into(),
                     }).into()
diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs
index cb5010a638d..4c1575cf589 100644
--- a/src/libsyntax/parse/parser.rs
+++ b/src/libsyntax/parse/parser.rs
@@ -358,7 +358,7 @@ impl TokenCursor {
 
         let body = TokenTree::Delimited(sp, Delimited {
             delim: token::Bracket,
-            tts: [TokenTree::Token(sp, token::Ident(ast::Ident::from_str("doc"))),
+            tts: [TokenTree::Token(sp, token::Ident(ast::Ident::from_str("doc"), false)),
                   TokenTree::Token(sp, token::Eq),
                   TokenTree::Token(sp, token::Literal(
                       token::StrRaw(Symbol::intern(&stripped), num_of_hashes), None))]
@@ -784,7 +784,7 @@ impl<'a> Parser<'a> {
 
     fn parse_ident_common(&mut self, recover: bool) -> PResult<'a, ast::Ident> {
         match self.token {
-            token::Ident(i) => {
+            token::Ident(i, _) => {
                 if self.token.is_reserved_ident() {
                     let mut err = self.expected_ident_found();
                     if recover {
@@ -1925,7 +1925,7 @@ impl<'a> Parser<'a> {
 
     pub fn parse_path_segment_ident(&mut self) -> PResult<'a, ast::Ident> {
         match self.token {
-            token::Ident(sid) if self.token.is_path_segment_keyword() => {
+            token::Ident(sid, _) if self.token.is_path_segment_keyword() => {
                 self.bump();
                 Ok(sid)
             }
@@ -2740,11 +2740,14 @@ impl<'a> Parser<'a> {
     }
 
     pub fn process_potential_macro_variable(&mut self) {
-        let ident = match self.token {
+        let (ident, is_raw) = match self.token {
             token::Dollar if self.span.ctxt() != syntax_pos::hygiene::SyntaxContext::empty() &&
                              self.look_ahead(1, |t| t.is_ident()) => {
                 self.bump();
-                let name = match self.token { token::Ident(ident) => ident, _ => unreachable!() };
+                let name = match self.token {
+                    token::Ident(ident, _) => ident,
+                    _ => unreachable!()
+                };
                 let mut err = self.fatal(&format!("unknown macro variable `{}`", name));
                 err.span_label(self.span, "unknown macro variable");
                 err.emit();
@@ -2753,13 +2756,13 @@ impl<'a> Parser<'a> {
             token::Interpolated(ref nt) => {
                 self.meta_var_span = Some(self.span);
                 match nt.0 {
-                    token::NtIdent(ident) => ident,
+                    token::NtIdent(ident, is_raw) => (ident, is_raw),
                     _ => return,
                 }
             }
             _ => return,
         };
-        self.token = token::Ident(ident.node);
+        self.token = token::Ident(ident.node, is_raw);
         self.span = ident.span;
     }
 
@@ -4245,7 +4248,7 @@ impl<'a> Parser<'a> {
                      -> PResult<'a, Option<P<Item>>> {
         let token_lo = self.span;
         let (ident, def) = match self.token {
-            token::Ident(ident) if ident.name == keywords::Macro.name() => {
+            token::Ident(ident, false) if ident.name == keywords::Macro.name() => {
                 self.bump();
                 let ident = self.parse_ident()?;
                 let tokens = if self.check(&token::OpenDelim(token::Brace)) {
@@ -4273,7 +4276,7 @@ impl<'a> Parser<'a> {
 
                 (ident, ast::MacroDef { tokens: tokens.into(), legacy: false })
             }
-            token::Ident(ident) if ident.name == "macro_rules" &&
+            token::Ident(ident, _) if ident.name == "macro_rules" &&
                                    self.look_ahead(1, |t| *t == token::Not) => {
                 let prev_span = self.prev_span;
                 self.complain_if_pub_macro(&vis.node, prev_span);
@@ -5078,7 +5081,9 @@ impl<'a> Parser<'a> {
     fn parse_self_arg(&mut self) -> PResult<'a, Option<Arg>> {
         let expect_ident = |this: &mut Self| match this.token {
             // Preserve hygienic context.
-            token::Ident(ident) => { let sp = this.span; this.bump(); codemap::respan(sp, ident) }
+            token::Ident(ident, _) => {
+                let sp = this.span; this.bump(); codemap::respan(sp, ident)
+            }
             _ => unreachable!()
         };
         let isolated_self = |this: &mut Self, n| {
@@ -5375,7 +5380,7 @@ impl<'a> Parser<'a> {
             VisibilityKind::Inherited => Ok(()),
             _ => {
                 let is_macro_rules: bool = match self.token {
-                    token::Ident(sid) => sid.name == Symbol::intern("macro_rules"),
+                    token::Ident(sid, _) => sid.name == Symbol::intern("macro_rules"),
                     _ => false,
                 };
                 if is_macro_rules {
@@ -7016,7 +7021,7 @@ impl<'a> Parser<'a> {
     fn parse_rename(&mut self) -> PResult<'a, Option<Ident>> {
         if self.eat_keyword(keywords::As) {
             match self.token {
-                token::Ident(ident) if ident.name == keywords::Underscore.name() => {
+                token::Ident(ident, false) if ident.name == keywords::Underscore.name() => {
                     self.bump(); // `_`
                     Ok(Some(Ident { name: ident.name.gensymed(), ..ident }))
                 }
diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs
index 4ada9e20f2c..6406651bcba 100644
--- a/src/libsyntax/parse/token.rs
+++ b/src/libsyntax/parse/token.rs
@@ -91,8 +91,8 @@ impl Lit {
     }
 }
 
-fn ident_can_begin_expr(ident: ast::Ident) -> bool {
-    let ident_token: Token = Ident(ident);
+fn ident_can_begin_expr(ident: ast::Ident, is_raw: bool) -> bool {
+    let ident_token: Token = Ident(ident, is_raw);
 
     !ident_token.is_reserved_ident() ||
     ident_token.is_path_segment_keyword() ||
@@ -116,8 +116,8 @@ fn ident_can_begin_expr(ident: ast::Ident) -> bool {
     ].contains(&ident.name)
 }
 
-fn ident_can_begin_type(ident: ast::Ident) -> bool {
-    let ident_token: Token = Ident(ident);
+fn ident_can_begin_type(ident: ast::Ident, is_raw: bool) -> bool {
+    let ident_token: Token = Ident(ident, is_raw);
 
     !ident_token.is_reserved_ident() ||
     ident_token.is_path_segment_keyword() ||
@@ -132,6 +132,37 @@ fn ident_can_begin_type(ident: ast::Ident) -> bool {
     ].contains(&ident.name)
 }
 
+pub fn is_path_segment_keyword(id: ast::Ident) -> bool {
+    id.name == keywords::Super.name() ||
+    id.name == keywords::SelfValue.name() ||
+    id.name == keywords::SelfType.name() ||
+    id.name == keywords::Extern.name() ||
+    id.name == keywords::Crate.name() ||
+    id.name == keywords::CrateRoot.name() ||
+    id.name == keywords::DollarCrate.name()
+}
+
+// Returns true for reserved identifiers used internally for elided lifetimes,
+// unnamed method parameters, crate root module, error recovery etc.
+pub fn is_special_ident(id: ast::Ident) -> bool {
+    id.name <= keywords::Underscore.name()
+}
+
+/// Returns `true` if the token is a keyword used in the language.
+pub fn is_used_keyword(id: ast::Ident) -> bool {
+    id.name >= keywords::As.name() && id.name <= keywords::While.name()
+}
+
+/// Returns `true` if the token is a keyword reserved for possible future use.
+pub fn is_unused_keyword(id: ast::Ident) -> bool {
+    id.name >= keywords::Abstract.name() && id.name <= keywords::Yield.name()
+}
+
+/// Returns `true` if the token is either a special identifier or a keyword.
+pub fn is_reserved_ident(id: ast::Ident) -> bool {
+    is_special_ident(id) || is_used_keyword(id) || is_unused_keyword(id)
+}
+
 #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug)]
 pub enum Token {
     /* Expression-operator symbols. */
@@ -175,7 +206,7 @@ pub enum Token {
     Literal(Lit, Option<ast::Name>),
 
     /* Name components */
-    Ident(ast::Ident),
+    Ident(ast::Ident, /* is_raw */ bool),
     Lifetime(ast::Ident),
 
     // The `LazyTokenStream` is a pure function of the `Nonterminal`,
@@ -203,6 +234,11 @@ impl Token {
         Token::Interpolated(Lrc::new((nt, LazyTokenStream::new())))
     }
 
+    /// Recovers a `Token` from an `ast::Ident`. This creates a raw identifier if necessary.
+    pub fn from_ast_ident(ident: ast::Ident) -> Token {
+        Ident(ident, is_reserved_ident(ident))
+    }
+
     /// Returns `true` if the token starts with '>'.
     pub fn is_like_gt(&self) -> bool {
         match *self {
@@ -214,7 +250,8 @@ impl Token {
     /// Returns `true` if the token can appear at the start of an expression.
     pub fn can_begin_expr(&self) -> bool {
         match *self {
-            Ident(ident)                => ident_can_begin_expr(ident), // value name or keyword
+            Ident(ident, is_raw)              =>
+                ident_can_begin_expr(ident, is_raw), // value name or keyword
             OpenDelim(..)                     | // tuple, array or block
             Literal(..)                       | // literal
             Not                               | // operator not
@@ -239,7 +276,8 @@ impl Token {
     /// Returns `true` if the token can appear at the start of a type.
     pub fn can_begin_type(&self) -> bool {
         match *self {
-            Ident(ident)                => ident_can_begin_type(ident), // type name or keyword
+            Ident(ident, is_raw)        =>
+                ident_can_begin_type(ident, is_raw), // type name or keyword
             OpenDelim(Paren)            | // tuple
             OpenDelim(Bracket)          | // array
             Not                         | // never
@@ -272,17 +310,32 @@ impl Token {
         }
     }
 
-    pub fn ident(&self) -> Option<ast::Ident> {
+    fn ident_common(&self, allow_raw: bool) -> Option<ast::Ident> {
         match *self {
-            Ident(ident) => Some(ident),
+            Ident(ident, is_raw) if !is_raw || allow_raw => Some(ident),
             Interpolated(ref nt) => match nt.0 {
-                NtIdent(ident) => Some(ident.node),
+                NtIdent(ident, is_raw) if !is_raw || allow_raw => Some(ident.node),
                 _ => None,
             },
             _ => None,
         }
     }
 
+    pub fn nonraw_ident(&self) -> Option<ast::Ident> {
+        self.ident_common(false)
+    }
+
+    pub fn is_raw_ident(&self) -> bool {
+        match *self {
+            Ident(_, true) => true,
+            _ => false,
+        }
+    }
+
+    pub fn ident(&self) -> Option<ast::Ident> {
+        self.ident_common(true)
+    }
+
     /// Returns `true` if the token is an identifier.
     pub fn is_ident(&self) -> bool {
         self.ident().is_some()
@@ -351,18 +404,12 @@ impl Token {
 
     /// Returns `true` if the token is a given keyword, `kw`.
     pub fn is_keyword(&self, kw: keywords::Keyword) -> bool {
-        self.ident().map(|ident| ident.name == kw.name()).unwrap_or(false)
+        self.nonraw_ident().map(|ident| ident.name == kw.name()).unwrap_or(false)
     }
 
     pub fn is_path_segment_keyword(&self) -> bool {
-        match self.ident() {
-            Some(id) => id.name == keywords::Super.name() ||
-                        id.name == keywords::SelfValue.name() ||
-                        id.name == keywords::SelfType.name() ||
-                        id.name == keywords::Extern.name() ||
-                        id.name == keywords::Crate.name() ||
-                        id.name == keywords::CrateRoot.name() ||
-                        id.name == keywords::DollarCrate.name(),
+        match self.nonraw_ident() {
+            Some(id) => is_path_segment_keyword(id),
             None => false,
         }
     }
@@ -370,24 +417,24 @@ impl Token {
     // Returns true for reserved identifiers used internally for elided lifetimes,
     // unnamed method parameters, crate root module, error recovery etc.
     pub fn is_special_ident(&self) -> bool {
-        match self.ident() {
-            Some(id) => id.name <= keywords::Underscore.name(),
+        match self.nonraw_ident() {
+            Some(id) => is_special_ident(id),
             _ => false,
         }
     }
 
     /// Returns `true` if the token is a keyword used in the language.
     pub fn is_used_keyword(&self) -> bool {
-        match self.ident() {
-            Some(id) => id.name >= keywords::As.name() && id.name <= keywords::While.name(),
+        match self.nonraw_ident() {
+            Some(id) => is_used_keyword(id),
             _ => false,
         }
     }
 
     /// Returns `true` if the token is a keyword reserved for possible future use.
     pub fn is_unused_keyword(&self) -> bool {
-        match self.ident() {
-            Some(id) => id.name >= keywords::Abstract.name() && id.name <= keywords::Yield.name(),
+        match self.nonraw_ident() {
+            Some(id) => is_unused_keyword(id),
             _ => false,
         }
     }
@@ -460,7 +507,10 @@ impl Token {
 
     /// Returns `true` if the token is either a special identifier or a keyword.
     pub fn is_reserved_ident(&self) -> bool {
-        self.is_special_ident() || self.is_used_keyword() || self.is_unused_keyword()
+        match self.nonraw_ident() {
+            Some(id) => is_reserved_ident(id),
+            _ => false,
+        }
     }
 
     pub fn interpolated_to_tokenstream(&self, sess: &ParseSess, span: Span)
@@ -496,8 +546,8 @@ impl Token {
             Nonterminal::NtImplItem(ref item) => {
                 tokens = prepend_attrs(sess, &item.attrs, item.tokens.as_ref(), span);
             }
-            Nonterminal::NtIdent(ident) => {
-                let token = Token::Ident(ident.node);
+            Nonterminal::NtIdent(ident, is_raw) => {
+                let token = Token::Ident(ident.node, is_raw);
                 tokens = Some(TokenTree::Token(ident.span, token).into());
             }
             Nonterminal::NtLifetime(lifetime) => {
@@ -529,7 +579,7 @@ pub enum Nonterminal {
     NtPat(P<ast::Pat>),
     NtExpr(P<ast::Expr>),
     NtTy(P<ast::Ty>),
-    NtIdent(ast::SpannedIdent),
+    NtIdent(ast::SpannedIdent, /* is_raw */ bool),
     /// Stuff inside brackets for attributes
     NtMeta(ast::MetaItem),
     NtPath(ast::Path),
-- 
cgit 1.4.1-3-g733a5


From 7d5c29b9eae5857c040bf6f1b2d729596c8af3ae Mon Sep 17 00:00:00 2001
From: Lymia Aluysia <lymia@lymiahugs.com>
Date: Wed, 14 Mar 2018 02:00:41 -0500
Subject: Feature gate raw identifiers.

---
 src/libsyntax/feature_gate.rs                        | 14 ++++++++++++++
 src/libsyntax/parse/lexer/mod.rs                     |  1 +
 src/libsyntax/parse/mod.rs                           |  4 ++++
 src/libsyntax/parse/parser.rs                        |  5 ++++-
 src/test/run-pass/rfc-2151-raw-identifiers/attr.rs   |  2 ++
 src/test/run-pass/rfc-2151-raw-identifiers/basic.rs  |  2 ++
 src/test/run-pass/rfc-2151-raw-identifiers/items.rs  |  2 ++
 src/test/run-pass/rfc-2151-raw-identifiers/macros.rs |  1 +
 src/test/ui/feature-gate-raw-identifiers.rs          | 14 ++++++++++++++
 src/test/ui/feature-gate-raw-identifiers.stderr      | 11 +++++++++++
 src/test/ui/raw-literal-keywords.rs                  |  1 +
 src/test/ui/raw-literal-keywords.stderr              |  6 +++---
 src/test/ui/raw-literal-self.rs                      |  2 ++
 src/test/ui/raw-literal-self.stderr                  |  2 +-
 14 files changed, 62 insertions(+), 5 deletions(-)
 create mode 100644 src/test/ui/feature-gate-raw-identifiers.rs
 create mode 100644 src/test/ui/feature-gate-raw-identifiers.stderr

(limited to 'src/libsyntax/parse')

diff --git a/src/libsyntax/feature_gate.rs b/src/libsyntax/feature_gate.rs
index fa600cd6860..153e42c8214 100644
--- a/src/libsyntax/feature_gate.rs
+++ b/src/libsyntax/feature_gate.rs
@@ -452,6 +452,9 @@ declare_features! (
 
     // `use path as _;` and `extern crate c as _;`
     (active, underscore_imports, "1.26.0", Some(48216), None),
+
+    // Raw identifiers allowing keyword names to be used
+    (active, raw_identifiers, "1.26.0", Some(48589), None),
 );
 
 declare_features! (
@@ -1932,6 +1935,17 @@ pub fn check_crate(krate: &ast::Crate,
         parse_sess: sess,
         plugin_attributes,
     };
+
+    if !features.raw_identifiers {
+        for &span in sess.raw_identifier_spans.borrow().iter() {
+            if !span.allows_unstable() {
+                gate_feature!(&ctx, raw_identifiers, span,
+                    "raw identifiers are experimental and subject to change"
+                );
+            }
+        }
+    }
+
     let visitor = &mut PostExpansionVisitor { context: &ctx };
     visitor.whole_crate_feature_gates(krate);
     visit::walk_crate(visitor, krate);
diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs
index 0596fb44abe..8e746ea69e7 100644
--- a/src/libsyntax/parse/lexer/mod.rs
+++ b/src/libsyntax/parse/lexer/mod.rs
@@ -1796,6 +1796,7 @@ mod tests {
             included_mod_stack: RefCell::new(Vec::new()),
             code_map: cm,
             missing_fragment_specifiers: RefCell::new(HashSet::new()),
+            raw_identifier_spans: RefCell::new(Vec::new()),
             registered_diagnostics: Lock::new(ErrorMap::new()),
             non_modrs_mods: RefCell::new(vec![]),
         }
diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs
index 4acfdab53c0..03ac1e8b5a9 100644
--- a/src/libsyntax/parse/mod.rs
+++ b/src/libsyntax/parse/mod.rs
@@ -48,6 +48,9 @@ pub struct ParseSess {
     pub unstable_features: UnstableFeatures,
     pub config: CrateConfig,
     pub missing_fragment_specifiers: RefCell<HashSet<Span>>,
+    /// Places where raw identifiers were used. This is used for feature gating
+    /// raw identifiers
+    pub raw_identifier_spans: RefCell<Vec<Span>>,
     /// The registered diagnostics codes
     pub registered_diagnostics: Lock<ErrorMap>,
     // Spans where a `mod foo;` statement was included in a non-mod.rs file.
@@ -74,6 +77,7 @@ impl ParseSess {
             unstable_features: UnstableFeatures::from_environment(),
             config: HashSet::new(),
             missing_fragment_specifiers: RefCell::new(HashSet::new()),
+            raw_identifier_spans: RefCell::new(Vec::new()),
             registered_diagnostics: Lock::new(ErrorMap::new()),
             included_mod_stack: RefCell::new(vec![]),
             code_map,
diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs
index 4c1575cf589..c2ee78e9e9d 100644
--- a/src/libsyntax/parse/parser.rs
+++ b/src/libsyntax/parse/parser.rs
@@ -784,7 +784,7 @@ impl<'a> Parser<'a> {
 
     fn parse_ident_common(&mut self, recover: bool) -> PResult<'a, ast::Ident> {
         match self.token {
-            token::Ident(i, _) => {
+            token::Ident(i, is_raw) => {
                 if self.token.is_reserved_ident() {
                     let mut err = self.expected_ident_found();
                     if recover {
@@ -793,6 +793,9 @@ impl<'a> Parser<'a> {
                         return Err(err);
                     }
                 }
+                if is_raw {
+                    self.sess.raw_identifier_spans.borrow_mut().push(self.span);
+                }
                 self.bump();
                 Ok(i)
             }
diff --git a/src/test/run-pass/rfc-2151-raw-identifiers/attr.rs b/src/test/run-pass/rfc-2151-raw-identifiers/attr.rs
index 2ef9fba2076..6cea75cf1d1 100644
--- a/src/test/run-pass/rfc-2151-raw-identifiers/attr.rs
+++ b/src/test/run-pass/rfc-2151-raw-identifiers/attr.rs
@@ -8,6 +8,8 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
+#![feature(raw_identifiers)]
+
 use std::mem;
 
 #[r#repr(r#C, r#packed)]
diff --git a/src/test/run-pass/rfc-2151-raw-identifiers/basic.rs b/src/test/run-pass/rfc-2151-raw-identifiers/basic.rs
index eefce3981be..5d495c4e9e5 100644
--- a/src/test/run-pass/rfc-2151-raw-identifiers/basic.rs
+++ b/src/test/run-pass/rfc-2151-raw-identifiers/basic.rs
@@ -8,6 +8,8 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
+#![feature(raw_identifiers)]
+
 fn r#fn(r#match: u32) -> u32 {
     r#match
 }
diff --git a/src/test/run-pass/rfc-2151-raw-identifiers/items.rs b/src/test/run-pass/rfc-2151-raw-identifiers/items.rs
index 4306ffe2042..256bd263d38 100644
--- a/src/test/run-pass/rfc-2151-raw-identifiers/items.rs
+++ b/src/test/run-pass/rfc-2151-raw-identifiers/items.rs
@@ -8,6 +8,8 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
+#![feature(raw_identifiers)]
+
 #[derive(Debug, PartialEq, Eq)]
 struct IntWrapper(u32);
 
diff --git a/src/test/run-pass/rfc-2151-raw-identifiers/macros.rs b/src/test/run-pass/rfc-2151-raw-identifiers/macros.rs
index 9e89b79266c..4bd16ded52f 100644
--- a/src/test/run-pass/rfc-2151-raw-identifiers/macros.rs
+++ b/src/test/run-pass/rfc-2151-raw-identifiers/macros.rs
@@ -9,6 +9,7 @@
 // except according to those terms.
 
 #![feature(decl_macro)]
+#![feature(raw_identifiers)]
 
 r#macro_rules! r#struct {
     ($r#struct:expr) => { $r#struct }
diff --git a/src/test/ui/feature-gate-raw-identifiers.rs b/src/test/ui/feature-gate-raw-identifiers.rs
new file mode 100644
index 00000000000..38024feb432
--- /dev/null
+++ b/src/test/ui/feature-gate-raw-identifiers.rs
@@ -0,0 +1,14 @@
+// Copyright 2018 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+fn main() {
+    let r#foo = 3; //~ ERROR raw identifiers are experimental and subject to change
+    println!("{}", foo);
+}
diff --git a/src/test/ui/feature-gate-raw-identifiers.stderr b/src/test/ui/feature-gate-raw-identifiers.stderr
new file mode 100644
index 00000000000..02eff7247c4
--- /dev/null
+++ b/src/test/ui/feature-gate-raw-identifiers.stderr
@@ -0,0 +1,11 @@
+error[E0658]: raw identifiers are experimental and subject to change (see issue #48589)
+  --> $DIR/feature-gate-raw-identifiers.rs:12:9
+   |
+LL |     let r#foo = 3; //~ ERROR raw identifiers are experimental and subject to change
+   |         ^^^^^
+   |
+   = help: add #![feature(raw_identifiers)] to the crate attributes to enable
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0658`.
diff --git a/src/test/ui/raw-literal-keywords.rs b/src/test/ui/raw-literal-keywords.rs
index 8b3747dbe15..9b28aa0b151 100644
--- a/src/test/ui/raw-literal-keywords.rs
+++ b/src/test/ui/raw-literal-keywords.rs
@@ -11,6 +11,7 @@
 // compile-flags: -Z parse-only
 
 #![feature(dyn_trait)]
+#![feature(raw_identifiers)]
 
 fn test_if() {
     r#if true { } //~ ERROR found `true`
diff --git a/src/test/ui/raw-literal-keywords.stderr b/src/test/ui/raw-literal-keywords.stderr
index 022f80ae8a4..3758568323c 100644
--- a/src/test/ui/raw-literal-keywords.stderr
+++ b/src/test/ui/raw-literal-keywords.stderr
@@ -1,17 +1,17 @@
 error: expected one of `!`, `.`, `::`, `;`, `?`, `{`, `}`, or an operator, found `true`
-  --> $DIR/raw-literal-keywords.rs:16:10
+  --> $DIR/raw-literal-keywords.rs:17:10
    |
 LL |     r#if true { } //~ ERROR found `true`
    |          ^^^^ expected one of 8 possible tokens here
 
 error: expected one of `!`, `.`, `::`, `;`, `?`, `{`, `}`, or an operator, found `Test`
-  --> $DIR/raw-literal-keywords.rs:20:14
+  --> $DIR/raw-literal-keywords.rs:21:14
    |
 LL |     r#struct Test; //~ ERROR found `Test`
    |              ^^^^ expected one of 8 possible tokens here
 
 error: expected one of `!`, `.`, `::`, `;`, `?`, `{`, `}`, or an operator, found `Test`
-  --> $DIR/raw-literal-keywords.rs:24:13
+  --> $DIR/raw-literal-keywords.rs:25:13
    |
 LL |     r#union Test; //~ ERROR found `Test`
    |             ^^^^ expected one of 8 possible tokens here
diff --git a/src/test/ui/raw-literal-self.rs b/src/test/ui/raw-literal-self.rs
index 17496d767b6..f88d6cf9a67 100644
--- a/src/test/ui/raw-literal-self.rs
+++ b/src/test/ui/raw-literal-self.rs
@@ -10,6 +10,8 @@
 
 // compile-flags: -Z parse-only
 
+#![feature(raw_identifiers)]
+
 fn self_test(r#self: u32) {
     //~^ ERROR `r#self` is not currently supported.
 }
diff --git a/src/test/ui/raw-literal-self.stderr b/src/test/ui/raw-literal-self.stderr
index f4b75937247..e3345847aa8 100644
--- a/src/test/ui/raw-literal-self.stderr
+++ b/src/test/ui/raw-literal-self.stderr
@@ -1,5 +1,5 @@
 error: `r#self` is not currently supported.
-  --> $DIR/raw-literal-self.rs:13:14
+  --> $DIR/raw-literal-self.rs:15:14
    |
 LL | fn self_test(r#self: u32) {
    |              ^^^^^^
-- 
cgit 1.4.1-3-g733a5


From d2e7953d1325b1a1fe1cef526dbe8d23fa3e00a1 Mon Sep 17 00:00:00 2001
From: Lymia Aluysia <lymia@lymiahugs.com>
Date: Sun, 18 Mar 2018 11:21:38 -0500
Subject: Move raw_identifiers check to the lexer.

---
 src/libsyntax/parse/lexer/mod.rs | 4 ++++
 src/libsyntax/parse/parser.rs    | 5 +----
 2 files changed, 5 insertions(+), 4 deletions(-)

(limited to 'src/libsyntax/parse')

diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs
index 8e746ea69e7..068929c8948 100644
--- a/src/libsyntax/parse/lexer/mod.rs
+++ b/src/libsyntax/parse/lexer/mod.rs
@@ -1155,6 +1155,10 @@ impl<'a> StringReader<'a> {
                             &format!("`r#{}` is not currently supported.", ident.name)
                         ).raise();
                     }
+                    if is_raw_ident {
+                        let span = self.mk_sp(raw_start, self.pos);
+                        self.sess.raw_identifier_spans.borrow_mut().push(span);
+                    }
                     token::Ident(ident, is_raw_ident)
                 }));
             }
diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs
index c2ee78e9e9d..4c1575cf589 100644
--- a/src/libsyntax/parse/parser.rs
+++ b/src/libsyntax/parse/parser.rs
@@ -784,7 +784,7 @@ impl<'a> Parser<'a> {
 
     fn parse_ident_common(&mut self, recover: bool) -> PResult<'a, ast::Ident> {
         match self.token {
-            token::Ident(i, is_raw) => {
+            token::Ident(i, _) => {
                 if self.token.is_reserved_ident() {
                     let mut err = self.expected_ident_found();
                     if recover {
@@ -793,9 +793,6 @@ impl<'a> Parser<'a> {
                         return Err(err);
                     }
                 }
-                if is_raw {
-                    self.sess.raw_identifier_spans.borrow_mut().push(self.span);
-                }
                 self.bump();
                 Ok(i)
             }
-- 
cgit 1.4.1-3-g733a5


From 5c3d6320deb20f78d83d12fb3a319b9dd6b15290 Mon Sep 17 00:00:00 2001
From: Lymia Aluysia <lymia@lymiahugs.com>
Date: Sun, 18 Mar 2018 12:16:02 -0500
Subject: Return a is_raw parameter from Token::ident rather than having
 separate methods.

---
 src/libsyntax/ext/tt/macro_parser.rs |  4 ++--
 src/libsyntax/ext/tt/quoted.rs       |  4 ++--
 src/libsyntax/parse/token.rs         | 45 ++++++++++++------------------------
 3 files changed, 19 insertions(+), 34 deletions(-)

(limited to 'src/libsyntax/parse')

diff --git a/src/libsyntax/ext/tt/macro_parser.rs b/src/libsyntax/ext/tt/macro_parser.rs
index 714b7ca981d..8cb331c65da 100644
--- a/src/libsyntax/ext/tt/macro_parser.rs
+++ b/src/libsyntax/ext/tt/macro_parser.rs
@@ -364,8 +364,8 @@ pub fn parse_failure_msg(tok: Token) -> String {
 
 /// Perform a token equality check, ignoring syntax context (that is, an unhygienic comparison)
 fn token_name_eq(t1: &Token, t2: &Token) -> bool {
-    if let (Some(id1), Some(id2)) = (t1.ident(), t2.ident()) {
-        id1.name == id2.name && t1.is_raw_ident() == t2.is_raw_ident()
+    if let (Some((id1, is_raw1)), Some((id2, is_raw2))) = (t1.ident(), t2.ident()) {
+        id1.name == id2.name && is_raw1 == is_raw2
     } else if let (&token::Lifetime(id1), &token::Lifetime(id2)) = (t1, t2) {
         id1.name == id2.name
     } else {
diff --git a/src/libsyntax/ext/tt/quoted.rs b/src/libsyntax/ext/tt/quoted.rs
index 11eb9b940ae..f324edeb117 100644
--- a/src/libsyntax/ext/tt/quoted.rs
+++ b/src/libsyntax/ext/tt/quoted.rs
@@ -200,7 +200,7 @@ pub fn parse(
                 let span = match trees.next() {
                     Some(tokenstream::TokenTree::Token(span, token::Colon)) => match trees.next() {
                         Some(tokenstream::TokenTree::Token(end_sp, ref tok)) => match tok.ident() {
-                            Some(kind) => {
+                            Some((kind, _)) => {
                                 let span = end_sp.with_lo(start_sp.lo());
                                 result.push(TokenTree::MetaVarDecl(span, ident, kind));
                                 continue;
@@ -289,7 +289,7 @@ where
             // `tree` is followed by an `ident`. This could be `$meta_var` or the `$crate` special
             // metavariable that names the crate of the invokation.
             Some(tokenstream::TokenTree::Token(ident_span, ref token)) if token.is_ident() => {
-                let ident = token.ident().unwrap();
+                let (ident, _) = token.ident().unwrap();
                 let span = ident_span.with_lo(span.lo());
                 if ident.name == keywords::Crate.name() {
                     let ident = ast::Ident {
diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs
index 6406651bcba..4e7a282adc5 100644
--- a/src/libsyntax/parse/token.rs
+++ b/src/libsyntax/parse/token.rs
@@ -310,32 +310,17 @@ impl Token {
         }
     }
 
-    fn ident_common(&self, allow_raw: bool) -> Option<ast::Ident> {
+    pub fn ident(&self) -> Option<(ast::Ident, bool)> {
         match *self {
-            Ident(ident, is_raw) if !is_raw || allow_raw => Some(ident),
+            Ident(ident, is_raw) => Some((ident, is_raw)),
             Interpolated(ref nt) => match nt.0 {
-                NtIdent(ident, is_raw) if !is_raw || allow_raw => Some(ident.node),
+                NtIdent(ident, is_raw) => Some((ident.node, is_raw)),
                 _ => None,
             },
             _ => None,
         }
     }
 
-    pub fn nonraw_ident(&self) -> Option<ast::Ident> {
-        self.ident_common(false)
-    }
-
-    pub fn is_raw_ident(&self) -> bool {
-        match *self {
-            Ident(_, true) => true,
-            _ => false,
-        }
-    }
-
-    pub fn ident(&self) -> Option<ast::Ident> {
-        self.ident_common(true)
-    }
-
     /// Returns `true` if the token is an identifier.
     pub fn is_ident(&self) -> bool {
         self.ident().is_some()
@@ -404,37 +389,37 @@ impl Token {
 
     /// Returns `true` if the token is a given keyword, `kw`.
     pub fn is_keyword(&self, kw: keywords::Keyword) -> bool {
-        self.nonraw_ident().map(|ident| ident.name == kw.name()).unwrap_or(false)
+        self.ident().map(|(ident, is_raw)| ident.name == kw.name() && !is_raw).unwrap_or(false)
     }
 
     pub fn is_path_segment_keyword(&self) -> bool {
-        match self.nonraw_ident() {
-            Some(id) => is_path_segment_keyword(id),
-            None => false,
+        match self.ident() {
+            Some((id, false)) => is_path_segment_keyword(id),
+            _ => false,
         }
     }
 
     // Returns true for reserved identifiers used internally for elided lifetimes,
     // unnamed method parameters, crate root module, error recovery etc.
     pub fn is_special_ident(&self) -> bool {
-        match self.nonraw_ident() {
-            Some(id) => is_special_ident(id),
+        match self.ident() {
+            Some((id, false)) => is_special_ident(id),
             _ => false,
         }
     }
 
     /// Returns `true` if the token is a keyword used in the language.
     pub fn is_used_keyword(&self) -> bool {
-        match self.nonraw_ident() {
-            Some(id) => is_used_keyword(id),
+        match self.ident() {
+            Some((id, false)) => is_used_keyword(id),
             _ => false,
         }
     }
 
     /// Returns `true` if the token is a keyword reserved for possible future use.
     pub fn is_unused_keyword(&self) -> bool {
-        match self.nonraw_ident() {
-            Some(id) => is_unused_keyword(id),
+        match self.ident() {
+            Some((id, false)) => is_unused_keyword(id),
             _ => false,
         }
     }
@@ -507,8 +492,8 @@ impl Token {
 
     /// Returns `true` if the token is either a special identifier or a keyword.
     pub fn is_reserved_ident(&self) -> bool {
-        match self.nonraw_ident() {
-            Some(id) => is_reserved_ident(id),
+        match self.ident() {
+            Some((id, false)) => is_reserved_ident(id),
             _ => false,
         }
     }
-- 
cgit 1.4.1-3-g733a5


From bfb94ac5f43ac7fb0736185421f6135fe7043a2e Mon Sep 17 00:00:00 2001
From: Lymia Aluysia <lymia@lymiahugs.com>
Date: Thu, 22 Mar 2018 10:34:51 -0500
Subject: Clean up raw identifier handling when recovering tokens from AST.

---
 src/libsyntax/ext/quote.rs         | 5 +++--
 src/libsyntax/ext/tt/transcribe.rs | 2 +-
 src/libsyntax/parse/token.rs       | 2 +-
 3 files changed, 5 insertions(+), 4 deletions(-)

(limited to 'src/libsyntax/parse')

diff --git a/src/libsyntax/ext/quote.rs b/src/libsyntax/ext/quote.rs
index 090b9a35446..540a03ff032 100644
--- a/src/libsyntax/ext/quote.rs
+++ b/src/libsyntax/ext/quote.rs
@@ -238,8 +238,9 @@ pub mod rt {
                 if i > 0 {
                     inner.push(TokenTree::Token(self.span, token::Colon).into());
                 }
-                inner.push(TokenTree::Token(self.span,
-                                            token::Ident(segment.identifier, false)).into());
+                inner.push(TokenTree::Token(
+                    self.span, token::Token::from_ast_ident(segment.identifier)
+                ).into());
             }
             inner.push(self.tokens.clone());
 
diff --git a/src/libsyntax/ext/tt/transcribe.rs b/src/libsyntax/ext/tt/transcribe.rs
index 0f5855a3225..3f01d5ec6dd 100644
--- a/src/libsyntax/ext/tt/transcribe.rs
+++ b/src/libsyntax/ext/tt/transcribe.rs
@@ -169,7 +169,7 @@ pub fn transcribe(cx: &ExtCtxt,
                         Ident { ctxt: ident.ctxt.apply_mark(cx.current_expansion.mark), ..ident };
                     sp = sp.with_ctxt(sp.ctxt().apply_mark(cx.current_expansion.mark));
                     result.push(TokenTree::Token(sp, token::Dollar).into());
-                    result.push(TokenTree::Token(sp, token::Ident(ident, false)).into());
+                    result.push(TokenTree::Token(sp, token::Token::from_ast_ident(ident)).into());
                 }
             }
             quoted::TokenTree::Delimited(mut span, delimited) => {
diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs
index 4e7a282adc5..7798a7a77ee 100644
--- a/src/libsyntax/parse/token.rs
+++ b/src/libsyntax/parse/token.rs
@@ -236,7 +236,7 @@ impl Token {
 
     /// Recovers a `Token` from an `ast::Ident`. This creates a raw identifier if necessary.
     pub fn from_ast_ident(ident: ast::Ident) -> Token {
-        Ident(ident, is_reserved_ident(ident))
+        Ident(ident, is_reserved_ident(ident) && !is_path_segment_keyword(ident))
     }
 
     /// Returns `true` if the token starts with '>'.
-- 
cgit 1.4.1-3-g733a5