Keep the original token in `ast::Lit`

7 files changed, 66 insertions, 42 deletions

diff --git a/src/libsyntax/ast.rs b/src/libsyntax/ast.rs
index 783792cf197..04bc146e145 100644
--- a/src/libsyntax/ast.rs
+++ b/src/libsyntax/ast.rs
@@ -6,6 +6,7 @@ pub use crate::symbol::{Ident, Symbol as Name};
 pub use crate::util::parser::ExprPrecedence;
 
 use crate::ext::hygiene::{Mark, SyntaxContext};
+use crate::parse::token;
 use crate::print::pprust;
 use crate::ptr::P;
 use crate::source_map::{dummy_spanned, respan, Spanned};
@@ -1354,6 +1355,8 @@ pub enum StrStyle {
 #[derive(Clone, RustcEncodable, RustcDecodable, Debug, Hash, PartialEq)]
 pub struct Lit {
     pub node: LitKind,
+    pub token: token::Lit,
+    pub suffix: Option<Symbol>,
     pub span: Span,
 }
 
diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs
index e331a263354..2b874581083 100644
--- a/src/libsyntax/attr/mod.rs
+++ b/src/libsyntax/attr/mod.rs
@@ -350,7 +350,9 @@ impl Attribute {
 /* Constructors */
 
 pub fn mk_name_value_item_str(ident: Ident, value: Spanned<Symbol>) -> MetaItem {
-    let value = Lit { node: LitKind::Str(value.node, ast::StrStyle::Cooked), span: value.span };
+    let node = LitKind::Str(value.node, ast::StrStyle::Cooked);
+    let (token, suffix) = node.lit_token();
+    let value = Lit { node, token, suffix, span: value.span };
     mk_name_value_item(ident.span.to(value.span), ident, value)
 }
 
@@ -417,7 +419,9 @@ pub fn mk_spanned_attr_outer(sp: Span, id: AttrId, item: MetaItem) -> Attribute
 
 pub fn mk_sugared_doc_attr(id: AttrId, text: Symbol, span: Span) -> Attribute {
     let style = doc_comment_style(&text.as_str());
-    let lit = Lit { node: LitKind::Str(text, ast::StrStyle::Cooked), span };
+    let node = LitKind::Str(text, ast::StrStyle::Cooked);
+    let (token, suffix) = node.lit_token();
+    let lit = Lit { node, token, suffix, span };
     Attribute {
         id,
         style,
@@ -562,7 +566,7 @@ impl MetaItemKind {
                 tokens.next();
                 return if let Some(TokenTree::Token(span, token)) = tokens.next() {
                     LitKind::from_token(token)
-                        .map(|node| MetaItemKind::NameValue(Lit { node, span }))
+                        .map(|(node, token, suffix)| MetaItemKind::NameValue(Lit { node, token, suffix, span }))
                 } else {
                     None
                 };
@@ -607,9 +611,9 @@ impl NestedMetaItem {
         where I: Iterator<Item = TokenTree>,
     {
         if let Some(TokenTree::Token(span, token)) = tokens.peek().cloned() {
-            if let Some(node) = LitKind::from_token(token) {
+            if let Some((node, token, suffix)) = LitKind::from_token(token) {
                 tokens.next();
-                return Some(NestedMetaItem::Literal(Lit { node, span }));
+                return Some(NestedMetaItem::Literal(Lit { node, token, suffix, span }));
             }
         }
 
@@ -625,28 +629,35 @@ impl Lit {
 
 impl LitKind {
     fn token(&self) -> Token {
+        match self.lit_token() {
+            (token::Bool(symbol), _) => Token::Ident(Ident::with_empty_ctxt(symbol), false),
+            (lit, suffix) => Token::Literal(lit, suffix),
+        }
+    }
+
+    pub(crate) fn lit_token(&self) -> (token::Lit, Option<Symbol>) {
         use std::ascii;
 
         match *self {
             LitKind::Str(string, ast::StrStyle::Cooked) => {
                 let escaped = string.as_str().escape_default().to_string();
-                Token::Literal(token::Lit::Str_(Symbol::intern(&escaped)), None)
+                (token::Lit::Str_(Symbol::intern(&escaped)), None)
             }
             LitKind::Str(string, ast::StrStyle::Raw(n)) => {
-                Token::Literal(token::Lit::StrRaw(string, n), None)
+                (token::Lit::StrRaw(string, n), None)
             }
             LitKind::ByteStr(ref bytes) => {
                 let string = bytes.iter().cloned().flat_map(ascii::escape_default)
                     .map(Into::<char>::into).collect::<String>();
-                Token::Literal(token::Lit::ByteStr(Symbol::intern(&string)), None)
+                (token::Lit::ByteStr(Symbol::intern(&string)), None)
             }
             LitKind::Byte(byte) => {
                 let string: String = ascii::escape_default(byte).map(Into::<char>::into).collect();
-                Token::Literal(token::Lit::Byte(Symbol::intern(&string)), None)
+                (token::Lit::Byte(Symbol::intern(&string)), None)
             }
             LitKind::Char(ch) => {
                 let string: String = ch.escape_default().map(Into::<char>::into).collect();
-                Token::Literal(token::Lit::Char(Symbol::intern(&string)), None)
+                (token::Lit::Char(Symbol::intern(&string)), None)
             }
             LitKind::Int(n, ty) => {
                 let suffix = match ty {
@@ -654,38 +665,39 @@ impl LitKind {
                     ast::LitIntType::Signed(ty) => Some(Symbol::intern(ty.ty_to_string())),
                     ast::LitIntType::Unsuffixed => None,
                 };
-                Token::Literal(token::Lit::Integer(Symbol::intern(&n.to_string())), suffix)
+                (token::Lit::Integer(Symbol::intern(&n.to_string())), suffix)
             }
             LitKind::Float(symbol, ty) => {
-                Token::Literal(token::Lit::Float(symbol), Some(Symbol::intern(ty.ty_to_string())))
+                (token::Lit::Float(symbol), Some(Symbol::intern(ty.ty_to_string())))
             }
-            LitKind::FloatUnsuffixed(symbol) => Token::Literal(token::Lit::Float(symbol), None),
-            LitKind::Bool(value) => Token::Ident(Ident::with_empty_ctxt(Symbol::intern(if value {
-                "true"
-            } else {
-                "false"
-            })), false),
-            LitKind::Err(val) => Token::Literal(token::Lit::Err(val), None),
+            LitKind::FloatUnsuffixed(symbol) => (token::Lit::Float(symbol), None),
+            LitKind::Bool(value) => {
+                let kw = if value { keywords::True } else { keywords::False };
+                (token::Lit::Bool(kw.name()), None)
+            }
+            LitKind::Err(val) => (token::Lit::Err(val), None),
         }
     }
 
-    fn from_token(token: Token) -> Option<LitKind> {
+    fn from_token(token: Token) -> Option<(LitKind, token::Lit, Option<Symbol>)> {
         match token {
-            Token::Ident(ident, false) if ident.name == "true" => Some(LitKind::Bool(true)),
-            Token::Ident(ident, false) if ident.name == "false" => Some(LitKind::Bool(false)),
+            Token::Ident(ident, false) if ident.name == keywords::True.name() =>
+                Some((LitKind::Bool(true), token::Bool(ident.name), None)),
+            Token::Ident(ident, false) if ident.name == keywords::False.name() =>
+                Some((LitKind::Bool(false), token::Bool(ident.name), None)),
             Token::Interpolated(nt) => match *nt {
                 token::NtExpr(ref v) | token::NtLiteral(ref v) => match v.node {
-                    ExprKind::Lit(ref lit) => Some(lit.node.clone()),
+                    ExprKind::Lit(ref lit) => Some((lit.node.clone(), lit.token, lit.suffix)),
                     _ => None,
                 },
                 _ => None,
             },
             Token::Literal(lit, suf) => {
                 let (suffix_illegal, result) = parse::lit_token(lit, suf, None);
-                if suffix_illegal && suf.is_some() {
+                if result.is_none() || suffix_illegal && suf.is_some() {
                     return None;
                 }
-                result
+                Some((result.unwrap(), lit, suf))
             }
             _ => None,
         }
diff --git a/src/libsyntax/ext/build.rs b/src/libsyntax/ext/build.rs
index 0fe85361b54..0f9977e8503 100644
--- a/src/libsyntax/ext/build.rs
+++ b/src/libsyntax/ext/build.rs
@@ -698,7 +698,8 @@ impl<'a> AstBuilder for ExtCtxt<'a> {
     }
 
     fn expr_lit(&self, span: Span, node: ast::LitKind) -> P<ast::Expr> {
-        self.expr(span, ast::ExprKind::Lit(ast::Lit { node, span }))
+        let (token, suffix) = node.lit_token();
+        self.expr(span, ast::ExprKind::Lit(ast::Lit { node, token, suffix, span }))
     }
     fn expr_usize(&self, span: Span, i: usize) -> P<ast::Expr> {
         self.expr_lit(span, ast::LitKind::Int(i as u128,
@@ -1166,8 +1167,9 @@ impl<'a> AstBuilder for ExtCtxt<'a> {
 
     fn meta_name_value(&self, span: Span, name: ast::Name, node: ast::LitKind)
                        -> ast::MetaItem {
+        let (token, suffix) = node.lit_token();
         attr::mk_name_value_item(span, Ident::with_empty_ctxt(name).with_span_pos(span),
-                                 ast::Lit { node, span })
+                                 ast::Lit { node, token, suffix, span })
     }
 
     fn item_use(&self, sp: Span,
diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs
index be44b964ba5..4d4e99009a9 100644
--- a/src/libsyntax/parse/mod.rs
+++ b/src/libsyntax/parse/mod.rs
@@ -376,6 +376,7 @@ crate fn lit_token(lit: token::Lit, suf: Option<Symbol>, diag: Option<(Span, &Ha
     use ast::LitKind;
 
     match lit {
+        token::Bool(_) => panic!("literal token contains `Lit::Bool`"),
         token::Byte(i) => {
             let lit_kind = match unescape_byte(&i.as_str()) {
                 Ok(c) => LitKind::Byte(c),
diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs
index 2b30d2db95e..b988cb1447d 100644
--- a/src/libsyntax/parse/parser.rs
+++ b/src/libsyntax/parse/parser.rs
@@ -2070,11 +2070,11 @@ impl<'a> Parser<'a> {
     }
 
     /// Matches `token_lit = LIT_INTEGER | ...`.
-    fn parse_lit_token(&mut self) -> PResult<'a, LitKind> {
+    fn parse_lit_token(&mut self) -> PResult<'a, (LitKind, token::Lit, Option<Symbol>)> {
         let out = match self.token {
             token::Interpolated(ref nt) => match **nt {
                 token::NtExpr(ref v) | token::NtLiteral(ref v) => match v.node {
-                    ExprKind::Lit(ref lit) => { lit.node.clone() }
+                    ExprKind::Lit(ref lit) => { (lit.node.clone(), lit.token, lit.suffix) }
                     _ => { return self.unexpected_last(&self.token); }
                 },
                 _ => { return self.unexpected_last(&self.token); }
@@ -2088,19 +2088,19 @@ impl<'a> Parser<'a> {
                     self.expect_no_suffix(sp, &format!("a {}", lit.literal_name()), suf)
                 }
 
-                result.unwrap()
+                (result.unwrap(), lit, suf)
             }
             token::Dot if self.look_ahead(1, |t| match t {
-                token::Literal(parse::token::Lit::Integer(_) , _) => true,
+                token::Literal(token::Lit::Integer(_) , _) => true,
                 _ => false,
             }) => { // recover from `let x = .4;`
                 let lo = self.span;
                 self.bump();
                 if let token::Literal(
-                    parse::token::Lit::Integer(val),
+                    token::Lit::Integer(val),
                     suffix,
                 ) = self.token {
-                    let suffix = suffix.and_then(|s| {
+                    let float_suffix = suffix.and_then(|s| {
                         let s = s.as_str();
                         if s == "f32" {
                             Some("f32")
@@ -2117,14 +2117,14 @@ impl<'a> Parser<'a> {
                     err.span_suggestion(
                         sp,
                         "must have an integer part",
-                        format!("0.{}{}", val, suffix),
+                        format!("0.{}{}", val, float_suffix),
                         Applicability::MachineApplicable,
                     );
                     err.emit();
-                    return Ok(match suffix {
-                        "f32" => ast::LitKind::Float(val, ast::FloatTy::F32),
-                        "f64" => ast::LitKind::Float(val, ast::FloatTy::F64),
-                        _ => ast::LitKind::FloatUnsuffixed(val),
+                    return Ok(match float_suffix {
+                        "f32" => (ast::LitKind::Float(val, ast::FloatTy::F32), token::Float(val), suffix),
+                        "f64" => (ast::LitKind::Float(val, ast::FloatTy::F64), token::Float(val), suffix),
+                        _ => (ast::LitKind::FloatUnsuffixed(val), token::Float(val), suffix),
                     });
                 } else {
                     unreachable!();
@@ -2140,14 +2140,14 @@ impl<'a> Parser<'a> {
     /// Matches `lit = true | false | token_lit`.
     crate fn parse_lit(&mut self) -> PResult<'a, Lit> {
         let lo = self.span;
-        let node = if self.eat_keyword(keywords::True) {
-            LitKind::Bool(true)
+        let (node, token, suffix) = if self.eat_keyword(keywords::True) {
+            (LitKind::Bool(true), token::Bool(keywords::True.name()), None)
         } else if self.eat_keyword(keywords::False) {
-            LitKind::Bool(false)
+            (LitKind::Bool(false), token::Bool(keywords::False.name()), None)
         } else {
             self.parse_lit_token()?
         };
-        Ok(Lit { node, span: lo.to(self.prev_span) })
+        Ok(Lit { node, token, suffix, span: lo.to(self.prev_span) })
     }
 
     /// Matches `'-' lit | lit` (cf. `ast_validation::AstValidator::check_expr_within_pat`).
diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs
index fd7a39c576d..48a949257ff 100644
--- a/src/libsyntax/parse/token.rs
+++ b/src/libsyntax/parse/token.rs
@@ -61,6 +61,7 @@ impl DelimToken {
 
 #[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)]
 pub enum Lit {
+    Bool(ast::Name), // AST only, must never appear in a `Token`
     Byte(ast::Name),
     Char(ast::Name),
     Err(ast::Name),
@@ -72,9 +73,13 @@ pub enum Lit {
     ByteStrRaw(ast::Name, u16), /* raw byte str delimited by n hash symbols */
 }
 
+#[cfg(target_arch = "x86_64")]
+static_assert!(MEM_SIZE_OF_LIT: mem::size_of::<Lit>() == 8);
+
 impl Lit {
     crate fn literal_name(&self) -> &'static str {
         match *self {
+            Bool(_) => panic!("literal token contains `Lit::Bool`"),
             Byte(_) => "byte literal",
             Char(_) => "char literal",
             Err(_) => "invalid literal",
diff --git a/src/libsyntax/print/pprust.rs b/src/libsyntax/print/pprust.rs
index 682621d40ab..0e93b857708 100644
--- a/src/libsyntax/print/pprust.rs
+++ b/src/libsyntax/print/pprust.rs
@@ -225,6 +225,7 @@ pub fn token_to_string(tok: &Token) -> String {
         /* Literals */
         token::Literal(lit, suf) => {
             let mut out = match lit {
+                token::Bool(_)          => panic!("literal token contains `Lit::Bool`"),
                 token::Byte(b)           => format!("b'{}'", b),
                 token::Char(c)           => format!("'{}'", c),
                 token::Err(c)            => format!("'{}'", c),