syntax: Turn `token::Lit` into a struct

4 files changed, 196 insertions, 175 deletions

diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs
index c97d8040761..deb76d6d70a 100644
--- a/src/libsyntax/parse/lexer/mod.rs
+++ b/src/libsyntax/parse/lexer/mod.rs
@@ -1,5 +1,6 @@
 use crate::ast::{self, Ident};
-use crate::parse::{token, ParseSess};
+use crate::parse::ParseSess;
+use crate::parse::token::{self, Token};
 use crate::symbol::Symbol;
 use crate::parse::unescape;
 use crate::parse::unescape_error_reporting::{emit_unescape_error, push_escaped_char};
@@ -21,7 +22,7 @@ mod unicode_chars;
 
 #[derive(Clone, Debug)]
 pub struct TokenAndSpan {
-    pub tok: token::Token,
+    pub tok: Token,
     pub sp: Span,
 }
 
@@ -55,7 +56,7 @@ pub struct StringReader<'a> {
     /// Stop reading src at this index.
     crate end_src_index: usize,
     // cached:
-    peek_tok: token::Token,
+    peek_tok: Token,
     peek_span: Span,
     peek_span_src_raw: Span,
     fatal_errs: Vec<DiagnosticBuilder<'a>>,
@@ -726,7 +727,7 @@ impl<'a> StringReader<'a> {
     }
 
     /// Lex a LIT_INTEGER or a LIT_FLOAT
-    fn scan_number(&mut self, c: char) -> token::Lit {
+    fn scan_number(&mut self, c: char) -> (token::LitKind, Symbol) {
         let mut base = 10;
         let start_bpos = self.pos;
         self.bump();
@@ -753,7 +754,7 @@ impl<'a> StringReader<'a> {
                 }
                 _ => {
                     // just a 0
-                    return token::Integer(self.name_from(start_bpos));
+                    return (token::Integer, self.name_from(start_bpos));
                 }
             }
         } else if c.is_digit(10) {
@@ -765,7 +766,7 @@ impl<'a> StringReader<'a> {
         if num_digits == 0 {
             self.err_span_(start_bpos, self.pos, "no valid digits found for number");
 
-            return token::Integer(Symbol::intern("0"));
+            return (token::Integer, Symbol::intern("0"));
         }
 
         // might be a float, but don't be greedy if this is actually an
@@ -783,17 +784,17 @@ impl<'a> StringReader<'a> {
             let pos = self.pos;
             self.check_float_base(start_bpos, pos, base);
 
-            token::Float(self.name_from(start_bpos))
+            (token::Float, self.name_from(start_bpos))
         } else {
             // it might be a float if it has an exponent
             if self.ch_is('e') || self.ch_is('E') {
                 self.scan_float_exponent();
                 let pos = self.pos;
                 self.check_float_base(start_bpos, pos, base);
-                return token::Float(self.name_from(start_bpos));
+                return (token::Float, self.name_from(start_bpos));
             }
             // but we certainly have an integer!
-            token::Integer(self.name_from(start_bpos))
+            (token::Integer, self.name_from(start_bpos))
         }
     }
 
@@ -846,7 +847,7 @@ impl<'a> StringReader<'a> {
         }
     }
 
-    fn binop(&mut self, op: token::BinOpToken) -> token::Token {
+    fn binop(&mut self, op: token::BinOpToken) -> Token {
         self.bump();
         if self.ch_is('=') {
             self.bump();
@@ -858,7 +859,7 @@ impl<'a> StringReader<'a> {
 
     /// Returns the next token from the string, advances the input past that
     /// token, and updates the interner
-    fn next_token_inner(&mut self) -> Result<token::Token, ()> {
+    fn next_token_inner(&mut self) -> Result<Token, ()> {
         let c = self.ch;
 
         if ident_start(c) {
@@ -912,10 +913,10 @@ impl<'a> StringReader<'a> {
         }
 
         if is_dec_digit(c) {
-            let num = self.scan_number(c.unwrap());
+            let (kind, symbol) = self.scan_number(c.unwrap());
             let suffix = self.scan_optional_raw_name();
-            debug!("next_token_inner: scanned number {:?}, {:?}", num, suffix);
-            return Ok(token::Literal(num, suffix));
+            debug!("next_token_inner: scanned number {:?}, {:?}, {:?}", kind, symbol, suffix);
+            return Ok(Token::lit(kind, symbol, suffix));
         }
 
         match c.expect("next_token_inner called at EOF") {
@@ -1073,10 +1074,10 @@ impl<'a> StringReader<'a> {
                     // lifetimes shouldn't end with a single quote
                     // if we find one, then this is an invalid character literal
                     if self.ch_is('\'') {
-                        let id = self.name_from(start);
+                        let symbol = self.name_from(start);
                         self.bump();
                         self.validate_char_escape(start_with_quote);
-                        return Ok(token::Literal(token::Char(id), None))
+                        return Ok(Token::lit(token::Char, symbol, None));
                     }
 
                     // Include the leading `'` in the real identifier, for macro
@@ -1098,43 +1099,43 @@ impl<'a> StringReader<'a> {
                     return Ok(token::Lifetime(ident));
                 }
                 let msg = "unterminated character literal";
-                let id = self.scan_single_quoted_string(start_with_quote, msg);
+                let symbol = self.scan_single_quoted_string(start_with_quote, msg);
                 self.validate_char_escape(start_with_quote);
                 let suffix = self.scan_optional_raw_name();
-                Ok(token::Literal(token::Char(id), suffix))
+                Ok(Token::lit(token::Char, symbol, suffix))
             }
             'b' => {
                 self.bump();
-                let lit = match self.ch {
+                let (kind, symbol) = match self.ch {
                     Some('\'') => {
                         let start_with_quote = self.pos;
                         self.bump();
                         let msg = "unterminated byte constant";
-                        let id = self.scan_single_quoted_string(start_with_quote, msg);
+                        let symbol = self.scan_single_quoted_string(start_with_quote, msg);
                         self.validate_byte_escape(start_with_quote);
-                        token::Byte(id)
+                        (token::Byte, symbol)
                     },
                     Some('"') => {
                         let start_with_quote = self.pos;
                         let msg = "unterminated double quote byte string";
-                        let id = self.scan_double_quoted_string(msg);
+                        let symbol = self.scan_double_quoted_string(msg);
                         self.validate_byte_str_escape(start_with_quote);
-                        token::ByteStr(id)
+                        (token::ByteStr, symbol)
                     },
                     Some('r') => self.scan_raw_byte_string(),
                     _ => unreachable!(),  // Should have been a token::Ident above.
                 };
                 let suffix = self.scan_optional_raw_name();
 
-                Ok(token::Literal(lit, suffix))
+                Ok(Token::lit(kind, symbol, suffix))
             }
             '"' => {
                 let start_with_quote = self.pos;
                 let msg = "unterminated double quote string";
-                let id = self.scan_double_quoted_string(msg);
+                let symbol = self.scan_double_quoted_string(msg);
                 self.validate_str_escape(start_with_quote);
                 let suffix = self.scan_optional_raw_name();
-                Ok(token::Literal(token::Str_(id), suffix))
+                Ok(Token::lit(token::Str, symbol, suffix))
             }
             'r' => {
                 let start_bpos = self.pos;
@@ -1205,14 +1206,14 @@ impl<'a> StringReader<'a> {
                 }
 
                 self.bump();
-                let id = if valid {
+                let symbol = if valid {
                     self.name_from_to(content_start_bpos, content_end_bpos)
                 } else {
                     Symbol::intern("??")
                 };
                 let suffix = self.scan_optional_raw_name();
 
-                Ok(token::Literal(token::StrRaw(id, hash_count), suffix))
+                Ok(Token::lit(token::StrRaw(hash_count), symbol, suffix))
             }
             '-' => {
                 if self.nextch_is('>') {
@@ -1366,7 +1367,7 @@ impl<'a> StringReader<'a> {
         id
     }
 
-    fn scan_raw_byte_string(&mut self) -> token::Lit {
+    fn scan_raw_byte_string(&mut self) -> (token::LitKind, Symbol) {
         let start_bpos = self.pos;
         self.bump();
         let mut hash_count = 0;
@@ -1423,7 +1424,7 @@ impl<'a> StringReader<'a> {
 
         self.bump();
 
-        token::ByteStrRaw(self.name_from_to(content_start_bpos, content_end_bpos), hash_count)
+        (token::ByteStrRaw(hash_count), self.name_from_to(content_start_bpos, content_end_bpos))
     }
 
     fn validate_char_escape(&self, start_with_quote: BytePos) {
@@ -1637,15 +1638,19 @@ mod tests {
 
     // check that the given reader produces the desired stream
     // of tokens (stop checking after exhausting the expected vec)
-    fn check_tokenization(mut string_reader: StringReader<'_>, expected: Vec<token::Token>) {
+    fn check_tokenization(mut string_reader: StringReader<'_>, expected: Vec<Token>) {
         for expected_tok in &expected {
             assert_eq!(&string_reader.next_token().tok, expected_tok);
         }
     }
 
     // make the identifier by looking up the string in the interner
-    fn mk_ident(id: &str) -> token::Token {
-        token::Token::from_ast_ident(Ident::from_str(id))
+    fn mk_ident(id: &str) -> Token {
+        Token::from_ast_ident(Ident::from_str(id))
+    }
+
+    fn mk_lit(kind: token::LitKind, symbol: &str, suffix: Option<&str>) -> Token {
+        Token::lit(kind, Symbol::intern(symbol), suffix.map(Symbol::intern))
     }
 
     #[test]
@@ -1694,7 +1699,7 @@ mod tests {
             let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
             let sh = mk_sess(sm.clone());
             assert_eq!(setup(&sm, &sh, "'a'".to_string()).next_token().tok,
-                    token::Literal(token::Char(Symbol::intern("a")), None));
+                       mk_lit(token::Char, "a", None));
         })
     }
 
@@ -1704,7 +1709,7 @@ mod tests {
             let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
             let sh = mk_sess(sm.clone());
             assert_eq!(setup(&sm, &sh, "' '".to_string()).next_token().tok,
-                    token::Literal(token::Char(Symbol::intern(" ")), None));
+                       mk_lit(token::Char, " ", None));
         })
     }
 
@@ -1714,7 +1719,7 @@ mod tests {
             let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
             let sh = mk_sess(sm.clone());
             assert_eq!(setup(&sm, &sh, "'\\n'".to_string()).next_token().tok,
-                    token::Literal(token::Char(Symbol::intern("\\n")), None));
+                       mk_lit(token::Char, "\\n", None));
         })
     }
 
@@ -1724,7 +1729,7 @@ mod tests {
             let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
             let sh = mk_sess(sm.clone());
             assert_eq!(setup(&sm, &sh, "'abc".to_string()).next_token().tok,
-                    token::Lifetime(Ident::from_str("'abc")));
+                       token::Lifetime(Ident::from_str("'abc")));
         })
     }
 
@@ -1733,10 +1738,8 @@ mod tests {
         with_default_globals(|| {
             let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
             let sh = mk_sess(sm.clone());
-            assert_eq!(setup(&sm, &sh, "r###\"\"#a\\b\x00c\"\"###".to_string())
-                        .next_token()
-                        .tok,
-                    token::Literal(token::StrRaw(Symbol::intern("\"#a\\b\x00c\""), 3), None));
+            assert_eq!(setup(&sm, &sh, "r###\"\"#a\\b\x00c\"\"###".to_string()).next_token().tok,
+                       mk_lit(token::StrRaw(3), "\"#a\\b\x00c\"", None));
         })
     }
 
@@ -1748,18 +1751,16 @@ mod tests {
             macro_rules! test {
                 ($input: expr, $tok_type: ident, $tok_contents: expr) => {{
                     assert_eq!(setup(&sm, &sh, format!("{}suffix", $input)).next_token().tok,
-                            token::Literal(token::$tok_type(Symbol::intern($tok_contents)),
-                                            Some(Symbol::intern("suffix"))));
+                               mk_lit(token::$tok_type, $tok_contents, Some("suffix")));
                     // with a whitespace separator:
                     assert_eq!(setup(&sm, &sh, format!("{} suffix", $input)).next_token().tok,
-                            token::Literal(token::$tok_type(Symbol::intern($tok_contents)),
-                                            None));
+                               mk_lit(token::$tok_type, $tok_contents, None));
                 }}
             }
 
             test!("'a'", Char, "a");
             test!("b'a'", Byte, "a");
-            test!("\"a\"", Str_, "a");
+            test!("\"a\"", Str, "a");
             test!("b\"a\"", ByteStr, "a");
             test!("1234", Integer, "1234");
             test!("0b101", Integer, "0b101");
@@ -1768,14 +1769,11 @@ mod tests {
             test!("1.0e10", Float, "1.0e10");
 
             assert_eq!(setup(&sm, &sh, "2us".to_string()).next_token().tok,
-                    token::Literal(token::Integer(Symbol::intern("2")),
-                                    Some(Symbol::intern("us"))));
+                       mk_lit(token::Integer, "2", Some("us")));
             assert_eq!(setup(&sm, &sh, "r###\"raw\"###suffix".to_string()).next_token().tok,
-                    token::Literal(token::StrRaw(Symbol::intern("raw"), 3),
-                                    Some(Symbol::intern("suffix"))));
+                       mk_lit(token::StrRaw(3), "raw", Some("suffix")));
             assert_eq!(setup(&sm, &sh, "br###\"raw\"###suffix".to_string()).next_token().tok,
-                    token::Literal(token::ByteStrRaw(Symbol::intern("raw"), 3),
-                                    Some(Symbol::intern("suffix"))));
+                       mk_lit(token::ByteStrRaw(3), "raw", Some("suffix")));
         })
     }
 
@@ -1796,8 +1794,7 @@ mod tests {
                 token::Comment => {}
                 _ => panic!("expected a comment!"),
             }
-            assert_eq!(lexer.next_token().tok,
-                    token::Literal(token::Char(Symbol::intern("a")), None));
+            assert_eq!(lexer.next_token().tok, mk_lit(token::Char, "a", None));
         })
     }
 
diff --git a/src/libsyntax/parse/literal.rs b/src/libsyntax/parse/literal.rs
index 4ddf3ddbcee..7554c711993 100644
--- a/src/libsyntax/parse/literal.rs
+++ b/src/libsyntax/parse/literal.rs
@@ -3,7 +3,7 @@
 use crate::ast::{self, Ident, Lit, LitKind};
 use crate::parse::parser::Parser;
 use crate::parse::PResult;
-use crate::parse::token;
+use crate::parse::token::{self, Token};
 use crate::parse::unescape::{unescape_str, unescape_char, unescape_byte_str, unescape_byte};
 use crate::print::pprust;
 use crate::symbol::{kw, Symbol};
@@ -27,14 +27,21 @@ crate enum LitError {
 }
 
 impl LitError {
-    crate fn report(&self, diag: &Handler, lit: token::Lit, suf: Option<Symbol>, span: Span) {
+    crate fn report(
+        &self,
+        diag: &Handler,
+        token::Lit { kind, suffix, .. }: token::Lit,
+        span: Span,
+    ) {
         match *self {
             LitError::NotLiteral | LitError::LexerError => {}
             LitError::InvalidSuffix => {
-                expect_no_suffix(diag, span, &format!("{} {}", lit.article(), lit.descr()), suf);
+                expect_no_suffix(
+                    diag, span, &format!("{} {}", kind.article(), kind.descr()), suffix
+                );
             }
             LitError::InvalidIntSuffix => {
-                let suf = suf.expect("suffix error with no suffix").as_str();
+                let suf = suffix.expect("suffix error with no suffix").as_str();
                 if looks_like_width_suffix(&['i', 'u'], &suf) {
                     // If it looks like a width, try to be helpful.
                     let msg = format!("invalid width `{}` for integer literal", &suf[1..]);
@@ -50,7 +57,7 @@ impl LitError {
                 }
             }
             LitError::InvalidFloatSuffix => {
-                let suf = suf.expect("suffix error with no suffix").as_str();
+                let suf = suffix.expect("suffix error with no suffix").as_str();
                 if looks_like_width_suffix(&['f'], &suf) {
                     // If it looks like a width, try to be helpful.
                     let msg = format!("invalid width `{}` for float literal", &suf[1..]);
@@ -84,43 +91,42 @@ impl LitKind {
     /// If diagnostic handler is passed, always returns `Some`,
     /// possibly after reporting non-fatal errors and recovery.
     fn from_lit_token(
-        lit: token::Lit,
-        suf: Option<Symbol>,
+        token::Lit { kind, symbol, suffix }: token::Lit,
     ) -> Result<LitKind, LitError> {
-        if suf.is_some() && !lit.may_have_suffix() {
+        if suffix.is_some() && !kind.may_have_suffix() {
             return Err(LitError::InvalidSuffix);
         }
 
-        Ok(match lit {
-            token::Bool(i) => {
-                assert!(i == kw::True || i == kw::False);
-                LitKind::Bool(i == kw::True)
+        Ok(match kind {
+            token::Bool => {
+                assert!(symbol == kw::True || symbol == kw::False);
+                LitKind::Bool(symbol == kw::True)
             }
-            token::Byte(i) => {
-                match unescape_byte(&i.as_str()) {
+            token::Byte => {
+                match unescape_byte(&symbol.as_str()) {
                     Ok(c) => LitKind::Byte(c),
-                    Err(_) => LitKind::Err(i),
+                    Err(_) => return Err(LitError::LexerError),
                 }
             },
-            token::Char(i) => {
-                match unescape_char(&i.as_str()) {
+            token::Char => {
+                match unescape_char(&symbol.as_str()) {
                     Ok(c) => LitKind::Char(c),
                     Err(_) => return Err(LitError::LexerError),
                 }
             },
-            token::Err(i) => LitKind::Err(i),
 
             // There are some valid suffixes for integer and float literals,
             // so all the handling is done internally.
-            token::Integer(s) => return integer_lit(s, suf),
-            token::Float(s) => return float_lit(s, suf),
+            token::Integer => return integer_lit(symbol, suffix),
+            token::Float => return float_lit(symbol, suffix),
 
-            token::Str_(mut sym) => {
+            token::Str => {
                 // If there are no characters requiring special treatment we can
                 // reuse the symbol from the token. Otherwise, we must generate a
                 // new symbol because the string in the LitKind is different to the
                 // string in the token.
                 let mut error = None;
+                let mut sym = symbol;
                 let s = &sym.as_str();
                 if s.as_bytes().iter().any(|&c| c == b'\\' || c == b'\r') {
                     let mut buf = String::with_capacity(s.len());
@@ -138,16 +144,17 @@ impl LitKind {
 
                 LitKind::Str(sym, ast::StrStyle::Cooked)
             }
-            token::StrRaw(mut sym, n) => {
+            token::StrRaw(n) => {
                 // Ditto.
+                let mut sym = symbol;
                 let s = &sym.as_str();
                 if s.contains('\r') {
                     sym = Symbol::intern(&raw_str_lit(s));
                 }
                 LitKind::Str(sym, ast::StrStyle::Raw(n))
             }
-            token::ByteStr(i) => {
-                let s = &i.as_str();
+            token::ByteStr => {
+                let s = &symbol.as_str();
                 let mut buf = Vec::with_capacity(s.len());
                 let mut error = None;
                 unescape_byte_str(s, &mut |_, unescaped_byte| {
@@ -162,36 +169,37 @@ impl LitKind {
                 buf.shrink_to_fit();
                 LitKind::ByteStr(Lrc::new(buf))
             }
-            token::ByteStrRaw(i, _) => {
-                LitKind::ByteStr(Lrc::new(i.to_string().into_bytes()))
+            token::ByteStrRaw(_) => {
+                LitKind::ByteStr(Lrc::new(symbol.to_string().into_bytes()))
             }
+            token::Err => LitKind::Err(symbol),
         })
     }
 
     /// Attempts to recover a token from semantic literal.
     /// This function is used when the original token doesn't exist (e.g. the literal is created
     /// by an AST-based macro) or unavailable (e.g. from HIR pretty-printing).
-    pub fn to_lit_token(&self) -> (token::Lit, Option<Symbol>) {
-        match *self {
+    pub fn to_lit_token(&self) -> token::Lit {
+        let (kind, symbol, suffix) = match *self {
             LitKind::Str(string, ast::StrStyle::Cooked) => {
                 let escaped = string.as_str().escape_default().to_string();
-                (token::Lit::Str_(Symbol::intern(&escaped)), None)
+                (token::Str, Symbol::intern(&escaped), None)
             }
             LitKind::Str(string, ast::StrStyle::Raw(n)) => {
-                (token::Lit::StrRaw(string, n), None)
+                (token::StrRaw(n), string, None)
             }
             LitKind::ByteStr(ref bytes) => {
                 let string = bytes.iter().cloned().flat_map(ascii::escape_default)
                     .map(Into::<char>::into).collect::<String>();
-                (token::Lit::ByteStr(Symbol::intern(&string)), None)
+                (token::ByteStr, Symbol::intern(&string), None)
             }
             LitKind::Byte(byte) => {
                 let string: String = ascii::escape_default(byte).map(Into::<char>::into).collect();
-                (token::Lit::Byte(Symbol::intern(&string)), None)
+                (token::Byte, Symbol::intern(&string), None)
             }
             LitKind::Char(ch) => {
                 let string: String = ch.escape_default().map(Into::<char>::into).collect();
-                (token::Lit::Char(Symbol::intern(&string)), None)
+                (token::Char, Symbol::intern(&string), None)
             }
             LitKind::Int(n, ty) => {
                 let suffix = match ty {
@@ -199,29 +207,33 @@ impl LitKind {
                     ast::LitIntType::Signed(ty) => Some(Symbol::intern(ty.ty_to_string())),
                     ast::LitIntType::Unsuffixed => None,
                 };
-                (token::Lit::Integer(Symbol::intern(&n.to_string())), suffix)
+                (token::Integer, Symbol::intern(&n.to_string()), suffix)
             }
             LitKind::Float(symbol, ty) => {
-                (token::Lit::Float(symbol), Some(Symbol::intern(ty.ty_to_string())))
+                (token::Float, symbol, Some(Symbol::intern(ty.ty_to_string())))
+            }
+            LitKind::FloatUnsuffixed(symbol) => {
+                (token::Float, symbol, None)
             }
-            LitKind::FloatUnsuffixed(symbol) => (token::Lit::Float(symbol), None),
             LitKind::Bool(value) => {
-                let kw = if value { kw::True } else { kw::False };
-                (token::Lit::Bool(kw), None)
+                let symbol = if value { kw::True } else { kw::False };
+                (token::Bool, symbol, None)
             }
-            LitKind::Err(val) => (token::Lit::Err(val), None),
-        }
+            LitKind::Err(symbol) => {
+                (token::Err, symbol, None)
+            }
+        };
+
+        token::Lit::new(kind, symbol, suffix)
     }
 }
 
 impl Lit {
     fn from_lit_token(
         token: token::Lit,
-        suffix: Option<Symbol>,
         span: Span,
     ) -> Result<Lit, LitError> {
-        let node = LitKind::from_lit_token(token, suffix)?;
-        Ok(Lit { node, token, suffix, span })
+        Ok(Lit { token, node: LitKind::from_lit_token(token)?, span })
     }
 
     /// Converts literal token with a suffix into an AST literal.
@@ -232,11 +244,11 @@ impl Lit {
         token: &token::Token,
         span: Span,
     ) -> Result<Lit, LitError> {
-        let (lit, suf) = match *token {
+        let lit = match *token {
             token::Ident(ident, false) if ident.name == kw::True || ident.name == kw::False =>
-                (token::Bool(ident.name), None),
-            token::Literal(token, suffix) =>
-                (token, suffix),
+                token::Lit::new(token::Bool, ident.name, None),
+            token::Literal(lit) =>
+                lit,
             token::Interpolated(ref nt) => {
                 if let token::NtExpr(expr) | token::NtLiteral(expr) = &**nt {
                     if let ast::ExprKind::Lit(lit) = &expr.node {
@@ -248,22 +260,21 @@ impl Lit {
             _ => return Err(LitError::NotLiteral)
         };
 
-        Lit::from_lit_token(lit, suf, span)
+        Lit::from_lit_token(lit, span)
     }
 
     /// Attempts to recover an AST literal from semantic literal.
     /// This function is used when the original token doesn't exist (e.g. the literal is created
     /// by an AST-based macro) or unavailable (e.g. from HIR pretty-printing).
     pub fn from_lit_kind(node: LitKind, span: Span) -> Lit {
-        let (token, suffix) = node.to_lit_token();
-        Lit { node, token, suffix, span }
+        Lit { token: node.to_lit_token(), node, span }
     }
 
     /// Losslessly convert an AST literal into a token stream.
     crate fn tokens(&self) -> TokenStream {
-        let token = match self.token {
-            token::Bool(symbol) => token::Ident(Ident::new(symbol, self.span), false),
-            token => token::Literal(token, self.suffix),
+        let token = match self.token.kind {
+            token::Bool => token::Ident(Ident::new(self.token.symbol, self.span), false),
+            _ => token::Literal(self.token),
         };
         TokenTree::Token(self.span, token).into()
     }
@@ -276,11 +287,11 @@ impl<'a> Parser<'a> {
         if self.token == token::Dot {
             // Attempt to recover `.4` as `0.4`.
             recovered = self.look_ahead(1, |t| {
-                if let token::Literal(token::Integer(val), suf) = *t {
+                if let token::Literal(token::Lit { kind: token::Integer, symbol, suffix }) = *t {
                     let next_span = self.look_ahead_span(1);
                     if self.span.hi() == next_span.lo() {
-                        let sym = String::from("0.") + &val.as_str();
-                        let token = token::Literal(token::Float(Symbol::intern(&sym)), suf);
+                        let s = String::from("0.") + &symbol.as_str();
+                        let token = Token::lit(token::Float, Symbol::intern(&s), suffix);
                         return Some((token, self.span.to(next_span)));
                     }
                 }
@@ -313,10 +324,11 @@ impl<'a> Parser<'a> {
                 return Err(self.span_fatal(span, &msg));
             }
             Err(err) => {
-                let (lit, suf) = token.expect_lit();
+                let lit = token.expect_lit();
                 self.bump();
-                err.report(&self.sess.span_diagnostic, lit, suf, span);
-                return Ok(Lit::from_lit_token(token::Err(lit.symbol()), suf, span).ok().unwrap());
+                err.report(&self.sess.span_diagnostic, lit, span);
+                let lit = token::Lit::new(token::Err, lit.symbol, lit.suffix);
+                return Ok(Lit::from_lit_token(lit, span).ok().unwrap());
             }
         }
     }
diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs
index 3d82344ac1e..ae3665c834b 100644
--- a/src/libsyntax/parse/parser.rs
+++ b/src/libsyntax/parse/parser.rs
@@ -352,10 +352,12 @@ impl TokenCursor {
         let body = TokenTree::Delimited(
             delim_span,
             token::Bracket,
-            [TokenTree::Token(sp, token::Ident(ast::Ident::with_empty_ctxt(sym::doc), false)),
-             TokenTree::Token(sp, token::Eq),
-             TokenTree::Token(sp, token::Literal(
-                token::StrRaw(Symbol::intern(&stripped), num_of_hashes), None))
+            [
+                TokenTree::Token(sp, token::Ident(ast::Ident::with_empty_ctxt(sym::doc), false)),
+                TokenTree::Token(sp, token::Eq),
+                TokenTree::Token(sp, token::Token::lit(
+                    token::StrRaw(num_of_hashes), Symbol::intern(&stripped), None
+                )),
             ]
             .iter().cloned().collect::<TokenStream>().into(),
         );
@@ -2241,10 +2243,10 @@ impl<'a> Parser<'a> {
     }
 
     fn parse_field_name(&mut self) -> PResult<'a, Ident> {
-        if let token::Literal(token::Integer(name), suffix) = self.token {
+        if let token::Literal(token::Lit { kind: token::Integer, symbol, suffix }) = self.token {
             self.expect_no_suffix(self.span, "a tuple index", suffix);
             self.bump();
-            Ok(Ident::new(name, self.prev_span))
+            Ok(Ident::new(symbol, self.prev_span))
         } else {
             self.parse_ident_common(false)
         }
@@ -3045,19 +3047,19 @@ impl<'a> Parser<'a> {
                     token::Ident(..) => {
                         e = self.parse_dot_suffix(e, lo)?;
                     }
-                    token::Literal(token::Integer(name), suffix) => {
+                    token::Literal(token::Lit { kind: token::Integer, symbol, suffix }) => {
                         let span = self.span;
                         self.bump();
-                        let field = ExprKind::Field(e, Ident::new(name, span));
+                        let field = ExprKind::Field(e, Ident::new(symbol, span));
                         e = self.mk_expr(lo.to(span), field, ThinVec::new());
 
                         self.expect_no_suffix(span, "a tuple index", suffix);
                     }
-                    token::Literal(token::Float(n), _suf) => {
+                    token::Literal(token::Lit { kind: token::Float, symbol, .. }) => {
                       self.bump();
-                      let fstr = n.as_str();
-                      let mut err = self.diagnostic()
-                          .struct_span_err(self.prev_span, &format!("unexpected token: `{}`", n));
+                      let fstr = symbol.as_str();
+                      let msg = format!("unexpected token: `{}`", symbol);
+                      let mut err = self.diagnostic().struct_span_err(self.prev_span, &msg);
                       err.span_label(self.prev_span, "unexpected token");
                       if fstr.chars().all(|x| "0123456789.".contains(x)) {
                           let float = match fstr.parse::<f64>().ok() {
@@ -7557,11 +7559,12 @@ impl<'a> Parser<'a> {
     /// the `extern` keyword, if one is found.
     fn parse_opt_abi(&mut self) -> PResult<'a, Option<Abi>> {
         match self.token {
-            token::Literal(token::Str_(s), suf) | token::Literal(token::StrRaw(s, _), suf) => {
+            token::Literal(token::Lit { kind: token::Str, symbol, suffix }) |
+            token::Literal(token::Lit { kind: token::StrRaw(..), symbol, suffix }) => {
                 let sp = self.span;
-                self.expect_no_suffix(sp, "an ABI spec", suf);
+                self.expect_no_suffix(sp, "an ABI spec", suffix);
                 self.bump();
-                match abi::lookup(&s.as_str()) {
+                match abi::lookup(&symbol.as_str()) {
                     Some(abi) => Ok(Some(abi)),
                     None => {
                         let prev_span = self.prev_span;
@@ -7570,7 +7573,7 @@ impl<'a> Parser<'a> {
                             prev_span,
                             E0703,
                             "invalid ABI: found `{}`",
-                            s);
+                            symbol);
                         err.span_label(prev_span, "invalid ABI");
                         err.help(&format!("valid ABIs: {}", abi::all_names().join(", ")));
                         err.emit();
@@ -8370,8 +8373,10 @@ impl<'a> Parser<'a> {
 
     pub fn parse_optional_str(&mut self) -> Option<(Symbol, ast::StrStyle, Option<ast::Name>)> {
         let ret = match self.token {
-            token::Literal(token::Str_(s), suf) => (s, ast::StrStyle::Cooked, suf),
-            token::Literal(token::StrRaw(s, n), suf) => (s, ast::StrStyle::Raw(n), suf),
+            token::Literal(token::Lit { kind: token::Str, symbol, suffix }) =>
+                (symbol, ast::StrStyle::Cooked, suffix),
+            token::Literal(token::Lit { kind: token::StrRaw(n), symbol, suffix }) =>
+                (symbol, ast::StrStyle::Raw(n), suffix),
             _ => return None
         };
         self.bump();
diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs
index fbc27d1999d..4711a156ab1 100644
--- a/src/libsyntax/parse/token.rs
+++ b/src/libsyntax/parse/token.rs
@@ -1,7 +1,7 @@
 pub use BinOpToken::*;
 pub use Nonterminal::*;
 pub use DelimToken::*;
-pub use Lit::*;
+pub use LitKind::*;
 pub use Token::*;
 
 use crate::ast::{self};
@@ -59,59 +59,62 @@ impl DelimToken {
     }
 }
 
-#[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)]
-pub enum Lit {
-    Bool(ast::Name), // AST only, must never appear in a `Token`
-    Byte(ast::Name),
-    Char(ast::Name),
-    Err(ast::Name),
-    Integer(ast::Name),
-    Float(ast::Name),
-    Str_(ast::Name),
-    StrRaw(ast::Name, u16), /* raw str delimited by n hash symbols */
-    ByteStr(ast::Name),
-    ByteStrRaw(ast::Name, u16), /* raw byte str delimited by n hash symbols */
+#[derive(Clone, Copy, PartialEq, RustcEncodable, RustcDecodable, Debug)]
+pub enum LitKind {
+    Bool, // AST only, must never appear in a `Token`
+    Byte,
+    Char,
+    Integer,
+    Float,
+    Str,
+    StrRaw(u16), // raw string delimited by `n` hash symbols
+    ByteStr,
+    ByteStrRaw(u16), // raw byte string delimited by `n` hash symbols
+    Err,
 }
 
-#[cfg(target_arch = "x86_64")]
-static_assert_size!(Lit, 8);
-
-impl Lit {
-    crate fn symbol(&self) -> Symbol {
-        match *self {
-            Bool(s) | Byte(s) | Char(s) | Integer(s) | Float(s) | Err(s) |
-            Str_(s) | StrRaw(s, _) | ByteStr(s) | ByteStrRaw(s, _) => s
-        }
-    }
+#[derive(Clone, Copy, PartialEq, RustcEncodable, RustcDecodable, Debug)]
+pub struct Lit {
+    pub kind: LitKind,
+    pub symbol: Symbol,
+    pub suffix: Option<Symbol>,
+}
 
-    crate fn article(&self) -> &'static str {
-        match *self {
-            Integer(_) | Err(_) => "an",
+impl LitKind {
+    crate fn article(self) -> &'static str {
+        match self {
+            Integer | Err => "an",
             _ => "a",
         }
     }
 
-    crate fn descr(&self) -> &'static str {
-        match *self {
-            Bool(_) => panic!("literal token contains `Lit::Bool`"),
-            Byte(_) => "byte literal",
-            Char(_) => "char literal",
-            Err(_) => "invalid literal",
-            Integer(_) => "integer literal",
-            Float(_) => "float literal",
-            Str_(_) | StrRaw(..) => "string literal",
-            ByteStr(_) | ByteStrRaw(..) => "byte string literal"
+    crate fn descr(self) -> &'static str {
+        match self {
+            Bool => panic!("literal token contains `Lit::Bool`"),
+            Byte => "byte literal",
+            Char => "char literal",
+            Integer => "integer literal",
+            Float => "float literal",
+            Str | StrRaw(..) => "string literal",
+            ByteStr | ByteStrRaw(..) => "byte string literal",
+            Err => "invalid literal",
         }
     }
 
-    crate fn may_have_suffix(&self) -> bool {
-        match *self {
-            Integer(..) | Float(..) | Err(..) => true,
+    crate fn may_have_suffix(self) -> bool {
+        match self {
+            Integer | Float | Err => true,
             _ => false,
         }
     }
 }
 
+impl Lit {
+    pub fn new(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> Lit {
+        Lit { kind, symbol, suffix }
+    }
+}
+
 pub(crate) fn ident_can_begin_expr(ident: ast::Ident, is_raw: bool) -> bool {
     let ident_token: Token = Ident(ident, is_raw);
 
@@ -201,7 +204,7 @@ pub enum Token {
     CloseDelim(DelimToken),
 
     /* Literals */
-    Literal(Lit, Option<ast::Name>),
+    Literal(Lit),
 
     /* Name components */
     Ident(ast::Ident, /* is_raw */ bool),
@@ -318,6 +321,10 @@ impl Token {
         self == &Question || self == &OpenDelim(Paren)
     }
 
+    pub fn lit(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> Token {
+        Literal(Lit::new(kind, symbol, suffix))
+    }
+
     /// Returns `true` if the token is any literal
     crate fn is_lit(&self) -> bool {
         match *self {
@@ -326,9 +333,9 @@ impl Token {
         }
     }
 
-    crate fn expect_lit(&self) -> (Lit, Option<Symbol>) {
+    crate fn expect_lit(&self) -> Lit {
         match *self {
-            Literal(lit, suf) => (lit, suf),
+            Literal(lit) => lit,
             _=> panic!("`expect_lit` called on non-literal"),
         }
     }
@@ -579,13 +586,13 @@ impl Token {
             (&DocComment(a), &DocComment(b)) |
             (&Shebang(a), &Shebang(b)) => a == b,
 
+            (&Literal(a), &Literal(b)) => a == b,
+
             (&Lifetime(a), &Lifetime(b)) => a.name == b.name,
             (&Ident(a, b), &Ident(c, d)) => b == d && (a.name == c.name ||
                                                        a.name == kw::DollarCrate ||
                                                        c.name == kw::DollarCrate),
 
-            (&Literal(a, b), &Literal(c, d)) => b == d && a == c,
-
             (&Interpolated(_), &Interpolated(_)) => false,
 
             _ => panic!("forgot to add a token?"),