Auto merge of #60679 - petrochenkov:lit2, r=matklad

Keep original literal tokens in AST

The original literal tokens (`token::Lit`) are kept in AST until lowering to HIR.

The tokens are kept together with their lowered "semantic" representation (`ast::LitKind`), so the size of `ast::Lit` is increased (this also increases the size of meta-item structs used for processing built-in attributes).
However, the size of `ast::Expr` stays the same.

The intent is to remove the "semantic" representation from AST eventually and keep literals as tokens until lowering to HIR (at least), and I'm going to work on that, but it would be good to land this sooner to unblock progress on the [lexer refactoring](https://github.com/rust-lang/rust/pull/59706).

Fixes a part of https://github.com/rust-lang/rust/issues/43081 (literal tokens that are passed to proc macros are always precise, including hexadecimal numbers, strings with their original escaping, etc)
Fixes a part of https://github.com/rust-lang/rust/issues/60495 (everything except for proc macro API doesn't need escaping anymore)
This also allows to eliminate a certain hack from the lexer (https://rust-lang.zulipchat.com/#narrow/stream/131828-t-compiler/topic/pretty-printing.20comments/near/165005357).

cc @matklad

7 files changed, 520 insertions, 473 deletions

diff --git a/src/libsyntax/parse/classify.rs b/src/libsyntax/parse/classify.rs
index b4103440e35..dfd6f451c28 100644
--- a/src/libsyntax/parse/classify.rs
+++ b/src/libsyntax/parse/classify.rs
@@ -25,16 +25,3 @@ pub fn expr_requires_semi_to_be_stmt(e: &ast::Expr) -> bool {
         _ => true,
     }
 }
-
-/// this statement requires a semicolon after it.
-/// note that in one case (`stmt_semi`), we've already
-/// seen the semicolon, and thus don't need another.
-pub fn stmt_ends_with_semi(stmt: &ast::StmtKind) -> bool {
-    match *stmt {
-        ast::StmtKind::Local(_) => true,
-        ast::StmtKind::Expr(ref e) => expr_requires_semi_to_be_stmt(e),
-        ast::StmtKind::Item(_) |
-        ast::StmtKind::Semi(..) |
-        ast::StmtKind::Mac(..) => false,
-    }
-}
diff --git a/src/libsyntax/parse/lexer/comments.rs b/src/libsyntax/parse/lexer/comments.rs
index 74fff3324ea..97d3fc002e9 100644
--- a/src/libsyntax/parse/lexer/comments.rs
+++ b/src/libsyntax/parse/lexer/comments.rs
@@ -3,8 +3,7 @@ pub use CommentStyle::*;
 use crate::ast;
 use crate::source_map::SourceMap;
 use crate::parse::lexer::{is_block_doc_comment, is_pattern_whitespace};
-use crate::parse::lexer::{self, ParseSess, StringReader, TokenAndSpan};
-use crate::print::pprust;
+use crate::parse::lexer::{self, ParseSess, StringReader};
 
 use syntax_pos::{BytePos, CharPos, Pos, FileName};
 use log::debug;
@@ -339,16 +338,9 @@ fn consume_comment(rdr: &mut StringReader<'_>,
     debug!("<<< consume comment");
 }
 
-#[derive(Clone)]
-pub struct Literal {
-    pub lit: String,
-    pub pos: BytePos,
-}
-
 // it appears this function is called only from pprust... that's
 // probably not a good thing.
-pub fn gather_comments_and_literals(sess: &ParseSess, path: FileName, srdr: &mut dyn Read)
-    -> (Vec<Comment>, Vec<Literal>)
+pub fn gather_comments(sess: &ParseSess, path: FileName, srdr: &mut dyn Read) -> Vec<Comment>
 {
     let mut src = String::new();
     srdr.read_to_string(&mut src).unwrap();
@@ -357,7 +349,6 @@ pub fn gather_comments_and_literals(sess: &ParseSess, path: FileName, srdr: &mut
     let mut rdr = lexer::StringReader::new_raw(sess, source_file, None);
 
     let mut comments: Vec<Comment> = Vec::new();
-    let mut literals: Vec<Literal> = Vec::new();
     let mut code_to_the_left = false; // Only code
     let mut anything_to_the_left = false; // Code or comments
 
@@ -382,26 +373,12 @@ pub fn gather_comments_and_literals(sess: &ParseSess, path: FileName, srdr: &mut
             }
         }
 
-        let bstart = rdr.pos;
         rdr.next_token();
-        // discard, and look ahead; we're working with internal state
-        let TokenAndSpan { tok, sp } = rdr.peek();
-        if tok.is_lit() {
-            rdr.with_str_from(bstart, |s| {
-                debug!("tok lit: {}", s);
-                literals.push(Literal {
-                    lit: s.to_string(),
-                    pos: sp.lo(),
-                });
-            })
-        } else {
-            debug!("tok: {}", pprust::token_to_string(&tok));
-        }
         code_to_the_left = true;
         anything_to_the_left = true;
     }
 
-    (comments, literals)
+    comments
 }
 
 #[cfg(test)]
diff --git a/src/libsyntax/parse/lexer/mod.rs b/src/libsyntax/parse/lexer/mod.rs
index 2882acb0e78..e76605cde32 100644
--- a/src/libsyntax/parse/lexer/mod.rs
+++ b/src/libsyntax/parse/lexer/mod.rs
@@ -262,18 +262,6 @@ impl<'a> StringReader<'a> {
         }
     }
 
-    pub fn new(sess: &'a ParseSess,
-               source_file: Lrc<syntax_pos::SourceFile>,
-               override_span: Option<Span>) -> Self {
-        let mut sr = StringReader::new_raw(sess, source_file, override_span);
-        if sr.advance_token().is_err() {
-            sr.emit_fatal_errors();
-            FatalError.raise();
-        }
-
-        sr
-    }
-
     pub fn new_or_buffered_errs(sess: &'a ParseSess,
                                 source_file: Lrc<syntax_pos::SourceFile>,
                                 override_span: Option<Span>) -> Result<Self, Vec<Diagnostic>> {
@@ -1627,7 +1615,12 @@ mod tests {
                  teststr: String)
                  -> StringReader<'a> {
         let sf = sm.new_source_file(PathBuf::from(teststr.clone()).into(), teststr);
-        StringReader::new(sess, sf, None)
+        let mut sr = StringReader::new_raw(sess, sf, None);
+        if sr.advance_token().is_err() {
+            sr.emit_fatal_errors();
+            FatalError.raise();
+        }
+        sr
     }
 
     #[test]
diff --git a/src/libsyntax/parse/literal.rs b/src/libsyntax/parse/literal.rs
new file mode 100644
index 00000000000..53195421ddc
--- /dev/null
+++ b/src/libsyntax/parse/literal.rs
@@ -0,0 +1,487 @@
+//! Code related to parsing literals.
+
+use crate::ast::{self, Ident, Lit, LitKind};
+use crate::parse::parser::Parser;
+use crate::parse::PResult;
+use crate::parse::token::{self, Token};
+use crate::parse::unescape::{unescape_str, unescape_char, unescape_byte_str, unescape_byte};
+use crate::print::pprust;
+use crate::symbol::{keywords, Symbol};
+use crate::tokenstream::{TokenStream, TokenTree};
+
+use errors::{Applicability, Handler};
+use log::debug;
+use rustc_data_structures::sync::Lrc;
+use syntax_pos::Span;
+
+use std::ascii;
+
+macro_rules! err {
+    ($opt_diag:expr, |$span:ident, $diag:ident| $($body:tt)*) => {
+        match $opt_diag {
+            Some(($span, $diag)) => { $($body)* }
+            None => return None,
+        }
+    }
+}
+
+impl LitKind {
+    /// Converts literal token with a suffix into a semantic literal.
+    /// Works speculatively and may return `None` if diagnostic handler is not passed.
+    /// If diagnostic handler is passed, always returns `Some`,
+    /// possibly after reporting non-fatal errors and recovery.
+    fn from_lit_token(
+        lit: token::Lit,
+        suf: Option<Symbol>,
+        diag: Option<(Span, &Handler)>
+    ) -> Option<LitKind> {
+        if suf.is_some() && !lit.may_have_suffix() {
+            err!(diag, |span, diag| {
+                expect_no_suffix(span, diag, &format!("a {}", lit.literal_name()), suf)
+            });
+        }
+
+        Some(match lit {
+            token::Bool(i) => {
+                assert!(i == keywords::True.name() || i == keywords::False.name());
+                LitKind::Bool(i == keywords::True.name())
+            }
+            token::Byte(i) => {
+                match unescape_byte(&i.as_str()) {
+                    Ok(c) => LitKind::Byte(c),
+                    Err(_) => LitKind::Err(i),
+                }
+            },
+            token::Char(i) => {
+                match unescape_char(&i.as_str()) {
+                    Ok(c) => LitKind::Char(c),
+                    Err(_) => LitKind::Err(i),
+                }
+            },
+            token::Err(i) => LitKind::Err(i),
+
+            // There are some valid suffixes for integer and float literals,
+            // so all the handling is done internally.
+            token::Integer(s) => return integer_lit(&s.as_str(), suf, diag),
+            token::Float(s) => return float_lit(&s.as_str(), suf, diag),
+
+            token::Str_(mut sym) => {
+                // If there are no characters requiring special treatment we can
+                // reuse the symbol from the Token. Otherwise, we must generate a
+                // new symbol because the string in the LitKind is different to the
+                // string in the Token.
+                let mut has_error = false;
+                let s = &sym.as_str();
+                if s.as_bytes().iter().any(|&c| c == b'\\' || c == b'\r') {
+                    let mut buf = String::with_capacity(s.len());
+                    unescape_str(s, &mut |_, unescaped_char| {
+                        match unescaped_char {
+                            Ok(c) => buf.push(c),
+                            Err(_) => has_error = true,
+                        }
+                    });
+                    if has_error {
+                        return Some(LitKind::Err(sym));
+                    }
+                    sym = Symbol::intern(&buf)
+                }
+
+                LitKind::Str(sym, ast::StrStyle::Cooked)
+            }
+            token::StrRaw(mut sym, n) => {
+                // Ditto.
+                let s = &sym.as_str();
+                if s.contains('\r') {
+                    sym = Symbol::intern(&raw_str_lit(s));
+                }
+                LitKind::Str(sym, ast::StrStyle::Raw(n))
+            }
+            token::ByteStr(i) => {
+                let s = &i.as_str();
+                let mut buf = Vec::with_capacity(s.len());
+                let mut has_error = false;
+                unescape_byte_str(s, &mut |_, unescaped_byte| {
+                    match unescaped_byte {
+                        Ok(c) => buf.push(c),
+                        Err(_) => has_error = true,
+                    }
+                });
+                if has_error {
+                    return Some(LitKind::Err(i));
+                }
+                buf.shrink_to_fit();
+                LitKind::ByteStr(Lrc::new(buf))
+            }
+            token::ByteStrRaw(i, _) => {
+                LitKind::ByteStr(Lrc::new(i.to_string().into_bytes()))
+            }
+        })
+    }
+
+    /// Attempts to recover a token from semantic literal.
+    /// This function is used when the original token doesn't exist (e.g. the literal is created
+    /// by an AST-based macro) or unavailable (e.g. from HIR pretty-printing).
+    pub fn to_lit_token(&self) -> (token::Lit, Option<Symbol>) {
+        match *self {
+            LitKind::Str(string, ast::StrStyle::Cooked) => {
+                let escaped = string.as_str().escape_default().to_string();
+                (token::Lit::Str_(Symbol::intern(&escaped)), None)
+            }
+            LitKind::Str(string, ast::StrStyle::Raw(n)) => {
+                (token::Lit::StrRaw(string, n), None)
+            }
+            LitKind::ByteStr(ref bytes) => {
+                let string = bytes.iter().cloned().flat_map(ascii::escape_default)
+                    .map(Into::<char>::into).collect::<String>();
+                (token::Lit::ByteStr(Symbol::intern(&string)), None)
+            }
+            LitKind::Byte(byte) => {
+                let string: String = ascii::escape_default(byte).map(Into::<char>::into).collect();
+                (token::Lit::Byte(Symbol::intern(&string)), None)
+            }
+            LitKind::Char(ch) => {
+                let string: String = ch.escape_default().map(Into::<char>::into).collect();
+                (token::Lit::Char(Symbol::intern(&string)), None)
+            }
+            LitKind::Int(n, ty) => {
+                let suffix = match ty {
+                    ast::LitIntType::Unsigned(ty) => Some(Symbol::intern(ty.ty_to_string())),
+                    ast::LitIntType::Signed(ty) => Some(Symbol::intern(ty.ty_to_string())),
+                    ast::LitIntType::Unsuffixed => None,
+                };
+                (token::Lit::Integer(Symbol::intern(&n.to_string())), suffix)
+            }
+            LitKind::Float(symbol, ty) => {
+                (token::Lit::Float(symbol), Some(Symbol::intern(ty.ty_to_string())))
+            }
+            LitKind::FloatUnsuffixed(symbol) => (token::Lit::Float(symbol), None),
+            LitKind::Bool(value) => {
+                let kw = if value { keywords::True } else { keywords::False };
+                (token::Lit::Bool(kw.name()), None)
+            }
+            LitKind::Err(val) => (token::Lit::Err(val), None),
+        }
+    }
+}
+
+impl Lit {
+    /// Converts literal token with a suffix into an AST literal.
+    /// Works speculatively and may return `None` if diagnostic handler is not passed.
+    /// If diagnostic handler is passed, may return `Some`,
+    /// possibly after reporting non-fatal errors and recovery, or `None` for irrecoverable errors.
+    crate fn from_token(
+        token: &token::Token,
+        span: Span,
+        diag: Option<(Span, &Handler)>,
+    ) -> Option<Lit> {
+        let (token, suffix) = match *token {
+            token::Ident(ident, false) if ident.name == keywords::True.name() ||
+                                          ident.name == keywords::False.name() =>
+                (token::Bool(ident.name), None),
+            token::Literal(token, suffix) =>
+                (token, suffix),
+            token::Interpolated(ref nt) => {
+                if let token::NtExpr(expr) | token::NtLiteral(expr) = &**nt {
+                    if let ast::ExprKind::Lit(lit) = &expr.node {
+                        return Some(lit.clone());
+                    }
+                }
+                return None;
+            }
+            _ => return None,
+        };
+
+        let node = LitKind::from_lit_token(token, suffix, diag)?;
+        Some(Lit { node, token, suffix, span })
+    }
+
+    /// Attempts to recover an AST literal from semantic literal.
+    /// This function is used when the original token doesn't exist (e.g. the literal is created
+    /// by an AST-based macro) or unavailable (e.g. from HIR pretty-printing).
+    pub fn from_lit_kind(node: LitKind, span: Span) -> Lit {
+        let (token, suffix) = node.to_lit_token();
+        Lit { node, token, suffix, span }
+    }
+
+    /// Losslessly convert an AST literal into a token stream.
+    crate fn tokens(&self) -> TokenStream {
+        let token = match self.token {
+            token::Bool(symbol) => Token::Ident(Ident::with_empty_ctxt(symbol), false),
+            token => Token::Literal(token, self.suffix),
+        };
+        TokenTree::Token(self.span, token).into()
+    }
+}
+
+impl<'a> Parser<'a> {
+    /// Matches `lit = true | false | token_lit`.
+    crate fn parse_lit(&mut self) -> PResult<'a, Lit> {
+        let diag = Some((self.span, &self.sess.span_diagnostic));
+        if let Some(lit) = Lit::from_token(&self.token, self.span, diag) {
+            self.bump();
+            return Ok(lit);
+        } else if self.token == token::Dot {
+            // Recover `.4` as `0.4`.
+            let recovered = self.look_ahead(1, |t| {
+                if let token::Literal(token::Integer(val), suf) = *t {
+                    let next_span = self.look_ahead_span(1);
+                    if self.span.hi() == next_span.lo() {
+                        let sym = String::from("0.") + &val.as_str();
+                        let token = token::Literal(token::Float(Symbol::intern(&sym)), suf);
+                        return Some((token, self.span.to(next_span)));
+                    }
+                }
+                None
+            });
+            if let Some((token, span)) = recovered {
+                self.diagnostic()
+                    .struct_span_err(span, "float literals must have an integer part")
+                    .span_suggestion(
+                        span,
+                        "must have an integer part",
+                        pprust::token_to_string(&token),
+                        Applicability::MachineApplicable,
+                    )
+                    .emit();
+                let diag = Some((span, &self.sess.span_diagnostic));
+                if let Some(lit) = Lit::from_token(&token, span, diag) {
+                    self.bump();
+                    self.bump();
+                    return Ok(lit);
+                }
+            }
+        }
+
+        Err(self.span_fatal(self.span, &format!("unexpected token: {}", self.this_token_descr())))
+    }
+}
+
+crate fn expect_no_suffix(sp: Span, diag: &Handler, kind: &str, suffix: Option<ast::Name>) {
+    match suffix {
+        None => {/* everything ok */}
+        Some(suf) => {
+            let text = suf.as_str();
+            if text.is_empty() {
+                diag.span_bug(sp, "found empty literal suffix in Some")
+            }
+            let mut err = if kind == "a tuple index" &&
+                ["i32", "u32", "isize", "usize"].contains(&text.to_string().as_str())
+            {
+                // #59553: warn instead of reject out of hand to allow the fix to percolate
+                // through the ecosystem when people fix their macros
+                let mut err = diag.struct_span_warn(
+                    sp,
+                    &format!("suffixes on {} are invalid", kind),
+                );
+                err.note(&format!(
+                    "`{}` is *temporarily* accepted on tuple index fields as it was \
+                        incorrectly accepted on stable for a few releases",
+                    text,
+                ));
+                err.help(
+                    "on proc macros, you'll want to use `syn::Index::from` or \
+                        `proc_macro::Literal::*_unsuffixed` for code that will desugar \
+                        to tuple field access",
+                );
+                err.note(
+                    "for more context, see https://github.com/rust-lang/rust/issues/60210",
+                );
+                err
+            } else {
+                diag.struct_span_err(sp, &format!("suffixes on {} are invalid", kind))
+            };
+            err.span_label(sp, format!("invalid suffix `{}`", text));
+            err.emit();
+        }
+    }
+}
+
+/// Parses a string representing a raw string literal into its final form. The
+/// only operation this does is convert embedded CRLF into a single LF.
+fn raw_str_lit(lit: &str) -> String {
+    debug!("raw_str_lit: given {}", lit.escape_default());
+    let mut res = String::with_capacity(lit.len());
+
+    let mut chars = lit.chars().peekable();
+    while let Some(c) = chars.next() {
+        if c == '\r' {
+            if *chars.peek().unwrap() != '\n' {
+                panic!("lexer accepted bare CR");
+            }
+            chars.next();
+            res.push('\n');
+        } else {
+            res.push(c);
+        }
+    }
+
+    res.shrink_to_fit();
+    res
+}
+
+// check if `s` looks like i32 or u1234 etc.
+fn looks_like_width_suffix(first_chars: &[char], s: &str) -> bool {
+    s.starts_with(first_chars) && s[1..].chars().all(|c| c.is_ascii_digit())
+}
+
+fn filtered_float_lit(data: Symbol, suffix: Option<Symbol>, diag: Option<(Span, &Handler)>)
+                      -> Option<LitKind> {
+    debug!("filtered_float_lit: {}, {:?}", data, suffix);
+    let suffix = match suffix {
+        Some(suffix) => suffix,
+        None => return Some(LitKind::FloatUnsuffixed(data)),
+    };
+
+    Some(match &*suffix.as_str() {
+        "f32" => LitKind::Float(data, ast::FloatTy::F32),
+        "f64" => LitKind::Float(data, ast::FloatTy::F64),
+        suf => {
+            err!(diag, |span, diag| {
+                if suf.len() >= 2 && looks_like_width_suffix(&['f'], suf) {
+                    // if it looks like a width, lets try to be helpful.
+                    let msg = format!("invalid width `{}` for float literal", &suf[1..]);
+                    diag.struct_span_err(span, &msg).help("valid widths are 32 and 64").emit()
+                } else {
+                    let msg = format!("invalid suffix `{}` for float literal", suf);
+                    diag.struct_span_err(span, &msg)
+                        .span_label(span, format!("invalid suffix `{}`", suf))
+                        .help("valid suffixes are `f32` and `f64`")
+                        .emit();
+                }
+            });
+
+            LitKind::FloatUnsuffixed(data)
+        }
+    })
+}
+fn float_lit(s: &str, suffix: Option<Symbol>, diag: Option<(Span, &Handler)>)
+                 -> Option<LitKind> {
+    debug!("float_lit: {:?}, {:?}", s, suffix);
+    // FIXME #2252: bounds checking float literals is deferred until trans
+
+    // Strip underscores without allocating a new String unless necessary.
+    let s2;
+    let s = if s.chars().any(|c| c == '_') {
+        s2 = s.chars().filter(|&c| c != '_').collect::<String>();
+        &s2
+    } else {
+        s
+    };
+
+    filtered_float_lit(Symbol::intern(s), suffix, diag)
+}
+
+fn integer_lit(s: &str, suffix: Option<Symbol>, diag: Option<(Span, &Handler)>)
+                   -> Option<LitKind> {
+    // s can only be ascii, byte indexing is fine
+
+    // Strip underscores without allocating a new String unless necessary.
+    let s2;
+    let mut s = if s.chars().any(|c| c == '_') {
+        s2 = s.chars().filter(|&c| c != '_').collect::<String>();
+        &s2
+    } else {
+        s
+    };
+
+    debug!("integer_lit: {}, {:?}", s, suffix);
+
+    let mut base = 10;
+    let orig = s;
+    let mut ty = ast::LitIntType::Unsuffixed;
+
+    if s.starts_with('0') && s.len() > 1 {
+        match s.as_bytes()[1] {
+            b'x' => base = 16,
+            b'o' => base = 8,
+            b'b' => base = 2,
+            _ => { }
+        }
+    }
+
+    // 1f64 and 2f32 etc. are valid float literals.
+    if let Some(suf) = suffix {
+        if looks_like_width_suffix(&['f'], &suf.as_str()) {
+            let err = match base {
+                16 => Some("hexadecimal float literal is not supported"),
+                8 => Some("octal float literal is not supported"),
+                2 => Some("binary float literal is not supported"),
+                _ => None,
+            };
+            if let Some(err) = err {
+                err!(diag, |span, diag| {
+                    diag.struct_span_err(span, err)
+                        .span_label(span, "not supported")
+                        .emit();
+                });
+            }
+            return filtered_float_lit(Symbol::intern(s), Some(suf), diag)
+        }
+    }
+
+    if base != 10 {
+        s = &s[2..];
+    }
+
+    if let Some(suf) = suffix {
+        if suf.as_str().is_empty() {
+            err!(diag, |span, diag| diag.span_bug(span, "found empty literal suffix in Some"));
+        }
+        ty = match &*suf.as_str() {
+            "isize" => ast::LitIntType::Signed(ast::IntTy::Isize),
+            "i8"  => ast::LitIntType::Signed(ast::IntTy::I8),
+            "i16" => ast::LitIntType::Signed(ast::IntTy::I16),
+            "i32" => ast::LitIntType::Signed(ast::IntTy::I32),
+            "i64" => ast::LitIntType::Signed(ast::IntTy::I64),
+            "i128" => ast::LitIntType::Signed(ast::IntTy::I128),
+            "usize" => ast::LitIntType::Unsigned(ast::UintTy::Usize),
+            "u8"  => ast::LitIntType::Unsigned(ast::UintTy::U8),
+            "u16" => ast::LitIntType::Unsigned(ast::UintTy::U16),
+            "u32" => ast::LitIntType::Unsigned(ast::UintTy::U32),
+            "u64" => ast::LitIntType::Unsigned(ast::UintTy::U64),
+            "u128" => ast::LitIntType::Unsigned(ast::UintTy::U128),
+            suf => {
+                // i<digits> and u<digits> look like widths, so lets
+                // give an error message along those lines
+                err!(diag, |span, diag| {
+                    if looks_like_width_suffix(&['i', 'u'], suf) {
+                        let msg = format!("invalid width `{}` for integer literal", &suf[1..]);
+                        diag.struct_span_err(span, &msg)
+                            .help("valid widths are 8, 16, 32, 64 and 128")
+                            .emit();
+                    } else {
+                        let msg = format!("invalid suffix `{}` for numeric literal", suf);
+                        diag.struct_span_err(span, &msg)
+                            .span_label(span, format!("invalid suffix `{}`", suf))
+                            .help("the suffix must be one of the integral types \
+                                   (`u32`, `isize`, etc)")
+                            .emit();
+                    }
+                });
+
+                ty
+            }
+        }
+    }
+
+    debug!("integer_lit: the type is {:?}, base {:?}, the new string is {:?}, the original \
+           string was {:?}, the original suffix was {:?}", ty, base, s, orig, suffix);
+
+    Some(match u128::from_str_radix(s, base) {
+        Ok(r) => LitKind::Int(r, ty),
+        Err(_) => {
+            // small bases are lexed as if they were base 10, e.g, the string
+            // might be `0b10201`. This will cause the conversion above to fail,
+            // but these cases have errors in the lexer: we don't want to emit
+            // two errors, and we especially don't want to emit this error since
+            // it isn't necessarily true.
+            let already_errored = base < 10 &&
+                s.chars().any(|c| c.to_digit(10).map_or(false, |d| d >= base));
+
+            if !already_errored {
+                err!(diag, |span, diag| diag.span_err(span, "int literal is too large"));
+            }
+            LitKind::Int(0, ty)
+        }
+    })
+}
diff --git a/src/libsyntax/parse/mod.rs b/src/libsyntax/parse/mod.rs
index be44b964ba5..526143b2875 100644
--- a/src/libsyntax/parse/mod.rs
+++ b/src/libsyntax/parse/mod.rs
@@ -5,7 +5,6 @@ use crate::early_buffered_lints::{BufferedEarlyLint, BufferedEarlyLintId};
 use crate::source_map::{SourceMap, FilePathMapping};
 use crate::feature_gate::UnstableFeatures;
 use crate::parse::parser::Parser;
-use crate::symbol::Symbol;
 use crate::syntax::parse::parser::emit_unclosed_delims;
 use crate::tokenstream::{TokenStream, TokenTree};
 use crate::diagnostics::plugin::ErrorMap;
@@ -14,7 +13,6 @@ use crate::print::pprust::token_to_string;
 use errors::{Applicability, FatalError, Level, Handler, ColorConfig, Diagnostic, DiagnosticBuilder};
 use rustc_data_structures::sync::{Lrc, Lock};
 use syntax_pos::{Span, SourceFile, FileName, MultiSpan};
-use log::debug;
 
 use rustc_data_structures::fx::{FxHashSet, FxHashMap};
 use std::borrow::Cow;
@@ -25,18 +23,15 @@ pub type PResult<'a, T> = Result<T, DiagnosticBuilder<'a>>;
 
 #[macro_use]
 pub mod parser;
-
+pub mod attr;
 pub mod lexer;
 pub mod token;
-pub mod attr;
-pub mod diagnostics;
-
-pub mod classify;
-
-pub(crate) mod unescape;
-use unescape::{unescape_str, unescape_char, unescape_byte_str, unescape_byte};
 
-pub(crate) mod unescape_error_reporting;
+crate mod classify;
+crate mod diagnostics;
+crate mod literal;
+crate mod unescape;
+crate mod unescape_error_reporting;
 
 /// Info about a parsing session.
 pub struct ParseSess {
@@ -334,284 +329,6 @@ pub fn stream_to_parser(sess: &ParseSess, stream: TokenStream) -> Parser<'_> {
     Parser::new(sess, stream, None, true, false)
 }
 
-/// Parses a string representing a raw string literal into its final form. The
-/// only operation this does is convert embedded CRLF into a single LF.
-fn raw_str_lit(lit: &str) -> String {
-    debug!("raw_str_lit: given {}", lit.escape_default());
-    let mut res = String::with_capacity(lit.len());
-
-    let mut chars = lit.chars().peekable();
-    while let Some(c) = chars.next() {
-        if c == '\r' {
-            if *chars.peek().unwrap() != '\n' {
-                panic!("lexer accepted bare CR");
-            }
-            chars.next();
-            res.push('\n');
-        } else {
-            res.push(c);
-        }
-    }
-
-    res.shrink_to_fit();
-    res
-}
-
-// check if `s` looks like i32 or u1234 etc.
-fn looks_like_width_suffix(first_chars: &[char], s: &str) -> bool {
-    s.starts_with(first_chars) && s[1..].chars().all(|c| c.is_ascii_digit())
-}
-
-macro_rules! err {
-    ($opt_diag:expr, |$span:ident, $diag:ident| $($body:tt)*) => {
-        match $opt_diag {
-            Some(($span, $diag)) => { $($body)* }
-            None => return None,
-        }
-    }
-}
-
-crate fn lit_token(lit: token::Lit, suf: Option<Symbol>, diag: Option<(Span, &Handler)>)
-                 -> (bool /* suffix illegal? */, Option<ast::LitKind>) {
-    use ast::LitKind;
-
-    match lit {
-        token::Byte(i) => {
-            let lit_kind = match unescape_byte(&i.as_str()) {
-                Ok(c) => LitKind::Byte(c),
-                Err(_) => LitKind::Err(i),
-            };
-            (true, Some(lit_kind))
-        },
-        token::Char(i) => {
-            let lit_kind = match unescape_char(&i.as_str()) {
-                Ok(c) => LitKind::Char(c),
-                Err(_) => LitKind::Err(i),
-            };
-            (true, Some(lit_kind))
-        },
-        token::Err(i) => (true, Some(LitKind::Err(i))),
-
-        // There are some valid suffixes for integer and float literals,
-        // so all the handling is done internally.
-        token::Integer(s) => (false, integer_lit(&s.as_str(), suf, diag)),
-        token::Float(s) => (false, float_lit(&s.as_str(), suf, diag)),
-
-        token::Str_(mut sym) => {
-            // If there are no characters requiring special treatment we can
-            // reuse the symbol from the Token. Otherwise, we must generate a
-            // new symbol because the string in the LitKind is different to the
-            // string in the Token.
-            let mut has_error = false;
-            let s = &sym.as_str();
-            if s.as_bytes().iter().any(|&c| c == b'\\' || c == b'\r') {
-                let mut buf = String::with_capacity(s.len());
-                unescape_str(s, &mut |_, unescaped_char| {
-                    match unescaped_char {
-                        Ok(c) => buf.push(c),
-                        Err(_) => has_error = true,
-                    }
-                });
-                if has_error {
-                    return (true, Some(LitKind::Err(sym)));
-                }
-                sym = Symbol::intern(&buf)
-            }
-
-            (true, Some(LitKind::Str(sym, ast::StrStyle::Cooked)))
-        }
-        token::StrRaw(mut sym, n) => {
-            // Ditto.
-            let s = &sym.as_str();
-            if s.contains('\r') {
-                sym = Symbol::intern(&raw_str_lit(s));
-            }
-            (true, Some(LitKind::Str(sym, ast::StrStyle::Raw(n))))
-        }
-        token::ByteStr(i) => {
-            let s = &i.as_str();
-            let mut buf = Vec::with_capacity(s.len());
-            let mut has_error = false;
-            unescape_byte_str(s, &mut |_, unescaped_byte| {
-                match unescaped_byte {
-                    Ok(c) => buf.push(c),
-                    Err(_) => has_error = true,
-                }
-            });
-            if has_error {
-                return (true, Some(LitKind::Err(i)));
-            }
-            buf.shrink_to_fit();
-            (true, Some(LitKind::ByteStr(Lrc::new(buf))))
-        }
-        token::ByteStrRaw(i, _) => {
-            (true, Some(LitKind::ByteStr(Lrc::new(i.to_string().into_bytes()))))
-        }
-    }
-}
-
-fn filtered_float_lit(data: Symbol, suffix: Option<Symbol>, diag: Option<(Span, &Handler)>)
-                      -> Option<ast::LitKind> {
-    debug!("filtered_float_lit: {}, {:?}", data, suffix);
-    let suffix = match suffix {
-        Some(suffix) => suffix,
-        None => return Some(ast::LitKind::FloatUnsuffixed(data)),
-    };
-
-    Some(match &*suffix.as_str() {
-        "f32" => ast::LitKind::Float(data, ast::FloatTy::F32),
-        "f64" => ast::LitKind::Float(data, ast::FloatTy::F64),
-        suf => {
-            err!(diag, |span, diag| {
-                if suf.len() >= 2 && looks_like_width_suffix(&['f'], suf) {
-                    // if it looks like a width, lets try to be helpful.
-                    let msg = format!("invalid width `{}` for float literal", &suf[1..]);
-                    diag.struct_span_err(span, &msg).help("valid widths are 32 and 64").emit()
-                } else {
-                    let msg = format!("invalid suffix `{}` for float literal", suf);
-                    diag.struct_span_err(span, &msg)
-                        .span_label(span, format!("invalid suffix `{}`", suf))
-                        .help("valid suffixes are `f32` and `f64`")
-                        .emit();
-                }
-            });
-
-            ast::LitKind::FloatUnsuffixed(data)
-        }
-    })
-}
-fn float_lit(s: &str, suffix: Option<Symbol>, diag: Option<(Span, &Handler)>)
-                 -> Option<ast::LitKind> {
-    debug!("float_lit: {:?}, {:?}", s, suffix);
-    // FIXME #2252: bounds checking float literals is deferred until trans
-
-    // Strip underscores without allocating a new String unless necessary.
-    let s2;
-    let s = if s.chars().any(|c| c == '_') {
-        s2 = s.chars().filter(|&c| c != '_').collect::<String>();
-        &s2
-    } else {
-        s
-    };
-
-    filtered_float_lit(Symbol::intern(s), suffix, diag)
-}
-
-fn integer_lit(s: &str, suffix: Option<Symbol>, diag: Option<(Span, &Handler)>)
-                   -> Option<ast::LitKind> {
-    // s can only be ascii, byte indexing is fine
-
-    // Strip underscores without allocating a new String unless necessary.
-    let s2;
-    let mut s = if s.chars().any(|c| c == '_') {
-        s2 = s.chars().filter(|&c| c != '_').collect::<String>();
-        &s2
-    } else {
-        s
-    };
-
-    debug!("integer_lit: {}, {:?}", s, suffix);
-
-    let mut base = 10;
-    let orig = s;
-    let mut ty = ast::LitIntType::Unsuffixed;
-
-    if s.starts_with('0') && s.len() > 1 {
-        match s.as_bytes()[1] {
-            b'x' => base = 16,
-            b'o' => base = 8,
-            b'b' => base = 2,
-            _ => { }
-        }
-    }
-
-    // 1f64 and 2f32 etc. are valid float literals.
-    if let Some(suf) = suffix {
-        if looks_like_width_suffix(&['f'], &suf.as_str()) {
-            let err = match base {
-                16 => Some("hexadecimal float literal is not supported"),
-                8 => Some("octal float literal is not supported"),
-                2 => Some("binary float literal is not supported"),
-                _ => None,
-            };
-            if let Some(err) = err {
-                err!(diag, |span, diag| {
-                    diag.struct_span_err(span, err)
-                        .span_label(span, "not supported")
-                        .emit();
-                });
-            }
-            return filtered_float_lit(Symbol::intern(s), Some(suf), diag)
-        }
-    }
-
-    if base != 10 {
-        s = &s[2..];
-    }
-
-    if let Some(suf) = suffix {
-        if suf.as_str().is_empty() {
-            err!(diag, |span, diag| diag.span_bug(span, "found empty literal suffix in Some"));
-        }
-        ty = match &*suf.as_str() {
-            "isize" => ast::LitIntType::Signed(ast::IntTy::Isize),
-            "i8"  => ast::LitIntType::Signed(ast::IntTy::I8),
-            "i16" => ast::LitIntType::Signed(ast::IntTy::I16),
-            "i32" => ast::LitIntType::Signed(ast::IntTy::I32),
-            "i64" => ast::LitIntType::Signed(ast::IntTy::I64),
-            "i128" => ast::LitIntType::Signed(ast::IntTy::I128),
-            "usize" => ast::LitIntType::Unsigned(ast::UintTy::Usize),
-            "u8"  => ast::LitIntType::Unsigned(ast::UintTy::U8),
-            "u16" => ast::LitIntType::Unsigned(ast::UintTy::U16),
-            "u32" => ast::LitIntType::Unsigned(ast::UintTy::U32),
-            "u64" => ast::LitIntType::Unsigned(ast::UintTy::U64),
-            "u128" => ast::LitIntType::Unsigned(ast::UintTy::U128),
-            suf => {
-                // i<digits> and u<digits> look like widths, so lets
-                // give an error message along those lines
-                err!(diag, |span, diag| {
-                    if looks_like_width_suffix(&['i', 'u'], suf) {
-                        let msg = format!("invalid width `{}` for integer literal", &suf[1..]);
-                        diag.struct_span_err(span, &msg)
-                            .help("valid widths are 8, 16, 32, 64 and 128")
-                            .emit();
-                    } else {
-                        let msg = format!("invalid suffix `{}` for numeric literal", suf);
-                        diag.struct_span_err(span, &msg)
-                            .span_label(span, format!("invalid suffix `{}`", suf))
-                            .help("the suffix must be one of the integral types \
-                                   (`u32`, `isize`, etc)")
-                            .emit();
-                    }
-                });
-
-                ty
-            }
-        }
-    }
-
-    debug!("integer_lit: the type is {:?}, base {:?}, the new string is {:?}, the original \
-           string was {:?}, the original suffix was {:?}", ty, base, s, orig, suffix);
-
-    Some(match u128::from_str_radix(s, base) {
-        Ok(r) => ast::LitKind::Int(r, ty),
-        Err(_) => {
-            // small bases are lexed as if they were base 10, e.g, the string
-            // might be `0b10201`. This will cause the conversion above to fail,
-            // but these cases have errors in the lexer: we don't want to emit
-            // two errors, and we especially don't want to emit this error since
-            // it isn't necessarily true.
-            let already_errored = base < 10 &&
-                s.chars().any(|c| c.to_digit(10).map_or(false, |d| d >= base));
-
-            if !already_errored {
-                err!(diag, |span, diag| diag.span_err(span, "int literal is too large"));
-            }
-            ast::LitKind::Int(0, ty)
-        }
-    })
-}
-
 /// A sequence separator.
 pub struct SeqSep {
     /// The seperator token.
diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs
index d97d1e2f0f4..f95981680b9 100644
--- a/src/libsyntax/parse/parser.rs
+++ b/src/libsyntax/parse/parser.rs
@@ -15,7 +15,7 @@ use crate::ast::{ForeignItem, ForeignItemKind, FunctionRetTy};
 use crate::ast::{GenericParam, GenericParamKind};
 use crate::ast::GenericArg;
 use crate::ast::{Ident, ImplItem, IsAsync, IsAuto, Item, ItemKind};
-use crate::ast::{Label, Lifetime, Lit, LitKind};
+use crate::ast::{Label, Lifetime};
 use crate::ast::{Local, LocalSource};
 use crate::ast::MacStmtStyle;
 use crate::ast::{Mac, Mac_, MacDelimiter};
@@ -35,7 +35,7 @@ use crate::ast::{RangeEnd, RangeSyntax};
 use crate::{ast, attr};
 use crate::ext::base::DummyResult;
 use crate::source_map::{self, SourceMap, Spanned, respan};
-use crate::parse::{self, SeqSep, classify, token};
+use crate::parse::{SeqSep, classify, literal, token};
 use crate::parse::lexer::{TokenAndSpan, UnmatchedBrace};
 use crate::parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration};
 use crate::parse::token::DelimToken;
@@ -46,7 +46,7 @@ use crate::ptr::P;
 use crate::parse::PResult;
 use crate::ThinVec;
 use crate::tokenstream::{self, DelimSpan, TokenTree, TokenStream, TreeAndJoint};
-use crate::symbol::{Symbol, keywords};
+use crate::symbol::{keywords, Symbol};
 
 use errors::{Applicability, DiagnosticBuilder, DiagnosticId, FatalError};
 use rustc_target::spec::abi::{self, Abi};
@@ -613,7 +613,7 @@ impl<'a> Parser<'a> {
         })
     }
 
-    fn this_token_descr(&self) -> String {
+    crate fn this_token_descr(&self) -> String {
         if let Some(prefix) = self.token_descr() {
             format!("{} `{}`", prefix, self.this_token_to_string())
         } else {
@@ -621,11 +621,6 @@ impl<'a> Parser<'a> {
         }
     }
 
-    fn unexpected_last<T>(&self, t: &token::Token) -> PResult<'a, T> {
-        let token_str = pprust::token_to_string(t);
-        Err(self.span_fatal(self.prev_span, &format!("unexpected token: `{}`", token_str)))
-    }
-
     crate fn unexpected<T>(&mut self) -> PResult<'a, T> {
         match self.expect_one_of(&[], &[]) {
             Err(e) => Err(e),
@@ -1109,43 +1104,7 @@ impl<'a> Parser<'a> {
     }
 
     fn expect_no_suffix(&self, sp: Span, kind: &str, suffix: Option<ast::Name>) {
-        match suffix {
-            None => {/* everything ok */}
-            Some(suf) => {
-                let text = suf.as_str();
-                if text.is_empty() {
-                    self.span_bug(sp, "found empty literal suffix in Some")
-                }
-                let mut err = if kind == "a tuple index" &&
-                    ["i32", "u32", "isize", "usize"].contains(&text.to_string().as_str())
-                {
-                    // #59553: warn instead of reject out of hand to allow the fix to percolate
-                    // through the ecosystem when people fix their macros
-                    let mut err = self.struct_span_warn(
-                        sp,
-                        &format!("suffixes on {} are invalid", kind),
-                    );
-                    err.note(&format!(
-                        "`{}` is *temporarily* accepted on tuple index fields as it was \
-                         incorrectly accepted on stable for a few releases",
-                        text,
-                    ));
-                    err.help(
-                        "on proc macros, you'll want to use `syn::Index::from` or \
-                         `proc_macro::Literal::*_unsuffixed` for code that will desugar \
-                         to tuple field access",
-                    );
-                    err.note(
-                        "for more context, see https://github.com/rust-lang/rust/issues/60210",
-                    );
-                    err
-                } else {
-                    self.struct_span_err(sp, &format!("suffixes on {} are invalid", kind))
-                };
-                err.span_label(sp, format!("invalid suffix `{}`", text));
-                err.emit();
-            }
-        }
+        literal::expect_no_suffix(sp, &self.sess.span_diagnostic, kind, suffix)
     }
 
     /// Attempts to consume a `<`. If `<<` is seen, replaces it with a single
@@ -1423,7 +1382,7 @@ impl<'a> Parser<'a> {
         })
     }
 
-    fn look_ahead_span(&self, dist: usize) -> Span {
+    crate fn look_ahead_span(&self, dist: usize) -> Span {
         if dist == 0 {
             return self.span
         }
@@ -1452,9 +1411,6 @@ impl<'a> Parser<'a> {
     crate fn struct_span_err<S: Into<MultiSpan>>(&self, sp: S, m: &str) -> DiagnosticBuilder<'a> {
         self.sess.span_diagnostic.struct_span_err(sp, m)
     }
-    fn struct_span_warn<S: Into<MultiSpan>>(&self, sp: S, m: &str) -> DiagnosticBuilder<'a> {
-        self.sess.span_diagnostic.struct_span_warn(sp, m)
-    }
     crate fn span_bug<S: Into<MultiSpan>>(&self, sp: S, m: &str) -> ! {
         self.sess.span_diagnostic.span_bug(sp, m)
     }
@@ -2069,88 +2025,6 @@ impl<'a> Parser<'a> {
         }
     }
 
-    /// Matches `token_lit = LIT_INTEGER | ...`.
-    fn parse_lit_token(&mut self) -> PResult<'a, LitKind> {
-        let out = match self.token {
-            token::Interpolated(ref nt) => match **nt {
-                token::NtExpr(ref v) | token::NtLiteral(ref v) => match v.node {
-                    ExprKind::Lit(ref lit) => { lit.node.clone() }
-                    _ => { return self.unexpected_last(&self.token); }
-                },
-                _ => { return self.unexpected_last(&self.token); }
-            },
-            token::Literal(lit, suf) => {
-                let diag = Some((self.span, &self.sess.span_diagnostic));
-                let (suffix_illegal, result) = parse::lit_token(lit, suf, diag);
-
-                if suffix_illegal {
-                    let sp = self.span;
-                    self.expect_no_suffix(sp, &format!("a {}", lit.literal_name()), suf)
-                }
-
-                result.unwrap()
-            }
-            token::Dot if self.look_ahead(1, |t| match t {
-                token::Literal(parse::token::Lit::Integer(_) , _) => true,
-                _ => false,
-            }) => { // recover from `let x = .4;`
-                let lo = self.span;
-                self.bump();
-                if let token::Literal(
-                    parse::token::Lit::Integer(val),
-                    suffix,
-                ) = self.token {
-                    let suffix = suffix.and_then(|s| {
-                        let s = s.as_str();
-                        if s == "f32" {
-                            Some("f32")
-                        } else if s == "f64" {
-                            Some("f64")
-                        } else {
-                            None
-                        }
-                    }).unwrap_or("");
-                    self.bump();
-                    let sp = lo.to(self.prev_span);
-                    let mut err = self.diagnostic()
-                        .struct_span_err(sp, "float literals must have an integer part");
-                    err.span_suggestion(
-                        sp,
-                        "must have an integer part",
-                        format!("0.{}{}", val, suffix),
-                        Applicability::MachineApplicable,
-                    );
-                    err.emit();
-                    return Ok(match suffix {
-                        "f32" => ast::LitKind::Float(val, ast::FloatTy::F32),
-                        "f64" => ast::LitKind::Float(val, ast::FloatTy::F64),
-                        _ => ast::LitKind::FloatUnsuffixed(val),
-                    });
-                } else {
-                    unreachable!();
-                };
-            }
-            _ => { return self.unexpected_last(&self.token); }
-        };
-
-        self.bump();
-        Ok(out)
-    }
-
-    /// Matches `lit = true | false | token_lit`.
-    crate fn parse_lit(&mut self) -> PResult<'a, Lit> {
-        let lo = self.span;
-        let lit = if self.eat_keyword(keywords::True) {
-            LitKind::Bool(true)
-        } else if self.eat_keyword(keywords::False) {
-            LitKind::Bool(false)
-        } else {
-            let lit = self.parse_lit_token()?;
-            lit
-        };
-        Ok(source_map::Spanned { node: lit, span: lo.to(self.prev_span) })
-    }
-
     /// Matches `'-' lit | lit` (cf. `ast_validation::AstValidator::check_expr_within_pat`).
     crate fn parse_literal_maybe_minus(&mut self) -> PResult<'a, P<Expr>> {
         maybe_whole_expr!(self);
diff --git a/src/libsyntax/parse/token.rs b/src/libsyntax/parse/token.rs
index fd7a39c576d..0c2ea70aa20 100644
--- a/src/libsyntax/parse/token.rs
+++ b/src/libsyntax/parse/token.rs
@@ -61,6 +61,7 @@ impl DelimToken {
 
 #[derive(Clone, PartialEq, RustcEncodable, RustcDecodable, Hash, Debug, Copy)]
 pub enum Lit {
+    Bool(ast::Name), // AST only, must never appear in a `Token`
     Byte(ast::Name),
     Char(ast::Name),
     Err(ast::Name),
@@ -72,9 +73,13 @@ pub enum Lit {
     ByteStrRaw(ast::Name, u16), /* raw byte str delimited by n hash symbols */
 }
 
+#[cfg(target_arch = "x86_64")]
+static_assert!(MEM_SIZE_OF_LIT: mem::size_of::<Lit>() == 8);
+
 impl Lit {
     crate fn literal_name(&self) -> &'static str {
         match *self {
+            Bool(_) => panic!("literal token contains `Lit::Bool`"),
             Byte(_) => "byte literal",
             Char(_) => "char literal",
             Err(_) => "invalid literal",
@@ -85,6 +90,13 @@ impl Lit {
         }
     }
 
+    crate fn may_have_suffix(&self) -> bool {
+        match *self {
+            Integer(..) | Float(..) => true,
+            _ => false,
+        }
+    }
+
     // See comments in `Nonterminal::to_tokenstream` for why we care about
     // *probably* equal here rather than actual equality
     fn probably_equal_for_proc_macro(&self, other: &Lit) -> bool {