Use `token::Lit` in `ast::ExprKind::Lit`.

Instead of `ast::Lit`.

Literal lowering now happens at two different times. Expression literals
are lowered when HIR is crated. Attribute literals are lowered during
parsing.

This commit changes the language very slightly. Some programs that used
to not compile now will compile. This is because some invalid literals
that are removed by `cfg` or attribute macros will no longer trigger
errors. See this comment for more details:
https://github.com/rust-lang/rust/pull/102944#issuecomment-1277476773

1 files changed, 97 insertions, 145 deletions

diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs
index b072573af23..c9629ea49e0 100644
--- a/compiler/rustc_parse/src/parser/expr.rs
+++ b/compiler/rustc_parse/src/parser/expr.rs
@@ -7,35 +7,30 @@ use super::{
 };
 use crate::errors::{
     ArrayBracketsInsteadOfSpaces, ArrayBracketsInsteadOfSpacesSugg, AsyncMoveOrderIncorrect,
-    BinaryFloatLiteralNotSupported, BracesForStructLiteral, CatchAfterTry, CommaAfterBaseStruct,
-    ComparisonInterpretedAsGeneric, ComparisonOrShiftInterpretedAsGenericSugg,
-    DoCatchSyntaxRemoved, DotDotDot, EqFieldInit, ExpectedElseBlock, ExpectedEqForLetExpr,
-    ExpectedExpressionFoundLet, FieldExpressionWithGeneric, FloatLiteralRequiresIntegerPart,
-    FoundExprWouldBeStmt, HexadecimalFloatLiteralNotSupported, IfExpressionMissingCondition,
-    IfExpressionMissingThenBlock, IfExpressionMissingThenBlockSub, IntLiteralTooLarge,
+    BracesForStructLiteral, CatchAfterTry, CommaAfterBaseStruct, ComparisonInterpretedAsGeneric,
+    ComparisonOrShiftInterpretedAsGenericSugg, DoCatchSyntaxRemoved, DotDotDot, EqFieldInit,
+    ExpectedElseBlock, ExpectedEqForLetExpr, ExpectedExpressionFoundLet,
+    FieldExpressionWithGeneric, FloatLiteralRequiresIntegerPart, FoundExprWouldBeStmt,
+    IfExpressionMissingCondition, IfExpressionMissingThenBlock, IfExpressionMissingThenBlockSub,
     InvalidBlockMacroSegment, InvalidComparisonOperator, InvalidComparisonOperatorSub,
-    InvalidFloatLiteralSuffix, InvalidFloatLiteralWidth, InvalidIntLiteralWidth,
-    InvalidInterpolatedExpression, InvalidLiteralSuffix, InvalidLiteralSuffixOnTupleIndex,
-    InvalidLogicalOperator, InvalidLogicalOperatorSub, InvalidNumLiteralBasePrefix,
-    InvalidNumLiteralSuffix, LabeledLoopInBreak, LeadingPlusNotSupported, LeftArrowOperator,
+    InvalidInterpolatedExpression, InvalidLiteralSuffixOnTupleIndex, InvalidLogicalOperator,
+    InvalidLogicalOperatorSub, LabeledLoopInBreak, LeadingPlusNotSupported, LeftArrowOperator,
     LifetimeInBorrowExpression, MacroInvocationWithQualifiedPath, MalformedLoopLabel,
     MatchArmBodyWithoutBraces, MatchArmBodyWithoutBracesSugg, MissingCommaAfterMatchArm,
     MissingDotDot, MissingInInForLoop, MissingInInForLoopSub, MissingSemicolonBeforeArray,
     NoFieldsForFnCall, NotAsNegationOperator, NotAsNegationOperatorSub,
-    OctalFloatLiteralNotSupported, OuterAttributeNotAllowedOnIfElse, ParenthesesWithStructFields,
+    OuterAttributeNotAllowedOnIfElse, ParenthesesWithStructFields,
     RequireColonAfterLabeledExpression, ShiftInterpretedAsGeneric, StructLiteralNotAllowedHere,
     StructLiteralNotAllowedHereSugg, TildeAsUnaryOperator, UnexpectedTokenAfterLabel,
     UnexpectedTokenAfterLabelSugg, WrapExpressionInParentheses,
 };
 use crate::maybe_recover_from_interpolated_ty_qpath;
-
 use core::mem;
 use rustc_ast::ptr::P;
 use rustc_ast::token::{self, Delimiter, Token, TokenKind};
 use rustc_ast::tokenstream::Spacing;
 use rustc_ast::util::case::Case;
 use rustc_ast::util::classify;
-use rustc_ast::util::literal::LitError;
 use rustc_ast::util::parser::{prec_let_scrutinee_needs_par, AssocOp, Fixity};
 use rustc_ast::visit::Visitor;
 use rustc_ast::{self as ast, AttrStyle, AttrVec, CaptureBy, ExprField, Lit, UnOp, DUMMY_NODE_ID};
@@ -47,7 +42,7 @@ use rustc_errors::{
     Applicability, Diagnostic, DiagnosticBuilder, ErrorGuaranteed, IntoDiagnostic, PResult,
     StashKey,
 };
-use rustc_session::errors::ExprParenthesesNeeded;
+use rustc_session::errors::{report_lit_error, ExprParenthesesNeeded};
 use rustc_session::lint::builtin::BREAK_WITH_LABEL_AND_LOOP;
 use rustc_session::lint::BuiltinLintDiagnostics;
 use rustc_span::source_map::{self, Span, Spanned};
@@ -1415,9 +1410,9 @@ impl<'a> Parser<'a> {
 
     fn parse_lit_expr(&mut self) -> PResult<'a, P<Expr>> {
         let lo = self.token.span;
-        match self.parse_opt_lit() {
-            Some(literal) => {
-                let expr = self.mk_expr(lo.to(self.prev_token.span), ExprKind::Lit(literal));
+        match self.parse_opt_token_lit() {
+            Some((token_lit, _)) => {
+                let expr = self.mk_expr(lo.to(self.prev_token.span), ExprKind::Lit(token_lit));
                 self.maybe_recover_from_bad_qpath(expr)
             }
             None => self.try_macro_suggestion(),
@@ -1548,7 +1543,7 @@ impl<'a> Parser<'a> {
                 })
             });
             consume_colon = false;
-            Ok(self.mk_expr(lo, ExprKind::Lit(lit)))
+            Ok(self.mk_expr(lo, ExprKind::Lit(lit.token_lit)))
         } else if !ate_colon
             && (self.check_noexpect(&TokenKind::Comma) || self.check_noexpect(&TokenKind::Gt))
         {
@@ -1625,9 +1620,9 @@ impl<'a> Parser<'a> {
 
     /// Emit an error when a char is parsed as a lifetime because of a missing quote
     pub(super) fn recover_unclosed_char(
-        &mut self,
+        &self,
         lifetime: Ident,
-        err: impl FnOnce(&mut Self) -> DiagnosticBuilder<'a, ErrorGuaranteed>,
+        err: impl FnOnce(&Self) -> DiagnosticBuilder<'a, ErrorGuaranteed>,
     ) -> ast::Lit {
         if let Some(mut diag) =
             self.sess.span_diagnostic.steal_diagnostic(lifetime.span, StashKey::LifetimeIsChar)
@@ -1649,9 +1644,10 @@ impl<'a> Parser<'a> {
                 )
                 .emit();
         }
+        let name = lifetime.without_first_quote().name;
         ast::Lit {
-            token_lit: token::Lit::new(token::LitKind::Char, lifetime.name, None),
-            kind: ast::LitKind::Char(lifetime.name.as_str().chars().next().unwrap_or('_')),
+            token_lit: token::Lit::new(token::LitKind::Char, name, None),
+            kind: ast::LitKind::Char(name.as_str().chars().next().unwrap_or('_')),
             span: lifetime.span,
         }
     }
@@ -1765,7 +1761,7 @@ impl<'a> Parser<'a> {
     /// In case of error returns `Some(lit)` if the next token is a literal with a wrong kind,
     /// and returns `None` if the next token is not literal at all.
     pub fn parse_str_lit(&mut self) -> Result<ast::StrLit, Option<Lit>> {
-        match self.parse_opt_lit() {
+        match self.parse_opt_ast_lit() {
             Some(lit) => match lit.kind {
                 ast::LitKind::Str(symbol_unescaped, style) => Ok(ast::StrLit {
                     style,
@@ -1780,41 +1776,47 @@ impl<'a> Parser<'a> {
         }
     }
 
-    pub(super) fn parse_lit(&mut self) -> PResult<'a, Lit> {
-        self.parse_opt_lit().ok_or(()).or_else(|()| {
-            if let token::Interpolated(inner) = &self.token.kind {
-                let expr = match inner.as_ref() {
-                    token::NtExpr(expr) => Some(expr),
-                    token::NtLiteral(expr) => Some(expr),
-                    _ => None,
-                };
-                if let Some(expr) = expr {
-                    if matches!(expr.kind, ExprKind::Err) {
-                        let mut err = InvalidInterpolatedExpression { span: self.token.span }
-                            .into_diagnostic(&self.sess.span_diagnostic);
-                        err.downgrade_to_delayed_bug();
-                        return Err(err);
-                    }
-                }
-            }
-            let token = self.token.clone();
-            let err = |self_: &mut Self| {
-                let msg = format!("unexpected token: {}", super::token_descr(&token));
-                self_.struct_span_err(token.span, &msg)
+    fn handle_missing_lit(&mut self) -> PResult<'a, Lit> {
+        if let token::Interpolated(inner) = &self.token.kind {
+            let expr = match inner.as_ref() {
+                token::NtExpr(expr) => Some(expr),
+                token::NtLiteral(expr) => Some(expr),
+                _ => None,
             };
-            // On an error path, eagerly consider a lifetime to be an unclosed character lit
-            if self.token.is_lifetime() {
-                let lt = self.expect_lifetime();
-                Ok(self.recover_unclosed_char(lt.ident, err))
-            } else {
-                Err(err(self))
+            if let Some(expr) = expr {
+                if matches!(expr.kind, ExprKind::Err) {
+                    let mut err = InvalidInterpolatedExpression { span: self.token.span }
+                        .into_diagnostic(&self.sess.span_diagnostic);
+                    err.downgrade_to_delayed_bug();
+                    return Err(err);
+                }
             }
-        })
+        }
+        let token = self.token.clone();
+        let err = |self_: &Self| {
+            let msg = format!("unexpected token: {}", super::token_descr(&token));
+            self_.struct_span_err(token.span, &msg)
+        };
+        // On an error path, eagerly consider a lifetime to be an unclosed character lit
+        if self.token.is_lifetime() {
+            let lt = self.expect_lifetime();
+            Ok(self.recover_unclosed_char(lt.ident, err))
+        } else {
+            Err(err(self))
+        }
     }
 
-    /// Matches `lit = true | false | token_lit`.
-    /// Returns `None` if the next token is not a literal.
-    pub(super) fn parse_opt_lit(&mut self) -> Option<Lit> {
+    pub(super) fn parse_token_lit(&mut self) -> PResult<'a, (token::Lit, Span)> {
+        self.parse_opt_token_lit()
+            .ok_or(())
+            .or_else(|()| self.handle_missing_lit().map(|lit| (lit.token_lit, lit.span)))
+    }
+
+    pub(super) fn parse_ast_lit(&mut self) -> PResult<'a, Lit> {
+        self.parse_opt_ast_lit().ok_or(()).or_else(|()| self.handle_missing_lit())
+    }
+
+    fn recover_after_dot(&mut self) -> Option<Token> {
         let mut recovered = None;
         if self.token == token::Dot {
             // Attempt to recover `.4` as `0.4`. We don't currently have any syntax where
@@ -1840,100 +1842,50 @@ impl<'a> Parser<'a> {
             }
         }
 
-        let token = recovered.as_ref().unwrap_or(&self.token);
-        match Lit::from_token(token) {
-            Ok(lit) => {
-                self.bump();
-                Some(lit)
-            }
-            Err(LitError::NotLiteral) => None,
-            Err(err) => {
-                let span = token.span;
-                let token::Literal(lit) = token.kind else {
-                    unreachable!();
-                };
-                self.bump();
-                self.report_lit_error(err, lit, span);
-                // Pack possible quotes and prefixes from the original literal into
-                // the error literal's symbol so they can be pretty-printed faithfully.
-                let suffixless_lit = token::Lit::new(lit.kind, lit.symbol, None);
-                let symbol = Symbol::intern(&suffixless_lit.to_string());
-                let lit = token::Lit::new(token::Err, symbol, lit.suffix);
-                Some(Lit::from_token_lit(lit, span).unwrap_or_else(|_| unreachable!()))
-            }
-        }
+        recovered
     }
 
-    fn report_lit_error(&self, err: LitError, lit: token::Lit, span: Span) {
-        // Checks if `s` looks like i32 or u1234 etc.
-        fn looks_like_width_suffix(first_chars: &[char], s: &str) -> bool {
-            s.len() > 1 && s.starts_with(first_chars) && s[1..].chars().all(|c| c.is_ascii_digit())
-        }
-
-        // Try to lowercase the prefix if it's a valid base prefix.
-        fn fix_base_capitalisation(s: &str) -> Option<String> {
-            if let Some(stripped) = s.strip_prefix('B') {
-                Some(format!("0b{stripped}"))
-            } else if let Some(stripped) = s.strip_prefix('O') {
-                Some(format!("0o{stripped}"))
-            } else if let Some(stripped) = s.strip_prefix('X') {
-                Some(format!("0x{stripped}"))
-            } else {
-                None
-            }
-        }
+    /// Matches `lit = true | false | token_lit`.
+    /// Returns `None` if the next token is not a literal.
+    pub(super) fn parse_opt_token_lit(&mut self) -> Option<(token::Lit, Span)> {
+        let recovered = self.recover_after_dot();
+        let token = recovered.as_ref().unwrap_or(&self.token);
+        let span = token.span;
+        token::Lit::from_token(token).map(|token_lit| {
+            self.bump();
+            (token_lit, span)
+        })
+    }
 
-        let token::Lit { kind, suffix, .. } = lit;
-        match err {
-            // `NotLiteral` is not an error by itself, so we don't report
-            // it and give the parser opportunity to try something else.
-            LitError::NotLiteral => {}
-            // `LexerError` *is* an error, but it was already reported
-            // by lexer, so here we don't report it the second time.
-            LitError::LexerError => {}
-            LitError::InvalidSuffix => {
-                if let Some(suffix) = suffix {
-                    self.sess.emit_err(InvalidLiteralSuffix {
-                        span,
-                        kind: format!("{}", kind.descr()),
-                        suffix,
-                    });
-                }
-            }
-            LitError::InvalidIntSuffix => {
-                let suf = suffix.expect("suffix error with no suffix");
-                let suf = suf.as_str();
-                if looks_like_width_suffix(&['i', 'u'], &suf) {
-                    // If it looks like a width, try to be helpful.
-                    self.sess.emit_err(InvalidIntLiteralWidth { span, width: suf[1..].into() });
-                } else if let Some(fixed) = fix_base_capitalisation(suf) {
-                    self.sess.emit_err(InvalidNumLiteralBasePrefix { span, fixed });
-                } else {
-                    self.sess.emit_err(InvalidNumLiteralSuffix { span, suffix: suf.to_string() });
-                }
-            }
-            LitError::InvalidFloatSuffix => {
-                let suf = suffix.expect("suffix error with no suffix");
-                let suf = suf.as_str();
-                if looks_like_width_suffix(&['f'], suf) {
-                    // If it looks like a width, try to be helpful.
-                    self.sess
-                        .emit_err(InvalidFloatLiteralWidth { span, width: suf[1..].to_string() });
-                } else {
-                    self.sess.emit_err(InvalidFloatLiteralSuffix { span, suffix: suf.to_string() });
+    /// Matches `lit = true | false | token_lit`.
+    /// Returns `None` if the next token is not a literal.
+    pub(super) fn parse_opt_ast_lit(&mut self) -> Option<Lit> {
+        let recovered = self.recover_after_dot();
+        let token = recovered.as_ref().unwrap_or(&self.token);
+        match token::Lit::from_token(token) {
+            Some(token_lit) => {
+                match Lit::from_token_lit(token_lit, token.span) {
+                    Ok(lit) => {
+                        self.bump();
+                        Some(lit)
+                    }
+                    Err(err) => {
+                        let span = token.span;
+                        let token::Literal(lit) = token.kind else {
+                            unreachable!();
+                        };
+                        self.bump();
+                        report_lit_error(&self.sess, err, lit, span);
+                        // Pack possible quotes and prefixes from the original literal into
+                        // the error literal's symbol so they can be pretty-printed faithfully.
+                        let suffixless_lit = token::Lit::new(lit.kind, lit.symbol, None);
+                        let symbol = Symbol::intern(&suffixless_lit.to_string());
+                        let lit = token::Lit::new(token::Err, symbol, lit.suffix);
+                        Some(Lit::from_token_lit(lit, span).unwrap_or_else(|_| unreachable!()))
+                    }
                 }
             }
-            LitError::NonDecimalFloat(base) => {
-                match base {
-                    16 => self.sess.emit_err(HexadecimalFloatLiteralNotSupported { span }),
-                    8 => self.sess.emit_err(OctalFloatLiteralNotSupported { span }),
-                    2 => self.sess.emit_err(BinaryFloatLiteralNotSupported { span }),
-                    _ => unreachable!(),
-                };
-            }
-            LitError::IntTooLarge => {
-                self.sess.emit_err(IntLiteralTooLarge { span });
-            }
+            None => None,
         }
     }
 
@@ -1958,8 +1910,8 @@ impl<'a> Parser<'a> {
 
         let lo = self.token.span;
         let minus_present = self.eat(&token::BinOp(token::Minus));
-        let lit = self.parse_lit()?;
-        let expr = self.mk_expr(lit.span, ExprKind::Lit(lit));
+        let (token_lit, span) = self.parse_token_lit()?;
+        let expr = self.mk_expr(span, ExprKind::Lit(token_lit));
 
         if minus_present {
             Ok(self.mk_expr(lo.to(self.prev_token.span), self.mk_unary(UnOp::Neg, expr)))