From 49ed25b5d2d5dc88f9fa3e268d9bd210acc875de Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Thu, 18 Apr 2024 21:31:17 +1000
Subject: Remove `NtExpr` and `NtLiteral`.

Notes about tests:
- tests/ui/rfcs/rfc-2294-if-let-guard/feature-gate.rs: some messages are
  now duplicated due to repeated parsing.

- tests/ui/rfcs/rfc-2497-if-let-chains/disallowed-positions.rs: ditto.

- `tests/ui/proc-macro/macro-rules-derive-cfg.rs`: the diff looks large
  but the only difference is the insertion of a single
  invisible-delimited group around a metavar.

- `tests/ui/attributes/nonterminal-expansion.rs`: a slight span
  degradation, somehow related to the recent massive attr parsing
  rewrite (#135726). I couldn't work out exactly what is going wrong,
  but I don't think it's worth holding things up for a single slightly
  suboptimal error message.
---
 compiler/rustc_parse/src/errors.rs             |   4 +-
 compiler/rustc_parse/src/parser/expr.rs        | 220 ++++++++++++++++---------
 compiler/rustc_parse/src/parser/item.rs        |  53 +++++-
 compiler/rustc_parse/src/parser/mod.rs         |  32 +++-
 compiler/rustc_parse/src/parser/nonterminal.rs |  16 +-
 compiler/rustc_parse/src/parser/pat.rs         |   2 +-
 6 files changed, 225 insertions(+), 102 deletions(-)

(limited to 'compiler/rustc_parse/src')

diff --git a/compiler/rustc_parse/src/errors.rs b/compiler/rustc_parse/src/errors.rs
index 55cb7687823..dfdef018bc3 100644
--- a/compiler/rustc_parse/src/errors.rs
+++ b/compiler/rustc_parse/src/errors.rs
@@ -1696,10 +1696,10 @@ pub(crate) struct SelfArgumentPointer {
 
 #[derive(Diagnostic)]
 #[diag(parse_unexpected_token_after_dot)]
-pub(crate) struct UnexpectedTokenAfterDot<'a> {
+pub(crate) struct UnexpectedTokenAfterDot {
     #[primary_span]
     pub span: Span,
-    pub actual: Cow<'a, str>,
+    pub actual: String,
 }
 
 #[derive(Diagnostic)]
diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs
index 00e75cc2c2f..424fee968e1 100644
--- a/compiler/rustc_parse/src/parser/expr.rs
+++ b/compiler/rustc_parse/src/parser/expr.rs
@@ -4,10 +4,10 @@ use core::mem;
 use core::ops::{Bound, ControlFlow};
 
 use ast::mut_visit::{self, MutVisitor};
-use ast::token::{IdentIsRaw, MetaVarKind};
+use ast::token::IdentIsRaw;
 use ast::{CoroutineKind, ForLoopKind, GenBlockKind, MatchKind, Pat, Path, PathSegment, Recovered};
 use rustc_ast::ptr::P;
-use rustc_ast::token::{self, Delimiter, Token, TokenKind};
+use rustc_ast::token::{self, Delimiter, InvisibleOrigin, MetaVarKind, Token, TokenKind};
 use rustc_ast::tokenstream::TokenTree;
 use rustc_ast::util::case::Case;
 use rustc_ast::util::classify;
@@ -19,7 +19,6 @@ use rustc_ast::{
     MetaItemLit, Movability, Param, RangeLimits, StmtKind, Ty, TyKind, UnOp, UnsafeBinderCastKind,
     YieldKind,
 };
-use rustc_ast_pretty::pprust;
 use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_errors::{Applicability, Diag, PResult, StashKey, Subdiagnostic};
 use rustc_lexer::unescape::unescape_char;
@@ -605,7 +604,7 @@ impl<'a> Parser<'a> {
             // can't continue an expression after an ident
             token::Ident(name, is_raw) => token::ident_can_begin_expr(name, t.span, is_raw),
             token::Literal(..) | token::Pound => true,
-            _ => t.is_whole_expr(),
+            _ => t.is_metavar_expr(),
         };
         self.token.is_ident_named(sym::not) && self.look_ahead(1, token_cannot_continue_expr)
     }
@@ -641,6 +640,13 @@ impl<'a> Parser<'a> {
             TokenKind::NtIdent(..) | TokenKind::NtLifetime(..) | TokenKind::Interpolated(..) => {
                 self.prev_token.span
             }
+            TokenKind::CloseDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(_))) => {
+                // `expr.span` is the interpolated span, because invisible open
+                // and close delims both get marked with the same span, one
+                // that covers the entire thing between them. (See
+                // `rustc_expand::mbe::transcribe::transcribe`.)
+                self.prev_token.span
+            }
             _ => expr.span,
         }
     }
@@ -979,12 +985,30 @@ impl<'a> Parser<'a> {
     }
 
     fn error_unexpected_after_dot(&self) {
-        let actual = pprust::token_to_string(&self.token);
+        let actual = super::token_descr(&self.token);
         let span = self.token.span;
         let sm = self.psess.source_map();
         let (span, actual) = match (&self.token.kind, self.subparser_name) {
-            (token::Eof, Some(_)) if let Ok(actual) = sm.span_to_snippet(sm.next_point(span)) => {
-                (span.shrink_to_hi(), actual.into())
+            (token::Eof, Some(_)) if let Ok(snippet) = sm.span_to_snippet(sm.next_point(span)) => {
+                (span.shrink_to_hi(), format!("`{}`", snippet))
+            }
+            (token::CloseDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(_))), _) => {
+                // No need to report an error. This case will only occur when parsing a pasted
+                // metavariable, and we should have emitted an error when parsing the macro call in
+                // the first place. E.g. in this code:
+                // ```
+                // macro_rules! m { ($e:expr) => { $e }; }
+                //
+                // fn main() {
+                //     let f = 1;
+                //     m!(f.);
+                // }
+                // ```
+                // we'll get an error "unexpected token: `)` when parsing the `m!(f.)`, so we don't
+                // want to issue a second error when parsing the expansion `«f.»` (where `«`/`»`
+                // represent the invisible delimiters).
+                self.dcx().span_delayed_bug(span, "bad dot expr in metavariable");
+                return;
             }
             _ => (span, actual),
         };
@@ -1364,17 +1388,31 @@ impl<'a> Parser<'a> {
         let span = self.token.span;
         if let token::Interpolated(nt) = &self.token.kind {
             match &**nt {
-                token::NtExpr(e) | token::NtLiteral(e) => {
-                    let e = e.clone();
-                    self.bump();
-                    return Ok(e);
-                }
                 token::NtBlock(block) => {
                     let block = block.clone();
                     self.bump();
                     return Ok(self.mk_expr(self.prev_token.span, ExprKind::Block(block, None)));
                 }
             };
+        } else if let Some(expr) = self.eat_metavar_seq_with_matcher(
+            |mv_kind| matches!(mv_kind, MetaVarKind::Expr { .. }),
+            |this| {
+                let expr = this.parse_expr();
+                // FIXME(nnethercote) Sometimes with expressions we get a trailing comma, possibly
+                // related to the FIXME in `collect_tokens_for_expr`. Examples are the multi-line
+                // `assert_eq!` calls involving arguments annotated with `#[rustfmt::skip]` in
+                // `compiler/rustc_index/src/bit_set/tests.rs`.
+                if this.token.kind == token::Comma {
+                    this.bump();
+                }
+                expr
+            },
+        ) {
+            return Ok(expr);
+        } else if let Some(lit) =
+            self.eat_metavar_seq(MetaVarKind::Literal, |this| this.parse_literal_maybe_minus())
+        {
+            return Ok(lit);
         } else if let Some(path) = self.eat_metavar_seq(MetaVarKind::Path, |this| {
             this.collect_tokens_no_attrs(|this| this.parse_path(PathStyle::Type))
         }) {
@@ -2062,87 +2100,107 @@ impl<'a> Parser<'a> {
             .or_else(|()| self.handle_missing_lit(Parser::mk_meta_item_lit_char))
     }
 
-    fn recover_after_dot(&mut self) -> Option<Token> {
-        let mut recovered = None;
+    fn recover_after_dot(&mut self) {
         if self.token == token::Dot {
             // Attempt to recover `.4` as `0.4`. We don't currently have any syntax where
             // dot would follow an optional literal, so we do this unconditionally.
-            recovered = self.look_ahead(1, |next_token| {
+            let recovered = self.look_ahead(1, |next_token| {
+                // If it's an integer that looks like a float, then recover as such.
+                //
+                // We will never encounter the exponent part of a floating
+                // point literal here, since there's no use of the exponent
+                // syntax that also constitutes a valid integer, so we need
+                // not check for that.
                 if let token::Literal(token::Lit { kind: token::Integer, symbol, suffix }) =
                     next_token.kind
+                    && suffix.is_none_or(|s| s == sym::f32 || s == sym::f64)
+                    && symbol.as_str().chars().all(|c| c.is_numeric() || c == '_')
+                    && self.token.span.hi() == next_token.span.lo()
                 {
-                    // If this integer looks like a float, then recover as such.
-                    //
-                    // We will never encounter the exponent part of a floating
-                    // point literal here, since there's no use of the exponent
-                    // syntax that also constitutes a valid integer, so we need
-                    // not check for that.
-                    if suffix.is_none_or(|s| s == sym::f32 || s == sym::f64)
-                        && symbol.as_str().chars().all(|c| c.is_numeric() || c == '_')
-                        && self.token.span.hi() == next_token.span.lo()
-                    {
-                        let s = String::from("0.") + symbol.as_str();
-                        let kind = TokenKind::lit(token::Float, Symbol::intern(&s), suffix);
-                        return Some(Token::new(kind, self.token.span.to(next_token.span)));
-                    }
+                    let s = String::from("0.") + symbol.as_str();
+                    let kind = TokenKind::lit(token::Float, Symbol::intern(&s), suffix);
+                    Some(Token::new(kind, self.token.span.to(next_token.span)))
+                } else {
+                    None
                 }
-                None
             });
-            if let Some(token) = &recovered {
-                self.bump();
+            if let Some(recovered) = recovered {
                 self.dcx().emit_err(errors::FloatLiteralRequiresIntegerPart {
-                    span: token.span,
-                    suggestion: token.span.shrink_to_lo(),
+                    span: recovered.span,
+                    suggestion: recovered.span.shrink_to_lo(),
                 });
+                self.bump();
+                self.token = recovered;
             }
         }
+    }
 
-        recovered
+    /// Keep this in sync with `Token::can_begin_literal_maybe_minus` and
+    /// `Lit::from_token` (excluding unary negation).
+    fn eat_token_lit(&mut self) -> Option<token::Lit> {
+        match self.token.uninterpolate().kind {
+            token::Ident(name, IdentIsRaw::No) if name.is_bool_lit() => {
+                self.bump();
+                Some(token::Lit::new(token::Bool, name, None))
+            }
+            token::Literal(token_lit) => {
+                self.bump();
+                Some(token_lit)
+            }
+            token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
+                MetaVarKind::Literal,
+            ))) => {
+                let lit = self
+                    .eat_metavar_seq(MetaVarKind::Literal, |this| this.parse_literal_maybe_minus())
+                    .expect("metavar seq literal");
+                let ast::ExprKind::Lit(token_lit) = lit.kind else {
+                    panic!("didn't reparse a literal");
+                };
+                Some(token_lit)
+            }
+            token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(
+                mv_kind @ MetaVarKind::Expr { can_begin_literal_maybe_minus: true, .. },
+            ))) => {
+                let expr = self
+                    .eat_metavar_seq(mv_kind, |this| this.parse_expr())
+                    .expect("metavar seq expr");
+                let ast::ExprKind::Lit(token_lit) = expr.kind else {
+                    panic!("didn't reparse an expr");
+                };
+                Some(token_lit)
+            }
+            _ => None,
+        }
     }
 
     /// Matches `lit = true | false | token_lit`.
     /// Returns `None` if the next token is not a literal.
-    pub(super) fn parse_opt_token_lit(&mut self) -> Option<(token::Lit, Span)> {
-        let recovered = self.recover_after_dot();
-        let token = recovered.as_ref().unwrap_or(&self.token);
-        let span = token.span;
-
-        token::Lit::from_token(token).map(|token_lit| {
-            self.bump();
-            (token_lit, span)
-        })
+    fn parse_opt_token_lit(&mut self) -> Option<(token::Lit, Span)> {
+        self.recover_after_dot();
+        let span = self.token.span;
+        self.eat_token_lit().map(|token_lit| (token_lit, span))
     }
 
     /// Matches `lit = true | false | token_lit`.
     /// Returns `None` if the next token is not a literal.
-    pub(super) fn parse_opt_meta_item_lit(&mut self) -> Option<MetaItemLit> {
-        let recovered = self.recover_after_dot();
-        let token = recovered.as_ref().unwrap_or(&self.token);
-        match token::Lit::from_token(token) {
-            Some(lit) => {
-                match MetaItemLit::from_token_lit(lit, token.span) {
-                    Ok(lit) => {
-                        self.bump();
-                        Some(lit)
-                    }
-                    Err(err) => {
-                        let span = token.uninterpolated_span();
-                        self.bump();
-                        let guar = report_lit_error(self.psess, err, lit, span);
-                        // Pack possible quotes and prefixes from the original literal into
-                        // the error literal's symbol so they can be pretty-printed faithfully.
-                        let suffixless_lit = token::Lit::new(lit.kind, lit.symbol, None);
-                        let symbol = Symbol::intern(&suffixless_lit.to_string());
-                        let lit = token::Lit::new(token::Err(guar), symbol, lit.suffix);
-                        Some(
-                            MetaItemLit::from_token_lit(lit, span)
-                                .unwrap_or_else(|_| unreachable!()),
-                        )
-                    }
+    fn parse_opt_meta_item_lit(&mut self) -> Option<MetaItemLit> {
+        self.recover_after_dot();
+        let span = self.token.span;
+        let uninterpolated_span = self.uninterpolated_token_span();
+        self.eat_token_lit().map(|token_lit| {
+            match MetaItemLit::from_token_lit(token_lit, span) {
+                Ok(lit) => lit,
+                Err(err) => {
+                    let guar = report_lit_error(&self.psess, err, token_lit, uninterpolated_span);
+                    // Pack possible quotes and prefixes from the original literal into
+                    // the error literal's symbol so they can be pretty-printed faithfully.
+                    let suffixless_lit = token::Lit::new(token_lit.kind, token_lit.symbol, None);
+                    let symbol = Symbol::intern(&suffixless_lit.to_string());
+                    let token_lit = token::Lit::new(token::Err(guar), symbol, token_lit.suffix);
+                    MetaItemLit::from_token_lit(token_lit, uninterpolated_span).unwrap()
                 }
             }
-            None => None,
-        }
+        })
     }
 
     pub(super) fn expect_no_tuple_index_suffix(&self, span: Span, suffix: Symbol) {
@@ -2166,9 +2224,10 @@ impl<'a> Parser<'a> {
     /// Matches `'-' lit | lit` (cf. `ast_validation::AstValidator::check_expr_within_pat`).
     /// Keep this in sync with `Token::can_begin_literal_maybe_minus`.
     pub fn parse_literal_maybe_minus(&mut self) -> PResult<'a, P<Expr>> {
-        if let token::Interpolated(nt) = &self.token.kind {
-            match &**nt {
-                // FIXME(nnethercote) The `NtExpr` case should only match if
+        if let Some(expr) = self.eat_metavar_seq_with_matcher(
+            |mv_kind| matches!(mv_kind, MetaVarKind::Expr { .. }),
+            |this| {
+                // FIXME(nnethercote) The `expr` case should only match if
                 // `e` is an `ExprKind::Lit` or an `ExprKind::Unary` containing
                 // an `UnOp::Neg` and an `ExprKind::Lit`, like how
                 // `can_begin_literal_maybe_minus` works. But this method has
@@ -2178,13 +2237,14 @@ impl<'a> Parser<'a> {
                 // `ExprKind::Path` must be accepted when parsing range
                 // patterns. That requires some care. So for now, we continue
                 // being less strict here than we should be.
-                token::NtExpr(e) | token::NtLiteral(e) => {
-                    let e = e.clone();
-                    self.bump();
-                    return Ok(e);
-                }
-                _ => {}
-            };
+                this.parse_expr()
+            },
+        ) {
+            return Ok(expr);
+        } else if let Some(lit) =
+            self.eat_metavar_seq(MetaVarKind::Literal, |this| this.parse_literal_maybe_minus())
+        {
+            return Ok(lit);
         }
 
         let lo = self.token.span;
diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs
index e93fb2473fb..20833a0f70f 100644
--- a/compiler/rustc_parse/src/parser/item.rs
+++ b/compiler/rustc_parse/src/parser/item.rs
@@ -1290,12 +1290,24 @@ impl<'a> Parser<'a> {
     }
 
     fn is_unsafe_foreign_mod(&self) -> bool {
-        self.token.is_keyword(kw::Unsafe)
-            && self.is_keyword_ahead(1, &[kw::Extern])
-            && self.look_ahead(
-                2 + self.look_ahead(2, |t| t.can_begin_string_literal() as usize),
-                |t| *t == token::OpenDelim(Delimiter::Brace),
-            )
+        // Look for `unsafe`.
+        if !self.token.is_keyword(kw::Unsafe) {
+            return false;
+        }
+        // Look for `extern`.
+        if !self.is_keyword_ahead(1, &[kw::Extern]) {
+            return false;
+        }
+
+        // Look for the optional ABI string literal.
+        let n = if self.look_ahead(2, |t| t.can_begin_string_literal()) { 3 } else { 2 };
+
+        // Look for the `{`. Use `tree_look_ahead` because the ABI (if present)
+        // might be a metavariable i.e. an invisible-delimited sequence, and
+        // `tree_look_ahead` will consider that a single element when looking
+        // ahead.
+        self.tree_look_ahead(n, |t| matches!(t, TokenTree::Delimited(_, _, Delimiter::Brace, _)))
+            == Some(true)
     }
 
     fn is_static_global(&mut self) -> bool {
@@ -2604,13 +2616,36 @@ impl<'a> Parser<'a> {
                 })
             // `extern ABI fn`
             || self.check_keyword_case(exp!(Extern), case)
+                // Use `tree_look_ahead` because `ABI` might be a metavariable,
+                // i.e. an invisible-delimited sequence, and `tree_look_ahead`
+                // will consider that a single element when looking ahead.
                 && self.look_ahead(1, |t| t.can_begin_string_literal())
-                && (self.look_ahead(2, |t| t.is_keyword_case(kw::Fn, case)) ||
+                && (self.tree_look_ahead(2, |tt| {
+                    match tt {
+                        TokenTree::Token(t, _) => t.is_keyword_case(kw::Fn, case),
+                        TokenTree::Delimited(..) => false,
+                    }
+                }) == Some(true) ||
                     // This branch is only for better diagnostics; `pub`, `unsafe`, etc. are not
                     // allowed here.
                     (self.may_recover()
-                        && self.look_ahead(2, |t| ALL_QUALS.iter().any(|exp| t.is_keyword(exp.kw)))
-                        && self.look_ahead(3, |t| t.is_keyword_case(kw::Fn, case))))
+                        && self.tree_look_ahead(2, |tt| {
+                            match tt {
+                                TokenTree::Token(t, _) =>
+                                    ALL_QUALS.iter().any(|exp| {
+                                        t.is_keyword(exp.kw)
+                                    }),
+                                TokenTree::Delimited(..) => false,
+                            }
+                        }) == Some(true)
+                        && self.tree_look_ahead(3, |tt| {
+                            match tt {
+                                TokenTree::Token(t, _) => t.is_keyword_case(kw::Fn, case),
+                                TokenTree::Delimited(..) => false,
+                            }
+                        }) == Some(true)
+                    )
+                )
     }
 
     /// Parses all the "front matter" (or "qualifiers") for a `fn` declaration,
diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs
index d865fd42764..a08fbde52a7 100644
--- a/compiler/rustc_parse/src/parser/mod.rs
+++ b/compiler/rustc_parse/src/parser/mod.rs
@@ -24,8 +24,8 @@ pub use pat::{CommaRecoveryMode, RecoverColon, RecoverComma};
 use path::PathStyle;
 use rustc_ast::ptr::P;
 use rustc_ast::token::{
-    self, Delimiter, IdentIsRaw, InvisibleOrigin, MetaVarKind, Nonterminal, NtPatKind, Token,
-    TokenKind,
+    self, Delimiter, IdentIsRaw, InvisibleOrigin, MetaVarKind, Nonterminal, NtExprKind, NtPatKind,
+    Token, TokenKind,
 };
 use rustc_ast::tokenstream::{AttrsTarget, Spacing, TokenStream, TokenTree};
 use rustc_ast::util::case::Case;
@@ -101,6 +101,7 @@ pub enum ForceCollect {
 #[macro_export]
 macro_rules! maybe_whole {
     ($p:expr, $constructor:ident, |$x:ident| $e:expr) => {
+        #[allow(irrefutable_let_patterns)] // FIXME: temporary
         if let token::Interpolated(nt) = &$p.token.kind
             && let token::$constructor(x) = &**nt
         {
@@ -299,6 +300,10 @@ impl TokenTreeCursor {
         self.stream.get(self.index)
     }
 
+    fn look_ahead(&self, n: usize) -> Option<&TokenTree> {
+        self.stream.get(self.index + n)
+    }
+
     #[inline]
     fn bump(&mut self) {
         self.index += 1;
@@ -1290,6 +1295,17 @@ impl<'a> Parser<'a> {
         looker(&token)
     }
 
+    /// Like `lookahead`, but skips over token trees rather than tokens. Useful
+    /// when looking past possible metavariable pasting sites.
+    pub fn tree_look_ahead<R>(
+        &self,
+        dist: usize,
+        looker: impl FnOnce(&TokenTree) -> R,
+    ) -> Option<R> {
+        assert_ne!(dist, 0);
+        self.token_cursor.curr.look_ahead(dist - 1).map(looker)
+    }
+
     /// Returns whether any of the given keywords are `dist` tokens ahead of the current one.
     pub(crate) fn is_keyword_ahead(&self, dist: usize, kws: &[Symbol]) -> bool {
         self.look_ahead(dist, |t| kws.iter().any(|&kw| t.is_keyword(kw)))
@@ -1706,6 +1722,16 @@ impl<'a> Parser<'a> {
     pub fn approx_token_stream_pos(&self) -> u32 {
         self.num_bump_calls
     }
+
+    pub fn uninterpolated_token_span(&self) -> Span {
+        match &self.token.kind {
+            token::Interpolated(nt) => nt.use_span(),
+            token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(_))) => {
+                self.look_ahead(1, |t| t.span)
+            }
+            _ => self.token.span,
+        }
+    }
 }
 
 pub(crate) fn make_unclosed_delims_error(
@@ -1758,6 +1784,8 @@ pub enum ParseNtResult {
     Item(P<ast::Item>),
     Stmt(P<ast::Stmt>),
     Pat(P<ast::Pat>, NtPatKind),
+    Expr(P<ast::Expr>, NtExprKind),
+    Literal(P<ast::Expr>),
     Ty(P<ast::Ty>),
     Meta(P<ast::AttrItem>),
     Path(P<ast::Path>),
diff --git a/compiler/rustc_parse/src/parser/nonterminal.rs b/compiler/rustc_parse/src/parser/nonterminal.rs
index 1123755ce00..b4e540d670d 100644
--- a/compiler/rustc_parse/src/parser/nonterminal.rs
+++ b/compiler/rustc_parse/src/parser/nonterminal.rs
@@ -48,10 +48,6 @@ impl<'a> Parser<'a> {
         /// Old variant of `may_be_ident`. Being phased out.
         fn nt_may_be_ident(nt: &Nonterminal) -> bool {
             match nt {
-                NtExpr(_)
-                | NtLiteral(_) // `true`, `false`
-                => true,
-
                 NtBlock(_) => false,
             }
         }
@@ -95,7 +91,7 @@ impl<'a> Parser<'a> {
                 token::OpenDelim(Delimiter::Brace) => true,
                 token::NtLifetime(..) => true,
                 token::Interpolated(nt) => match &**nt {
-                    NtBlock(_) | NtExpr(_) | NtLiteral(_) => true,
+                    NtBlock(_) => true,
                 },
                 token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(k))) => match k {
                     MetaVarKind::Block
@@ -179,10 +175,14 @@ impl<'a> Parser<'a> {
                     pat_kind,
                 ));
             }
-            NonterminalKind::Expr(_) => NtExpr(self.parse_expr_force_collect()?),
+            NonterminalKind::Expr(expr_kind) => {
+                return Ok(ParseNtResult::Expr(self.parse_expr_force_collect()?, expr_kind));
+            }
             NonterminalKind::Literal => {
-                // The `:literal` matcher does not support attributes
-                NtLiteral(self.collect_tokens_no_attrs(|this| this.parse_literal_maybe_minus())?)
+                // The `:literal` matcher does not support attributes.
+                return Ok(ParseNtResult::Literal(
+                    self.collect_tokens_no_attrs(|this| this.parse_literal_maybe_minus())?,
+                ));
             }
             NonterminalKind::Ty => {
                 return Ok(ParseNtResult::Ty(
diff --git a/compiler/rustc_parse/src/parser/pat.rs b/compiler/rustc_parse/src/parser/pat.rs
index 174cc929fa7..9612f71b2af 100644
--- a/compiler/rustc_parse/src/parser/pat.rs
+++ b/compiler/rustc_parse/src/parser/pat.rs
@@ -1252,7 +1252,7 @@ impl<'a> Parser<'a> {
                 || *t == token::Dot // e.g. `.5` for recovery;
                 || matches!(t.kind, token::Literal(..) | token::Minus)
                 || t.is_bool_lit()
-                || t.is_whole_expr()
+                || t.is_metavar_expr()
                 || t.is_lifetime() // recover `'a` instead of `'a'`
                 || (self.may_recover() // recover leading `(`
                     && *t == token::OpenDelim(Delimiter::Parenthesis)
-- 
cgit 1.4.1-3-g733a5


From d59b17c5cdbf345d0088d90fc03faf2501a39757 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Wed, 26 Mar 2025 15:54:14 +1100
Subject: Remove `Token::uninterpolated_span`.

In favour of the similar method on `Parser`, which works on things
other than identifiers and lifetimes.
---
 compiler/rustc_ast/src/token.rs         | 27 ++++++-------------------
 compiler/rustc_parse/src/parser/expr.rs | 14 ++++++-------
 compiler/rustc_parse/src/parser/item.rs | 10 +++++-----
 compiler/rustc_parse/src/parser/mod.rs  | 35 +++++++++++++++++++++++++--------
 compiler/rustc_parse/src/parser/ty.rs   |  6 +++---
 5 files changed, 48 insertions(+), 44 deletions(-)

(limited to 'compiler/rustc_parse/src')

diff --git a/compiler/rustc_ast/src/token.rs b/compiler/rustc_ast/src/token.rs
index 2081777fff3..d57a369eebf 100644
--- a/compiler/rustc_ast/src/token.rs
+++ b/compiler/rustc_ast/src/token.rs
@@ -448,8 +448,9 @@ pub enum TokenKind {
 
     /// Identifier token.
     /// Do not forget about `NtIdent` when you want to match on identifiers.
-    /// It's recommended to use `Token::(ident,uninterpolate,uninterpolated_span)` to
-    /// treat regular and interpolated identifiers in the same way.
+    /// It's recommended to use `Token::{ident,uninterpolate}` and
+    /// `Parser::token_uninterpolated_span` to treat regular and interpolated
+    /// identifiers in the same way.
     Ident(Symbol, IdentIsRaw),
     /// This identifier (and its span) is the identifier passed to the
     /// declarative macro. The span in the surrounding `Token` is the span of
@@ -458,8 +459,9 @@ pub enum TokenKind {
 
     /// Lifetime identifier token.
     /// Do not forget about `NtLifetime` when you want to match on lifetime identifiers.
-    /// It's recommended to use `Token::(lifetime,uninterpolate,uninterpolated_span)` to
-    /// treat regular and interpolated lifetime identifiers in the same way.
+    /// It's recommended to use `Token::{ident,uninterpolate}` and
+    /// `Parser::token_uninterpolated_span` to treat regular and interpolated
+    /// identifiers in the same way.
     Lifetime(Symbol, IdentIsRaw),
     /// This identifier (and its span) is the lifetime passed to the
     /// declarative macro. The span in the surrounding `Token` is the span of
@@ -585,23 +587,6 @@ impl Token {
         Token::new(Ident(ident.name, ident.is_raw_guess().into()), ident.span)
     }
 
-    /// For interpolated tokens, returns a span of the fragment to which the interpolated
-    /// token refers. For all other tokens this is just a regular span.
-    /// It is particularly important to use this for identifiers and lifetimes
-    /// for which spans affect name resolution and edition checks.
-    /// Note that keywords are also identifiers, so they should use this
-    /// if they keep spans or perform edition checks.
-    //
-    // Note: `Parser::uninterpolated_token_span` may give better information
-    // than this method does.
-    pub fn uninterpolated_span(&self) -> Span {
-        match self.kind {
-            NtIdent(ident, _) | NtLifetime(ident, _) => ident.span,
-            Interpolated(ref nt) => nt.use_span(),
-            _ => self.span,
-        }
-    }
-
     pub fn is_range_separator(&self) -> bool {
         [DotDot, DotDotDot, DotDotEq].contains(&self.kind)
     }
diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs
index 424fee968e1..1c2df3ffcd1 100644
--- a/compiler/rustc_parse/src/parser/expr.rs
+++ b/compiler/rustc_parse/src/parser/expr.rs
@@ -1318,7 +1318,7 @@ impl<'a> Parser<'a> {
 
     /// Assuming we have just parsed `.`, continue parsing into an expression.
     fn parse_dot_suffix(&mut self, self_arg: P<Expr>, lo: Span) -> PResult<'a, P<Expr>> {
-        if self.token.uninterpolated_span().at_least_rust_2018() && self.eat_keyword(exp!(Await)) {
+        if self.token_uninterpolated_span().at_least_rust_2018() && self.eat_keyword(exp!(Await)) {
             return Ok(self.mk_await_expr(self_arg, lo));
         }
 
@@ -1509,9 +1509,9 @@ impl<'a> Parser<'a> {
                 this.parse_expr_let(restrictions)
             } else if this.eat_keyword(exp!(Underscore)) {
                 Ok(this.mk_expr(this.prev_token.span, ExprKind::Underscore))
-            } else if this.token.uninterpolated_span().at_least_rust_2018() {
+            } else if this.token_uninterpolated_span().at_least_rust_2018() {
                 // `Span::at_least_rust_2018()` is somewhat expensive; don't get it repeatedly.
-                if this.token.uninterpolated_span().at_least_rust_2024()
+                if this.token_uninterpolated_span().at_least_rust_2024()
                     // check for `gen {}` and `gen move {}`
                     // or `async gen {}` and `async gen move {}`
                     && (this.is_gen_block(kw::Gen, 0)
@@ -2186,7 +2186,7 @@ impl<'a> Parser<'a> {
     fn parse_opt_meta_item_lit(&mut self) -> Option<MetaItemLit> {
         self.recover_after_dot();
         let span = self.token.span;
-        let uninterpolated_span = self.uninterpolated_token_span();
+        let uninterpolated_span = self.token_uninterpolated_span();
         self.eat_token_lit().map(|token_lit| {
             match MetaItemLit::from_token_lit(token_lit, span) {
                 Ok(lit) => lit,
@@ -2390,7 +2390,7 @@ impl<'a> Parser<'a> {
         let movability =
             if self.eat_keyword(exp!(Static)) { Movability::Static } else { Movability::Movable };
 
-        let coroutine_kind = if self.token.uninterpolated_span().at_least_rust_2018() {
+        let coroutine_kind = if self.token_uninterpolated_span().at_least_rust_2018() {
             self.parse_coroutine_kind(Case::Sensitive)
         } else {
             None
@@ -2939,7 +2939,7 @@ impl<'a> Parser<'a> {
     /// Parses `for await? <src_pat> in <src_expr> <src_loop_block>` (`for` token already eaten).
     fn parse_expr_for(&mut self, opt_label: Option<Label>, lo: Span) -> PResult<'a, P<Expr>> {
         let is_await =
-            self.token.uninterpolated_span().at_least_rust_2018() && self.eat_keyword(exp!(Await));
+            self.token_uninterpolated_span().at_least_rust_2018() && self.eat_keyword(exp!(Await));
 
         if is_await {
             self.psess.gated_spans.gate(sym::async_for_loop, self.prev_token.span);
@@ -3529,7 +3529,7 @@ impl<'a> Parser<'a> {
         self.token.is_keyword(kw::Try)
             && self
                 .look_ahead(1, |t| *t == token::OpenDelim(Delimiter::Brace) || t.is_whole_block())
-            && self.token.uninterpolated_span().at_least_rust_2018()
+            && self.token_uninterpolated_span().at_least_rust_2018()
     }
 
     /// Parses an `async move? {...}` or `gen move? {...}` expression.
diff --git a/compiler/rustc_parse/src/parser/item.rs b/compiler/rustc_parse/src/parser/item.rs
index 20833a0f70f..a8208e4717b 100644
--- a/compiler/rustc_parse/src/parser/item.rs
+++ b/compiler/rustc_parse/src/parser/item.rs
@@ -591,7 +591,7 @@ impl<'a> Parser<'a> {
         }
 
         // Parse stray `impl async Trait`
-        if (self.token.uninterpolated_span().at_least_rust_2018()
+        if (self.token_uninterpolated_span().at_least_rust_2018()
             && self.token.is_keyword(kw::Async))
             || self.is_kw_followed_by_ident(kw::Async)
         {
@@ -877,7 +877,7 @@ impl<'a> Parser<'a> {
             && self.look_ahead(1, |t| t.is_non_raw_ident_where(|i| i.name != kw::As))
         {
             self.bump(); // `default`
-            Defaultness::Default(self.prev_token.uninterpolated_span())
+            Defaultness::Default(self.prev_token_uninterpolated_span())
         } else {
             Defaultness::Final
         }
@@ -1208,7 +1208,7 @@ impl<'a> Parser<'a> {
         attrs: &mut AttrVec,
         mut safety: Safety,
     ) -> PResult<'a, ItemKind> {
-        let extern_span = self.prev_token.uninterpolated_span();
+        let extern_span = self.prev_token_uninterpolated_span();
         let abi = self.parse_abi(); // ABI?
         // FIXME: This recovery should be tested better.
         if safety == Safety::Default
@@ -2781,7 +2781,7 @@ impl<'a> Parser<'a> {
                             .expect("Span extracted directly from keyword should always work");
 
                         err.span_suggestion(
-                            self.token.uninterpolated_span(),
+                            self.token_uninterpolated_span(),
                             format!("`{original_kw}` already used earlier, remove this one"),
                             "",
                             Applicability::MachineApplicable,
@@ -2792,7 +2792,7 @@ impl<'a> Parser<'a> {
                     else if let Some(WrongKw::Misplaced(correct_pos_sp)) = wrong_kw {
                         let correct_pos_sp = correct_pos_sp.to(self.prev_token.span);
                         if let Ok(current_qual) = self.span_to_snippet(correct_pos_sp) {
-                            let misplaced_qual_sp = self.token.uninterpolated_span();
+                            let misplaced_qual_sp = self.token_uninterpolated_span();
                             let misplaced_qual = self.span_to_snippet(misplaced_qual_sp).unwrap();
 
                             err.span_suggestion(
diff --git a/compiler/rustc_parse/src/parser/mod.rs b/compiler/rustc_parse/src/parser/mod.rs
index a08fbde52a7..392a1c1057a 100644
--- a/compiler/rustc_parse/src/parser/mod.rs
+++ b/compiler/rustc_parse/src/parser/mod.rs
@@ -1313,14 +1313,14 @@ impl<'a> Parser<'a> {
 
     /// Parses asyncness: `async` or nothing.
     fn parse_coroutine_kind(&mut self, case: Case) -> Option<CoroutineKind> {
-        let span = self.token.uninterpolated_span();
+        let span = self.token_uninterpolated_span();
         if self.eat_keyword_case(exp!(Async), case) {
             // FIXME(gen_blocks): Do we want to unconditionally parse `gen` and then
             // error if edition <= 2024, like we do with async and edition <= 2018?
-            if self.token.uninterpolated_span().at_least_rust_2024()
+            if self.token_uninterpolated_span().at_least_rust_2024()
                 && self.eat_keyword_case(exp!(Gen), case)
             {
-                let gen_span = self.prev_token.uninterpolated_span();
+                let gen_span = self.prev_token_uninterpolated_span();
                 Some(CoroutineKind::AsyncGen {
                     span: span.to(gen_span),
                     closure_id: DUMMY_NODE_ID,
@@ -1333,7 +1333,7 @@ impl<'a> Parser<'a> {
                     return_impl_trait_id: DUMMY_NODE_ID,
                 })
             }
-        } else if self.token.uninterpolated_span().at_least_rust_2024()
+        } else if self.token_uninterpolated_span().at_least_rust_2024()
             && self.eat_keyword_case(exp!(Gen), case)
         {
             Some(CoroutineKind::Gen {
@@ -1349,9 +1349,9 @@ impl<'a> Parser<'a> {
     /// Parses fn unsafety: `unsafe`, `safe` or nothing.
     fn parse_safety(&mut self, case: Case) -> Safety {
         if self.eat_keyword_case(exp!(Unsafe), case) {
-            Safety::Unsafe(self.prev_token.uninterpolated_span())
+            Safety::Unsafe(self.prev_token_uninterpolated_span())
         } else if self.eat_keyword_case(exp!(Safe), case) {
-            Safety::Safe(self.prev_token.uninterpolated_span())
+            Safety::Safe(self.prev_token_uninterpolated_span())
         } else {
             Safety::Default
         }
@@ -1378,7 +1378,7 @@ impl<'a> Parser<'a> {
                 .look_ahead(1, |t| *t == token::OpenDelim(Delimiter::Brace) || t.is_whole_block())
             && self.eat_keyword_case(exp!(Const), case)
         {
-            Const::Yes(self.prev_token.uninterpolated_span())
+            Const::Yes(self.prev_token_uninterpolated_span())
         } else {
             Const::No
         }
@@ -1723,8 +1723,15 @@ impl<'a> Parser<'a> {
         self.num_bump_calls
     }
 
-    pub fn uninterpolated_token_span(&self) -> Span {
+    /// For interpolated `self.token`, returns a span of the fragment to which
+    /// the interpolated token refers. For all other tokens this is just a
+    /// regular span. It is particularly important to use this for identifiers
+    /// and lifetimes for which spans affect name resolution and edition
+    /// checks. Note that keywords are also identifiers, so they should use
+    /// this if they keep spans or perform edition checks.
+    pub fn token_uninterpolated_span(&self) -> Span {
         match &self.token.kind {
+            token::NtIdent(ident, _) | token::NtLifetime(ident, _) => ident.span,
             token::Interpolated(nt) => nt.use_span(),
             token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(_))) => {
                 self.look_ahead(1, |t| t.span)
@@ -1732,6 +1739,18 @@ impl<'a> Parser<'a> {
             _ => self.token.span,
         }
     }
+
+    /// Like `token_uninterpolated_span`, but works on `self.prev_token`.
+    pub fn prev_token_uninterpolated_span(&self) -> Span {
+        match &self.prev_token.kind {
+            token::NtIdent(ident, _) | token::NtLifetime(ident, _) => ident.span,
+            token::Interpolated(nt) => nt.use_span(),
+            token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(_))) => {
+                self.look_ahead(0, |t| t.span)
+            }
+            _ => self.prev_token.span,
+        }
+    }
 }
 
 pub(crate) fn make_unclosed_delims_error(
diff --git a/compiler/rustc_parse/src/parser/ty.rs b/compiler/rustc_parse/src/parser/ty.rs
index b45ebae079c..93705da22c4 100644
--- a/compiler/rustc_parse/src/parser/ty.rs
+++ b/compiler/rustc_parse/src/parser/ty.rs
@@ -775,7 +775,7 @@ impl<'a> Parser<'a> {
     /// Is a `dyn B0 + ... + Bn` type allowed here?
     fn is_explicit_dyn_type(&mut self) -> bool {
         self.check_keyword(exp!(Dyn))
-            && (self.token.uninterpolated_span().at_least_rust_2018()
+            && (self.token_uninterpolated_span().at_least_rust_2018()
                 || self.look_ahead(1, |t| {
                     (can_begin_dyn_bound_in_edition_2015(t) || *t == TokenKind::Star)
                         && !can_continue_type_after_non_fn_ident(t)
@@ -998,13 +998,13 @@ impl<'a> Parser<'a> {
             BoundConstness::Never
         };
 
-        let asyncness = if self.token.uninterpolated_span().at_least_rust_2018()
+        let asyncness = if self.token_uninterpolated_span().at_least_rust_2018()
             && self.eat_keyword(exp!(Async))
         {
             self.psess.gated_spans.gate(sym::async_trait_bounds, self.prev_token.span);
             BoundAsyncness::Async(self.prev_token.span)
         } else if self.may_recover()
-            && self.token.uninterpolated_span().is_rust_2015()
+            && self.token_uninterpolated_span().is_rust_2015()
             && self.is_kw_followed_by_ident(kw::Async)
         {
             self.bump(); // eat `async`
-- 
cgit 1.4.1-3-g733a5


From 592d113ff24b91290022d9f1aae271598ff235fa Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <n.nethercote@gmail.com>
Date: Mon, 31 Mar 2025 16:32:55 +1100
Subject: Fix problem causing `rusqlite` compilation to OOM.

This makes the expression re-parsing more like how it's originally done
in `parse_nonterminal`.
---
 compiler/rustc_parse/src/parser/expr.rs    |   4 +-
 tests/ui/attributes/dont-dup-expr-attrs.rs | 133 +++++++++++++++++++++++++++++
 2 files changed, 136 insertions(+), 1 deletion(-)
 create mode 100644 tests/ui/attributes/dont-dup-expr-attrs.rs

(limited to 'compiler/rustc_parse/src')

diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs
index 1c2df3ffcd1..e1e6b93abf3 100644
--- a/compiler/rustc_parse/src/parser/expr.rs
+++ b/compiler/rustc_parse/src/parser/expr.rs
@@ -1397,7 +1397,9 @@ impl<'a> Parser<'a> {
         } else if let Some(expr) = self.eat_metavar_seq_with_matcher(
             |mv_kind| matches!(mv_kind, MetaVarKind::Expr { .. }),
             |this| {
-                let expr = this.parse_expr();
+                // Force collection (as opposed to just `parse_expr`) is required to avoid the
+                // attribute duplication seen in #138478.
+                let expr = this.parse_expr_force_collect();
                 // FIXME(nnethercote) Sometimes with expressions we get a trailing comma, possibly
                 // related to the FIXME in `collect_tokens_for_expr`. Examples are the multi-line
                 // `assert_eq!` calls involving arguments annotated with `#[rustfmt::skip]` in
diff --git a/tests/ui/attributes/dont-dup-expr-attrs.rs b/tests/ui/attributes/dont-dup-expr-attrs.rs
new file mode 100644
index 00000000000..21da6820a53
--- /dev/null
+++ b/tests/ui/attributes/dont-dup-expr-attrs.rs
@@ -0,0 +1,133 @@
+//@ check-pass
+//
+// During development of #124141 at one point expression on attributes were
+// being duplicated and `m1` caused an exponential blowup that caused OOM.
+// The number of recursive calls depends on the number of doc comments on the
+// expr block. On each recursive call, the `#[allow(deprecated)]` attribute(s) on
+// the `0` somehow get duplicated, resulting in 1, 2, 4, 8, ... identical
+// attributes.
+//
+// After the fix, the code compiles quickly and normally.
+
+macro_rules! m1 {
+    ($(#[$meta:meta])* { $e:expr }) => {
+        m1! { expr: { $e }, unprocessed: [$(#[$meta])*] }
+    };
+
+    (expr: { $e:expr }, unprocessed: [ #[$meta:meta] $($metas:tt)* ]) => {
+        m1! { expr: { $e }, unprocessed: [ $($metas)* ] }
+    };
+
+    (expr: { $e:expr }, unprocessed: []) => {
+        { $e }
+    }
+}
+
+macro_rules! m2 {
+    ($(#[$meta:meta])* { $e:stmt }) => {
+        m2! { stmt: { $e }, unprocessed: [$(#[$meta])*] }
+    };
+
+    (stmt: { $e:stmt }, unprocessed: [ #[$meta:meta] $($metas:tt)* ]) => {
+        m2! { stmt: { $e }, unprocessed: [ $($metas)* ] }
+    };
+
+    (stmt: { $e:stmt }, unprocessed: []) => {
+        { $e }
+    }
+}
+
+macro_rules! m3 {
+    ($(#[$meta:meta])* { $e:item }) => {
+        m3! { item: { $e }, unprocessed: [$(#[$meta])*] }
+    };
+
+    (item: { $e:item }, unprocessed: [ #[$meta:meta] $($metas:tt)* ]) => {
+        m3! { item: { $e }, unprocessed: [ $($metas)* ] }
+    };
+
+    (item: { $e:item }, unprocessed: []) => {
+        { $e }
+    }
+}
+
+fn main() {
+    // Each additional doc comment line doubles the compile time.
+    m1!(
+        /// a1
+        /// a2
+        /// a3
+        /// a4
+        /// a5
+        /// a6
+        /// a7
+        /// a8
+        /// a9
+        /// a10
+        /// a11
+        /// a12
+        /// a13
+        /// a14
+        /// a15
+        /// a16
+        /// a17
+        /// a18
+        /// a19
+        /// a20
+        {
+            #[allow(deprecated)] 0
+        }
+    );
+
+    m2!(
+        /// a1
+        /// a2
+        /// a3
+        /// a4
+        /// a5
+        /// a6
+        /// a7
+        /// a8
+        /// a9
+        /// a10
+        /// a11
+        /// a12
+        /// a13
+        /// a14
+        /// a15
+        /// a16
+        /// a17
+        /// a18
+        /// a19
+        /// a20
+        {
+            #[allow(deprecated)] let x = 5
+        }
+    );
+
+    m3!(
+        /// a1
+        /// a2
+        /// a3
+        /// a4
+        /// a5
+        /// a6
+        /// a7
+        /// a8
+        /// a9
+        /// a10
+        /// a11
+        /// a12
+        /// a13
+        /// a14
+        /// a15
+        /// a16
+        /// a17
+        /// a18
+        /// a19
+        /// a20
+        {
+            #[allow(deprecated)] struct S;
+        }
+    );
+}
-- 
cgit 1.4.1-3-g733a5